Fixes in "new" mmap

* mem.c (sys_mmap): Ensure unsigned expansion of tcp->u_arg[5].
Add page shift of offset for I386.
Use tcp->ext_arg[5] as offset for X32.
(sys_old_mmap): [X32] Remove this function, X32 doesn't use is.

Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>

mem.c

@@ -280,13 +280,8 @@ int sys_old_mmap(struct tcb *tcp)
 int
 sys_mmap(struct tcb *tcp)
 {
-	long long offset = tcp->u_arg[5];
+	unsigned long long offset = (unsigned long) tcp->u_arg[5];
 
-	/* FIXME: why only SH64? i386 mmap2 syscall ends up
-	 * in this function, but does not convert offset
-	 * from pages to bytes. See test/mmap_offset_decode.c
-	 * Why SH64 and i386 are handled differently?
-	 */
 #if defined(SH64)
 	/*
 	 * Old mmap differs from new mmap in specifying the
@@ -295,8 +290,12 @@ sys_mmap(struct tcb *tcp)
 	 * sees bytes in the printout.
 	 */
 	offset <<= PAGE_SHIFT;
-#endif
+#elif defined(I386)
-#if defined(LINUX_MIPSN32)
+	/* Try test/mmap_offset_decode.c */
+	offset <<= 12; /* 4096 byte pages */
+#elif defined(LINUX_MIPSN32) || defined(X32)
+	/* Try test/x32_mmap.c */
+	/* At least for X32 it definitely should not be page-shifted! */
 	offset = tcp->ext_arg[5];
 #endif
 	return print_mmap(tcp, tcp->u_arg, offset);
@@ -304,40 +303,6 @@ sys_mmap(struct tcb *tcp)
 #endif /* !HAVE_LONG_LONG_OFF_T */
 
 #if _LFS64_LARGEFILE || HAVE_LONG_LONG_OFF_T
-# if defined(X32)
-int sys_old_mmap(struct tcb *tcp)
-{
-	long u_arg[6];
-	if (umoven(tcp, tcp->u_arg[0], sizeof(u_arg), (char *) u_arg) == -1)
-		return 0;
-	if (entering(tcp)) {
-		/* addr */
-		if (!u_arg[0])
-			tprints("NULL, ");
-		else
-			tprintf("%#lx, ", u_arg[0]);
-		/* len */
-		tprintf("%lu, ", u_arg[1]);
-		/* prot */
-		printflags(mmap_prot, u_arg[2], "PROT_???");
-		tprints(", ");
-		/* flags */
-#  ifdef MAP_TYPE
-		printxval(mmap_flags, u_arg[3] & MAP_TYPE, "MAP_???");
-		addflags(mmap_flags, u_arg[3] & ~MAP_TYPE);
-#  else
-		printflags(mmap_flags, u_arg[3], "MAP_???");
-#  endif
-		/* fd */
-		tprints(", ");
-		printfd(tcp, u_arg[4]);
-		/* offset */
-		tprintf(", %#lx", u_arg[5]);
-	}
-	return RVAL_HEX;
-}
-# endif
-
 /* TODO: comment which arches use this routine.
  * For one, does ALPHA on Linux use this??
  * From code it seems that it might use 7 or 8 registers,

test/x32_mmap.c

@@ -0,0 +1,49 @@
+// Test program which explores whether mmap's ofs parameter
+// is 64-bit, and whether it needs to be shifted << PAGE_SHIFT.
+// Apparently it is 64-bit and isn't shifted.
+//
+// Build: x86_64-gcc -static -Wall -ox32_mmap x32_mmap.c
+// Typical output:
+// 7f9390696000-7f93906a6000 r--s 12345670000 08:06 2224545 /etc/passwd
+//                                ^^^^^^^^^^^
+#define _GNU_SOURCE
+#include <sys/types.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/syscall.h>
+// Ensure we are compiling to 64 bits
+struct bug { int t[sizeof(long) > 4 ? 1 : -1]; };
+int main(int argc, char **argv)
+{
+	long ofs = 0x12345670000; // fails if not page-aligned
+	errno = 0;
+	close(0);
+	if (open("/etc/passwd", O_RDONLY))
+		return 1;
+	long r = syscall(
+		(long) (__NR_mmap | 0x40000000), // make x32 call
+		(long) (0),		// start
+		(long) (0x10000),	// len
+		(long) (PROT_READ),	// prot
+		(long) (MAP_SHARED),	// flags
+		(long) (0),		// fd
+		(long) (ofs)		// ofs
+	);
+	printf("ret:0x%lx errno:%m\n", r);
+
+	char buf[16*1024];
+	sprintf(buf, "/proc/%d/maps", getpid());
+	int fd = open(buf, O_RDONLY);
+	if (fd > 0) {
+		int sz = read(fd, buf, sizeof(buf));
+		if (sz > 0)
+			write(1, buf, sz);
+	}
+
+	return 0;
+}