Regression test portability: batch of bugs (volume 2)

Fix various regression test portability in tests/bugs.

bug-861542.t
- Avoid syntax specific to GNU sed.

bug-860663.t
- Command argument length is system dependent, and specifying 1000 file
  path may overflow it. Use a C program to do the job in a portable and
  efficient way.
- Add a test that we created the specified amount of files.

bug-858242.c, bug-808400-fcntl.c, bug-808400-flock.c
- fstat64() is Linux-specific. Define it as fstat for other systems.

bug-823081.t
- Use portable tail -n instead of tail --lines

In many tests:
- Do not assume python interpreter name. Use $PYTHON as defined
  in env.rc by configure.

utils/libcxattr.py
- If python version is 2.6 or higher, use a portable mechanism to
  recover errno. The original version is retained for python version
  2.5 and earlier but it only works on Linux.

BUG: 1129939
Change-Id: If2fea1ffec5cc6ab2de426fb200e884450afe61b
Signed-off-by: Emmanuel Dreyfus <manu@netbsd.org>
Reviewed-on: http://review.gluster.org/9097
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

tests/bugs/bug-1111490.t

@@ -16,7 +16,8 @@ TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 --aux-gfid-mount
 
 # create file with specific gfid
 uuid=`uuidgen`
-EXPECT "File creation OK" gfid-access.py $M0 ROOT file0 $uuid file
+EXPECT "File creation OK" $PYTHON $(dirname $0)/../utils/gfid-access.py \
+                                  $M0 ROOT file0 $uuid file
 
 # check gfid
 EXPECT "$uuid" getfattr --only-values -n glusterfs.gfid.string $M0/file0
@@ -27,6 +28,7 @@ TEST umount $M0
 TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0 --aux-gfid-mount
 
 # touch the file again (gfid-access.py handles errno)
-EXPECT "File creation OK" gfid-access.py $M0 ROOT file0 $uuid file
+EXPECT "File creation OK" $PYTHON  $(dirname $0)/../utils/gfid-access.py \
+                                   $M0 ROOT file0 $uuid file
 
 cleanup;

tests/bugs/bug-808400-fcntl.c

@@ -7,6 +7,10 @@
 #include <fcntl.h>
 #include <sys/wait.h>
 
+#ifndef linux
+#define fstat64(fd, st) fstat(fd, st)
+#endif
+
 int
 run_child (char *filename)
 {

tests/bugs/bug-808400-flock.c

@@ -7,6 +7,10 @@
 #include <fcntl.h>
 #include <sys/wait.h>
 
+#ifndef linux
+#define fstat64(fd, st) fstat(fd, st)
+#endif
+
 int
 run_child (char *filename)
 {

tests/bugs/bug-823081.t

@@ -22,20 +22,20 @@ function set_tail ()
 set_tail $V0;
 
 TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
-tail=`tail --lines=1 $logdir/$cmd_log_history | cut -d " " -f 5-`
+tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 5-`
 TEST [[ \"$tail\" == \"$tail_success\" ]]
 
 TEST ! $CLI volume create $V0 $H0:$B0/${V0}{1,2};
-tail=`tail --lines=1 $logdir/$cmd_log_history | cut -d " " -f 5-`
+tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 5-`
 TEST [[ \"$tail\" == \"$tail_failure\" ]]
 
 set_tail $V1;
 TEST gluster volume create $V1 $H0:$B0/${V1}{1,2} force;
-tail=`tail --lines=1 $logdir/$cmd_log_history | cut -d " " -f 5-`
+tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 5-`
 TEST [[ \"$tail\" == \"$tail_success_force\" ]]
 
 TEST ! gluster volume create $V1 $H0:$B0/${V1}{1,2} force;
-tail=`tail --lines=1 $logdir/$cmd_log_history | cut -d " " -f 5-`
+tail=`tail -n 1 $logdir/$cmd_log_history | cut -d " " -f 5-`
 TEST [[ \"$tail\" == \"$tail_failure_force\" ]]
 
 cleanup;

tests/bugs/bug-853258.t

@@ -38,7 +38,7 @@ for i in $(seq 0 3); do
 	xattrs="$xattrs $(dht_get_layout $B0/${V0}$i)"
 done
 
-overlap=$(python2 $(dirname $0)/overlap.py $xattrs)
+overlap=$( $PYTHON $(dirname $0)/overlap.py $xattrs)
 # 2863311531 = 0xaaaaaaab = 2/3 overlap
 TEST [ "$overlap" -ge 2863311531 ]
 

tests/bugs/bug-857330/normal.t

@@ -14,7 +14,8 @@ TEST $CLI volume start $V0;
 
 TEST glusterfs -s $H0 --volfile-id=$V0 $M0;
 
-TEST create-files.py --multi -b 10 -d 10  -n 10 $M0;
+TEST $PYTHON $(dirname $0)/../../utils/create-files.py \
+             --multi -b 10 -d 10  -n 10 $M0;
 
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 

tests/bugs/bug-857330/xml.t

@@ -15,7 +15,8 @@ TEST $CLI volume start $V0;
 
 TEST glusterfs -s $H0 --volfile-id=$V0 $M0;
 
-TEST create-files.py --multi -b 10 -d 10  -n 10 $M0;
+TEST $PYTHON $(dirname $0)/../../utils/create-files.py \
+             --multi -b 10 -d 10  -n 10 $M0;
 
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 

tests/bugs/bug-858242.c

@@ -7,6 +7,10 @@
 #include <stdlib.h>
 #include <unistd.h>
 
+#ifndef linux
+#define fstat64(fd, st) fstat(fd, st)
+#endif
+
 int
 main (int argc, char *argv[])
 {

tests/bugs/bug-860663.c

@@ -0,0 +1,42 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <string.h>
+#include <err.h>
+#include <sys/param.h>
+
+int
+main(argc, argv)
+        int argc;
+        char **argv;
+{
+        char *basepath;
+        char path[MAXPATHLEN + 1];
+        unsigned int count;
+        int i, fd;
+
+        if (argc != 3)
+                errx(1, "usage: %s path count", argv[0]);
+
+        basepath = argv[1];
+        count = atoi(argv[2]);
+
+        if (count > 999999)
+                errx(1, "count too big");
+
+        if (strlen(basepath) > MAXPATHLEN - 6)
+                errx(1, "path too long");
+
+        for (i = 0; i < count; i++) {
+                (void)sprintf(path, "%s%06d", basepath, i);
+
+                if ((fd = open(path, O_CREAT|O_RDWR, 0644)) == -1)
+                        err(1, "create %s failed", path);
+
+                if (close(fd) != 0)
+                        warn("close %s failed", path);
+        }
+
+        return 0;
+}

tests/bugs/bug-860663.t

@@ -17,6 +17,8 @@ function file_count()
 
 BRICK_COUNT=3
 
+build_tester $(dirname $0)/bug-860663.c
+
 TEST glusterd
 TEST pidof glusterd
 
@@ -26,9 +28,10 @@ TEST $CLI volume start $V0
 ## Mount FUSE
 TEST glusterfs -s $H0 --volfile-id $V0 $M0;
 
-TEST touch $M0/files{1..10000};
+TEST $(dirname $0)/bug-860663 $M0/files 10000
 
 ORIG_FILE_COUNT=`ls -l $M0 | wc -l`;
+TEST [ $ORIG_FILE_COUNT -ge 10000 ]
 
 # Kill a brick process
 kill -9 `cat $GLUSTERD_WORKDIR/vols/$V0/run/$H0-d-backends-${V0}1.pid`;
@@ -37,7 +40,7 @@ TEST $CLI volume rebalance $V0 fix-layout start
 
 sleep 30;
 
-TEST ! touch $M0/files{1..10000};
+TEST ! $(dirname $0)/bug-860663 $M0/files 10000
 
 TEST $CLI volume start $V0 force
 
@@ -47,4 +50,5 @@ NEW_FILE_COUNT=`ls -l $M0 | wc -l`;
 
 EXPECT "0" file_count $ORIG_FILE_COUNT $NEW_FILE_COUNT
 
+rm -f $(dirname $0)/bug-860663
 cleanup;

tests/bugs/bug-861542.t

@@ -30,8 +30,7 @@ function xml_port_field()
     local opt=$2;
         $CLI --xml volume status $vol $opt | tr -d '\n' |\
 #Find the first occurrence of the string between <port> and </port>
-        sed -r 's/<port>/&\n/;s/<\/port>/\n&/;s/^.*\n(.*)\n.*$/\1/'| \
-        grep -v xml | tr -d '\n';
+        sed -rn 's/<port>/&###/;s/<\/port>/###&/;s/^.*###(.*)###.*$/\1/p'
 }
 
 TEST $CLI volume status $V0;

tests/bugs/bug-902610.t

@@ -27,7 +27,7 @@ function get_layout()
 	fi
 
 	# Figure out where the join point is.
-	target=$(python -c "print '%08x' % (0x$layout1_e + 1)")
+	target=$( $PYTHON -c "print '%08x' % (0x$layout1_e + 1)")
 	#echo "target for layout2 = $target" > /dev/tty
 
 	# The second layout should cover everything that the first doesn't.

tests/env.rc.in

@@ -12,3 +12,9 @@ export GLUSTERD_WORKDIR
 
 PYTHONPATH=@BUILD_PYTHON_SITE_PACKAGES@:$PYTHON_PATH
 export PYTHONPATH
+
+PYTHON=@PYTHON@
+export PYTHON
+
+PYTHONPATH=@BUILD_PYTHON_SITE_PACKAGES@:$PYTHON_PATH
+export PYTHONPATH

tests/include.rc

@@ -301,7 +301,7 @@ which killall > /dev/null || {
 
 which pidof > /dev/null || {
   pidof() {
-    pidof.py $@
+    $PYTHON pidof.py $@
   }
 }
 
@@ -612,13 +612,13 @@ which md5sum > /dev/null || {
 
 which setfattr > /dev/null || {
   setfattr() {
-    setfattr.py $@
+    $PYTHON setfattr.py $@
   }
 }
 
 which getfattr > /dev/null || {
   getfattr() {
-    getfattr.py $@
+    $PYTHON getfattr.py $@
   }
 }
 
@@ -659,6 +659,26 @@ useradd --help 2>/dev/null | grep -q -- '--no-create-home' || {
   }
 }
 
+userdel --help 2>/dev/null | grep -q -- '--force' || {
+  userdel() {
+    if [ "x$1" = "x--force" ]; then
+      user=$2
+    else
+      user=$1
+    fi
+    eval "$( which userdel ) $user"
+  }
+}
+
+useradd --help 2>/dev/null | grep -q -- '--no-create-home' || {
+  useradd() {
+    # Just remove -M (do not create home) which is the default
+    # other options are identical
+    args=`echo $*|sed 's/-M//'`
+    eval "$( which useradd ) $args"
+  }
+}
+
 alias EXPECT='_EXPECT $LINENO'
 alias EXPECT_NOT='_EXPECT_NOT $LINENO'
 alias TEST='_TEST $LINENO'
@@ -827,6 +847,156 @@ function UMOUNT_LOOP ()
   esac
 }
 
+function SETUP_LOOP ()
+{
+  if [ $# != 1 ] ; then
+    echo "SETUP_LOOP usage" >&2
+    exit 1;
+  fi
+
+  backend=$1
+
+  case ${OSTYPE} in
+  Linux)
+    losetup --find --show ${backend}
+    ;;
+  NetBSD)
+    vnd=`vnconfig -l|awk -F: '/not in use/{print $1; exit}'`
+    if [ "x${vnd}" = "x" ] ; then
+      echo "no more vnd" >&2
+      exit 1;
+    fi
+    vnconfig ${vnd} ${backend}
+    echo ${vnd}
+    ;;
+  *)
+    echo "Please define SETUP_LOOP for ${OSTYPE} in include.rc" >&2
+    exit 1;
+    ;;
+  esac
+}
+
+function MKFS_LOOP ()
+{
+  args=`getopt i: $*`
+  if [ $? -ne 0 ] ; then
+    echo "MKFS_LOOP usage" >&2
+    exit 1;
+  fi
+  set -- ${args}
+
+  isize=""
+  while test $# -gt 0; do
+    case "$1" in
+    -i)         isize=$2; shift ;;
+    --)         shift; break ;;
+    esac
+    shift
+  done
+
+  dev=$1
+
+  case ${OSTYPE} in
+  Linux)
+    test "x${isize}" != "x" && isize="-i size=${isize}"
+    mkfs.xfs  -f ${isize} ${dev}
+    ;;
+  NetBSD)
+    test "x${isize}" != "x" && isize="-i ${isize}"
+
+    echo ${dev} | grep -q '^vnd'
+    if [ $? -ne 0 ] ; then
+      vnd=`vnconfig -l|awk -F: '/not in use/{print $1; exit}'`
+      if [ "x${vnd}" = "x" ] ; then
+        echo "no more vnd" >&2
+        exit 1;
+      fi
+      vnconfig ${vnd} ${dev}
+    else
+      vnd=${dev}
+    fi
+    newfs ${isize} /dev/r${vnd}a
+    ;;
+  *)
+    echo "Please define MKFS_LOOP for ${OSTYPE} in include.rc" >&2
+    exit 1;
+    ;;
+  esac
+}
+
+function MOUNT_LOOP ()
+{
+  if [ $# != 2 ] ; then
+    echo "MOUNT_LOOP usage" >&2
+    exit 1;
+  fi
+
+  dev=$1
+  target=$2
+
+  case ${OSTYPE} in
+  Linux)
+    echo ${dev} | grep -q '^/dev/loop'
+    if [ $? -eq 0 ] ; then
+      mount -t xfs  ${dev} ${target}
+    else
+      mount -o loop  ${dev} ${target}
+    fi
+    ;;
+  NetBSD)
+    echo ${dev} | grep -q '^vnd'
+    if [ $? -ne 0 ] ; then
+      ino=`/usr/bin/stat -f %i ${dev}`
+      dev=`vnconfig -l | awk -v ino=${ino} -F'[: ]*' '($5 == ino) {print $1}'`
+    fi
+
+    mount /dev/${dev}a ${target} >&2
+    if [ $? -ne 0 ] ; then
+      echo "failed to mount  /dev/${dev}a on  ${target}" >&2
+      exit 1
+    fi
+
+    mkdir -p ${target}/.attribute/system  ${target}/.attribute/user
+    mount -u -o extattr ${target} >&2
+
+    ;;
+  *)
+    echo "Please define MOUNT_LOOP for ${OSTYPE} in include.rc" >&2
+    exit 1;
+    ;;
+  esac
+}
+
+function UMOUNT_LOOP ()
+{
+  case ${OSTYPE} in
+  Linux)
+    force_umount $*
+    ;;
+  NetBSD)
+    for target in $* ; do
+      dev=`mount | awk -v target=${target} '($3 == target) {print $1}'`
+      force_umount ${target}
+      echo ${dev} | grep -q '^/dev/vnd'
+      if [ $? -eq 0 ] ; then
+        dev=`echo ${dev} | sed 's|^/dev/||; s|a$||'`
+        vnconfig -u ${dev}
+      else
+        ino=`/usr/bin/stat -f %i ${dev}`
+        dev=`vnconfig -l | awk -v ino=${ino} -F'[: ]*' '($5 == ino) {print $1}'`
+        if [ "x${dev}" != "x" ] ; then
+          vnconfig -u ${dev}
+        fi
+      fi
+    done
+    ;;
+  *)
+    echo "Please define UMOUNT_LOOP for ${OSTYPE} in include.rc" >&2
+    exit 1;
+    ;;
+  esac
+}
+
 function STAT()
 {
         stat $1

tests/utils/libcxattr.py

@@ -9,6 +9,7 @@
 #
 
 import os
+import sys
 from ctypes import CDLL, c_int, create_string_buffer
 from ctypes.util import find_library
 
@@ -25,10 +26,18 @@ class Xattr(object):
          sizes we expect
     """
 
-    libc = CDLL(find_library("libc"))
+    if sys.hexversion >= 0x02060000:
+        from ctypes import DEFAULT_MODE
+        libc = CDLL(find_library("libc"), DEFAULT_MODE, None, True)
+    else:
+        libc = CDLL(find_library("libc"))
 
     @classmethod
     def geterrno(cls):
+        if sys.hexversion >= 0x02060000:
+            from ctypes import get_errno
+            return get_errno()
+        # breaks on NetBSD
         return c_int.in_dll(cls.libc, 'errno').value
 
     @classmethod