save stderr to file in case it is useful

do better recording of when stderr has content in it
the failed/passed count was not kept in the right place
and the failure-outputs need to be dumped from the right place
record status code better to file, and if it exists, compare the stderr as well
sanitizing the first line for filename path

diff --git a/tests/.gitignore b/tests/.gitignore
index 98d697f7c97dbfdd59e61439b2205e9713af6e2f..543951f468b54c48219045e00b23f0fbfd075004 100644 (file)
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -2,3 +2,4 @@
 .passed
 DIFF
 NEW
+*.out.raw.stderr

diff --git a/tests/TESTonce b/tests/TESTonce
index 4f4acd9d06342b58394868601dd7dc65153e17a8..d60c01ff8dda90868728603ef11b57d22885face 100755 (executable)
--- a/tests/TESTonce
+++ b/tests/TESTonce
@@ -17,6 +17,13 @@ $options=$ARGV[3];
 my $r;
 
 $outputbase = basename($output);
+my $coredump = false;
+my $status = 0;
+my $linecount = 0;
+my $rawstderrlog = "NEW/${outputbase}.raw.stderr";
+my $stderrlog = "NEW/${outputbase}.stderr";
+my $diffstat = 0;
+my $errdiffstat = 0;
 
 if ($^O eq 'MSWin32') {
     $r = system "..\\windump -n -t -r $input $options 2>NUL | sed 's/\\r//' | tee NEW/$outputbase | diff $output - >DIFF/$outputbase.diff";
@@ -25,28 +32,84 @@ if ($^O eq 'MSWin32') {
 else {
     # we used to do this as a nice pipeline, but the problem is that $r fails to
     # to be set properly if the tcpdump core dumps.
-    $r = system "../tcpdump 2>/dev/null -n -t -r $input $options >NEW/$outputbase";
+    $r = system "$TCPDUMP 2>${rawstderrlog} -# -n -r $input $options >NEW/${outputbase}";
+    if($r == -1) {
+        # failed to start due to error.
+        $status = $!;
+    }
     if($r != 0) {
-        my $coredump = false;
-        my $status = 0;
+        $coredump = false;
+        $status = 0;
         # this means tcpdump failed.
         open(OUTPUT, ">>"."NEW/$outputbase") || die "fail to open $outputbase\n";
-        printf OUTPUT "EXIT CODE %08x\n", $r;
+        if( $r & 128 ) {
+            $coredump = $r & 127;
+        }
+        if( WIFEXITED($r)) {
+            $status = WEXITSTATUS($r);
+        }
+
+        if($coredump || $status) {
+            printf OUTPUT "EXIT CODE %08x: dump:%d code: %d\n", $r, $coredump, $status;
+        } else {
+            printf OUTPUT "EXIT CODE %08x\n", $r;
+        }
         close(OUTPUT);
         $r = 0;
     }
     if($r == 0) {
         $r = system "cat NEW/$outputbase | diff $output - >DIFF/$outputbase.diff";
+        $diffstat = WEXITSTATUS($r);
+    }
+
+    # process the file, sanitize "reading from" line, and count lines
+    $linecount = 0;
+    open(ERRORRAW, "<" . $rawstderrlog);
+    open(ERROROUT, ">" . $stderrlog);
+    while(<ERRORRAW>) {
+        next if /^$/;  # blank lines are boring
+        if(/^(reading from file )(.*)(,.*)$/) {
+            my $filename = basename($2);
+            print ERROROUT "${1}${filename}${3}\n";
+            next;
+        }
+        print ERROROUT;
+        $linecount++;
+    }
+    close(ERROROUT);
+    close(ERRORRAW);
+
+    if ( -f "$output.stderr" ) {
+        $nr = system "cat $stderrlog | diff $output.stderr - >DIFF/$outputbase.stderr.diff";
+        if($r == 0) {
+            $r = $nr;
+        }
+        $errdiffstat = WEXITSTATUS($nr);
     }
+
+    if($r == 0) {
+        if($linecount == 0 && $status == 0) {
+            unlink($stderrlog);
+        } else {
+            $errdiffstat = 1;
+        }
+    }
+
     #print sprintf("END: %08x\n", $r);
 }
 
 if($r == 0) {
-  printf "    %-35s: passed\n", $name;
-  unlink "DIFF/$outputbase.diff";
-  exit 0;
+    my $stderrlog="";
+    if($linecount > 0) {
+        $stderrlog=sprintf("-- %d lines extra in stderr", $linecount);
+    }
+    printf "    %-35s: passed%s\n", $name, $stderrlog;
+    unlink "DIFF/$outputbase.diff";
+    exit 0;
 }
-printf "    %-35s: TEST FAILED(%s)", $name, $r == -1 ? $! : "exit $?";
+
+# must have failed!
+printf "    %-35s: TEST FAILED(exit core=%d/diffstat=%d,%d/r=%d)", $name, $coredump, $diffstat, $errdiffstat, $r;
 open FOUT, '>>failure-outputs.txt';
 printf FOUT "\nFailed test: $name\n\n";
 close FOUT;

diff --git a/tests/TESTrun.sh b/tests/TESTrun.sh
index 3cbe4d7fc66a3a3a065c91bd5f0d4bbd795b0d29..3301830bc350c8a54395e8856f3db29db9868fa7 100755 (executable)
--- a/tests/TESTrun.sh
+++ b/tests/TESTrun.sh
@@ -4,9 +4,25 @@ srcdir=${SRCDIR-..}
 
 echo RUNNING from ${srcdir}
 
+<<<<<<< HEAD
 mkdir -p NEW
 mkdir -p DIFF
-cat /dev/null > failure-outputs.txt
+
+# make it absolute
+srcdir=$(cd $srcdir && pwd)
+
+# this should be run from the compiled build directory,
+# with srcdir= set to wherever the source code is.
+# not from the tests directory.
+echo RUNNING from ${srcdir}
+
+passedfile=$(pwd)/tests/.passed
+failedfile=$(pwd)/tests/.failed
+failureoutput=$(pwd)/tests/failure-outputs.txt
+mkdir -p tests/NEW
+mkdir -p tests/DIFF
+
+cat /dev/null > ${failureoutput}
 
 runComplexTests()
 {
@@ -15,8 +31,8 @@ runComplexTests()
     case $i in ${srcdir}/tests/TEST*.sh) continue;; esac
     sh $i ${srcdir}
   done
-  passed=`cat .passed`
-  failed=`cat .failed`
+  passed=`cat ${passedfile}`
+  failed=`cat ${failedfile}`
 }
 
 runSimpleTests()
@@ -31,28 +47,34 @@ runSimpleTests()
     rm -f core
     [ "$only" != "" -a "$name" != "$only" ] && continue
     export SRCDIR=${srcdir}
+    # I hate shells with their stupid, useless subshells.
+    passed=`cat ${passedfile}`
+    failed=`cat ${failedfile}`
+    (cd tests  # run TESTonce in tests directory
+
     if ${srcdir}/tests/TESTonce $name ${srcdir}/tests/$input ${srcdir}/tests/$output "$options"
     then
       passed=`expr $passed + 1`
-      echo $passed >.passed
+      echo $passed >${passedfile}
     else
       failed=`expr $failed + 1`
-      echo $failed >.failed
+      echo $failed >${failedfile}
     fi
     if [ -d COREFILES ]; then
         if [ -f core ]; then mv core COREFILES/$name.core; fi
-    fi
+    fi)
+
     [ "$only" != "" -a "$name" = "$only" ] && break
   done
   # I hate shells with their stupid, useless subshells.
-  passed=`cat .passed`
-  failed=`cat .failed`
+  passed=`cat ${passedfile}`
+  failed=`cat ${failedfile}`
 }
 
 passed=0
 failed=0
-echo $passed >.passed
-echo $failed >.failed
+echo $passed >${passedfile}
+echo $failed >${failedfile}
 if [ $# -eq 0 ]
 then
   runComplexTests
@@ -70,7 +92,7 @@ echo '------------------------------------------------'
 printf "%4u tests failed\n" $failed
 printf "%4u tests passed\n" $passed
 echo
-cat failure-outputs.txt
+cat ${failureoutput}
 echo
 echo
 exit $failed