record status code better to file, and if it exists, compare the stderr as well

sanitizing the first line for filename path

diff --git a/tests/.gitignore b/tests/.gitignore
index 98d697f7c97dbfdd59e61439b2205e9713af6e2f..543951f468b54c48219045e00b23f0fbfd075004 100644 (file)
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -2,3 +2,4 @@
 .passed
 DIFF
 NEW
+*.out.raw.stderr

diff --git a/tests/TESTonce b/tests/TESTonce
index 7bd09b9e2b66ee6037435271e1f16264bfebac2a..d499c07f8b68d253b5e8122f01fe588f09047688 100755 (executable)
--- a/tests/TESTonce
+++ b/tests/TESTonce
@@ -23,8 +23,10 @@ $outputbase = basename($output);
 my $coredump = false;
 my $status = 0;
 my $linecount = 0;
+my $rawstderrlog = "NEW/${outputbase}.raw.stderr";
 my $stderrlog = "NEW/${outputbase}.stderr";
 my $diffstat = 0;
+my $errdiffstat = 0;
 
 if ($^O eq 'MSWin32') {
     $r = system "..\\windump -# -n -r $input $options 2>NUL | sed 's/\\r//' | tee NEW/$outputbase | diff $output - >DIFF/$outputbase.diff";
@@ -33,7 +35,7 @@ if ($^O eq 'MSWin32') {
 else {
     # we used to do this as a nice pipeline, but the problem is that $r fails to
     # to be set properly if the tcpdump core dumps.
-    $r = system "$TCPDUMP 2>${stderrlog} -# -n -r $input $options >NEW/${outputbase}";
+    $r = system "$TCPDUMP 2>${rawstderrlog} -# -n -r $input $options >NEW/${outputbase}";
     if($r == -1) {
         # failed to start due to error.
         $status = $!;
@@ -50,7 +52,11 @@ else {
             $status = WEXITSTATUS($r);
         }
 
-        printf OUTPUT "EXIT CODE %08x: dump:%d code: %d\n", $r, $coredump, $status;
+        if($coredump || $status) {
+            printf OUTPUT "EXIT CODE %08x: dump:%d code: %d\n", $r, $coredump, $status;
+        } else {
+            printf OUTPUT "EXIT CODE %08x\n", $r;
+        }
         close(OUTPUT);
         $r = 0;
     }
@@ -58,16 +64,37 @@ else {
         $r = system "cat NEW/$outputbase | diff $output - >DIFF/$outputbase.diff";
         $diffstat = WEXITSTATUS($r);
     }
-    if($r == 0) {
-        $linecount = 0;
-        open(ERROROUT, "<" . $stderrlog);
-        while(<ERROROUT>) {
-            next if /^$/;  # blank lines are boring
-            next if /^reading from file/;  # stock output boring too
-            $linecount++;
+
+    # process the file, sanitize "reading from" line, and count lines
+    $linecount = 0;
+    open(ERRORRAW, "<" . $rawstderrlog);
+    open(ERROROUT, ">" . $stderrlog);
+    while(<ERRORRAW>) {
+        next if /^$/;  # blank lines are boring
+        if(/^(reading from file )(.*)(,.*)$/) {
+            my $filename = basename($2);
+            print ERROROUT "${1}${filename}${3}\n";
+            next;
         }
-        if($linecount == 0) {
+        print ERROROUT;
+        $linecount++;
+    }
+    close(ERROROUT);
+    close(ERRORRAW);
+
+    if ( -f "$output.stderr" ) {
+        $nr = system "cat $stderrlog | diff $output.stderr - >DIFF/$outputbase.stderr.diff";
+        if($r == 0) {
+            $r = $nr;
+        }
+        $errdiffstat = WEXITSTATUS($nr);
+    }
+
+    if($r == 0) {
+        if($linecount == 0 && $status == 0) {
             unlink($stderrlog);
+        } else {
+            $errdiffstat = 1;
         }
     }
 
@@ -77,14 +104,14 @@ else {
 if($r == 0) {
     my $stderrlog="";
     if($linecount > 0) {
-        $stderrlog=sprintf("%d lines in stderr", $linecount);
+        $stderrlog=sprintf("-- %d lines extra in stderr", $linecount);
     }
     printf "    %-35s: passed%s\n", $name, $stderrlog;
     unlink "DIFF/$outputbase.diff";
     exit 0;
 }
 # must have failed!
-printf "    %-35s: TEST FAILED(exit core=%d/diffstat=%d/r=%d)", $name, $coredump, $diffstat, $r;
+printf "    %-35s: TEST FAILED(exit core=%d/diffstat=%d,%d/r=%d)", $name, $coredump, $diffstat, $errdiffstat, $r;
 open FOUT, '>>failure-outputs.txt';
 printf FOUT "\nFailed test: $name\n\n";
 close FOUT;

diff --git a/tests/TESTrun.sh b/tests/TESTrun.sh
index 1c1178f385e240aa026bac2042faa1155e5e5f3f..0b5ee4b284c318433971494af01b0b2d484c9f4e 100755 (executable)
--- a/tests/TESTrun.sh
+++ b/tests/TESTrun.sh
@@ -15,9 +15,10 @@ echo RUNNING from ${srcdir}
 
 passedfile=$(pwd)/tests/.passed
 failedfile=$(pwd)/tests/.failed
+failureoutput=$(pwd)/tests/failure-outputs.txt
 mkdir -p tests/NEW
 mkdir -p tests/DIFF
-cat /dev/null > failure-outputs.txt
+cat /dev/null > ${failureoutput}
 
 runComplexTests()
 {
@@ -86,7 +87,7 @@ echo '------------------------------------------------'
 printf "%4u tests failed\n" $failed
 printf "%4u tests passed\n" $passed
 echo
-cat tests/failure-outputs.txt
+cat ${failureoutput}
 echo
 echo
 exit $failed