From: Denis Ovsienko <denis@ovsienko.info>
Date: Fri, 23 Jul 2021 00:52:50 +0000 (+0100)
Subject: CI: Import recent build matrix improvements. [skip appveyor]
X-Git-Url: https://git.tcpdump.org/tcpdump/commitdiff_plain/507a4c91b5bee98f64656afe9fbe6d314dbff21a

CI: Import recent build matrix improvements. [skip appveyor]

Start using build_common.sh, make default assignments early and
uniformly, use Solaris-compatible command substitution and exports,
install bc on linux-amd64, deduplicate and reformat some long commands.
Run "make -s" without CFLAGS=-Werror when warnings are expected.

This implements initial support for AIX and Solaris in the tcpdump build
matrix scripts.

* AIX 7.1
M4='/opt/freeware/bin/m4' \
MATRIX_CC='gcc' \
MATRIX_CMAKE=no \
MATRIX_BUILD_LIBPCAP=yes \
./build_matrix.sh

* Solaris 9
MATRIX_CC=gcc \
MATRIX_CMAKE=no \
MATRIX_BUILD_LIBPCAP=no \
./build_matrix.sh

* Solaris 10 and 11
MATRIX_CC='gcc' \
MATRIX_CMAKE=no \
./build_matrix.sh
---

diff --git a/.cirrus.yml b/.cirrus.yml
index 0a3e3d32..40ef91f8 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -47,7 +47,7 @@ linux_task:
       - MATRIX_CC: clang
   script:
     - apt-get -qy update
-    - apt-get -qy install git autoconf make cmake clang gcc
+    - apt-get -qy install git autoconf make cmake clang gcc bc
     - apt-get -qy install flex bison libdbus-1-dev libbluetooth-dev libnl-genl-3-dev libibverbs-dev # for libpcap
     - apt-get -qy install libssl-dev libsmi2-dev libcap-ng-dev libpcap-dev
     - apt-get -qy install sudo # for some tcpdump commands
diff --git a/build.sh b/build.sh
index e909337e..7f3931ba 100755
--- a/build.sh
+++ b/build.sh
@@ -1,115 +1,101 @@
 #!/bin/sh -e
 
 # This script runs one build with setup environment variables: BUILD_LIBPCAP,
-# REMOTE, CC, CMAKE, CRYPTO and SMB
-# (default: BUILD_LIBPCAP=no, REMOTE=no, CC=gcc, CMAKE=no, CRYPTO=no, SMB=no).
+# REMOTE, CC, CMAKE, CRYPTO and SMB.
 
-# BUILD_LIBPCAP: no or yes
-BUILD_LIBPCAP=${BUILD_LIBPCAP:-no}
-# REMOTE: no or yes
-REMOTE=${REMOTE:-no}
-# CC: gcc or clang
-CC=${CC:-gcc}
-# GCC and Clang recognize --version and print to stdout. Sun compilers
-# recognize -V and print to stderr.
-"$CC" --version 2>/dev/null || "$CC" -V || :
-# CMAKE: no or yes
-CMAKE=${CMAKE:-no}
-# CRYPTO: no or yes
-CRYPTO=${CRYPTO:-no}
-# SMB: no or yes
-SMB=${SMB:-no}
+: "${BUILD_LIBPCAP:=no}"
+: "${REMOTE:=no}"
+: "${CC:=gcc}"
+: "${CMAKE:=no}"
+: "${CRYPTO:=no}"
+: "${SMB:=no}"
+
+. ./build_common.sh
 # Install directory prefix
 if [ -z "$PREFIX" ]; then
-    PREFIX=$(mktemp -d -t tcpdump_build_XXXXXXXX)
+    PREFIX=`mktempdir tcpdump_build`
     echo "PREFIX set to '$PREFIX'"
 fi
 # For TESTrun
-export TCPDUMP_BIN="$PREFIX/bin/tcpdump"
-
-# Run a command after displaying it
-run_after_echo() {
-    printf '$ '
-    echo "$@"
-    # shellcheck disable=SC2068
-    $@
-}
+TCPDUMP_BIN="$PREFIX/bin/tcpdump"
+export TCPDUMP_BIN
 
+print_cc_version
 if [ "$CMAKE" = no ]; then
-    echo '$ ./configure [...]'
     if [ "$BUILD_LIBPCAP" = yes ]; then
         echo "Using PKG_CONFIG_PATH=$PKG_CONFIG_PATH"
-        ./configure --with-crypto="$CRYPTO" --enable-smb="$SMB" --prefix="$PREFIX"
-        export LD_LIBRARY_PATH="$PREFIX/lib"
+        run_after_echo ./configure --with-crypto="$CRYPTO" \
+            --enable-smb="$SMB" --prefix="$PREFIX"
+        LD_LIBRARY_PATH="$PREFIX/lib"
+        export LD_LIBRARY_PATH
     else
-        ./configure --disable-local-libpcap --with-crypto="$CRYPTO" --enable-smb="$SMB" --prefix="$PREFIX"
+        run_after_echo ./configure --with-crypto="$CRYPTO" \
+            --enable-smb="$SMB" --prefix="$PREFIX" --disable-local-libpcap
     fi
 else
-    rm -rf build
-    mkdir build
-    cd build
-    echo '$ cmake [...]'
+    run_after_echo rm -rf build
+    run_after_echo mkdir build
+    run_after_echo cd build
     if [ "$BUILD_LIBPCAP" = yes ]; then
-        cmake -DWITH_CRYPTO="$CRYPTO" -DENABLE_SMB="$SMB" -DCMAKE_PREFIX_PATH="$PREFIX" -DCMAKE_INSTALL_PREFIX="$PREFIX" ..
-        export LD_LIBRARY_PATH="$PREFIX/lib"
+        run_after_echo cmake -DWITH_CRYPTO="$CRYPTO" -DENABLE_SMB="$SMB" \
+            -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_PREFIX_PATH="$PREFIX" ..
+        LD_LIBRARY_PATH="$PREFIX/lib"
+        export LD_LIBRARY_PATH
     else
-        cmake -DWITH_CRYPTO="$CRYPTO" -DENABLE_SMB="$SMB" -DCMAKE_INSTALL_PREFIX="$PREFIX" ..
+        run_after_echo cmake -DWITH_CRYPTO="$CRYPTO" -DENABLE_SMB="$SMB" \
+            -DCMAKE_INSTALL_PREFIX="$PREFIX" ..
     fi
 fi
-run_after_echo "make -s clean"
-run_after_echo "make -s CFLAGS=-Werror"
-echo '$ make install'
-make install
-run_after_echo "$TCPDUMP_BIN --version"
-run_after_echo "$TCPDUMP_BIN -h"
-run_after_echo "$TCPDUMP_BIN -D"
-system=$(uname -s)
-case "$system" in
-Linux|FreeBSD|NetBSD|OpenBSD)
-    run_after_echo "ldd $TCPDUMP_BIN"
-    ;;
-Darwin)
-    run_after_echo "otool -L $TCPDUMP_BIN"
-    ;;
+run_after_echo make -s clean
+# The norm is to compile without any warnings, but tcpdump builds on some OSes
+# are not warning-free for one or another reason. If you manage to fix one of
+# these cases, please remember to raise the bar here so if the warnings appear
+# again, it will trigger an error.
+case `uname -s` in
+    AIX)
+        CFLAGS=
+        ;;
+    SunOS)
+        case `uname -r` in
+        5.10|5.11)
+            CFLAGS=-Werror
+            ;;
+        *)
+            CFLAGS=
+            ;;
+        esac
+        ;;
+    *)
+        CFLAGS=-Werror
+        ;;
 esac
+run_after_echo make -s ${CFLAGS:+CFLAGS="$CFLAGS"}
+run_after_echo make install
+run_after_echo "$TCPDUMP_BIN" --version
+run_after_echo "$TCPDUMP_BIN" -h
+run_after_echo "$TCPDUMP_BIN" -D
+print_so_deps "$TCPDUMP_BIN"
 if [ "$CIRRUS_CI" = true ]; then
-    if [ -n "$LD_LIBRARY_PATH" ]; then
-        run_after_echo "sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH $TCPDUMP_BIN -J"
-        run_after_echo "sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH $TCPDUMP_BIN -L"
-    else
-        run_after_echo "sudo $TCPDUMP_BIN -J"
-        run_after_echo "sudo $TCPDUMP_BIN -L"
-    fi
+    run_after_echo sudo \
+        ${LD_LIBRARY_PATH:+LD_LIBRARY_PATH="$LD_LIBRARY_PATH"} \
+        "$TCPDUMP_BIN" -J
+    run_after_echo sudo \
+        ${LD_LIBRARY_PATH:+LD_LIBRARY_PATH="$LD_LIBRARY_PATH"} \
+        "$TCPDUMP_BIN" -L
 fi
 if [ "$BUILD_LIBPCAP" = yes ]; then
-    run_after_echo "make check"
+    run_after_echo make check
 fi
 if [ "$CMAKE" = no ]; then
-    system=$(uname -s)
-    run_after_echo "make releasetar"
+    run_after_echo make releasetar
 fi
 if [ "$CIRRUS_CI" = true ]; then
-    if [ -n "$LD_LIBRARY_PATH" ]; then
-        run_after_echo "sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH $TCPDUMP_BIN -#n -c 10"
-    else
-        run_after_echo "sudo $TCPDUMP_BIN -#n -c 10"
-    fi
-fi
-# Beware that setting MATRIX_DEBUG will produce A LOT of additional output
-# here and in any nested libpcap builds. Multiplied by the matrix size, the
-# full output log size might exceed limits of some CI systems (as previously
-# happened with Travis CI). Use with caution on a reduced matrix.
-if [ "$MATRIX_DEBUG" = true ]; then
-    echo '$ cat Makefile [...]'
-    sed '/DO NOT DELETE THIS LINE -- mkdep uses it/q' < Makefile
-    echo '$ cat config.h'
-    cat config.h
-    if [ "$CMAKE" = no ]; then
-        echo '$ cat config.log'
-        cat config.log
-    fi
+    run_after_echo sudo \
+        ${LD_LIBRARY_PATH:+LD_LIBRARY_PATH="$LD_LIBRARY_PATH"} \
+        "$TCPDUMP_BIN" -#n -c 10
 fi
+handle_matrix_debug
 if [ "$DELETE_PREFIX" = yes ]; then
-    rm -rf "$PREFIX"
+    run_after_echo rm -rf "$PREFIX"
 fi
 # vi: set tabstop=4 softtabstop=0 expandtab shiftwidth=4 smarttab autoindent :
diff --git a/build_common.sh b/build_common.sh
new file mode 100644
index 00000000..0209806e
--- /dev/null
+++ b/build_common.sh
@@ -0,0 +1,149 @@
+#!/bin/sh -e
+
+# The only purpose of the above shebang is to orient shellcheck right.
+# To make CI scripts maintenance simpler, copies of this file in the
+# libpcap, tcpdump and tcpslice git repositories should be identical.
+# Please mind that Solaris /bin/sh before 11 does not support the $()
+# command substitution syntax, hence the SC2006 directives.
+
+# A poor man's mktemp(1) for OSes that don't have one (e.g. AIX 7, Solaris 9).
+mktempdir_diy() {
+    while true; do
+        # /bin/sh implements $RANDOM in AIX 7, but not in Solaris before 11,
+        # thus use dd and od instead.
+        # shellcheck disable=SC2006
+        mktempdir_diy_suffix=`dd if=/dev/urandom bs=1 count=4 2>/dev/null | od -t x -A n | head -1 | tr -d '\t '`
+        [ -z "$mktempdir_diy_suffix" ] && return 1
+        mktempdir_diy_path="${TMPDIR:-/tmp}/${1:?}.${mktempdir_diy_suffix}"
+        # "test -e" would be more appropriate, but it is not available in
+        # Solaris /bin/sh before 11.
+        if [ ! -d "$mktempdir_diy_path" ]; then
+            mkdir "$mktempdir_diy_path"
+            chmod go= "$mktempdir_diy_path"
+            echo "$mktempdir_diy_path"
+            break
+        fi
+        # Try again (AIX /dev/urandom returns zeroes quite often).
+    done
+}
+
+mktempdir() {
+    mktempdir_prefix=${1:-tmp}
+    # shellcheck disable=SC2006
+    case `uname -s` in
+    Darwin|FreeBSD|NetBSD)
+        # In these operating systems mktemp(1) always appends an implicit
+        # ".XXXXXXXX" suffix to the requested template when creating a
+        # temporary directory.
+        mktemp -d -t "$mktempdir_prefix"
+        ;;
+    AIX)
+        mktempdir_diy "$mktempdir_prefix"
+        ;;
+    SunOS)
+        # shellcheck disable=SC2006
+        case `uname -r` in
+        5.10|5.11)
+            mktemp -d -t "${mktempdir_prefix}.XXXXXXXX"
+            ;;
+        *)
+            mktempdir_diy "$mktempdir_prefix"
+            ;;
+        esac
+        ;;
+    *)
+        # At least Linux and OpenBSD implementations require explicit trailing
+        # X'es in the template, so make it the same suffix as above.
+        mktemp -d -t "${mktempdir_prefix}.XXXXXXXX"
+        ;;
+    esac
+}
+
+print_sysinfo() {
+    uname -a
+    date
+}
+
+print_cc_version() {
+    # shellcheck disable=SC2006
+    case `basename "$CC"` in
+    gcc*|clang*)
+        # GCC and Clang recognize --version, print to stdout and exit with 0.
+        "$CC" --version
+        ;;
+    xl*)
+        # XL C for AIX recognizes -qversion, prints to stdout and exits with 0,
+        # but on an unknown command-line flag displays its man page and waits.
+        "$CC" -qversion
+        ;;
+    sun*)
+        # Sun compilers recognize -V, print to stderr and exit with an error.
+        "$CC" -V 2>&1 || :
+        ;;
+    *)
+        "$CC" --version || "$CC" -V || :
+        ;;
+    esac
+}
+
+increment() {
+    # No arithmetic expansion in Solaris /bin/sh before 11.
+    echo "${1:?} + 1" | bc
+}
+
+# Display text in magenta.
+echo_magenta() {
+    # ANSI magenta, the imploded text, ANSI reset, newline.
+    printf '\033[35;1m%s\033[0m\n' "$*"
+}
+
+# Run a command after displaying it.
+run_after_echo() {
+    : "${1:?}" # Require at least one argument.
+    printf '$ %s\n' "$*"
+    "$@"
+}
+
+print_so_deps() {
+    case `uname -s` in
+    Darwin)
+        run_after_echo otool -L "${1:?}"
+        ;;
+    *)
+        run_after_echo ldd "${1:?}"
+        ;;
+    esac
+}
+
+# Beware that setting MATRIX_DEBUG for tcpdump or tcpslice will produce A LOT
+# of additional output there and in any nested libpcap builds. Multiplied by
+# the matrix size, the full output log size might exceed limits of some CI
+# systems (as it had previously happened with Travis CI). Use with caution on
+# a reduced matrix.
+handle_matrix_debug() {
+    [ "$MATRIX_DEBUG" != yes ] && return
+    echo '$ cat Makefile [...]'
+    sed '/^# DO NOT DELETE THIS LINE -- mkdep uses it.$/q' <Makefile
+    run_after_echo cat config.h
+    [ "$CMAKE" = yes ] || run_after_echo cat config.log
+}
+
+purge_directory() {
+    # shellcheck disable=SC2006
+    if [ "`uname -s`" = SunOS ] && [ "`uname -r`" = 5.11 ]; then
+        # In Solaris 11 /bin/sh the pathname expansion of "*" always includes
+        # "." and "..", so the straightforward rm would always fail.
+        (
+            cd "${1:?}"
+            for pd_each in *; do
+                if [ "$pd_each" != . ] && [ "$pd_each" != .. ]; then
+                    rm -rf "$pd_each"
+                fi
+            done
+        )
+    else
+        rm -rf "${1:?}"/*
+    fi
+}
+
+# vi: set tabstop=4 softtabstop=0 expandtab shiftwidth=4 smarttab autoindent :
diff --git a/build_matrix.sh b/build_matrix.sh
index 5c5df5ea..2ee596b9 100755
--- a/build_matrix.sh
+++ b/build_matrix.sh
@@ -2,57 +2,57 @@
 
 # This script executes the matrix loops, exclude tests and cleaning.
 # It calls the build.sh script which runs one build with setup environment
-# variables: BUILD_LIBPCAP, REMOTE, CC, CMAKE, CRYPTO and SMB
-# (default: BUILD_LIBPCAP=no, REMOTE=no, CC=gcc, CMAKE=no, CRYPTO=no, SMB=no).
+# variables: BUILD_LIBPCAP, REMOTE, CC, CMAKE, CRYPTO and SMB.
 # The matrix can be configured with environment variables
 # MATRIX_BUILD_LIBPCAP, MATRIX_REMOTE, MATRIX_CC, MATRIX_CMAKE, MATRIX_CRYPTO
-# and MATRIX_SMB
-# (default: MATRIX_BUILD_LIBPCAP='no yes', MATRIX_REMOTE='no yes',
-# MATRIX_CC='gcc clang', MATRIX_CMAKE='no yes', MATRIX_CRYPTO='no yes',
-# MATRIX_SMB='no yes').
+# and MATRIX_SMB.
 
-uname -a
-date
+: "${MATRIX_BUILD_LIBPCAP:=no yes}"
+: "${MATRIX_REMOTE:=no}"
+: "${MATRIX_CC:=gcc clang}"
+: "${MATRIX_CMAKE:=no yes}"
+: "${MATRIX_CRYPTO:=no yes}"
+: "${MATRIX_SMB:=no yes}"
+
+. ./build_common.sh
+print_sysinfo
 # Install directory prefix
 if [ -z "$PREFIX" ]; then
-    PREFIX=$(mktemp -d -t tcpdump_build_matrix_XXXXXXXX)
+    PREFIX=`mktempdir tcpdump_build_matrix`
     echo "PREFIX set to '$PREFIX'"
     export PREFIX
 fi
 COUNT=0
 
-# Display text in magenta
-echo_magenta() {
-    printf '\033[35;1m' # ANSI magenta
-    echo "$@"
-    printf '\033[0m' # ANSI reset
-}
-
 build_tcpdump() {
-    for CC in ${MATRIX_CC:-gcc clang}; do
+    for CC in $MATRIX_CC; do
         export CC
         # Exclude gcc on macOS (it is just an alias for clang).
-        if [ "$CC" = gcc ] && [ "$(uname -s)" = Darwin ]; then
+        if [ "$CC" = gcc ] && [ "`uname -s`" = Darwin ]; then
             echo '(skipped)'
             continue
         fi
-        for CMAKE in ${MATRIX_CMAKE:-no yes}; do
+        for CMAKE in $MATRIX_CMAKE; do
             export CMAKE
-            for CRYPTO in ${MATRIX_CRYPTO:-no yes}; do
+            for CRYPTO in $MATRIX_CRYPTO; do
                 export CRYPTO
-                for SMB in ${MATRIX_SMB:-no yes}; do
+                for SMB in $MATRIX_SMB; do
                     export SMB
-                    COUNT=$((COUNT+1))
+                    COUNT=`increment $COUNT`
                     echo_magenta "===== SETUP $COUNT: BUILD_LIBPCAP=$BUILD_LIBPCAP REMOTE=${REMOTE:-?} CC=$CC CMAKE=$CMAKE CRYPTO=$CRYPTO SMB=$SMB ====="
                     # Run one build with setup environment variables:
                     # BUILD_LIBPCAP, REMOTE, CC, CMAKE, CRYPTO and SMB
-                    ./build.sh
+                    run_after_echo ./build.sh
                     echo 'Cleaning...'
-                    if [ "$CMAKE" = yes ]; then rm -rf build; else make distclean; fi
-                    rm -rf "$PREFIX"/bin/tcpdump*
-                    git status -suall
+                    if [ "$CMAKE" = yes ]; then
+                        run_after_echo rm -rf build
+                    else
+                        run_after_echo make distclean
+                    fi
+                    run_after_echo rm -rf "$PREFIX"/bin/tcpdump*
+                    run_after_echo git status -suall
                     # Cancel changes in configure
-                    git checkout configure
+                    run_after_echo git checkout configure
                 done
             done
         done
@@ -60,28 +60,29 @@ build_tcpdump() {
 }
 
 touch .devel configure
-for BUILD_LIBPCAP in ${MATRIX_BUILD_LIBPCAP:-no yes}; do
+for BUILD_LIBPCAP in $MATRIX_BUILD_LIBPCAP; do
     export BUILD_LIBPCAP
     if [ "$BUILD_LIBPCAP" = yes ]; then
-        for REMOTE in ${MATRIX_REMOTE:-no}; do
+        for REMOTE in $MATRIX_REMOTE; do
             export REMOTE
             # Build libpcap with Autoconf.
             echo_magenta "Build libpcap (CMAKE=no REMOTE=$REMOTE)"
             (cd ../libpcap && CMAKE=no ./build.sh)
             # Set PKG_CONFIG_PATH for configure when building libpcap
             if [ "$CMAKE" != no ]; then
-                export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig"
+                PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig"
+                export PKG_CONFIG_PATH
             fi
             build_tcpdump
         done
     else
         echo_magenta 'Use system libpcap'
-        rm -rf "${PREFIX:?}"/*
-        make -C ../libpcap distclean || :
+        purge_directory "$PREFIX"
+        (cd ../libpcap; make distclean || echo '(Ignoring the make error.)')
         build_tcpdump
     fi
 done
 
-rm -rf "$PREFIX"
+run_after_echo rm -rf "$PREFIX"
 echo_magenta "Tested setup count: $COUNT"
 # vi: set tabstop=4 softtabstop=0 expandtab shiftwidth=4 smarttab autoindent :