#!/bin/bash # Copyright (c) Meta Platforms, Inc. and affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. show_help() { printf "Usage: runtests.sh [-hlqrv] [-d DIR] [-t TEST] [-b] FLOW_BINARY [[-f] TEST_FILTER]\n\n" printf "Runs Flow's tests.\n\n" echo " [-b] FLOW_BINARY" echo " path to Flow binary (the -b is optional)" echo " [-f] TEST_FILTER" echo " optional regular expression to choose test directories to run" echo " -l" echo " lists the tests that will be run" echo " -d DIR" echo " run tests in DIR/tests/" echo " -t TEST" echo " run the test DIR/tests/TEST, equivalent to a filter of \"^TEST$\"" echo " -c" echo " only run check tests" echo " -r" echo " re-record failing tests to update expected output" echo " -q" echo " quiet output (hides status, just prints results)" echo " -s" echo " test saved state" echo " -L" echo " test long-lived workers" echo " -v" echo " verbose output (shows skipped tests)" echo " -h" echo " display this help and exit" } export IN_FLOW_TEST=1 export FLOW_LOG_LEVEL=debug export FLOW_NODE_BINARY=${FLOW_NODE_BINARY:-${NODE_BINARY:-$(which node)}} # find git. to skip git tests even when git is available, # set FLOW_GIT_BINARY to empty string if [ -n "${FLOW_GIT_BINARY-unset}" ]; then export FLOW_GIT_BINARY=${FLOW_GIT_BINARY:-$(which git)} fi OPTIND=1 record=0 saved_state=0 verbose=0 quiet=0 relative="." check_only=0 long_lived_workers=0 export saved_state filter check_only long_lived_workers while getopts "b:d:f:clqxrsiLt:vh?" opt; do case "$opt" in b) FLOW="$OPTARG" ;; d) relative="$OPTARG" ;; f) filter="$OPTARG" ;; c) check_only=1 ;; l) list_tests=1 ;; t) specific_test="$OPTARG" ;; q) quiet=1 ;; r) record=1 ;; s) saved_state=1 printf "Testing saved state by running all tests using saved state\\n" ;; L) long_lived_workers=1 ;; v) verbose=1 ;; h|\?) show_help exit 0 ;; esac done shift $((OPTIND-1)) [ "$1" = "--" ] && shift if [ -n "$specific_test" ]; then if [[ "$saved_state" -eq 1 ]]; then specific_test=$(echo $specific_test | sed 's/\(.*\)-saved-state$/\1/') fi if [[ "$long_lived_workers" -eq 1 ]]; then specific_test=$(echo $specific_test | sed 's/\(.*\)-long-lived-workers$/\1/') fi filter="^$specific_test$" fi FLOW="${FLOW:-$1}" filter="${filter:-$2}" if [[ "$OSTYPE" == "darwin"* ]]; then _canonicalize_file_path() { local dir file dir=$(dirname -- "$1") file=$(basename -- "$1") (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") } FLOW=$(_canonicalize_file_path "$FLOW") else FLOW=$(readlink -f "$FLOW") fi VERSION=$("$FLOW" version --semver) export FLOW VERSION if [ -t 1 ]; then COLOR_RESET="\x1b[0m" COLOR_DEFAULT="\x1b[39;49;0m" COLOR_DEFAULT_BOLD="\x1b[39;49;1m" COLOR_RED_BOLD="\x1b[31;1m" COLOR_GREEN_BOLD="\x1b[32;1m" COLOR_YELLOW_BOLD="\x1b[33;1m" COLOR_MAGENTA_BOLD="\x1b[35;1m" COLOR_WHITE_ON_RED_BOLD="\x1b[37;41;1m" else COLOR_RESET="" COLOR_DEFAULT="" COLOR_DEFAULT_BOLD="" COLOR_RED_BOLD="" COLOR_GREEN_BOLD="" COLOR_YELLOW_BOLD="" COLOR_MAGENTA_BOLD="" COLOR_WHITE_ON_RED_BOLD="" fi print_failure() { dir=$1 name=${dir%*/} name=${name##*/} if [[ "$record" -eq 1 ]]; then printf "%b[✗] UPDATED:%b %s%b\n" \ "$COLOR_MAGENTA_BOLD" "$COLOR_DEFAULT" "$name" "$COLOR_RESET" else printf "%b[✗] FAILED:%b %s%b\n" \ "$COLOR_RED_BOLD" "$COLOR_DEFAULT" "$name" "$COLOR_RESET" fi diff_file="${dir}${name}.diff" err_file="${dir}${name}.err" [ -f "$err_file" ] && cat "$err_file" if [ -f "$diff_file" ]; then if [ -t 1 ] ; then esc=$(echo -e "\x1b") sed \ "s/^-/${esc}[31m-/;s/^+/${esc}[32m+/;s/^@/${esc}[35m@/;s/$/${esc}[0m/" \ < "$diff_file" else cat "$diff_file" fi fi # Default expected file extension is .exp ext=".exp" if [[ "$record" -eq 1 ]]; then # Copy .out to .exp, replacing the current version, if present, with # <VERSION>, so that the .exp doesn't have to be updated on each release. sed 's/'"${VERSION//./\\.}"'/<VERSION>/g' "${dir}${name}.out" > "${dir}${name}${ext}" rm "${dir}${name}.out" rm -f "$err_file" rm "$diff_file" fi } print_error() { dir=$1 name=${dir%*/} name=${name##*/} printf "%b[✗] ERRORED:%b %s%b\n" \ "$COLOR_RED_BOLD" "$COLOR_DEFAULT" "$name" "$COLOR_RESET" out_file="${dir}${name}.out" [ -f "$out_file" ] && cat "$out_file" err_file="${dir}${name}.err" [ -f "$err_file" ] && printf "\n\nStderr:\n" && cat "$err_file" monitor_log_file="${dir}${name}.monitor_log" [ -f "$monitor_log_file" ] && printf "\n\nServer monitor log:\n" && cat "$monitor_log_file" log_file="${dir}${name}.log" [ -f "$log_file" ] && printf "\n\nServer log:\n" && cat "$log_file" } print_skip() { name=$1 name=${name%*/} name=${name##*/} verbose=$2 if [[ "$verbose" -eq 1 ]]; then printf "%b[-] SKIPPED:%b %s%b\n" \ "$COLOR_YELLOW_BOLD" "$COLOR_DEFAULT" "$name" "$COLOR_RESET" elif [[ -t 1 ]]; then printf " %*s\r" ${#name} " " fi } print_success() { name=$1 name=${name%*/} name=${name##*/} printf "%b[✓] PASSED:%b %s%b\n" \ "$COLOR_GREEN_BOLD" "$COLOR_DEFAULT" "$name" "$COLOR_RESET" } print_run() { if [[ -t 1 && "$quiet" -eq 0 ]]; then name=$1 name=${name%*/} name=${name##*/} printf "%b[ ] RUNNING:%b %s%b\r" \ "$COLOR_DEFAULT_BOLD" "$COLOR_DEFAULT" "$name" "$COLOR_RESET" fi } cd "$relative" || exit 1 passed=0 failed=0 skipped=0 errored=0 # shellcheck source=fbcode/flow/scripts/lib/runtests-common.sh . "./scripts/lib/runtests-common.sh" if [ -z "$FLOW_MAX_WORKERS" ]; then export FLOW_MAX_WORKERS=2 fi num_to_run_in_parallel=${FLOW_RUNTESTS_PARALLELISM-16} if [[ "$quiet" -eq 0 ]]; then printf "Running up to %d test(s) in parallel\n" "$num_to_run_in_parallel" fi # Index N of results should correspond to the test at index N of dirs dirs=(tests/*/) results=() if [[ "$list_tests" -eq 1 ]]; then for dir in "${dirs[@]}"; do dir=${dir%*/} name=${dir##*/} if [[ -z $filter || $name =~ $filter ]]; then if [[ "$saved_state" -eq 1 ]]; then echo "$name-saved-state" elif [[ "$long_lived_workers" -eq 1 ]]; then echo "$name-long-lived-workers" else echo "$name" fi fi done exit 0 fi # Run all tests, num_to_run_in_parallel at a time parallel_run_tests() { ( i=0 for dir in "${dirs[@]}"; do echo "$(( i++ )):$dir" done ) | xargs -P"$num_to_run_in_parallel" -n1 \ "./scripts/run-one-test" } # Kick off all tests, reporting results over a pipe to our fd 3 exec 3< <(parallel_run_tests) # Print a test's result, given its index in dirs and results print_result() { local test_to_print=$1 code=${results[$test_to_print]} testname=${dirs[$test_to_print]} case $code in $RUNTEST_SUCCESS ) (( passed++ )) print_success "$testname" ;; $RUNTEST_FAILURE ) (( failed++ )) print_failure "$testname" ;; $RUNTEST_SKIP ) (( skipped++ )) print_skip "$testname" "$verbose" ;; $RUNTEST_MISSING_FILES ) (( errored++ )) print_error "$testname" name=${testname%*/} name=${name##*/} printf "Missing %s.exp file or .flowconfig file\n" "$name" ;; $RUNTEST_MISSING_ALL_OPTION ) (( errored++ )) print_error "$testname" echo 'You are required to set either `all=true` or `all=false` in your test `.flowconfig`.' ;; $RUNTEST_ERROR | '') # '' means the parallel runner stopped before giving a result (( errored++ )) print_error "$testname" ;; esac } # Collect the results in the order the tests finish next_test_to_print=0 print_run "${dirs[$next_test_to_print]}" while read -ru 3 index code; do results[$index]=$code # Print the results in order in a pretty way while [ -n "${results[$next_test_to_print]}" ]; do print_result "$next_test_to_print" (( next_test_to_print++ )) done if (( next_test_to_print < ${#dirs[@]} )); then print_run "${dirs[$next_test_to_print]}" fi done while (( next_test_to_print < ${#dirs[@]} )); do # If the parallel runner stopped early, print the remaining tests # (as errors, except any where we already have a result) print_result "$next_test_to_print" (( next_test_to_print++ )) done if [ "$failed" -eq 0 ]; then FAILED_COLOR="$COLOR_DEFAULT_BOLD" else FAILED_COLOR="$COLOR_WHITE_ON_RED_BOLD" fi if [ "$errored" -eq 0 ]; then ERRORED_COLOR="$COLOR_DEFAULT_BOLD" else ERRORED_COLOR="$COLOR_WHITE_ON_RED_BOLD" fi if [[ "$quiet" -eq 0 ]]; then echo printf "%bPassed: %d, %bFailed: %d%b, Skipped: %d, %bErrored: %d%b\n" \ "$COLOR_DEFAULT_BOLD" "$passed" \ "$FAILED_COLOR" "$failed" \ "$COLOR_DEFAULT_BOLD" "$skipped" \ "$ERRORED_COLOR" "$errored" \ "$COLOR_RESET" fi if [[ "$failed" -gt 0 || "$errored" -gt 0 ]]; then exit 1 fi