From a0345c9f92611ad9e72c7abdf45424b0ee4f15a9 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sun, 6 Oct 2024 17:08:27 +0200 Subject: [PATCH 01/31] feat: add -p|--parallel option --- bashunit | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bashunit b/bashunit index 640b38bc..e6ecc2c5 100755 --- a/bashunit +++ b/bashunit @@ -63,6 +63,9 @@ while [[ $# -gt 0 ]]; do -S|--stop-on-failure) export BASHUNIT_STOP_ON_FAILURE=true ;; + -p|--parallel) + export BASHUNIT_PARALLEL_RUN=true + ;; -e|--env|--load) # shellcheck disable=SC1090 source "$2" From 81ffb9bbef7cdc0a3811f59177809c484d9c3348 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sun, 6 Oct 2024 17:09:09 +0200 Subject: [PATCH 02/31] feat: call_test_functions & when is_parallel_run_enabled --- src/main.sh | 4 ++++ src/runner.sh | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main.sh b/src/main.sh index c1536c89..8ffad17d 100644 --- a/src/main.sh +++ b/src/main.sh @@ -17,6 +17,10 @@ function main::exec_tests() { console_header::print_version_with_env "$filter" "${test_files[@]}" runner::load_test_files "$filter" "${test_files[@]}" + if env::is_parallel_run_enabled; then + wait + fi + console_results::print_failing_tests_and_reset console_results::render_result exit_code=$? diff --git a/src/runner.sh b/src/runner.sh index 3676ba3c..10db7b55 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -13,9 +13,10 @@ function runner::load_test_files() { source "$test_file" runner::run_set_up_before_script - runner::call_test_functions "$test_file" "$filter" - if [ "$BASHUNIT_PARALLEL_RUN" = true ] ; then - wait + if env::is_parallel_run_enabled; then + runner::call_test_functions "$test_file" "$filter" & + else + runner::call_test_functions "$test_file" "$filter" fi runner::run_tear_down_after_script runner::clean_set_up_and_tear_down_after_script From 0a37c348d5b0a66e5e293c2c4cdabc26d65aac9a Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sun, 6 Oct 2024 17:10:25 +0200 Subject: [PATCH 03/31] feat: hide "Runing script" text when running parallel --- src/runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runner.sh b/src/runner.sh index 10db7b55..d28e8297 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -56,7 +56,7 @@ function runner::call_test_functions() { functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions")) if [[ "${#functions_to_run[@]}" -gt 0 ]]; then - if ! env::is_simple_output_enabled; then + if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then echo "Running $script" fi From a6c153958370a9b6264de6dd487582c55bf33b5d Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 20:01:49 +0200 Subject: [PATCH 04/31] feat: invoke parallel::call_test_functions & when is_parallel --- src/runner.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/runner.sh b/src/runner.sh index d28e8297..f7a21757 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -2,7 +2,9 @@ function runner::load_test_files() { local filter=$1 - local files=("${@:2}") # Store all arguments starting from the second as an array + shift + local files=("${@}") + local pids=() for test_file in "${files[@]}"; do if [[ ! -f $test_file ]]; then @@ -14,13 +16,21 @@ function runner::load_test_files() { runner::run_set_up_before_script if env::is_parallel_run_enabled; then - runner::call_test_functions "$test_file" "$filter" & + parallel::call_test_functions "$test_file" "$filter" & + pids+=($!) else runner::call_test_functions "$test_file" "$filter" fi runner::run_tear_down_after_script runner::clean_set_up_and_tear_down_after_script done + + # Wait for all background processes to finish + if env::is_parallel_run_enabled; then + for pid in "${pids[@]}"; do + wait "$pid" || echo "Test with PID $pid failed" + done + fi } function runner::functions_for_script() { @@ -56,7 +66,7 @@ function runner::call_test_functions() { functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions")) if [[ "${#functions_to_run[@]}" -gt 0 ]]; then - if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then + if ! env::is_simple_output_enabled; then echo "Running $script" fi From a0f85724a223abdf2d182f1e5640ac3f32d97cb8 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 22:12:47 +0200 Subject: [PATCH 05/31] feat: create parallel.sh --- bashunit | 1 + src/env.sh | 1 + src/parallel.sh | 292 ++++++++++++++++++++++++++++++++++++++++++++++++ src/runner.sh | 6 + 4 files changed, 300 insertions(+) create mode 100755 src/parallel.sh diff --git a/bashunit b/bashunit index e6ecc2c5..0b8127e9 100755 --- a/bashunit +++ b/bashunit @@ -14,6 +14,7 @@ source "$BASHUNIT_ROOT_DIR/src/globals.sh" source "$BASHUNIT_ROOT_DIR/src/dependencies.sh" source "$BASHUNIT_ROOT_DIR/src/io.sh" source "$BASHUNIT_ROOT_DIR/src/math.sh" +source "$BASHUNIT_ROOT_DIR/src/parallel.sh" source "$BASHUNIT_ROOT_DIR/src/env.sh" source "$BASHUNIT_ROOT_DIR/src/check_os.sh" source "$BASHUNIT_ROOT_DIR/src/clock.sh" diff --git a/src/env.sh b/src/env.sh index 98aba2c4..51f130f3 100644 --- a/src/env.sh +++ b/src/env.sh @@ -73,6 +73,7 @@ function env::find_terminal_width() { echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}" } +TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel" TERMINAL_WIDTH="$(env::find_terminal_width)" FAILURES_OUTPUT_PATH=$(mktemp) CAT="$(which cat)" diff --git a/src/parallel.sh b/src/parallel.sh new file mode 100755 index 00000000..895aa546 --- /dev/null +++ b/src/parallel.sh @@ -0,0 +1,292 @@ +#!/bin/bash + +# this function will be invoke with & to force running in a subprocess +function parallel::call_test_functions() { + local script="$1" + local filter="$2" + local prefix="test" + # Use declare -F to list all function names + local all_function_names + all_function_names=$(declare -F | awk '{print $3}') + local filtered_functions + # shellcheck disable=SC2207 + filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names") + + local functions_to_run + # shellcheck disable=SC2207 + functions_to_run=($(parallel::functions_for_script "$script" "$filtered_functions")) + + if [[ "${#functions_to_run[@]}" -gt 0 ]]; then + helper::check_duplicate_functions "$script" || true + + for function_name in "${functions_to_run[@]}"; do + local provider_data=() + while IFS=" " read -r line; do + provider_data+=("$line") + done <<< "$(helper::get_provider_data "$function_name" "$script")" + + # No data provider found + if [[ "${#provider_data[@]}" -eq 0 ]]; then + parallel::run_test "$script" "$function_name" + unset function_name + continue + fi + + # Execute the test function for each line of data + for data in "${provider_data[@]}"; do + IFS=" " read -r -a args <<< "$data" + if [ "${#args[@]}" -gt 1 ]; then + parallel::run_test "$script" "$function_name" "${args[@]}" + else + parallel::run_test "$script" "$function_name" "$data" + fi + done + unset function_name + done + fi +} + +function parallel::functions_for_script() { + local script="$1" + local all_function_names="$2" + + # Filter the names down to the ones defined in the script, sort them by line number + shopt -s extdebug + for f in $all_function_names; do + declare -F "$f" | grep "$script" + done | sort -k2 -n | awk '{print $1}' + shopt -u extdebug +} + +function parallel::run_test() { + local start_time + start_time=$(clock::now) + + local test_file="$1" + shift + local function_name="$1" + shift + local current_assertions_failed + current_assertions_failed="$(state::get_assertions_failed)" + local current_assertions_snapshot + current_assertions_snapshot="$(state::get_assertions_snapshot)" + local current_assertions_incomplete + current_assertions_incomplete="$(state::get_assertions_incomplete)" + local current_assertions_skipped + current_assertions_skipped="$(state::get_assertions_skipped)" + + # (FD = File Descriptor) + # Duplicate the current std-output (FD 1) and assigns it to FD 3. + # This means that FD 3 now points to wherever the std-output was pointing. + exec 3>&1 + + local test_execution_result + test_execution_result=$( + state::initialize_assertions_count + parallel::run_set_up + + # 2>&1: Redirects the std-error (FD 2) to the std-output (FD 1). + # points to the original std-output. + "$function_name" "$@" 2>&1 + + parallel::run_tear_down + parallel::clear_mocks + state::export_subshell_context + ) + + # Closes FD 3, which was used temporarily to hold the original stdout. + exec 3>&- + + # shellcheck disable=SC2155 + local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" + mkdir -p "$test_suite_dir" + echo "$test_execution_result" > "${test_suite_dir}/${function_name}.result" + + runner::parse_execution_result "$test_execution_result" + + local subshell_output + subshell_output=$(\ + echo "$test_execution_result" |\ + tail -n 1 |\ + sed -E -e 's/.*##TEST_OUTPUT=(.*)##.*/\1/g' |\ + base64 -d + ) + + if [[ -n "$subshell_output" ]]; then + # Formatted as "[type]line" @see `state::print_line()` + local type="${subshell_output%%]*}" # Remove everything after "]" + type="${type#[}" # Remove the leading "[" + local line="${subshell_output#*]}" # Remove everything before and including "]" + state::print_line "$type" "$line" + + subshell_output=$line + fi + + local runtime_output + runtime_output="${test_execution_result%%##ASSERTIONS*}" + + local runtime_error="" + for error in "command not found" "unbound variable" "permission denied" \ + "no such file or directory" "syntax error" "bad substitution" \ + "division by 0" "cannot allocate memory" "bad file descriptor" \ + "segmentation fault" "illegal option" "argument list too long" \ + "readonly variable" "missing keyword" "killed" \ + "cannot execute binary file"; do + if [[ "$runtime_output" == *"$error"* ]]; then + runtime_error=$(echo "${runtime_output#*: }" | tr -d '\n') + break + fi + done + + local total_assertions + total_assertions="$(state::calculate_total_assertions "$test_execution_result")" + + local end_time duration_ns duration + end_time=$(clock::now) + duration_ns=$(math::calculate "($end_time - $start_time) ") + duration=$(math::calculate "$duration_ns / 1000000") + + if [[ -n $runtime_error ]]; then + state::add_tests_failed + console_results::print_error_test "$function_name" "$runtime_error" + logger::test_failed "$test_file" "$function_name" "$duration" "$total_assertions" + runner::write_failure_result_output "$test_file" "$runtime_error" + return + fi + + if [[ "$current_assertions_failed" != "$(state::get_assertions_failed)" ]]; then + state::add_tests_failed + logger::test_failed "$test_file" "$function_name" "$duration" "$total_assertions" + runner::write_failure_result_output "$test_file" "$subshell_output" + if env::is_stop_on_failure_enabled; then + exit 1 + fi + return + fi + + if [[ "$current_assertions_snapshot" != "$(state::get_assertions_snapshot)" ]]; then + state::add_tests_snapshot + console_results::print_snapshot_test "$function_name" + logger::test_snapshot "$test_file" "$function_name" "$duration" "$total_assertions" + return + fi + + if [[ "$current_assertions_incomplete" != "$(state::get_assertions_incomplete)" ]]; then + state::add_tests_incomplete + logger::test_incomplete "$test_file" "$function_name" "$duration" "$total_assertions" + return + fi + + if [[ "$current_assertions_skipped" != "$(state::get_assertions_skipped)" ]]; then + state::add_tests_skipped + logger::test_skipped "$test_file" "$function_name" "$duration" "$total_assertions" + return + fi + + local label + label="$(helper::normalize_test_function_name "$function_name")" + + console_results::print_successful_test "${label}" "$duration" "$@" + state::add_tests_passed + logger::test_passed "$test_file" "$function_name" "$duration" "$total_assertions" +## +} + +function parallel::aggregate_test_results() { + local total_failed=0 + local total_passed=0 + local total_skipped=0 + local total_incomplete=0 + local total_snapshot=0 + + for script_dir in "$TEMP_DIR_PARALLEL_TEST_SUITE"/*; do + for result_file in "$script_dir"/*.result; do + while IFS= read -r line; do + # Extract assertion counts from the result lines using sed + failed=$(echo "$line" | sed -n 's/.*##ASSERTIONS_FAILED=\([0-9]*\).*/\1/p') + passed=$(echo "$line" | sed -n 's/.*##ASSERTIONS_PASSED=\([0-9]*\).*/\1/p') + skipped=$(echo "$line" | sed -n 's/.*##ASSERTIONS_SKIPPED=\([0-9]*\).*/\1/p') + incomplete=$(echo "$line" | sed -n 's/.*##ASSERTIONS_INCOMPLETE=\([0-9]*\).*/\1/p') + snapshot=$(echo "$line" | sed -n 's/.*##ASSERTIONS_SNAPSHOT=\([0-9]*\).*/\1/p') + + # Default to 0 if no match is found + failed=${failed:-0} + passed=${passed:-0} + skipped=${skipped:-0} + incomplete=${incomplete:-0} + snapshot=${snapshot:-0} + + # Add to the total counts + total_failed=$((total_failed + failed)) + total_passed=$((total_passed + passed)) + total_skipped=$((total_skipped + skipped)) + total_incomplete=$((total_incomplete + incomplete)) + total_snapshot=$((total_snapshot + snapshot)) + done < "$result_file" + + # Check and update test state based on the parsed results + if [ "$passed" -gt 0 ]; then + state::add_tests_passed + fi + + if [ "$failed" -gt 0 ]; then + state::add_tests_failed + fi + + if [ "$skipped" -gt 0 ]; then + state::add_tests_skipped + fi + + if [ "$incomplete" -gt 0 ]; then + state::add_tests_incomplete + fi + + if [ "$snapshot" -gt 0 ]; then + state::add_tests_snapshot + fi + done + done + + export _ASSERTIONS_FAILED=$total_failed + export _ASSERTIONS_PASSED=$total_passed + export _ASSERTIONS_SKIPPED=$total_skipped + export _ASSERTIONS_INCOMPLETE=$total_incomplete + export _ASSERTIONS_SNAPSHOT=$total_snapshot +} + + +function parallel::write_failure_result_output() { + local test_file=$1 + local error_msg=$2 + + echo -e "$(state::get_tests_failed)) $test_file\n$error_msg" >> "$FAILURES_OUTPUT_PATH" +} + +function parallel::run_set_up() { + helper::execute_function_if_exists 'set_up' +} + +function parallel::run_set_up_before_script() { + helper::execute_function_if_exists 'set_up_before_script' +} + +function parallel::run_tear_down() { + helper::execute_function_if_exists 'tear_down' +} + +function parallel::clear_mocks() { + for i in "${!MOCKED_FUNCTIONS[@]}"; do + unmock "${MOCKED_FUNCTIONS[$i]}" + done +} + +function parallel::run_tear_down_after_script() { + helper::execute_function_if_exists 'tear_down_after_script' +} + +function parallel::clean_set_up_and_tear_down_after_script() { + helper::unset_if_exists 'set_up' + helper::unset_if_exists 'tear_down' + helper::unset_if_exists 'set_up_before_script' + helper::unset_if_exists 'tear_down_after_script' +} diff --git a/src/runner.sh b/src/runner.sh index f7a21757..d2527812 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -6,6 +6,10 @@ function runner::load_test_files() { local files=("${@}") local pids=() + if env::is_parallel_run_enabled; then + rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE" + fi + for test_file in "${files[@]}"; do if [[ ! -f $test_file ]]; then continue @@ -30,6 +34,8 @@ function runner::load_test_files() { for pid in "${pids[@]}"; do wait "$pid" || echo "Test with PID $pid failed" done + + parallel::aggregate_test_results fi } From f5a5f13d2588ec95ed939d1c1fd2405120caf9d8 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 22:19:28 +0200 Subject: [PATCH 06/31] refactor: reuse runner functions when possible simplify parallel.sh --- src/parallel.sh | 242 ++---------------------------------------------- src/runner.sh | 11 ++- 2 files changed, 16 insertions(+), 237 deletions(-) diff --git a/src/parallel.sh b/src/parallel.sh index 895aa546..39631f0b 100755 --- a/src/parallel.sh +++ b/src/parallel.sh @@ -1,205 +1,15 @@ #!/bin/bash -# this function will be invoke with & to force running in a subprocess -function parallel::call_test_functions() { - local script="$1" - local filter="$2" - local prefix="test" - # Use declare -F to list all function names - local all_function_names - all_function_names=$(declare -F | awk '{print $3}') - local filtered_functions - # shellcheck disable=SC2207 - filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names") - - local functions_to_run - # shellcheck disable=SC2207 - functions_to_run=($(parallel::functions_for_script "$script" "$filtered_functions")) - - if [[ "${#functions_to_run[@]}" -gt 0 ]]; then - helper::check_duplicate_functions "$script" || true - - for function_name in "${functions_to_run[@]}"; do - local provider_data=() - while IFS=" " read -r line; do - provider_data+=("$line") - done <<< "$(helper::get_provider_data "$function_name" "$script")" - - # No data provider found - if [[ "${#provider_data[@]}" -eq 0 ]]; then - parallel::run_test "$script" "$function_name" - unset function_name - continue - fi - - # Execute the test function for each line of data - for data in "${provider_data[@]}"; do - IFS=" " read -r -a args <<< "$data" - if [ "${#args[@]}" -gt 1 ]; then - parallel::run_test "$script" "$function_name" "${args[@]}" - else - parallel::run_test "$script" "$function_name" "$data" - fi - done - unset function_name - done - fi -} - -function parallel::functions_for_script() { - local script="$1" - local all_function_names="$2" - - # Filter the names down to the ones defined in the script, sort them by line number - shopt -s extdebug - for f in $all_function_names; do - declare -F "$f" | grep "$script" - done | sort -k2 -n | awk '{print $1}' - shopt -u extdebug -} - -function parallel::run_test() { - local start_time - start_time=$(clock::now) - - local test_file="$1" - shift - local function_name="$1" - shift - local current_assertions_failed - current_assertions_failed="$(state::get_assertions_failed)" - local current_assertions_snapshot - current_assertions_snapshot="$(state::get_assertions_snapshot)" - local current_assertions_incomplete - current_assertions_incomplete="$(state::get_assertions_incomplete)" - local current_assertions_skipped - current_assertions_skipped="$(state::get_assertions_skipped)" - - # (FD = File Descriptor) - # Duplicate the current std-output (FD 1) and assigns it to FD 3. - # This means that FD 3 now points to wherever the std-output was pointing. - exec 3>&1 - - local test_execution_result - test_execution_result=$( - state::initialize_assertions_count - parallel::run_set_up - - # 2>&1: Redirects the std-error (FD 2) to the std-output (FD 1). - # points to the original std-output. - "$function_name" "$@" 2>&1 - - parallel::run_tear_down - parallel::clear_mocks - state::export_subshell_context - ) - - # Closes FD 3, which was used temporarily to hold the original stdout. - exec 3>&- - - # shellcheck disable=SC2155 - local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" - mkdir -p "$test_suite_dir" - echo "$test_execution_result" > "${test_suite_dir}/${function_name}.result" - - runner::parse_execution_result "$test_execution_result" - - local subshell_output - subshell_output=$(\ - echo "$test_execution_result" |\ - tail -n 1 |\ - sed -E -e 's/.*##TEST_OUTPUT=(.*)##.*/\1/g' |\ - base64 -d - ) - - if [[ -n "$subshell_output" ]]; then - # Formatted as "[type]line" @see `state::print_line()` - local type="${subshell_output%%]*}" # Remove everything after "]" - type="${type#[}" # Remove the leading "[" - local line="${subshell_output#*]}" # Remove everything before and including "]" - state::print_line "$type" "$line" - - subshell_output=$line - fi - - local runtime_output - runtime_output="${test_execution_result%%##ASSERTIONS*}" - - local runtime_error="" - for error in "command not found" "unbound variable" "permission denied" \ - "no such file or directory" "syntax error" "bad substitution" \ - "division by 0" "cannot allocate memory" "bad file descriptor" \ - "segmentation fault" "illegal option" "argument list too long" \ - "readonly variable" "missing keyword" "killed" \ - "cannot execute binary file"; do - if [[ "$runtime_output" == *"$error"* ]]; then - runtime_error=$(echo "${runtime_output#*: }" | tr -d '\n') - break - fi - done - - local total_assertions - total_assertions="$(state::calculate_total_assertions "$test_execution_result")" - - local end_time duration_ns duration - end_time=$(clock::now) - duration_ns=$(math::calculate "($end_time - $start_time) ") - duration=$(math::calculate "$duration_ns / 1000000") - - if [[ -n $runtime_error ]]; then - state::add_tests_failed - console_results::print_error_test "$function_name" "$runtime_error" - logger::test_failed "$test_file" "$function_name" "$duration" "$total_assertions" - runner::write_failure_result_output "$test_file" "$runtime_error" - return - fi - - if [[ "$current_assertions_failed" != "$(state::get_assertions_failed)" ]]; then - state::add_tests_failed - logger::test_failed "$test_file" "$function_name" "$duration" "$total_assertions" - runner::write_failure_result_output "$test_file" "$subshell_output" - if env::is_stop_on_failure_enabled; then - exit 1 - fi - return - fi - - if [[ "$current_assertions_snapshot" != "$(state::get_assertions_snapshot)" ]]; then - state::add_tests_snapshot - console_results::print_snapshot_test "$function_name" - logger::test_snapshot "$test_file" "$function_name" "$duration" "$total_assertions" - return - fi - - if [[ "$current_assertions_incomplete" != "$(state::get_assertions_incomplete)" ]]; then - state::add_tests_incomplete - logger::test_incomplete "$test_file" "$function_name" "$duration" "$total_assertions" - return - fi - - if [[ "$current_assertions_skipped" != "$(state::get_assertions_skipped)" ]]; then - state::add_tests_skipped - logger::test_skipped "$test_file" "$function_name" "$duration" "$total_assertions" - return - fi - - local label - label="$(helper::normalize_test_function_name "$function_name")" - - console_results::print_successful_test "${label}" "$duration" "$@" - state::add_tests_passed - logger::test_passed "$test_file" "$function_name" "$duration" "$total_assertions" -## -} - function parallel::aggregate_test_results() { + local temp_dir_parallel_test_suite=$1 + local total_failed=0 local total_passed=0 local total_skipped=0 local total_incomplete=0 local total_snapshot=0 - for script_dir in "$TEMP_DIR_PARALLEL_TEST_SUITE"/*; do + for script_dir in "$temp_dir_parallel_test_suite"/*; do for result_file in "$script_dir"/*.result; do while IFS= read -r line; do # Extract assertion counts from the result lines using sed @@ -224,11 +34,6 @@ function parallel::aggregate_test_results() { total_snapshot=$((total_snapshot + snapshot)) done < "$result_file" - # Check and update test state based on the parsed results - if [ "$passed" -gt 0 ]; then - state::add_tests_passed - fi - if [ "$failed" -gt 0 ]; then state::add_tests_failed fi @@ -244,6 +49,10 @@ function parallel::aggregate_test_results() { if [ "$snapshot" -gt 0 ]; then state::add_tests_snapshot fi + + if [ "$passed" -gt 0 ]; then + state::add_tests_passed + fi done done @@ -253,40 +62,3 @@ function parallel::aggregate_test_results() { export _ASSERTIONS_INCOMPLETE=$total_incomplete export _ASSERTIONS_SNAPSHOT=$total_snapshot } - - -function parallel::write_failure_result_output() { - local test_file=$1 - local error_msg=$2 - - echo -e "$(state::get_tests_failed)) $test_file\n$error_msg" >> "$FAILURES_OUTPUT_PATH" -} - -function parallel::run_set_up() { - helper::execute_function_if_exists 'set_up' -} - -function parallel::run_set_up_before_script() { - helper::execute_function_if_exists 'set_up_before_script' -} - -function parallel::run_tear_down() { - helper::execute_function_if_exists 'tear_down' -} - -function parallel::clear_mocks() { - for i in "${!MOCKED_FUNCTIONS[@]}"; do - unmock "${MOCKED_FUNCTIONS[$i]}" - done -} - -function parallel::run_tear_down_after_script() { - helper::execute_function_if_exists 'tear_down_after_script' -} - -function parallel::clean_set_up_and_tear_down_after_script() { - helper::unset_if_exists 'set_up' - helper::unset_if_exists 'tear_down' - helper::unset_if_exists 'set_up_before_script' - helper::unset_if_exists 'tear_down_after_script' -} diff --git a/src/runner.sh b/src/runner.sh index d2527812..3b3deb13 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -20,7 +20,7 @@ function runner::load_test_files() { runner::run_set_up_before_script if env::is_parallel_run_enabled; then - parallel::call_test_functions "$test_file" "$filter" & + runner::call_test_functions "$test_file" "$filter" & pids+=($!) else runner::call_test_functions "$test_file" "$filter" @@ -35,7 +35,7 @@ function runner::load_test_files() { wait "$pid" || echo "Test with PID $pid failed" done - parallel::aggregate_test_results + parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE" fi } @@ -189,6 +189,13 @@ function runner::run_test() { # Closes FD 3, which was used temporarily to hold the original stdout. exec 3>&- + if env::is_parallel_run_enabled; then + # shellcheck disable=SC2155 + local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" + mkdir -p "$test_suite_dir" + echo "$test_execution_result" > "${test_suite_dir}/${function_name}.result" + fi + runner::parse_execution_result "$test_execution_result" local subshell_output From 18b72e5491e98b34400e196b2ec0bcfc7d4ec35b Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 22:35:20 +0200 Subject: [PATCH 07/31] feat: add trap main::cleanup SIGINT --- src/main.sh | 10 ++++++++++ src/runner.sh | 8 ++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main.sh b/src/main.sh index 8ffad17d..f5a2bd7b 100644 --- a/src/main.sh +++ b/src/main.sh @@ -15,6 +15,9 @@ function main::exec_tests() { exit 1 fi + # Trap SIGINT (Ctrl-C) and call the cleanup function + trap main::cleanup SIGINT + console_header::print_version_with_env "$filter" "${test_files[@]}" runner::load_test_files "$filter" "${test_files[@]}" if env::is_parallel_run_enabled; then @@ -38,6 +41,13 @@ function main::exec_tests() { exit $exit_code } +function main::cleanup() { + printf "%sCaught Ctrl-C, killing all child processes...%s\n" "${_COLOR_FAILED}" "${_COLOR_DEFAULT}" + # Kill all child processes of this script + pkill -P $$ + exit 1 +} + function main::exec_assert() { local original_assert_fn=$1 local args=("${@:2}") diff --git a/src/runner.sh b/src/runner.sh index 3b3deb13..9e158cdf 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -190,10 +190,10 @@ function runner::run_test() { exec 3>&- if env::is_parallel_run_enabled; then - # shellcheck disable=SC2155 - local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" - mkdir -p "$test_suite_dir" - echo "$test_execution_result" > "${test_suite_dir}/${function_name}.result" + # shellcheck disable=SC2155 + local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" + mkdir -p "$test_suite_dir" + echo "$test_execution_result" > "${test_suite_dir}/${function_name}.result" fi runner::parse_execution_result "$test_execution_result" From dfd8967bc2f3ec3e00e1cb86c4fcac1aab99535a Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 23:26:28 +0200 Subject: [PATCH 08/31] feat: add --no-parallel the subshells within acceptance tests --- bashunit | 3 +++ src/parallel.sh | 10 +++++----- .../acceptance/bashunit_execution_error_test.sh | 6 +++--- tests/acceptance/bashunit_fail_test.sh | 16 ++++++++-------- .../bashunit_find_tests_command_line_test.sh | 8 ++++---- tests/acceptance/bashunit_log_junit_test.sh | 4 ++-- tests/acceptance/bashunit_pass_test.sh | 16 ++++++++-------- tests/acceptance/bashunit_path_test.sh | 16 ++++++++-------- tests/acceptance/bashunit_report_html_test.sh | 4 ++-- .../acceptance/bashunit_stop_on_failure_test.sh | 12 ++++++------ tests/acceptance/bashunit_test.sh | 8 ++++---- 11 files changed, 53 insertions(+), 50 deletions(-) diff --git a/bashunit b/bashunit index 0b8127e9..3f302505 100755 --- a/bashunit +++ b/bashunit @@ -67,6 +67,9 @@ while [[ $# -gt 0 ]]; do -p|--parallel) export BASHUNIT_PARALLEL_RUN=true ;; + --no-parallel) + export BASHUNIT_PARALLEL_RUN=false + ;; -e|--env|--load) # shellcheck disable=SC1090 source "$2" diff --git a/src/parallel.sh b/src/parallel.sh index 39631f0b..14e606ce 100755 --- a/src/parallel.sh +++ b/src/parallel.sh @@ -34,23 +34,23 @@ function parallel::aggregate_test_results() { total_snapshot=$((total_snapshot + snapshot)) done < "$result_file" - if [ "$failed" -gt 0 ]; then + if [ "${failed:-0}" -gt 0 ]; then state::add_tests_failed fi - if [ "$skipped" -gt 0 ]; then + if [ "${skipped:-0}" -gt 0 ]; then state::add_tests_skipped fi - if [ "$incomplete" -gt 0 ]; then + if [ "${incomplete:-0}" -gt 0 ]; then state::add_tests_incomplete fi - if [ "$snapshot" -gt 0 ]; then + if [ "${snapshot:-0}" -gt 0 ]; then state::add_tests_snapshot fi - if [ "$passed" -gt 0 ]; then + if [ "${passed:-0}" -gt 0 ]; then state::add_tests_passed fi done diff --git a/tests/acceptance/bashunit_execution_error_test.sh b/tests/acceptance/bashunit_execution_error_test.sh index 929bbedf..3f95c002 100644 --- a/tests/acceptance/bashunit_execution_error_test.sh +++ b/tests/acceptance/bashunit_execution_error_test.sh @@ -53,7 +53,7 @@ function test_bashunit_when_a_execution_error() { todo "Add snapshots with regex to assert this test (part of the error message is localized)" todo "Add snapshots with simple/verbose modes as in bashunit_pass_test and bashunit_fail_test" - assert_contains "$fixture_start" "$(./bashunit --env "$TEST_ENV_FILE" "$test_file")" - assert_contains "$fixture_end" "$(./bashunit --env "$TEST_ENV_FILE" "$test_file")" - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE" "$test_file")" + assert_contains "$fixture_start" "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file")" + assert_contains "$fixture_end" "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file")" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file")" } diff --git a/tests/acceptance/bashunit_fail_test.sh b/tests/acceptance/bashunit_fail_test.sh index d881aaf7..08b23fec 100644 --- a/tests/acceptance/bashunit_fail_test.sh +++ b/tests/acceptance/bashunit_fail_test.sh @@ -9,15 +9,15 @@ function set_up_before_script() { function test_bashunit_when_a_test_fail_verbose_output_env() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_a_test_fail.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" "$test_file")" - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE" "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file")" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file")" } function test_bashunit_when_a_test_fail_verbose_output_option() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_a_test_fail.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_SIMPLE" "$test_file" --verbose)" - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE_SIMPLE" "$test_file" --verbose)" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_SIMPLE" "$test_file" --verbose)" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_SIMPLE" "$test_file" --verbose)" } function test_different_verbose_snapshots_matches() { @@ -32,9 +32,9 @@ function test_bashunit_when_a_test_fail_simple_output_env() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_a_test_fail.sh # shellcheck disable=SC2317 - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_SIMPLE" "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_SIMPLE" "$test_file")" # shellcheck disable=SC2317 - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE_SIMPLE" "$test_file")" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_SIMPLE" "$test_file")" } function test_bashunit_when_a_test_fail_simple_output_option() { @@ -45,9 +45,9 @@ function test_bashunit_when_a_test_fail_simple_output_option() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_a_test_fail.sh # shellcheck disable=SC2317 - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" "$test_file" --simple)" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file" --simple)" # shellcheck disable=SC2317 - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE" "$test_file" --simple)" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file" --simple)" } function test_different_simple_snapshots_matches() { diff --git a/tests/acceptance/bashunit_find_tests_command_line_test.sh b/tests/acceptance/bashunit_find_tests_command_line_test.sh index fee87666..3162fc8e 100644 --- a/tests/acceptance/bashunit_find_tests_command_line_test.sh +++ b/tests/acceptance/bashunit_find_tests_command_line_test.sh @@ -8,23 +8,23 @@ function set_up_before_script() { function test_all_tests_files_within_a_directory() { local path="./tests/acceptance/fixtures" - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" "$path")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$path")" } function test_all_tests_files_within_a_file() { local path="./tests/acceptance/fixtures/tests_path/a_test.sh" - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" "$path")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$path")" } function test_all_tests_files_with_wildcard() { local path='./tests/acceptance/fixtures/tests_path/*' - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" "$path")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$path")" } function test_error_when_no_tests_found() { local path="./non_existing_path" - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE" "$path")" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$path")" } diff --git a/tests/acceptance/bashunit_log_junit_test.sh b/tests/acceptance/bashunit_log_junit_test.sh index 9bbcf151..45966984 100644 --- a/tests/acceptance/bashunit_log_junit_test.sh +++ b/tests/acceptance/bashunit_log_junit_test.sh @@ -8,7 +8,7 @@ function set_up_before_script() { function test_bashunit_when_log_junit_option() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_log_junit.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" --log-junit custom.xml "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --log-junit custom.xml "$test_file")" assert_file_exists custom.xml rm custom.xml } @@ -16,7 +16,7 @@ function test_bashunit_when_log_junit_option() { function test_bashunit_when_log_junit_env() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_log_junit.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_BASHUNIT_LOG_JUNIT" "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_BASHUNIT_LOG_JUNIT" "$test_file")" assert_file_exists log-junit.xml rm log-junit.xml } diff --git a/tests/acceptance/bashunit_pass_test.sh b/tests/acceptance/bashunit_pass_test.sh index 1234f861..4d71a6f3 100644 --- a/tests/acceptance/bashunit_pass_test.sh +++ b/tests/acceptance/bashunit_pass_test.sh @@ -9,15 +9,15 @@ function set_up_before_script() { function test_bashunit_when_a_test_passes_verbose_output_env() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_a_test_passes.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" "$test_file")" - assert_successful_code "$(./bashunit --env "$TEST_ENV_FILE" "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file")" + assert_successful_code "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file")" } function test_bashunit_when_a_test_passes_verbose_output_option() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_a_test_passes.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_SIMPLE" "$test_file" --verbose)" - assert_successful_code "$(./bashunit --env "$TEST_ENV_FILE_SIMPLE" "$test_file" --verbose)" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_SIMPLE" "$test_file" --verbose)" + assert_successful_code "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_SIMPLE" "$test_file" --verbose)" } function test_different_verbose_snapshots_matches() { @@ -27,15 +27,15 @@ function test_different_verbose_snapshots_matches() { function test_bashunit_when_a_test_passes_simple_output_env() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_a_test_passes.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_SIMPLE" "$test_file")" - assert_successful_code "$(./bashunit --env "$TEST_ENV_FILE_SIMPLE" "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_SIMPLE" "$test_file")" + assert_successful_code "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_SIMPLE" "$test_file")" } function test_bashunit_when_a_test_passes_simple_output_option() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_a_test_passes.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" "$test_file" --simple)" - assert_successful_code "$(./bashunit --env "$TEST_ENV_FILE" "$test_file" --simple)" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file" --simple)" + assert_successful_code "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file" --simple)" } function test_different_simple_snapshots_matches() { diff --git a/tests/acceptance/bashunit_path_test.sh b/tests/acceptance/bashunit_path_test.sh index 856a158d..e5ab4513 100644 --- a/tests/acceptance/bashunit_path_test.sh +++ b/tests/acceptance/bashunit_path_test.sh @@ -7,25 +7,25 @@ function set_up_before_script() { } function test_bashunit_without_path_env_nor_argument() { - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE")" - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE")" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE")" } function test_bashunit_with_argument_path() { local path="tests/acceptance/fixtures/tests_path" - assert_match_snapshot "$(./bashunit "$path" --env "$TEST_ENV_FILE")" - assert_successful_code "$(./bashunit "$path" --env "$TEST_ENV_FILE")" + assert_match_snapshot "$(./bashunit --no-parallel "$path" --env "$TEST_ENV_FILE")" + assert_successful_code "$(./bashunit --no-parallel "$path" --env "$TEST_ENV_FILE")" } function test_bashunit_with_env_default_path() { - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_WITH_PATH")" - assert_successful_code "$(./bashunit --env "$TEST_ENV_FILE_WITH_PATH")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_WITH_PATH")" + assert_successful_code "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_WITH_PATH")" } function test_bashunit_argument_overloads_default_path() { local path="tests/acceptance/fixtures/wrong_path" - assert_match_snapshot "$(./bashunit "$path" --env "$TEST_ENV_FILE_WITH_PATH")" - assert_general_error "$(./bashunit "$path" --env "$TEST_ENV_FILE_WITH_PATH")" + assert_match_snapshot "$(./bashunit --no-parallel "$path" --env "$TEST_ENV_FILE_WITH_PATH")" + assert_general_error "$(./bashunit --no-parallel "$path" --env "$TEST_ENV_FILE_WITH_PATH")" } diff --git a/tests/acceptance/bashunit_report_html_test.sh b/tests/acceptance/bashunit_report_html_test.sh index 2cee8a84..eecb9c15 100644 --- a/tests/acceptance/bashunit_report_html_test.sh +++ b/tests/acceptance/bashunit_report_html_test.sh @@ -9,7 +9,7 @@ function set_up_before_script() { function test_bashunit_when_report_html_option() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_report_html.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" --report-html custom.html "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --report-html custom.html "$test_file")" assert_file_exists custom.html if [[ -f custom.html ]]; then @@ -20,7 +20,7 @@ function test_bashunit_when_report_html_option() { function test_bashunit_when_report_html_env() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_report_html.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_REPORT_HTML" "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_REPORT_HTML" "$test_file")" assert_file_exists report.html if [[ -f custom.html ]]; then diff --git a/tests/acceptance/bashunit_stop_on_failure_test.sh b/tests/acceptance/bashunit_stop_on_failure_test.sh index 9a6a5923..fa24ab32 100644 --- a/tests/acceptance/bashunit_stop_on_failure_test.sh +++ b/tests/acceptance/bashunit_stop_on_failure_test.sh @@ -9,15 +9,15 @@ function set_up_before_script() { function test_bashunit_when_stop_on_failure_option() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" --stop-on-failure "$test_file")" - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE" --stop-on-failure "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --stop-on-failure "$test_file")" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --stop-on-failure "$test_file")" } function test_bashunit_when_stop_on_failure_env() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_STOP_ON_FAILURE" "$test_file")" - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE_STOP_ON_FAILURE" "$test_file")" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_STOP_ON_FAILURE" "$test_file")" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_STOP_ON_FAILURE" "$test_file")" } function test_different_snapshots_matches() { @@ -32,7 +32,7 @@ function test_bashunit_when_stop_on_failure_env_simple_output() { local test_file=./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh # shellcheck disable=SC2317 - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE_STOP_ON_FAILURE" "$test_file" --simple)" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_STOP_ON_FAILURE" "$test_file" --simple)" # shellcheck disable=SC2317 - assert_general_error "$(./bashunit --env "$TEST_ENV_FILE_STOP_ON_FAILURE" "$test_file" --simple)" + assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE_STOP_ON_FAILURE" "$test_file" --simple)" } diff --git a/tests/acceptance/bashunit_test.sh b/tests/acceptance/bashunit_test.sh index 81b96dc8..4c37f874 100644 --- a/tests/acceptance/bashunit_test.sh +++ b/tests/acceptance/bashunit_test.sh @@ -10,11 +10,11 @@ function test_bashunit_should_display_version() { fixture=$(printf "\e[1m\e[32mbashunit\e[0m - %s" "$BASHUNIT_VERSION") todo "Add snapshots with regex to assert this test (part of the output changes every version)" - assert_contains "$fixture" "$(./bashunit --env "$TEST_ENV_FILE" --version)" - assert_successful_code "$(./bashunit --env "$TEST_ENV_FILE" --version)" + assert_contains "$fixture" "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --version)" + assert_successful_code "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --version)" } function test_bashunit_should_display_help() { - assert_match_snapshot "$(./bashunit --env "$TEST_ENV_FILE" --help)" - assert_successful_code "$(./bashunit --env "$TEST_ENV_FILE" --help)" + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --help)" + assert_successful_code "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --help)" } From d5536a97821f7585f442ec11bd636faa44f30f59 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 23:34:33 +0200 Subject: [PATCH 09/31] feat: do not show Running "script" when is_parallel --- src/runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runner.sh b/src/runner.sh index 9e158cdf..7bef5d51 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -72,7 +72,7 @@ function runner::call_test_functions() { functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions")) if [[ "${#functions_to_run[@]}" -gt 0 ]]; then - if ! env::is_simple_output_enabled; then + if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then echo "Running $script" fi From 80fa5ef6aa4a68015d039d1fbdfe6f5d33ddc039 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 23:37:13 +0200 Subject: [PATCH 10/31] tests: add parallel running to CI --- .github/workflows/tests.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 53bcbf25..5a2c068c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -74,3 +74,29 @@ jobs: - name: Run Tests run: | ./bashunit -e tests/bootstrap.sh --simple tests/ + + simple-output-parallel: + name: "Simple output in parallel" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Tests + run: | + ./bashunit -e tests/bootstrap.sh --parallel --simple tests/ + + extended-output-parallel: + name: "Extended output in parallel" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Tests + run: | + ./bashunit -e tests/bootstrap.sh --parallel tests/ From fd871364e63eb9c255d34898748fc72e832c7bfd Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 23:44:00 +0200 Subject: [PATCH 11/31] fix: bashunit_direct_fn_call_test using --no-parallel on subshells --- tests/acceptance/bashunit_direct_fn_call_test.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/acceptance/bashunit_direct_fn_call_test.sh b/tests/acceptance/bashunit_direct_fn_call_test.sh index e0d14803..99ae8cdf 100644 --- a/tests/acceptance/bashunit_direct_fn_call_test.sh +++ b/tests/acceptance/bashunit_direct_fn_call_test.sh @@ -78,20 +78,20 @@ function test_bashunit_direct_fn_call_failure() { local expected="foo" local actual="bar" - assert_match_snapshot "$(./bashunit -a assert_same --env "$TEST_ENV_FILE" "$expected" $actual 2>&1)" - assert_general_error "$(./bashunit -a assert_same --env "$TEST_ENV_FILE" "$expected" $actual)" + assert_match_snapshot "$(./bashunit --no-parallel -a assert_same --env "$TEST_ENV_FILE" "$expected" $actual 2>&1)" + assert_general_error "$(./bashunit --no-parallel -a assert_same --env "$TEST_ENV_FILE" "$expected" $actual)" } function test_bashunit_direct_fn_call_non_existing_fn() { - assert_match_snapshot "$(./bashunit -a non_existing_fn --env "$TEST_ENV_FILE" 2>&1)" - assert_command_not_found "$(./bashunit -a non_existing_fn --env "$TEST_ENV_FILE")" + assert_match_snapshot "$(./bashunit --no-parallel -a non_existing_fn --env "$TEST_ENV_FILE" 2>&1)" + assert_command_not_found "$(./bashunit --no-parallel -a non_existing_fn --env "$TEST_ENV_FILE")" } # shellcheck disable=SC2155 function test_bashunit_assert_exit_code_successful_with_inner_func() { local temp=$(temp_file) # shellcheck disable=SC2116 - local output="$(./bashunit -a exit_code "0" "$(echo "unknown command")" 2> "$temp")" + local output="$(./bashunit --no-parallel -a exit_code "0" "$(echo "unknown command")" 2> "$temp")" assert_empty "$output" assert_file_contains "$temp" "Command not found: unknown command" @@ -101,7 +101,7 @@ function test_bashunit_assert_exit_code_successful_with_inner_func() { function test_bashunit_assert_exit_code_error_with_inner_func() { local temp=$(temp_file) # shellcheck disable=SC2116 - local output="$(./bashunit -a exit_code "1" "$(echo "unknown command")" 2> "$temp")" + local output="$(./bashunit --no-parallel -a exit_code "1" "$(echo "unknown command")" 2> "$temp")" assert_empty "$output" @@ -122,7 +122,7 @@ function test_bashunit_assert_exit_code_str_general_error() { # shellcheck disable=SC2155 function test_bashunit_assert_exit_code_str_successful_but_exit_code_error() { local temp=$(temp_file) - local output="$(./bashunit -a exit_code "1" "echo something to stdout" 2> "$temp")" + local output="$(./bashunit --no-parallel -a exit_code "1" "echo something to stdout" 2> "$temp")" assert_same "something to stdout" "$output" @@ -133,7 +133,7 @@ function test_bashunit_assert_exit_code_str_successful_but_exit_code_error() { # shellcheck disable=SC2155 function test_bashunit_assert_exit_code_str_successful_and_exit_code_ok() { local temp=$(temp_file) - local output="$(./bashunit -a exit_code "0" "echo something to stdout" 2> "$temp")" + local output="$(./bashunit --no-parallel -a exit_code "0" "echo something to stdout" 2> "$temp")" assert_same "something to stdout" "$output" assert_empty "$(cat "$temp")" From be2ba64feede22c650cc9ef4ac938615b5269dc7 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Wed, 9 Oct 2024 23:52:29 +0200 Subject: [PATCH 12/31] fix: cleanup_temp_files from different contexts when running in parallel --- src/globals.sh | 10 +++++----- tests/unit/directory_test.sh | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/globals.sh b/src/globals.sh index 8bbb5ca8..8b4f3250 100644 --- a/src/globals.sh +++ b/src/globals.sh @@ -25,17 +25,17 @@ function random_str() { } function temp_file() { - mkdir -p /tmp/bashunit-tmp && chmod -R 777 /tmp/bashunit-tmp - mktemp /tmp/bashunit-tmp/bashunit.XXXXXXX + mkdir -p /tmp/bashunit/tmp && chmod -R 777 /tmp/bashunit/tmp + mktemp /tmp/bashunit/tmp/bashunit.XXXXXXX } function temp_dir() { - mkdir -p /tmp/bashunit-tmp && chmod -R 777 /tmp/bashunit-tmp - mktemp -d /tmp/bashunit-tmp/bashunit.XXXXXXX + mkdir -p /tmp/bashunit/tmp && chmod -R 777 /tmp/bashunit/tmp + mktemp -d /tmp/bashunit/tmp/bashunit.XXXXXXX } function cleanup_temp_files() { - rm -rf /tmp/bashunit-tmp/* + rm -rf /tmp/bashunit/tmp/* } # shellcheck disable=SC2145 diff --git a/tests/unit/directory_test.sh b/tests/unit/directory_test.sh index 591b9d15..a6c5c581 100644 --- a/tests/unit/directory_test.sh +++ b/tests/unit/directory_test.sh @@ -65,7 +65,7 @@ function test_unsuccessful_assert_is_directory_when_a_file_is_given() { } function test_successful_assert_is_directory_empty() { - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) assert_empty "$(assert_is_directory_empty "$a_directory")" } @@ -86,7 +86,7 @@ function test_successful_assert_is_directory_not_empty() { } function test_unsuccessful_assert_is_directory_not_empty() { - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) assert_same\ "$(console_results::print_failed_test \ @@ -95,7 +95,7 @@ function test_unsuccessful_assert_is_directory_not_empty() { } function test_successful_assert_is_directory_readable() { - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) assert_empty "$(assert_is_directory_readable "$a_directory")" } @@ -115,7 +115,7 @@ function test_unsuccessful_assert_is_directory_readable_without_execution_permis return fi - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) chmod a-x "$a_directory" assert_same\ @@ -130,7 +130,7 @@ function test_unsuccessful_assert_is_directory_readable_without_read_permission( return fi - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) chmod a-r "$a_directory" assert_same\ @@ -145,7 +145,7 @@ function test_successful_assert_is_directory_not_readable_without_read_permissio return fi - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) chmod a-r "$a_directory" assert_empty "$(assert_is_directory_not_readable "$a_directory")" @@ -156,14 +156,14 @@ function test_successful_assert_is_directory_not_readable_without_execution_perm return fi - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) chmod a-x "$a_directory" assert_empty "$(assert_is_directory_not_readable "$a_directory")" } function test_unsuccessful_assert_is_directory_not_readable() { - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) assert_same\ "$(console_results::print_failed_test \ @@ -172,7 +172,7 @@ function test_unsuccessful_assert_is_directory_not_readable() { } function test_successful_assert_is_directory_writable() { - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) assert_empty "$(assert_is_directory_writable "$a_directory")" } @@ -182,7 +182,7 @@ function test_unsuccessful_assert_is_directory_writable() { return fi - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) chmod a-w "$a_directory" assert_same\ @@ -206,14 +206,14 @@ function test_successful_assert_is_directory_not_writable() { return fi - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) chmod a-w "$a_directory" assert_empty "$(assert_is_directory_not_writable "$a_directory")" } function test_unsuccessful_assert_is_directory_not_writable() { - local a_directory=$(temp_dir) + local a_directory=$(mktemp -d) assert_same\ "$(console_results::print_failed_test\ From 17b5ba2d7edcdc510e4ad50289a77f9c625bb3ed Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Thu, 10 Oct 2024 00:20:41 +0200 Subject: [PATCH 13/31] feat: add spinner loading while parallel::aggregate_test_results --- CHANGELOG.md | 1 + src/runner.sh | 24 +++++++++++++++---- .../bashunit_direct_fn_call_test.sh | 8 +++---- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d50843c..348f7477 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased](https://github.com/TypedDevs/bashunit/compare/0.17.0...main) +- Added `-p|--parallel` to enable running tests in parallel - Added `assert_file_contains` and `assert_file_not_contains` - Added `assert_true` and `assert_false` - Added `BASHUNIT_LOG_PATH` diff --git a/src/runner.sh b/src/runner.sh index 7bef5d51..c624c470 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -29,16 +29,32 @@ function runner::load_test_files() { runner::clean_set_up_and_tear_down_after_script done - # Wait for all background processes to finish if env::is_parallel_run_enabled; then - for pid in "${pids[@]}"; do - wait "$pid" || echo "Test with PID $pid failed" - done + wait + + runner::spinner & + local spinner_pid=$! parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE" + + # Kill the spinner once the aggregation finishes + disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null + printf "\r " # Clear the spinner output fi } +function runner::spinner() { + printf "\n" + local delay=0.1 + local spin_chars="|/-\\" + while true; do + for ((i=0; i<${#spin_chars}; i++)); do + printf "\r%s" "${spin_chars:$i:1}" + sleep "$delay" + done + done +} + function runner::functions_for_script() { local script="$1" local all_function_names="$2" diff --git a/tests/acceptance/bashunit_direct_fn_call_test.sh b/tests/acceptance/bashunit_direct_fn_call_test.sh index 99ae8cdf..b6abdfcb 100644 --- a/tests/acceptance/bashunit_direct_fn_call_test.sh +++ b/tests/acceptance/bashunit_direct_fn_call_test.sh @@ -89,7 +89,7 @@ function test_bashunit_direct_fn_call_non_existing_fn() { # shellcheck disable=SC2155 function test_bashunit_assert_exit_code_successful_with_inner_func() { - local temp=$(temp_file) + local temp=$(mktemp) # shellcheck disable=SC2116 local output="$(./bashunit --no-parallel -a exit_code "0" "$(echo "unknown command")" 2> "$temp")" @@ -99,7 +99,7 @@ function test_bashunit_assert_exit_code_successful_with_inner_func() { # shellcheck disable=SC2155 function test_bashunit_assert_exit_code_error_with_inner_func() { - local temp=$(temp_file) + local temp=$(mktemp) # shellcheck disable=SC2116 local output="$(./bashunit --no-parallel -a exit_code "1" "$(echo "unknown command")" 2> "$temp")" @@ -121,7 +121,7 @@ function test_bashunit_assert_exit_code_str_general_error() { # shellcheck disable=SC2155 function test_bashunit_assert_exit_code_str_successful_but_exit_code_error() { - local temp=$(temp_file) + local temp=$(mktemp) local output="$(./bashunit --no-parallel -a exit_code "1" "echo something to stdout" 2> "$temp")" assert_same "something to stdout" "$output" @@ -132,7 +132,7 @@ function test_bashunit_assert_exit_code_str_successful_but_exit_code_error() { # shellcheck disable=SC2155 function test_bashunit_assert_exit_code_str_successful_and_exit_code_ok() { - local temp=$(temp_file) + local temp=$(mktemp) local output="$(./bashunit --no-parallel -a exit_code "0" "echo something to stdout" 2> "$temp")" assert_same "something to stdout" "$output" From 88e17a299a030da55915322a9dc63131f1d75ecd Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Thu, 10 Oct 2024 00:33:28 +0200 Subject: [PATCH 14/31] docs: add Parallel docs to command and website --- docs/command-line.md | 20 +++++++++++++++++++ src/console_header.sh | 6 ++++++ ...nit_without_path_env_nor_argument.snapshot | 6 ++++++ ...test_bashunit_should_display_help.snapshot | 6 ++++++ 4 files changed, 38 insertions(+) diff --git a/docs/command-line.md b/docs/command-line.md index f84c74c9..355b51a9 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -99,6 +99,26 @@ Creates a report XML file that follows the JUnit XML format and contains informa ``` ::: +## Parallel + +> `bashunit -p|--parallel` + +bashunit provides an option to run each test in a separate child process, allowing you to parallelize the test execution and potentially speed up the testing process. When running in parallel mode, the execution order of tests is randomized. + +::: code-group +```bash [Example] +./bashunit ./tests --parallel +``` +::: + +This runs the tests in child processes with randomized execution, which may improve overall testing speed, especially for larger test suites. + +### Disabling Parallel Testing + +> `bashunit --no-parallel` + +If parallel testing is enabled by default or within a script, you can disable it using the --no-parallel option. This is useful if you need to run tests in sequence or if parallel execution is causing issues during debugging. + ## Report > `bashunit -r|--report-html ` diff --git a/src/console_header.sh b/src/console_header.sh index 01dc2146..d7ff1472 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -76,6 +76,12 @@ Options: -l|--log-junit Create a report JUnit XML file that contains information about the test results. + -p|--parallel + Run each test in child process, randomizing the tests execution order. + + --no-parallel + Disable the --parallel option. Util to disable parallel tests from within another test. + -r|--report-html Create a report HTML file that contains information about the test results. diff --git a/tests/acceptance/snapshots/bashunit_path_test_sh.test_bashunit_without_path_env_nor_argument.snapshot b/tests/acceptance/snapshots/bashunit_path_test_sh.test_bashunit_without_path_env_nor_argument.snapshot index cfc0b727..c0945093 100644 --- a/tests/acceptance/snapshots/bashunit_path_test_sh.test_bashunit_without_path_env_nor_argument.snapshot +++ b/tests/acceptance/snapshots/bashunit_path_test_sh.test_bashunit_without_path_env_nor_argument.snapshot @@ -23,6 +23,12 @@ Options: -l|--log-junit Create a report JUnit XML file that contains information about the test results. + -p|--parallel + Run each test in child process, randomizing the tests execution order. + + --no-parallel + Disable the --parallel option. Util to disable parallel tests from within another test. + -r|--report-html Create a report HTML file that contains information about the test results. diff --git a/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_help.snapshot b/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_help.snapshot index 6566ab4d..76737976 100644 --- a/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_help.snapshot +++ b/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_help.snapshot @@ -22,6 +22,12 @@ Options: -l|--log-junit Create a report JUnit XML file that contains information about the test results. + -p|--parallel + Run each test in child process, randomizing the tests execution order. + + --no-parallel + Disable the --parallel option. Util to disable parallel tests from within another test. + -r|--report-html Create a report HTML file that contains information about the test results. From 5515320291e64feda1448d618a6df99c93b9bf29 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Thu, 10 Oct 2024 00:42:46 +0200 Subject: [PATCH 15/31] fix: do not break new lines when rendering simple output in parallel --- src/state.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/state.sh b/src/state.sh index 1a31b23d..86ce427c 100644 --- a/src/state.sh +++ b/src/state.sh @@ -198,9 +198,13 @@ function state::print_line() { *) char="?" ;; esac - if (( _TOTAL_TESTS_COUNT % 50 == 0 )); then - printf "%s\n" "$char" + if env::is_parallel_run_enabled; then + printf "%s" "$char" else - printf "%s" "$char" + if (( _TOTAL_TESTS_COUNT % 50 == 0 )); then + printf "%s\n" "$char" + else + printf "%s" "$char" + fi fi } From 588d85e91bd39f40490c620ccce2237d4b29c834 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Thu, 10 Oct 2024 01:08:47 +0200 Subject: [PATCH 16/31] fix: test_result_file on parallel consider data providers --- src/runner.sh | 7 ++++++- tests/functional/provider_test.sh | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/runner.sh b/src/runner.sh index c624c470..b56a4b52 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -209,7 +209,12 @@ function runner::run_test() { # shellcheck disable=SC2155 local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" mkdir -p "$test_suite_dir" - echo "$test_execution_result" > "${test_suite_dir}/${function_name}.result" + + local test_result_file + test_result_file=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//') + test_result_file="${function_name}-${test_result_file}.result" + + echo "$test_execution_result" > "${test_suite_dir}/${test_result_file}" fi runner::parse_execution_result "$test_execution_result" diff --git a/tests/functional/provider_test.sh b/tests/functional/provider_test.sh index bfa0a211..8ba4f9fa 100644 --- a/tests/functional/provider_test.sh +++ b/tests/functional/provider_test.sh @@ -15,7 +15,13 @@ function test_multiple_values_from_data_provider() { function provide_multiples_values() { echo "aa" "bb" - echo "aa" "bb" +} + +# data_provider provide_single_values +function test_single_values_from_data_provider() { + local data="$1" + + assert_not_equals "zero" "$data" } function provide_single_values() { From 2aed30988deb34af97e16531038609844e9f2a07 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Thu, 10 Oct 2024 01:18:04 +0200 Subject: [PATCH 17/31] fix: add continue after counting each test state --- src/parallel.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/parallel.sh b/src/parallel.sh index 14e606ce..a7c9c4db 100755 --- a/src/parallel.sh +++ b/src/parallel.sh @@ -36,18 +36,22 @@ function parallel::aggregate_test_results() { if [ "${failed:-0}" -gt 0 ]; then state::add_tests_failed + continue fi - if [ "${skipped:-0}" -gt 0 ]; then - state::add_tests_skipped + if [ "${snapshot:-0}" -gt 0 ]; then + state::add_tests_snapshot + continue fi if [ "${incomplete:-0}" -gt 0 ]; then state::add_tests_incomplete + continue fi - if [ "${snapshot:-0}" -gt 0 ]; then - state::add_tests_snapshot + if [ "${skipped:-0}" -gt 0 ]; then + state::add_tests_skipped + continue fi if [ "${passed:-0}" -gt 0 ]; then From 3892c1091398786a70e0872e767a7c78c204009e Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Thu, 10 Oct 2024 01:30:07 +0200 Subject: [PATCH 18/31] fix: Check if the file exists, append a counter if necessary to ensure uniqueness --- src/parallel.sh | 4 +--- src/runner.sh | 12 ++++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/parallel.sh b/src/parallel.sh index a7c9c4db..3ea6bcf3 100755 --- a/src/parallel.sh +++ b/src/parallel.sh @@ -54,9 +54,7 @@ function parallel::aggregate_test_results() { continue fi - if [ "${passed:-0}" -gt 0 ]; then - state::add_tests_passed - fi + state::add_tests_passed done done diff --git a/src/runner.sh b/src/runner.sh index b56a4b52..1470da66 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -7,7 +7,7 @@ function runner::load_test_files() { local pids=() if env::is_parallel_run_enabled; then - rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE" + rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE" fi for test_file in "${files[@]}"; do @@ -214,7 +214,15 @@ function runner::run_test() { test_result_file=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//') test_result_file="${function_name}-${test_result_file}.result" - echo "$test_execution_result" > "${test_suite_dir}/${test_result_file}" + local unique_test_result_file="${test_suite_dir}/${test_result_file}" + local count=1 + + while [ -e "$unique_test_result_file" ]; do + unique_test_result_file="${test_suite_dir}/${test_result_file%.result}-$count.result" + count=$((count + 1)) + done + + echo "$test_execution_result" > "$unique_test_result_file" fi runner::parse_execution_result "$test_execution_result" From 8d314cd90c6501e214054fa9fd3cd8f0e3647f63 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Thu, 10 Oct 2024 01:53:52 +0200 Subject: [PATCH 19/31] feat: improve cleanup_temp_files on interruption ctrl+c --- src/main.sh | 4 ++-- src/runner.sh | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main.sh b/src/main.sh index f5a2bd7b..8a31df9e 100644 --- a/src/main.sh +++ b/src/main.sh @@ -37,14 +37,14 @@ function main::exec_tests() { fi cleanup_temp_files - exit $exit_code } function main::cleanup() { - printf "%sCaught Ctrl-C, killing all child processes...%s\n" "${_COLOR_FAILED}" "${_COLOR_DEFAULT}" + printf "%sCaught Ctrl-C, killing all child processes...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}" # Kill all child processes of this script pkill -P $$ + cleanup_temp_files exit 1 } diff --git a/src/runner.sh b/src/runner.sh index 1470da66..4eb33aed 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -20,7 +20,7 @@ function runner::load_test_files() { runner::run_set_up_before_script if env::is_parallel_run_enabled; then - runner::call_test_functions "$test_file" "$filter" & + runner::call_test_functions "$test_file" "$filter" 2>/dev/null & pids+=($!) else runner::call_test_functions "$test_file" "$filter" @@ -73,6 +73,8 @@ function run_test() { } function runner::call_test_functions() { + trap 'exit' SIGTERM + local script="$1" local filter="$2" local prefix="test" @@ -212,7 +214,11 @@ function runner::run_test() { local test_result_file test_result_file=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//') - test_result_file="${function_name}-${test_result_file}.result" + if [[ -z "$test_result_file" ]]; then + test_result_file="${function_name}.result" + else + test_result_file="${function_name}-${test_result_file}.result" + fi local unique_test_result_file="${test_suite_dir}/${test_result_file}" local count=1 From 6f8b0afa1b466197840566c6f23e181c9549b3df Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 18:01:48 +0200 Subject: [PATCH 20/31] refactor: simplify runner::run_test() --- src/runner.sh | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/src/runner.sh b/src/runner.sh index 4eb33aed..179a83b3 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -168,6 +168,7 @@ function runner::parse_execution_result() { ((_ASSERTIONS_SNAPSHOT += assertions_snapshot)) || true } +# shellcheck disable=SC2155 function runner::run_test() { local start_time start_time=$(clock::now) @@ -176,22 +177,17 @@ function runner::run_test() { shift local function_name="$1" shift - local current_assertions_failed - current_assertions_failed="$(state::get_assertions_failed)" - local current_assertions_snapshot - current_assertions_snapshot="$(state::get_assertions_snapshot)" - local current_assertions_incomplete - current_assertions_incomplete="$(state::get_assertions_incomplete)" - local current_assertions_skipped - current_assertions_skipped="$(state::get_assertions_skipped)" + local current_assertions_failed="$(state::get_assertions_failed)" + local current_assertions_snapshot="$(state::get_assertions_snapshot)" + local current_assertions_incomplete="$(state::get_assertions_incomplete)" + local current_assertions_skipped="$(state::get_assertions_skipped)" # (FD = File Descriptor) # Duplicate the current std-output (FD 1) and assigns it to FD 3. # This means that FD 3 now points to wherever the std-output was pointing. exec 3>&1 - local test_execution_result - test_execution_result=$( + local test_execution_result=$( state::initialize_assertions_count runner::run_set_up @@ -212,8 +208,7 @@ function runner::run_test() { local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" mkdir -p "$test_suite_dir" - local test_result_file - test_result_file=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//') + local test_result_file=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//') if [[ -z "$test_result_file" ]]; then test_result_file="${function_name}.result" else @@ -233,8 +228,7 @@ function runner::run_test() { runner::parse_execution_result "$test_execution_result" - local subshell_output - subshell_output=$(\ + local subshell_output=$(\ echo "$test_execution_result" |\ tail -n 1 |\ sed -E -e 's/.*##TEST_OUTPUT=(.*)##.*/\1/g' |\ @@ -251,8 +245,7 @@ function runner::run_test() { subshell_output=$line fi - local runtime_output - runtime_output="${test_execution_result%%##ASSERTIONS*}" + local runtime_output="${test_execution_result%%##ASSERTIONS*}" local runtime_error="" for error in "command not found" "unbound variable" "permission denied" \ @@ -267,13 +260,11 @@ function runner::run_test() { fi done - local total_assertions - total_assertions="$(state::calculate_total_assertions "$test_execution_result")" + local total_assertions="$(state::calculate_total_assertions "$test_execution_result")" - local end_time duration_ns duration - end_time=$(clock::now) - duration_ns=$(math::calculate "($end_time - $start_time) ") - duration=$(math::calculate "$duration_ns / 1000000") + local end_time=$(clock::now) + local duration_ns=$(math::calculate "($end_time - $start_time) ") + local duration=$(math::calculate "$duration_ns / 1000000") if [[ -n $runtime_error ]]; then state::add_tests_failed @@ -312,8 +303,7 @@ function runner::run_test() { return fi - local label - label="$(helper::normalize_test_function_name "$function_name")" + local label="$(helper::normalize_test_function_name "$function_name")" console_results::print_successful_test "${label}" "$duration" "$@" state::add_tests_passed From ecccc222a952833bfcc97a55b0c83973f6a4fd4d Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 18:02:51 +0200 Subject: [PATCH 21/31] feat: use --simple and --parallel on make test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ed321e2..c36e3f8c 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ test/list: @echo $(TEST_SCRIPTS) | tr ' ' '\n' test: $(TEST_SCRIPTS) - @./bashunit $(TEST_SCRIPTS) -e tests/bootstrap.sh + @./bashunit $(TEST_SCRIPTS) --simple --parallel test/watch: $(TEST_SCRIPTS) @./bashunit $(TEST_SCRIPTS) From dbb4351cd2392cebd7c279923c75e5cc71c9544a Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 18:16:11 +0200 Subject: [PATCH 22/31] docs: add parallel testing ADR --- adrs/adr-003-parallel-testing.md | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 adrs/adr-003-parallel-testing.md diff --git a/adrs/adr-003-parallel-testing.md b/adrs/adr-003-parallel-testing.md new file mode 100644 index 00000000..bc4bdebb --- /dev/null +++ b/adrs/adr-003-parallel-testing.md @@ -0,0 +1,50 @@ +# Title: Parallel testing + +* Status: accepted +* Authors: @Chemaclass +* Date: 2024-10-11 + +Technical Story: +- Pull Request: [TypedDevs/bashunit#358](https://github.com/TypedDevs/bashunit/pull/358) + +## Context and Problem Statement + +We aim to enhance testing performance by running tests in parallel processes while capturing and aggregating results effectively. + +## Considered Options + +- Implement parallel execution using subprocesses. +- Aggregate test results from temporary files. +- Use a spinner for user feedback during result aggregation. + +## Decision Outcome + +- Implemented parallel test execution using subprocesses. +- Each test creates a temporary directory to store results, later aggregated. + +### Positive Consequences + +- Reduced test execution time considerably. +- Clear feedback via a spinner during aggregation. + +### Negative Consequences + +- Potential complexity + - with handling temporary files during interruptions. + - in handling temporary files and managing subprocesses. + +## Technical Details + +When the `--parallel` flag is used, each test is run in its own subprocess by calling: + +> runner::call_test_functions "$test_file" "$filter" 2>/dev/null & + +Each test script creates a temporary directory and stores individual test results in temp files. +After all tests finish, the results are aggregated by traversing these directories and files. +This approach ensures isolation of test execution while improving performance by running tests concurrently. + +The aggregation (which collects all test outcomes into a final result set) is handled by the function: + +> parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE" + + From d2cdf30336a37c29d23e8a755d1506a4eead5fff Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 20:27:46 +0200 Subject: [PATCH 23/31] docs: add Parallel configuration --- .env.example | 1 + Makefile | 2 +- docs/command-line.md | 2 ++ docs/configuration.md | 9 +++++++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 6d4f11b3..574df050 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,7 @@ BASHUNIT_REPORT_HTML= BASHUNIT_LOAD_FILE= # Booleans +BASHUNIT_PARALLEL_RUN= BASHUNIT_SHOW_HEADER= BASHUNIT_HEADER_ASCII_ART= BASHUNIT_SIMPLE_OUTPUT= diff --git a/Makefile b/Makefile index c36e3f8c..a4605d0b 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ test/list: @echo $(TEST_SCRIPTS) | tr ' ' '\n' test: $(TEST_SCRIPTS) - @./bashunit $(TEST_SCRIPTS) --simple --parallel + @./bashunit $(TEST_SCRIPTS) test/watch: $(TEST_SCRIPTS) @./bashunit $(TEST_SCRIPTS) diff --git a/docs/command-line.md b/docs/command-line.md index 355b51a9..c228fc6c 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -113,6 +113,8 @@ bashunit provides an option to run each test in a separate child process, allowi This runs the tests in child processes with randomized execution, which may improve overall testing speed, especially for larger test suites. +You can use `BASHUNIT_PARALLEL_RUN` option in your [configuration](/configuration#parallel). + ### Disabling Parallel Testing > `bashunit --no-parallel` diff --git a/docs/configuration.md b/docs/configuration.md index f58f780c..32e409e3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -62,6 +62,15 @@ BASHUNIT_SIMPLE_OUTPUT=false ``` ::: +## Parallel + +> `BASHUNIT_PARALLEL_RUN=true|false` + +Runs the tests in child processes with randomized execution, which may improve overall testing speed, especially for larger test suites. + +Similar as using `-p|--parallel` option on the [command line](/command-line#parallel). + + ## Stop on failure > `BASHUNIT_STOP_ON_FAILURE=true|false` From 45782d940f812773735580315ca3802d9716c4d8 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 21:23:57 +0200 Subject: [PATCH 24/31] refactor: extract parse_result from runner --- src/globals.sh | 4 ++ src/runner.sh | 150 ++++++++++++++++++++++++++----------------------- 2 files changed, 85 insertions(+), 69 deletions(-) diff --git a/src/globals.sh b/src/globals.sh index 8b4f3250..eb5a76d2 100644 --- a/src/globals.sh +++ b/src/globals.sh @@ -15,6 +15,10 @@ function current_timestamp() { date +"%Y-%m-%d %H:%M:%S" } +function is_number() { + [[ -n "$1" && "$1" =~ ^-?[0-9]+$ ]] +} + function is_command_available() { command -v "$1" >/dev/null 2>&1 } diff --git a/src/runner.sh b/src/runner.sh index 179a83b3..8c9729ce 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -123,51 +123,6 @@ function runner::call_test_functions() { fi } -function runner::parse_execution_result() { - local execution_result=$1 - - local assertions_failed - assertions_failed=$(\ - echo "$execution_result" |\ - tail -n 1 |\ - sed -E -e 's/.*##ASSERTIONS_FAILED=([0-9]*)##.*/\1/g'\ - ) - - local assertions_passed - assertions_passed=$(\ - echo "$execution_result" |\ - tail -n 1 |\ - sed -E -e 's/.*##ASSERTIONS_PASSED=([0-9]*)##.*/\1/g'\ - ) - - local assertions_skipped - assertions_skipped=$(\ - echo "$execution_result" |\ - tail -n 1 |\ - sed -E -e 's/.*##ASSERTIONS_SKIPPED=([0-9]*)##.*/\1/g'\ - ) - - local assertions_incomplete - assertions_incomplete=$(\ - echo "$execution_result" |\ - tail -n 1 |\ - sed -E -e 's/.*##ASSERTIONS_INCOMPLETE=([0-9]*)##.*/\1/g'\ - ) - - local assertions_snapshot - assertions_snapshot=$(\ - echo "$execution_result" |\ - tail -n 1 |\ - sed -E -e 's/.*##ASSERTIONS_SNAPSHOT=([0-9]*)##.*/\1/g'\ - ) - - ((_ASSERTIONS_PASSED += assertions_passed)) || true - ((_ASSERTIONS_FAILED += assertions_failed)) || true - ((_ASSERTIONS_SKIPPED += assertions_skipped)) || true - ((_ASSERTIONS_INCOMPLETE += assertions_incomplete)) || true - ((_ASSERTIONS_SNAPSHOT += assertions_snapshot)) || true -} - # shellcheck disable=SC2155 function runner::run_test() { local start_time @@ -203,30 +158,7 @@ function runner::run_test() { # Closes FD 3, which was used temporarily to hold the original stdout. exec 3>&- - if env::is_parallel_run_enabled; then - # shellcheck disable=SC2155 - local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" - mkdir -p "$test_suite_dir" - - local test_result_file=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//') - if [[ -z "$test_result_file" ]]; then - test_result_file="${function_name}.result" - else - test_result_file="${function_name}-${test_result_file}.result" - fi - - local unique_test_result_file="${test_suite_dir}/${test_result_file}" - local count=1 - - while [ -e "$unique_test_result_file" ]; do - unique_test_result_file="${test_suite_dir}/${test_result_file%.result}-$count.result" - count=$((count + 1)) - done - - echo "$test_execution_result" > "$unique_test_result_file" - fi - - runner::parse_execution_result "$test_execution_result" + runner::parse_result "$test_execution_result" "$@" local subshell_output=$(\ echo "$test_execution_result" |\ @@ -310,6 +242,86 @@ function runner::run_test() { logger::test_passed "$test_file" "$function_name" "$duration" "$total_assertions" } +function runner::parse_result() { + local execution_result=$1 + shift + local args=("$@") + + if env::is_parallel_run_enabled; then + runner::parse_result_parallel "$execution_result" "${args[@]}" + else + runner::parse_result_sync "$execution_result" + fi +} + +# shellcheck disable=SC2155 +function runner::parse_result_parallel() { + local execution_result=$1 + shift + local args=("$@") + + local test_suite_dir="${TEMP_DIR_PARALLEL_TEST_SUITE}/$(basename "$test_file" .sh)" + mkdir -p "$test_suite_dir" + + local test_result_file=$(echo "${args[@]}" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//') + if [[ -z "$test_result_file" ]]; then + test_result_file="${function_name}.result" + else + test_result_file="${function_name}-${test_result_file}.result" + fi + + local unique_test_result_file="${test_suite_dir}/${test_result_file}" + local count=1 + + while [ -e "$unique_test_result_file" ]; do + unique_test_result_file="${test_suite_dir}/${test_result_file%.result}-$count.result" + count=$((count + 1)) + done + + echo "$execution_result" > "$unique_test_result_file" +} + +# shellcheck disable=SC2155 +function runner::parse_result_sync() { + local execution_result=$1 + + local assertions_failed=$(\ + echo "$execution_result" |\ + tail -n 1 |\ + sed -E -e 's/.*##ASSERTIONS_FAILED=([0-9]*)##.*/\1/g'\ + ) + + local assertions_passed=$(\ + echo "$execution_result" |\ + tail -n 1 |\ + sed -E -e 's/.*##ASSERTIONS_PASSED=([0-9]*)##.*/\1/g'\ + ) + + local assertions_skipped=$(\ + echo "$execution_result" |\ + tail -n 1 |\ + sed -E -e 's/.*##ASSERTIONS_SKIPPED=([0-9]*)##.*/\1/g'\ + ) + + local assertions_incomplete=$(\ + echo "$execution_result" |\ + tail -n 1 |\ + sed -E -e 's/.*##ASSERTIONS_INCOMPLETE=([0-9]*)##.*/\1/g'\ + ) + + local assertions_snapshot=$(\ + echo "$execution_result" |\ + tail -n 1 |\ + sed -E -e 's/.*##ASSERTIONS_SNAPSHOT=([0-9]*)##.*/\1/g'\ + ) + + ((_ASSERTIONS_PASSED += assertions_passed)) || true + ((_ASSERTIONS_FAILED += assertions_failed)) || true + ((_ASSERTIONS_SKIPPED += assertions_skipped)) || true + ((_ASSERTIONS_INCOMPLETE += assertions_incomplete)) || true + ((_ASSERTIONS_SNAPSHOT += assertions_snapshot)) || true +} + function runner::write_failure_result_output() { local test_file=$1 local error_msg=$2 From 4301a55d01e36e293a7bd832fd0298bc803d5acc Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 21:51:25 +0200 Subject: [PATCH 25/31] refactor: improve parallel sed regex --- src/parallel.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/parallel.sh b/src/parallel.sh index 3ea6bcf3..e25250b7 100755 --- a/src/parallel.sh +++ b/src/parallel.sh @@ -13,11 +13,11 @@ function parallel::aggregate_test_results() { for result_file in "$script_dir"/*.result; do while IFS= read -r line; do # Extract assertion counts from the result lines using sed - failed=$(echo "$line" | sed -n 's/.*##ASSERTIONS_FAILED=\([0-9]*\).*/\1/p') - passed=$(echo "$line" | sed -n 's/.*##ASSERTIONS_PASSED=\([0-9]*\).*/\1/p') - skipped=$(echo "$line" | sed -n 's/.*##ASSERTIONS_SKIPPED=\([0-9]*\).*/\1/p') - incomplete=$(echo "$line" | sed -n 's/.*##ASSERTIONS_INCOMPLETE=\([0-9]*\).*/\1/p') - snapshot=$(echo "$line" | sed -n 's/.*##ASSERTIONS_SNAPSHOT=\([0-9]*\).*/\1/p') + failed=$(echo "$line" | sed -n 's/.*##ASSERTIONS_FAILED=\([0-9]*\)##.*/\1/p') + passed=$(echo "$line" | sed -n 's/.*##ASSERTIONS_PASSED=\([0-9]*\)##.*/\1/p') + skipped=$(echo "$line" | sed -n 's/.*##ASSERTIONS_SKIPPED=\([0-9]*\)##.*/\1/p') + incomplete=$(echo "$line" | sed -n 's/.*##ASSERTIONS_INCOMPLETE=\([0-9]*\)##.*/\1/p') + snapshot=$(echo "$line" | sed -n 's/.*##ASSERTIONS_SNAPSHOT=\([0-9]*\)##.*/\1/p') # Default to 0 if no match is found failed=${failed:-0} From 170f064ac6a4a5aba2ced386a122f06df17da604 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 22:31:53 +0200 Subject: [PATCH 26/31] feat: add ID=random-str to each assertion --- src/runner.sh | 15 +++++++++++---- src/state.sh | 3 ++- tests/unit/state_test.sh | 15 +++++++++++---- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/runner.sh b/src/runner.sh index 8c9729ce..88a9d3f9 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -158,7 +158,7 @@ function runner::run_test() { # Closes FD 3, which was used temporarily to hold the original stdout. exec 3>&- - runner::parse_result "$test_execution_result" "$@" + runner::parse_result "$function_name" "$test_execution_result" "$@" local subshell_output=$(\ echo "$test_execution_result" |\ @@ -243,19 +243,23 @@ function runner::run_test() { } function runner::parse_result() { + local function_name=$1 + shift local execution_result=$1 shift local args=("$@") if env::is_parallel_run_enabled; then - runner::parse_result_parallel "$execution_result" "${args[@]}" + runner::parse_result_parallel "$function_name" "$execution_result" "${args[@]}" else - runner::parse_result_sync "$execution_result" + runner::parse_result_sync "$function_name" "$execution_result" fi } # shellcheck disable=SC2155 function runner::parse_result_parallel() { + local function_name=$1 + shift local execution_result=$1 shift local args=("$@") @@ -283,7 +287,8 @@ function runner::parse_result_parallel() { # shellcheck disable=SC2155 function runner::parse_result_sync() { - local execution_result=$1 + local function_name=$1 + local execution_result=$2 local assertions_failed=$(\ echo "$execution_result" |\ @@ -315,6 +320,8 @@ function runner::parse_result_sync() { sed -E -e 's/.*##ASSERTIONS_SNAPSHOT=([0-9]*)##.*/\1/g'\ ) + log "$function_name" "$execution_result" + ((_ASSERTIONS_PASSED += assertions_passed)) || true ((_ASSERTIONS_FAILED += assertions_failed)) || true ((_ASSERTIONS_SKIPPED += assertions_skipped)) || true diff --git a/src/state.sh b/src/state.sh index 86ce427c..982742ea 100644 --- a/src/state.sh +++ b/src/state.sh @@ -148,7 +148,8 @@ function state::export_subshell_context() { encoded_test_output=$(echo -n "$_TEST_OUTPUT" | base64) fi - echo "##ASSERTIONS_FAILED=$_ASSERTIONS_FAILED\ + echo "##ID=$(random_str)\ +##ASSERTIONS_FAILED=$_ASSERTIONS_FAILED\ ##ASSERTIONS_PASSED=$_ASSERTIONS_PASSED\ ##ASSERTIONS_SKIPPED=$_ASSERTIONS_SKIPPED\ ##ASSERTIONS_INCOMPLETE=$_ASSERTIONS_INCOMPLETE\ diff --git a/tests/unit/state_test.sh b/tests/unit/state_test.sh index 9fa4fda1..7deffde0 100644 --- a/tests/unit/state_test.sh +++ b/tests/unit/state_test.sh @@ -234,6 +234,8 @@ function test_set_duplicated_functions_merged() { } function test_initialize_assertions_count() { + mock random_str echo "abc123" + local export_assertions_count export_assertions_count=$( _ASSERTIONS_PASSED=10 @@ -247,7 +249,8 @@ function test_initialize_assertions_count() { ) assert_same\ - "##ASSERTIONS_FAILED=0\ + "##ID=abc123\ +##ASSERTIONS_FAILED=0\ ##ASSERTIONS_PASSED=0\ ##ASSERTIONS_SKIPPED=0\ ##ASSERTIONS_INCOMPLETE=0\ @@ -258,6 +261,8 @@ function test_initialize_assertions_count() { } function test_export_assertions_count() { + mock random_str echo "abc123" + local export_assertions_count export_assertions_count=$( _ASSERTIONS_PASSED=10 @@ -272,7 +277,8 @@ function test_export_assertions_count() { ) assert_same\ - "##ASSERTIONS_FAILED=5\ + "##ID=abc123\ +##ASSERTIONS_FAILED=5\ ##ASSERTIONS_PASSED=10\ ##ASSERTIONS_SKIPPED=42\ ##ASSERTIONS_INCOMPLETE=12\ @@ -282,12 +288,13 @@ function test_export_assertions_count() { } function test_calculate_total_assertions() { - local input="##ASSERTIONS_FAILED=1\ + local input="##ID=abc123\ + ##ASSERTIONS_FAILED=1\ ##ASSERTIONS_PASSED=2\ ##ASSERTIONS_SKIPPED=3\ ##ASSERTIONS_INCOMPLETE=4\ ##ASSERTIONS_SNAPSHOT=5\ - ##TEST_OUTPUT=##" + ##TEST_OUTPUT=3zhbEncodedBase64##" assert_same 15 "$(state::calculate_total_assertions "$input")" } From 0f452a89d59c10382a68073c1882a21d1828151a Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 22:40:04 +0200 Subject: [PATCH 27/31] feat: add DEV_MODE --- .env.example | 1 + src/env.sh | 6 ++++++ src/runner.sh | 9 ++++++++- src/state.sh | 2 +- tests/unit/state_test.sh | 6 +++--- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 574df050..6e3108a6 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,4 @@ BASHUNIT_HEADER_ASCII_ART= BASHUNIT_SIMPLE_OUTPUT= BASHUNIT_STOP_ON_FAILURE= BASHUNIT_SHOW_EXECUTION_TIME= +BASHUNIT_DEV_MODE= diff --git a/src/env.sh b/src/env.sh index 5c6fc4f9..4c5056af 100644 --- a/src/env.sh +++ b/src/env.sh @@ -27,6 +27,7 @@ _DEFAULT_HEADER_ASCII_ART="false" _DEFAULT_SIMPLE_OUTPUT="false" _DEFAULT_STOP_ON_FAILURE="false" _DEFAULT_SHOW_EXECUTION_TIME="true" +_DEFAULT_DEV_MODE="false" : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_DEFAULT_PARALLEL_RUN}}" : "${BASHUNIT_SHOW_HEADER:=${SHOW_HEADER:=$_DEFAULT_SHOW_HEADER}}" @@ -34,6 +35,7 @@ _DEFAULT_SHOW_EXECUTION_TIME="true" : "${BASHUNIT_SIMPLE_OUTPUT:=${SIMPLE_OUTPUT:=$_DEFAULT_SIMPLE_OUTPUT}}" : "${BASHUNIT_STOP_ON_FAILURE:=${STOP_ON_FAILURE:=$_DEFAULT_STOP_ON_FAILURE}}" : "${BASHUNIT_SHOW_EXECUTION_TIME:=${SHOW_EXECUTION_TIME:=$_DEFAULT_SHOW_EXECUTION_TIME}}" +: "${BASHUNIT_DEV_MODE:=${DEV_MODE:=$_DEFAULT_DEV_MODE}}" function env::is_parallel_run_enabled() { [[ "$BASHUNIT_PARALLEL_RUN" == "true" ]] @@ -59,6 +61,10 @@ function env::is_show_execution_time_enabled() { [[ "$BASHUNIT_SHOW_EXECUTION_TIME" == "true" ]] } +function env::is_dev_mode_enabled() { + [[ "$BASHUNIT_DEV_MODE" == "true" ]] +} + function env::find_terminal_width() { local cols="" diff --git a/src/runner.sh b/src/runner.sh index 88a9d3f9..b256ac91 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -320,7 +320,14 @@ function runner::parse_result_sync() { sed -E -e 's/.*##ASSERTIONS_SNAPSHOT=([0-9]*)##.*/\1/g'\ ) - log "$function_name" "$execution_result" + if env::is_dev_mode_enabled; then + local test_id=$(\ + echo "$execution_result" |\ + tail -n 1 |\ + sed -E -e 's/.*##TEST_ID=([0-9]*)##.*/\1/g'\ + ) + log "$test_id" "$function_name" "$execution_result" + fi ((_ASSERTIONS_PASSED += assertions_passed)) || true ((_ASSERTIONS_FAILED += assertions_failed)) || true diff --git a/src/state.sh b/src/state.sh index 982742ea..a0a14a51 100644 --- a/src/state.sh +++ b/src/state.sh @@ -148,7 +148,7 @@ function state::export_subshell_context() { encoded_test_output=$(echo -n "$_TEST_OUTPUT" | base64) fi - echo "##ID=$(random_str)\ + echo "##TEST_ID=$(random_str)\ ##ASSERTIONS_FAILED=$_ASSERTIONS_FAILED\ ##ASSERTIONS_PASSED=$_ASSERTIONS_PASSED\ ##ASSERTIONS_SKIPPED=$_ASSERTIONS_SKIPPED\ diff --git a/tests/unit/state_test.sh b/tests/unit/state_test.sh index 7deffde0..578414c2 100644 --- a/tests/unit/state_test.sh +++ b/tests/unit/state_test.sh @@ -249,7 +249,7 @@ function test_initialize_assertions_count() { ) assert_same\ - "##ID=abc123\ + "##TEST_ID=abc123\ ##ASSERTIONS_FAILED=0\ ##ASSERTIONS_PASSED=0\ ##ASSERTIONS_SKIPPED=0\ @@ -277,7 +277,7 @@ function test_export_assertions_count() { ) assert_same\ - "##ID=abc123\ + "##TEST_ID=abc123\ ##ASSERTIONS_FAILED=5\ ##ASSERTIONS_PASSED=10\ ##ASSERTIONS_SKIPPED=42\ @@ -288,7 +288,7 @@ function test_export_assertions_count() { } function test_calculate_total_assertions() { - local input="##ID=abc123\ + local input="##TEST_ID=abc123\ ##ASSERTIONS_FAILED=1\ ##ASSERTIONS_PASSED=2\ ##ASSERTIONS_SKIPPED=3\ From 155e6943d1a78653591273fa53d52c18a8bd931e Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 11 Oct 2024 23:07:52 +0200 Subject: [PATCH 28/31] feat: add logs on dev_mode --- bashunit | 2 +- src/env.sh | 2 +- src/runner.sh | 19 ++++++++++++++----- src/state.sh | 4 ++-- tests/unit/state_test.sh | 4 ++-- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/bashunit b/bashunit index 3f302505..33a80dd3 100755 --- a/bashunit +++ b/bashunit @@ -9,6 +9,7 @@ declare -r BASHUNIT_ROOT_DIR="$(dirname "${BASH_SOURCE[0]}")" export BASHUNIT_ROOT_DIR source "$BASHUNIT_ROOT_DIR/src/dev/debug.sh" +source "$BASHUNIT_ROOT_DIR/src/check_os.sh" source "$BASHUNIT_ROOT_DIR/src/str.sh" source "$BASHUNIT_ROOT_DIR/src/globals.sh" source "$BASHUNIT_ROOT_DIR/src/dependencies.sh" @@ -16,7 +17,6 @@ source "$BASHUNIT_ROOT_DIR/src/io.sh" source "$BASHUNIT_ROOT_DIR/src/math.sh" source "$BASHUNIT_ROOT_DIR/src/parallel.sh" source "$BASHUNIT_ROOT_DIR/src/env.sh" -source "$BASHUNIT_ROOT_DIR/src/check_os.sh" source "$BASHUNIT_ROOT_DIR/src/clock.sh" source "$BASHUNIT_ROOT_DIR/src/state.sh" source "$BASHUNIT_ROOT_DIR/src/colors.sh" diff --git a/src/env.sh b/src/env.sh index 4c5056af..3ed96928 100644 --- a/src/env.sh +++ b/src/env.sh @@ -79,7 +79,7 @@ function env::find_terminal_width() { echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}" } -TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel" +TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}" TERMINAL_WIDTH="$(env::find_terminal_width)" FAILURES_OUTPUT_PATH=$(mktemp) CAT="$(which cat)" diff --git a/src/runner.sh b/src/runner.sh index b256ac91..c3f37480 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -158,8 +158,6 @@ function runner::run_test() { # Closes FD 3, which was used temporarily to hold the original stdout. exec 3>&- - runner::parse_result "$function_name" "$test_execution_result" "$@" - local subshell_output=$(\ echo "$test_execution_result" |\ tail -n 1 |\ @@ -185,13 +183,15 @@ function runner::run_test() { "division by 0" "cannot allocate memory" "bad file descriptor" \ "segmentation fault" "illegal option" "argument list too long" \ "readonly variable" "missing keyword" "killed" \ - "cannot execute binary file"; do + "cannot execute binary file" "invalid arithmetic operator"; do if [[ "$runtime_output" == *"$error"* ]]; then runtime_error=$(echo "${runtime_output#*: }" | tr -d '\n') break fi done + runner::parse_result "$function_name" "$test_execution_result" "$@" + local total_assertions="$(state::calculate_total_assertions "$test_execution_result")" local end_time=$(clock::now) @@ -282,6 +282,15 @@ function runner::parse_result_parallel() { count=$((count + 1)) done + if env::is_dev_mode_enabled; then + local test_id=$(\ + echo "$execution_result" |\ + tail -n 1 |\ + sed -E -e 's/.*##TEST_ID=([a-zA-Z0-9]*)##.*/\1/g'\ + ) + log "debug" "[PARA] test_id:$test_id" "function_name:$function_name" "execution_result:$execution_result" + fi + echo "$execution_result" > "$unique_test_result_file" } @@ -324,9 +333,9 @@ function runner::parse_result_sync() { local test_id=$(\ echo "$execution_result" |\ tail -n 1 |\ - sed -E -e 's/.*##TEST_ID=([0-9]*)##.*/\1/g'\ + sed -E -e 's/.*##TEST_ID=([a-zA-Z0-9]*)##.*/\1/g'\ ) - log "$test_id" "$function_name" "$execution_result" + log "debug" "[SYNC] test_id:$test_id" "function_name:$function_name" "execution_result:$execution_result" fi ((_ASSERTIONS_PASSED += assertions_passed)) || true diff --git a/src/state.sh b/src/state.sh index a0a14a51..83e68ce3 100644 --- a/src/state.sh +++ b/src/state.sh @@ -148,7 +148,7 @@ function state::export_subshell_context() { encoded_test_output=$(echo -n "$_TEST_OUTPUT" | base64) fi - echo "##TEST_ID=$(random_str)\ + echo "##TEST_ID=$(LC_ALL=C tr -dc A-Za-z0-9 Date: Sat, 12 Oct 2024 00:05:36 +0200 Subject: [PATCH 29/31] refactor: remove unnecesary code --- src/globals.sh | 4 ---- src/runner.sh | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/globals.sh b/src/globals.sh index eb5a76d2..8b4f3250 100644 --- a/src/globals.sh +++ b/src/globals.sh @@ -15,10 +15,6 @@ function current_timestamp() { date +"%Y-%m-%d %H:%M:%S" } -function is_number() { - [[ -n "$1" && "$1" =~ ^-?[0-9]+$ ]] -} - function is_command_available() { command -v "$1" >/dev/null 2>&1 } diff --git a/src/runner.sh b/src/runner.sh index c3f37480..18a35d93 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -73,8 +73,6 @@ function run_test() { } function runner::call_test_functions() { - trap 'exit' SIGTERM - local script="$1" local filter="$2" local prefix="test" @@ -269,9 +267,9 @@ function runner::parse_result_parallel() { local test_result_file=$(echo "${args[@]}" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//') if [[ -z "$test_result_file" ]]; then - test_result_file="${function_name}.result" + test_result_file="${function_name}.$$.result" else - test_result_file="${function_name}-${test_result_file}.result" + test_result_file="${function_name}-${test_result_file}.$$.result" fi local unique_test_result_file="${test_suite_dir}/${test_result_file}" From 24edeed57b022499b195e27d30a3169c19b335e4 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sat, 12 Oct 2024 00:26:54 +0200 Subject: [PATCH 30/31] refactor: improve subshell_output --- src/runner.sh | 17 ++++++++++------- src/state.sh | 14 ++++++++++---- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/runner.sh b/src/runner.sh index 18a35d93..63b3746c 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -156,12 +156,15 @@ function runner::run_test() { # Closes FD 3, which was used temporarily to hold the original stdout. exec 3>&- - local subshell_output=$(\ - echo "$test_execution_result" |\ - tail -n 1 |\ - sed -E -e 's/.*##TEST_OUTPUT=(.*)##.*/\1/g' |\ - base64 -d - ) + local test_output_base64="${test_execution_result##*##TEST_OUTPUT=}" + test_output_base64="${test_output_base64%%##*}" + + local subshell_output + if command -v base64 >/dev/null; then + subshell_output=$(echo "$test_output_base64" | base64 -d) + else + subshell_output=$(echo "$test_output_base64" | openssl enc -d -base64) + fi if [[ -n "$subshell_output" ]]; then # Formatted as "[type]line" @see `state::print_line()` @@ -173,7 +176,7 @@ function runner::run_test() { subshell_output=$line fi - local runtime_output="${test_execution_result%%##ASSERTIONS*}" + local runtime_output="${test_execution_result%%##TEST_ID=*}" local runtime_error="" for error in "command not found" "unbound variable" "permission denied" \ diff --git a/src/state.sh b/src/state.sh index 83e68ce3..03078121 100644 --- a/src/state.sh +++ b/src/state.sh @@ -140,22 +140,28 @@ function state::initialize_assertions_count() { function state::export_subshell_context() { local encoded_test_output + if base64 --help 2>&1 | grep -q -- "-w"; then - # Alpine needs -w 0 to avoid line wrapping + # Alpine requires the -w 0 option to avoid wrapping encoded_test_output=$(echo -n "$_TEST_OUTPUT" | base64 -w 0) else - # macOS and others don't need -w 0 + # macOS and others: default base64 without wrapping encoded_test_output=$(echo -n "$_TEST_OUTPUT" | base64) fi - echo "##TEST_ID=$(LC_ALL=C tr -dc A-Za-z0-9 Date: Sat, 12 Oct 2024 00:41:25 +0200 Subject: [PATCH 31/31] docs: add warning notice for alpine when running parallel --- src/main.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main.sh b/src/main.sh index 8a31df9e..bca384aa 100644 --- a/src/main.sh +++ b/src/main.sh @@ -18,6 +18,12 @@ function main::exec_tests() { # Trap SIGINT (Ctrl-C) and call the cleanup function trap main::cleanup SIGINT + if env::is_parallel_run_enabled && check_os::is_alpine; then + printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}" + printf "in a beta stage.\nThis means there may be unresolved issues, " + printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}" + fi + console_header::print_version_with_env "$filter" "${test_files[@]}" runner::load_test_files "$filter" "${test_files[@]}" if env::is_parallel_run_enabled; then