Stop on failure when running on parallel#367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chemaclass merged 6 commits into from
Oct 12, 2024
Merged

Stop on failure when running on parallel #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f3599d
feat: display "totals" even on stop on failure
Chemaclass Oct 12, 2024
ab22ed4
feat: add Stop on failure enabled text
Chemaclass Oct 12, 2024
e8a94a9
feat: create .stop-on-failure shared on parallel tests
Chemaclass Oct 12, 2024
c1f1609
refactor: runner::call_test_functions
Chemaclass Oct 12, 2024
498a500
refactor: remove unused function run_test
Chemaclass Oct 12, 2024
ef1b601
docs: update changelog
Chemaclass Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
- `LOG_PATH="out.log"`
- `LOAD_FILE="tests/bootstrap.sh"`
- Add check that git is installed to `install.sh`
- Fixed `-S|--stop-on-failure` behaviour
- Improved time taken display
- Improved clean up temporal files and directories
- Improved CI test speed by running them in parallel
Expand Down
5 changes: 3 additions & 2 deletions src/env.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
_DEFAULT_LOAD_FILE="tests/bootstrap.sh"
_DEFAULT_LOG_JUNIT=""
_DEFAULT_REPORT_HTML=""
_DEFAULT_TERMINAL_WIDTH=100

: "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
: "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
Expand DownExpand Up@@ -76,10 +75,12 @@ function env::find_terminal_width() {
fi

# Directly echo the value with fallback
echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
echo "${cols:-100}"
}

EXIT_CODE_STOP_ON_FAILURE=4
TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
TERMINAL_WIDTH="$(env::find_terminal_width)"
FAILURES_OUTPUT_PATH=$(mktemp)
CAT="$(which cat)"
19 changes: 18 additions & 1 deletion src/main.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,20 +16,29 @@ function main::exec_tests() {
fi

# Trap SIGINT (Ctrl-C) and call the cleanup function
trap main::cleanup SIGINT
trap 'main::cleanup' SIGINT
trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT

if env::is_parallel_run_enabled && check_os::is_alpine; then
printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
printf "in a beta stage.\nThis means there may be unresolved issues, "
printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
fi

if env::is_parallel_run_enabled; then
parallel::reset
fi

console_header::print_version_with_env "$filter" "${test_files[@]}"
runner::load_test_files "$filter" "${test_files[@]}"
if env::is_parallel_run_enabled; then
wait
fi

if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
fi

console_results::print_failing_tests_and_reset
console_results::render_result
exit_code=$?
Expand All@@ -54,6 +63,14 @@ function main::cleanup() {
exit 1
}

function main::handle_stop_on_failure_sync() {
printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
console_results::print_failing_tests_and_reset
console_results::render_result
cleanup_temp_files
exit 1
}

function main::exec_assert() {
local original_assert_fn=$1
local args=("${@:2}")
Expand Down
14 changes: 14 additions & 0 deletions src/parallel.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
export _ASSERTIONS_INCOMPLETE=$total_incomplete
export _ASSERTIONS_SNAPSHOT=$total_snapshot
}

function parallel::mark_stop_on_failure() {
touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}

function parallel::must_stop_on_failure() {
[[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
}

function parallel::reset() {
# shellcheck disable=SC2153
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}
92 changes: 41 additions & 51 deletions src/runner.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,20 @@
#!/bin/bash
# shellcheck disable=SC2155

function runner::load_test_files() {
local filter=$1
shift
local files=("${@}")
local pids=()

if env::is_parallel_run_enabled; then
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
fi

for test_file in "${files[@]}"; do
if [[ ! -f $test_file ]]; then
continue
fi

# shellcheck source=/dev/null
source "$test_file"

runner::run_set_up_before_script
if env::is_parallel_run_enabled; then
runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
pids+=($!)
else
runner::call_test_functions "$test_file" "$filter"
fi
Expand All@@ -31,12 +24,9 @@ function runner::load_test_files() {

if env::is_parallel_run_enabled; then
wait

runner::spinner &
local spinner_pid=$!

parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"

# Kill the spinner once the aggregation finishes
disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
printf "\r " # Clear the spinner output
Expand DownExpand Up@@ -70,59 +60,56 @@ function runner::functions_for_script() {
shopt -u extdebug
}

# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
function run_test() {
runner::run_test "testing-fn" "$function_name" "$@"
}

function runner::call_test_functions() {
local script="$1"
local filter="$2"
local prefix="test"
# Use declare -F to list all function names
local all_function_names
all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions
filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")

local all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
# shellcheck disable=SC2207
local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))

if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi
if [[ "${#functions_to_run[@]}" -le 0 ]]; then
return
fi

helper::check_duplicate_functions "$script" || true
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi

for function_name in "${functions_to_run[@]}"; do
local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"
helper::check_duplicate_functions "$script" || true

# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi
for function_name in "${functions_to_run[@]}"; do
if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
break
fi

local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
fi
unset function_name
done
}

# shellcheck disable=SC2155
function runner::run_test() {
local start_time
start_time=$(clock::now)
Expand DownExpand Up@@ -204,8 +191,13 @@ function runner::run_test() {
state::add_tests_failed
reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
runner::write_failure_result_output "$test_file" "$subshell_output"

if env::is_stop_on_failure_enabled; then
exit 1
if env::is_parallel_run_enabled; then
parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
Expand DownExpand Up@@ -264,7 +256,6 @@ function runner::parse_result() {
fi
}

# shellcheck disable=SC2155
function runner::parse_result_parallel() {
local function_name=$1
shift
Expand DownExpand Up@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
echo "$execution_result" > "$unique_test_result_file"
}

# shellcheck disable=SC2155
function runner::parse_result_sync() {
local function_name=$1
local execution_result=$2
Expand Down
14 changes: 14 additions & 0 deletions ...napshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
14 changes: 14 additions & 0 deletions ...shots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all \u003cpre\u003e\u003ccode\u003e blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks"); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + '
Skip to content

Stop on failure when running on parallel#367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chemaclass merged 6 commits into from
Oct 12, 2024
Merged

Stop on failure when running on parallel #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f3599d
feat: display "totals" even on stop on failure
Chemaclass Oct 12, 2024
ab22ed4
feat: add Stop on failure enabled text
Chemaclass Oct 12, 2024
e8a94a9
feat: create .stop-on-failure shared on parallel tests
Chemaclass Oct 12, 2024
c1f1609
refactor: runner::call_test_functions
Chemaclass Oct 12, 2024
498a500
refactor: remove unused function run_test
Chemaclass Oct 12, 2024
ef1b601
docs: update changelog
Chemaclass Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
- `LOG_PATH="out.log"`
- `LOAD_FILE="tests/bootstrap.sh"`
- Add check that git is installed to `install.sh`
- Fixed `-S|--stop-on-failure` behaviour
- Improved time taken display
- Improved clean up temporal files and directories
- Improved CI test speed by running them in parallel
Expand Down
5 changes: 3 additions & 2 deletions src/env.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
_DEFAULT_LOAD_FILE="tests/bootstrap.sh"
_DEFAULT_LOG_JUNIT=""
_DEFAULT_REPORT_HTML=""
_DEFAULT_TERMINAL_WIDTH=100

: "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
: "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
Expand DownExpand Up@@ -76,10 +75,12 @@ function env::find_terminal_width() {
fi

# Directly echo the value with fallback
echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
echo "${cols:-100}"
}

EXIT_CODE_STOP_ON_FAILURE=4
TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
TERMINAL_WIDTH="$(env::find_terminal_width)"
FAILURES_OUTPUT_PATH=$(mktemp)
CAT="$(which cat)"
19 changes: 18 additions & 1 deletion src/main.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,20 +16,29 @@ function main::exec_tests() {
fi

# Trap SIGINT (Ctrl-C) and call the cleanup function
trap main::cleanup SIGINT
trap 'main::cleanup' SIGINT
trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT

if env::is_parallel_run_enabled && check_os::is_alpine; then
printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
printf "in a beta stage.\nThis means there may be unresolved issues, "
printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
fi

if env::is_parallel_run_enabled; then
parallel::reset
fi

console_header::print_version_with_env "$filter" "${test_files[@]}"
runner::load_test_files "$filter" "${test_files[@]}"
if env::is_parallel_run_enabled; then
wait
fi

if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
fi

console_results::print_failing_tests_and_reset
console_results::render_result
exit_code=$?
Expand All@@ -54,6 +63,14 @@ function main::cleanup() {
exit 1
}

function main::handle_stop_on_failure_sync() {
printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
console_results::print_failing_tests_and_reset
console_results::render_result
cleanup_temp_files
exit 1
}

function main::exec_assert() {
local original_assert_fn=$1
local args=("${@:2}")
Expand Down
14 changes: 14 additions & 0 deletions src/parallel.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
export _ASSERTIONS_INCOMPLETE=$total_incomplete
export _ASSERTIONS_SNAPSHOT=$total_snapshot
}

function parallel::mark_stop_on_failure() {
touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}

function parallel::must_stop_on_failure() {
[[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
}

function parallel::reset() {
# shellcheck disable=SC2153
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}
92 changes: 41 additions & 51 deletions src/runner.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,20 @@
#!/bin/bash
# shellcheck disable=SC2155

function runner::load_test_files() {
local filter=$1
shift
local files=("${@}")
local pids=()

if env::is_parallel_run_enabled; then
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
fi

for test_file in "${files[@]}"; do
if [[ ! -f $test_file ]]; then
continue
fi

# shellcheck source=/dev/null
source "$test_file"

runner::run_set_up_before_script
if env::is_parallel_run_enabled; then
runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
pids+=($!)
else
runner::call_test_functions "$test_file" "$filter"
fi
Expand All@@ -31,12 +24,9 @@ function runner::load_test_files() {

if env::is_parallel_run_enabled; then
wait

runner::spinner &
local spinner_pid=$!

parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"

# Kill the spinner once the aggregation finishes
disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
printf "\r " # Clear the spinner output
Expand DownExpand Up@@ -70,59 +60,56 @@ function runner::functions_for_script() {
shopt -u extdebug
}

# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
function run_test() {
runner::run_test "testing-fn" "$function_name" "$@"
}

function runner::call_test_functions() {
local script="$1"
local filter="$2"
local prefix="test"
# Use declare -F to list all function names
local all_function_names
all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions
filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")

local all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
# shellcheck disable=SC2207
local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))

if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi
if [[ "${#functions_to_run[@]}" -le 0 ]]; then
return
fi

helper::check_duplicate_functions "$script" || true
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi

for function_name in "${functions_to_run[@]}"; do
local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"
helper::check_duplicate_functions "$script" || true

# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi
for function_name in "${functions_to_run[@]}"; do
if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
break
fi

local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
fi
unset function_name
done
}

# shellcheck disable=SC2155
function runner::run_test() {
local start_time
start_time=$(clock::now)
Expand DownExpand Up@@ -204,8 +191,13 @@ function runner::run_test() {
state::add_tests_failed
reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
runner::write_failure_result_output "$test_file" "$subshell_output"

if env::is_stop_on_failure_enabled; then
exit 1
if env::is_parallel_run_enabled; then
parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
Expand DownExpand Up@@ -264,7 +256,6 @@ function runner::parse_result() {
fi
}

# shellcheck disable=SC2155
function runner::parse_result_parallel() {
local function_name=$1
shift
Expand DownExpand Up@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
echo "$execution_result" > "$unique_test_result_file"
}

# shellcheck disable=SC2155
function runner::parse_result_sync() {
local function_name=$1
local execution_result=$2
Expand Down
14 changes: 14 additions & 0 deletions ...napshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
14 changes: 14 additions & 0 deletions ...shots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Stop on failure when running on parallel#367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chemaclass merged 6 commits into from
Oct 12, 2024
Merged

Stop on failure when running on parallel #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f3599d
feat: display "totals" even on stop on failure
Chemaclass Oct 12, 2024
ab22ed4
feat: add Stop on failure enabled text
Chemaclass Oct 12, 2024
e8a94a9
feat: create .stop-on-failure shared on parallel tests
Chemaclass Oct 12, 2024
c1f1609
refactor: runner::call_test_functions
Chemaclass Oct 12, 2024
498a500
refactor: remove unused function run_test
Chemaclass Oct 12, 2024
ef1b601
docs: update changelog
Chemaclass Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
- `LOG_PATH="out.log"`
- `LOAD_FILE="tests/bootstrap.sh"`
- Add check that git is installed to `install.sh`
- Fixed `-S|--stop-on-failure` behaviour
- Improved time taken display
- Improved clean up temporal files and directories
- Improved CI test speed by running them in parallel
Expand Down
5 changes: 3 additions & 2 deletions src/env.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
_DEFAULT_LOAD_FILE="tests/bootstrap.sh"
_DEFAULT_LOG_JUNIT=""
_DEFAULT_REPORT_HTML=""
_DEFAULT_TERMINAL_WIDTH=100

: "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
: "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
Expand DownExpand Up@@ -76,10 +75,12 @@ function env::find_terminal_width() {
fi

# Directly echo the value with fallback
echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
echo "${cols:-100}"
}

EXIT_CODE_STOP_ON_FAILURE=4
TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
TERMINAL_WIDTH="$(env::find_terminal_width)"
FAILURES_OUTPUT_PATH=$(mktemp)
CAT="$(which cat)"
19 changes: 18 additions & 1 deletion src/main.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,20 +16,29 @@ function main::exec_tests() {
fi

# Trap SIGINT (Ctrl-C) and call the cleanup function
trap main::cleanup SIGINT
trap 'main::cleanup' SIGINT
trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT

if env::is_parallel_run_enabled && check_os::is_alpine; then
printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
printf "in a beta stage.\nThis means there may be unresolved issues, "
printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
fi

if env::is_parallel_run_enabled; then
parallel::reset
fi

console_header::print_version_with_env "$filter" "${test_files[@]}"
runner::load_test_files "$filter" "${test_files[@]}"
if env::is_parallel_run_enabled; then
wait
fi

if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
fi

console_results::print_failing_tests_and_reset
console_results::render_result
exit_code=$?
Expand All@@ -54,6 +63,14 @@ function main::cleanup() {
exit 1
}

function main::handle_stop_on_failure_sync() {
printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
console_results::print_failing_tests_and_reset
console_results::render_result
cleanup_temp_files
exit 1
}

function main::exec_assert() {
local original_assert_fn=$1
local args=("${@:2}")
Expand Down
14 changes: 14 additions & 0 deletions src/parallel.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
export _ASSERTIONS_INCOMPLETE=$total_incomplete
export _ASSERTIONS_SNAPSHOT=$total_snapshot
}

function parallel::mark_stop_on_failure() {
touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}

function parallel::must_stop_on_failure() {
[[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
}

function parallel::reset() {
# shellcheck disable=SC2153
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}
92 changes: 41 additions & 51 deletions src/runner.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,20 @@
#!/bin/bash
# shellcheck disable=SC2155

function runner::load_test_files() {
local filter=$1
shift
local files=("${@}")
local pids=()

if env::is_parallel_run_enabled; then
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
fi

for test_file in "${files[@]}"; do
if [[ ! -f $test_file ]]; then
continue
fi

# shellcheck source=/dev/null
source "$test_file"

runner::run_set_up_before_script
if env::is_parallel_run_enabled; then
runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
pids+=($!)
else
runner::call_test_functions "$test_file" "$filter"
fi
Expand All@@ -31,12 +24,9 @@ function runner::load_test_files() {

if env::is_parallel_run_enabled; then
wait

runner::spinner &
local spinner_pid=$!

parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"

# Kill the spinner once the aggregation finishes
disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
printf "\r " # Clear the spinner output
Expand DownExpand Up@@ -70,59 +60,56 @@ function runner::functions_for_script() {
shopt -u extdebug
}

# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
function run_test() {
runner::run_test "testing-fn" "$function_name" "$@"
}

function runner::call_test_functions() {
local script="$1"
local filter="$2"
local prefix="test"
# Use declare -F to list all function names
local all_function_names
all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions
filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")

local all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
# shellcheck disable=SC2207
local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))

if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi
if [[ "${#functions_to_run[@]}" -le 0 ]]; then
return
fi

helper::check_duplicate_functions "$script" || true
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi

for function_name in "${functions_to_run[@]}"; do
local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"
helper::check_duplicate_functions "$script" || true

# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi
for function_name in "${functions_to_run[@]}"; do
if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
break
fi

local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
fi
unset function_name
done
}

# shellcheck disable=SC2155
function runner::run_test() {
local start_time
start_time=$(clock::now)
Expand DownExpand Up@@ -204,8 +191,13 @@ function runner::run_test() {
state::add_tests_failed
reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
runner::write_failure_result_output "$test_file" "$subshell_output"

if env::is_stop_on_failure_enabled; then
exit 1
if env::is_parallel_run_enabled; then
parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
Expand DownExpand Up@@ -264,7 +256,6 @@ function runner::parse_result() {
fi
}

# shellcheck disable=SC2155
function runner::parse_result_parallel() {
local function_name=$1
shift
Expand DownExpand Up@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
echo "$execution_result" > "$unique_test_result_file"
}

# shellcheck disable=SC2155
function runner::parse_result_sync() {
local function_name=$1
local execution_result=$2
Expand Down
14 changes: 14 additions & 0 deletions ...napshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
14 changes: 14 additions & 0 deletions ...shots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length \u003e 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Stop on failure when running on parallel#367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chemaclass merged 6 commits into from
Oct 12, 2024
Merged

Stop on failure when running on parallel #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f3599d
feat: display "totals" even on stop on failure
Chemaclass Oct 12, 2024
ab22ed4
feat: add Stop on failure enabled text
Chemaclass Oct 12, 2024
e8a94a9
feat: create .stop-on-failure shared on parallel tests
Chemaclass Oct 12, 2024
c1f1609
refactor: runner::call_test_functions
Chemaclass Oct 12, 2024
498a500
refactor: remove unused function run_test
Chemaclass Oct 12, 2024
ef1b601
docs: update changelog
Chemaclass Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
- `LOG_PATH="out.log"`
- `LOAD_FILE="tests/bootstrap.sh"`
- Add check that git is installed to `install.sh`
- Fixed `-S|--stop-on-failure` behaviour
- Improved time taken display
- Improved clean up temporal files and directories
- Improved CI test speed by running them in parallel
Expand Down
5 changes: 3 additions & 2 deletions src/env.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
_DEFAULT_LOAD_FILE="tests/bootstrap.sh"
_DEFAULT_LOG_JUNIT=""
_DEFAULT_REPORT_HTML=""
_DEFAULT_TERMINAL_WIDTH=100

: "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
: "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
Expand DownExpand Up@@ -76,10 +75,12 @@ function env::find_terminal_width() {
fi

# Directly echo the value with fallback
echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
echo "${cols:-100}"
}

EXIT_CODE_STOP_ON_FAILURE=4
TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
TERMINAL_WIDTH="$(env::find_terminal_width)"
FAILURES_OUTPUT_PATH=$(mktemp)
CAT="$(which cat)"
19 changes: 18 additions & 1 deletion src/main.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,20 +16,29 @@ function main::exec_tests() {
fi

# Trap SIGINT (Ctrl-C) and call the cleanup function
trap main::cleanup SIGINT
trap 'main::cleanup' SIGINT
trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT

if env::is_parallel_run_enabled && check_os::is_alpine; then
printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
printf "in a beta stage.\nThis means there may be unresolved issues, "
printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
fi

if env::is_parallel_run_enabled; then
parallel::reset
fi

console_header::print_version_with_env "$filter" "${test_files[@]}"
runner::load_test_files "$filter" "${test_files[@]}"
if env::is_parallel_run_enabled; then
wait
fi

if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
fi

console_results::print_failing_tests_and_reset
console_results::render_result
exit_code=$?
Expand All@@ -54,6 +63,14 @@ function main::cleanup() {
exit 1
}

function main::handle_stop_on_failure_sync() {
printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
console_results::print_failing_tests_and_reset
console_results::render_result
cleanup_temp_files
exit 1
}

function main::exec_assert() {
local original_assert_fn=$1
local args=("${@:2}")
Expand Down
14 changes: 14 additions & 0 deletions src/parallel.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
export _ASSERTIONS_INCOMPLETE=$total_incomplete
export _ASSERTIONS_SNAPSHOT=$total_snapshot
}

function parallel::mark_stop_on_failure() {
touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}

function parallel::must_stop_on_failure() {
[[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
}

function parallel::reset() {
# shellcheck disable=SC2153
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}
92 changes: 41 additions & 51 deletions src/runner.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,20 @@
#!/bin/bash
# shellcheck disable=SC2155

function runner::load_test_files() {
local filter=$1
shift
local files=("${@}")
local pids=()

if env::is_parallel_run_enabled; then
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
fi

for test_file in "${files[@]}"; do
if [[ ! -f $test_file ]]; then
continue
fi

# shellcheck source=/dev/null
source "$test_file"

runner::run_set_up_before_script
if env::is_parallel_run_enabled; then
runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
pids+=($!)
else
runner::call_test_functions "$test_file" "$filter"
fi
Expand All@@ -31,12 +24,9 @@ function runner::load_test_files() {

if env::is_parallel_run_enabled; then
wait

runner::spinner &
local spinner_pid=$!

parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"

# Kill the spinner once the aggregation finishes
disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
printf "\r " # Clear the spinner output
Expand DownExpand Up@@ -70,59 +60,56 @@ function runner::functions_for_script() {
shopt -u extdebug
}

# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
function run_test() {
runner::run_test "testing-fn" "$function_name" "$@"
}

function runner::call_test_functions() {
local script="$1"
local filter="$2"
local prefix="test"
# Use declare -F to list all function names
local all_function_names
all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions
filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")

local all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
# shellcheck disable=SC2207
local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))

if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi
if [[ "${#functions_to_run[@]}" -le 0 ]]; then
return
fi

helper::check_duplicate_functions "$script" || true
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi

for function_name in "${functions_to_run[@]}"; do
local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"
helper::check_duplicate_functions "$script" || true

# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi
for function_name in "${functions_to_run[@]}"; do
if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
break
fi

local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
fi
unset function_name
done
}

# shellcheck disable=SC2155
function runner::run_test() {
local start_time
start_time=$(clock::now)
Expand DownExpand Up@@ -204,8 +191,13 @@ function runner::run_test() {
state::add_tests_failed
reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
runner::write_failure_result_output "$test_file" "$subshell_output"

if env::is_stop_on_failure_enabled; then
exit 1
if env::is_parallel_run_enabled; then
parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
Expand DownExpand Up@@ -264,7 +256,6 @@ function runner::parse_result() {
fi
}

# shellcheck disable=SC2155
function runner::parse_result_parallel() {
local function_name=$1
shift
Expand DownExpand Up@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
echo "$execution_result" > "$unique_test_result_file"
}

# shellcheck disable=SC2155
function runner::parse_result_sync() {
local function_name=$1
local execution_result=$2
Expand Down
14 changes: 14 additions & 0 deletions ...napshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
14 changes: 14 additions & 0 deletions ...shots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content

Stop on failure when running on parallel#367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chemaclass merged 6 commits into from
Oct 12, 2024
Merged

Stop on failure when running on parallel #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f3599d
feat: display "totals" even on stop on failure
Chemaclass Oct 12, 2024
ab22ed4
feat: add Stop on failure enabled text
Chemaclass Oct 12, 2024
e8a94a9
feat: create .stop-on-failure shared on parallel tests
Chemaclass Oct 12, 2024
c1f1609
refactor: runner::call_test_functions
Chemaclass Oct 12, 2024
498a500
refactor: remove unused function run_test
Chemaclass Oct 12, 2024
ef1b601
docs: update changelog
Chemaclass Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
- `LOG_PATH="out.log"`
- `LOAD_FILE="tests/bootstrap.sh"`
- Add check that git is installed to `install.sh`
- Fixed `-S|--stop-on-failure` behaviour
- Improved time taken display
- Improved clean up temporal files and directories
- Improved CI test speed by running them in parallel
Expand Down
5 changes: 3 additions & 2 deletions src/env.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
_DEFAULT_LOAD_FILE="tests/bootstrap.sh"
_DEFAULT_LOG_JUNIT=""
_DEFAULT_REPORT_HTML=""
_DEFAULT_TERMINAL_WIDTH=100

: "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
: "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
Expand DownExpand Up@@ -76,10 +75,12 @@ function env::find_terminal_width() {
fi

# Directly echo the value with fallback
echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
echo "${cols:-100}"
}

EXIT_CODE_STOP_ON_FAILURE=4
TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
TERMINAL_WIDTH="$(env::find_terminal_width)"
FAILURES_OUTPUT_PATH=$(mktemp)
CAT="$(which cat)"
19 changes: 18 additions & 1 deletion src/main.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,20 +16,29 @@ function main::exec_tests() {
fi

# Trap SIGINT (Ctrl-C) and call the cleanup function
trap main::cleanup SIGINT
trap 'main::cleanup' SIGINT
trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT

if env::is_parallel_run_enabled && check_os::is_alpine; then
printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
printf "in a beta stage.\nThis means there may be unresolved issues, "
printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
fi

if env::is_parallel_run_enabled; then
parallel::reset
fi

console_header::print_version_with_env "$filter" "${test_files[@]}"
runner::load_test_files "$filter" "${test_files[@]}"
if env::is_parallel_run_enabled; then
wait
fi

if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
fi

console_results::print_failing_tests_and_reset
console_results::render_result
exit_code=$?
Expand All@@ -54,6 +63,14 @@ function main::cleanup() {
exit 1
}

function main::handle_stop_on_failure_sync() {
printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
console_results::print_failing_tests_and_reset
console_results::render_result
cleanup_temp_files
exit 1
}

function main::exec_assert() {
local original_assert_fn=$1
local args=("${@:2}")
Expand Down
14 changes: 14 additions & 0 deletions src/parallel.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
export _ASSERTIONS_INCOMPLETE=$total_incomplete
export _ASSERTIONS_SNAPSHOT=$total_snapshot
}

function parallel::mark_stop_on_failure() {
touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}

function parallel::must_stop_on_failure() {
[[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
}

function parallel::reset() {
# shellcheck disable=SC2153
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}
92 changes: 41 additions & 51 deletions src/runner.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,20 @@
#!/bin/bash
# shellcheck disable=SC2155

function runner::load_test_files() {
local filter=$1
shift
local files=("${@}")
local pids=()

if env::is_parallel_run_enabled; then
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
fi

for test_file in "${files[@]}"; do
if [[ ! -f $test_file ]]; then
continue
fi

# shellcheck source=/dev/null
source "$test_file"

runner::run_set_up_before_script
if env::is_parallel_run_enabled; then
runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
pids+=($!)
else
runner::call_test_functions "$test_file" "$filter"
fi
Expand All@@ -31,12 +24,9 @@ function runner::load_test_files() {

if env::is_parallel_run_enabled; then
wait

runner::spinner &
local spinner_pid=$!

parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"

# Kill the spinner once the aggregation finishes
disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
printf "\r " # Clear the spinner output
Expand DownExpand Up@@ -70,59 +60,56 @@ function runner::functions_for_script() {
shopt -u extdebug
}

# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
function run_test() {
runner::run_test "testing-fn" "$function_name" "$@"
}

function runner::call_test_functions() {
local script="$1"
local filter="$2"
local prefix="test"
# Use declare -F to list all function names
local all_function_names
all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions
filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")

local all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
# shellcheck disable=SC2207
local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))

if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi
if [[ "${#functions_to_run[@]}" -le 0 ]]; then
return
fi

helper::check_duplicate_functions "$script" || true
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi

for function_name in "${functions_to_run[@]}"; do
local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"
helper::check_duplicate_functions "$script" || true

# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi
for function_name in "${functions_to_run[@]}"; do
if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
break
fi

local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
fi
unset function_name
done
}

# shellcheck disable=SC2155
function runner::run_test() {
local start_time
start_time=$(clock::now)
Expand DownExpand Up@@ -204,8 +191,13 @@ function runner::run_test() {
state::add_tests_failed
reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
runner::write_failure_result_output "$test_file" "$subshell_output"

if env::is_stop_on_failure_enabled; then
exit 1
if env::is_parallel_run_enabled; then
parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
Expand DownExpand Up@@ -264,7 +256,6 @@ function runner::parse_result() {
fi
}

# shellcheck disable=SC2155
function runner::parse_result_parallel() {
local function_name=$1
shift
Expand DownExpand Up@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
echo "$execution_result" > "$unique_test_result_file"
}

# shellcheck disable=SC2155
function runner::parse_result_sync() {
local function_name=$1
local execution_result=$2
Expand Down
14 changes: 14 additions & 0 deletions ...napshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
14 changes: 14 additions & 0 deletions ...shots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Stop on failure when running on parallel#367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chemaclass merged 6 commits into from
Oct 12, 2024
Merged

Stop on failure when running on parallel #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f3599d
feat: display "totals" even on stop on failure
Chemaclass Oct 12, 2024
ab22ed4
feat: add Stop on failure enabled text
Chemaclass Oct 12, 2024
e8a94a9
feat: create .stop-on-failure shared on parallel tests
Chemaclass Oct 12, 2024
c1f1609
refactor: runner::call_test_functions
Chemaclass Oct 12, 2024
498a500
refactor: remove unused function run_test
Chemaclass Oct 12, 2024
ef1b601
docs: update changelog
Chemaclass Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
- `LOG_PATH="out.log"`
- `LOAD_FILE="tests/bootstrap.sh"`
- Add check that git is installed to `install.sh`
- Fixed `-S|--stop-on-failure` behaviour
- Improved time taken display
- Improved clean up temporal files and directories
- Improved CI test speed by running them in parallel
Expand Down
5 changes: 3 additions & 2 deletions src/env.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
_DEFAULT_LOAD_FILE="tests/bootstrap.sh"
_DEFAULT_LOG_JUNIT=""
_DEFAULT_REPORT_HTML=""
_DEFAULT_TERMINAL_WIDTH=100

: "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
: "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
Expand DownExpand Up@@ -76,10 +75,12 @@ function env::find_terminal_width() {
fi

# Directly echo the value with fallback
echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
echo "${cols:-100}"
}

EXIT_CODE_STOP_ON_FAILURE=4
TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
TERMINAL_WIDTH="$(env::find_terminal_width)"
FAILURES_OUTPUT_PATH=$(mktemp)
CAT="$(which cat)"
19 changes: 18 additions & 1 deletion src/main.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,20 +16,29 @@ function main::exec_tests() {
fi

# Trap SIGINT (Ctrl-C) and call the cleanup function
trap main::cleanup SIGINT
trap 'main::cleanup' SIGINT
trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT

if env::is_parallel_run_enabled && check_os::is_alpine; then
printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
printf "in a beta stage.\nThis means there may be unresolved issues, "
printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
fi

if env::is_parallel_run_enabled; then
parallel::reset
fi

console_header::print_version_with_env "$filter" "${test_files[@]}"
runner::load_test_files "$filter" "${test_files[@]}"
if env::is_parallel_run_enabled; then
wait
fi

if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
fi

console_results::print_failing_tests_and_reset
console_results::render_result
exit_code=$?
Expand All@@ -54,6 +63,14 @@ function main::cleanup() {
exit 1
}

function main::handle_stop_on_failure_sync() {
printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
console_results::print_failing_tests_and_reset
console_results::render_result
cleanup_temp_files
exit 1
}

function main::exec_assert() {
local original_assert_fn=$1
local args=("${@:2}")
Expand Down
14 changes: 14 additions & 0 deletions src/parallel.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
export _ASSERTIONS_INCOMPLETE=$total_incomplete
export _ASSERTIONS_SNAPSHOT=$total_snapshot
}

function parallel::mark_stop_on_failure() {
touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}

function parallel::must_stop_on_failure() {
[[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
}

function parallel::reset() {
# shellcheck disable=SC2153
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}
92 changes: 41 additions & 51 deletions src/runner.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,20 @@
#!/bin/bash
# shellcheck disable=SC2155

function runner::load_test_files() {
local filter=$1
shift
local files=("${@}")
local pids=()

if env::is_parallel_run_enabled; then
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
fi

for test_file in "${files[@]}"; do
if [[ ! -f $test_file ]]; then
continue
fi

# shellcheck source=/dev/null
source "$test_file"

runner::run_set_up_before_script
if env::is_parallel_run_enabled; then
runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
pids+=($!)
else
runner::call_test_functions "$test_file" "$filter"
fi
Expand All@@ -31,12 +24,9 @@ function runner::load_test_files() {

if env::is_parallel_run_enabled; then
wait

runner::spinner &
local spinner_pid=$!

parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"

# Kill the spinner once the aggregation finishes
disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
printf "\r " # Clear the spinner output
Expand DownExpand Up@@ -70,59 +60,56 @@ function runner::functions_for_script() {
shopt -u extdebug
}

# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
function run_test() {
runner::run_test "testing-fn" "$function_name" "$@"
}

function runner::call_test_functions() {
local script="$1"
local filter="$2"
local prefix="test"
# Use declare -F to list all function names
local all_function_names
all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions
filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")

local all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
# shellcheck disable=SC2207
local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))

if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi
if [[ "${#functions_to_run[@]}" -le 0 ]]; then
return
fi

helper::check_duplicate_functions "$script" || true
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi

for function_name in "${functions_to_run[@]}"; do
local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"
helper::check_duplicate_functions "$script" || true

# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi
for function_name in "${functions_to_run[@]}"; do
if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
break
fi

local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
fi
unset function_name
done
}

# shellcheck disable=SC2155
function runner::run_test() {
local start_time
start_time=$(clock::now)
Expand DownExpand Up@@ -204,8 +191,13 @@ function runner::run_test() {
state::add_tests_failed
reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
runner::write_failure_result_output "$test_file" "$subshell_output"

if env::is_stop_on_failure_enabled; then
exit 1
if env::is_parallel_run_enabled; then
parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
Expand DownExpand Up@@ -264,7 +256,6 @@ function runner::parse_result() {
fi
}

# shellcheck disable=SC2155
function runner::parse_result_parallel() {
local function_name=$1
shift
Expand DownExpand Up@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
echo "$execution_result" > "$unique_test_result_file"
}

# shellcheck disable=SC2155
function runner::parse_result_sync() {
local function_name=$1
local execution_result=$2
Expand Down
14 changes: 14 additions & 0 deletions ...napshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
14 changes: 14 additions & 0 deletions ...shots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

Stop on failure when running on parallel#367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chemaclass merged 6 commits into from
Oct 12, 2024
Merged

Stop on failure when running on parallel #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f3599d
feat: display "totals" even on stop on failure
Chemaclass Oct 12, 2024
ab22ed4
feat: add Stop on failure enabled text
Chemaclass Oct 12, 2024
e8a94a9
feat: create .stop-on-failure shared on parallel tests
Chemaclass Oct 12, 2024
c1f1609
refactor: runner::call_test_functions
Chemaclass Oct 12, 2024
498a500
refactor: remove unused function run_test
Chemaclass Oct 12, 2024
ef1b601
docs: update changelog
Chemaclass Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
- `LOG_PATH="out.log"`
- `LOAD_FILE="tests/bootstrap.sh"`
- Add check that git is installed to `install.sh`
- Fixed `-S|--stop-on-failure` behaviour
- Improved time taken display
- Improved clean up temporal files and directories
- Improved CI test speed by running them in parallel
Expand Down
5 changes: 3 additions & 2 deletions src/env.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
_DEFAULT_LOAD_FILE="tests/bootstrap.sh"
_DEFAULT_LOG_JUNIT=""
_DEFAULT_REPORT_HTML=""
_DEFAULT_TERMINAL_WIDTH=100

: "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
: "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
Expand DownExpand Up@@ -76,10 +75,12 @@ function env::find_terminal_width() {
fi

# Directly echo the value with fallback
echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
echo "${cols:-100}"
}

EXIT_CODE_STOP_ON_FAILURE=4
TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
TERMINAL_WIDTH="$(env::find_terminal_width)"
FAILURES_OUTPUT_PATH=$(mktemp)
CAT="$(which cat)"
19 changes: 18 additions & 1 deletion src/main.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,20 +16,29 @@ function main::exec_tests() {
fi

# Trap SIGINT (Ctrl-C) and call the cleanup function
trap main::cleanup SIGINT
trap 'main::cleanup' SIGINT
trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT

if env::is_parallel_run_enabled && check_os::is_alpine; then
printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
printf "in a beta stage.\nThis means there may be unresolved issues, "
printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
fi

if env::is_parallel_run_enabled; then
parallel::reset
fi

console_header::print_version_with_env "$filter" "${test_files[@]}"
runner::load_test_files "$filter" "${test_files[@]}"
if env::is_parallel_run_enabled; then
wait
fi

if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
fi

console_results::print_failing_tests_and_reset
console_results::render_result
exit_code=$?
Expand All@@ -54,6 +63,14 @@ function main::cleanup() {
exit 1
}

function main::handle_stop_on_failure_sync() {
printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
console_results::print_failing_tests_and_reset
console_results::render_result
cleanup_temp_files
exit 1
}

function main::exec_assert() {
local original_assert_fn=$1
local args=("${@:2}")
Expand Down
14 changes: 14 additions & 0 deletions src/parallel.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
export _ASSERTIONS_INCOMPLETE=$total_incomplete
export _ASSERTIONS_SNAPSHOT=$total_snapshot
}

function parallel::mark_stop_on_failure() {
touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}

function parallel::must_stop_on_failure() {
[[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
}

function parallel::reset() {
# shellcheck disable=SC2153
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}
92 changes: 41 additions & 51 deletions src/runner.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,20 @@
#!/bin/bash
# shellcheck disable=SC2155

function runner::load_test_files() {
local filter=$1
shift
local files=("${@}")
local pids=()

if env::is_parallel_run_enabled; then
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
fi

for test_file in "${files[@]}"; do
if [[ ! -f $test_file ]]; then
continue
fi

# shellcheck source=/dev/null
source "$test_file"

runner::run_set_up_before_script
if env::is_parallel_run_enabled; then
runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
pids+=($!)
else
runner::call_test_functions "$test_file" "$filter"
fi
Expand All@@ -31,12 +24,9 @@ function runner::load_test_files() {

if env::is_parallel_run_enabled; then
wait

runner::spinner &
local spinner_pid=$!

parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"

# Kill the spinner once the aggregation finishes
disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
printf "\r " # Clear the spinner output
Expand DownExpand Up@@ -70,59 +60,56 @@ function runner::functions_for_script() {
shopt -u extdebug
}

# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
function run_test() {
runner::run_test "testing-fn" "$function_name" "$@"
}

function runner::call_test_functions() {
local script="$1"
local filter="$2"
local prefix="test"
# Use declare -F to list all function names
local all_function_names
all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions
filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")

local all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
# shellcheck disable=SC2207
local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))

if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi
if [[ "${#functions_to_run[@]}" -le 0 ]]; then
return
fi

helper::check_duplicate_functions "$script" || true
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi

for function_name in "${functions_to_run[@]}"; do
local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"
helper::check_duplicate_functions "$script" || true

# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi
for function_name in "${functions_to_run[@]}"; do
if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
break
fi

local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
fi
unset function_name
done
}

# shellcheck disable=SC2155
function runner::run_test() {
local start_time
start_time=$(clock::now)
Expand DownExpand Up@@ -204,8 +191,13 @@ function runner::run_test() {
state::add_tests_failed
reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
runner::write_failure_result_output "$test_file" "$subshell_output"

if env::is_stop_on_failure_enabled; then
exit 1
if env::is_parallel_run_enabled; then
parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
Expand DownExpand Up@@ -264,7 +256,6 @@ function runner::parse_result() {
fi
}

# shellcheck disable=SC2155
function runner::parse_result_parallel() {
local function_name=$1
shift
Expand DownExpand Up@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
echo "$execution_result" > "$unique_test_result_file"
}

# shellcheck disable=SC2155
function runner::parse_result_sync() {
local function_name=$1
local execution_result=$2
Expand Down
14 changes: 14 additions & 0 deletions ...napshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
14 changes: 14 additions & 0 deletions ...shots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content

Stop on failure when running on parallel#367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chemaclass merged 6 commits into from
Oct 12, 2024
Merged

Stop on failure when running on parallel #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1f3599d
feat: display "totals" even on stop on failure
Chemaclass Oct 12, 2024
ab22ed4
feat: add Stop on failure enabled text
Chemaclass Oct 12, 2024
e8a94a9
feat: create .stop-on-failure shared on parallel tests
Chemaclass Oct 12, 2024
c1f1609
refactor: runner::call_test_functions
Chemaclass Oct 12, 2024
498a500
refactor: remove unused function run_test
Chemaclass Oct 12, 2024
ef1b601
docs: update changelog
Chemaclass Oct 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
- `LOG_PATH="out.log"`
- `LOAD_FILE="tests/bootstrap.sh"`
- Add check that git is installed to `install.sh`
- Fixed `-S|--stop-on-failure` behaviour
- Improved time taken display
- Improved clean up temporal files and directories
- Improved CI test speed by running them in parallel
Expand Down
5 changes: 3 additions & 2 deletions src/env.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
_DEFAULT_LOAD_FILE="tests/bootstrap.sh"
_DEFAULT_LOG_JUNIT=""
_DEFAULT_REPORT_HTML=""
_DEFAULT_TERMINAL_WIDTH=100

: "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
: "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
Expand DownExpand Up@@ -76,10 +75,12 @@ function env::find_terminal_width() {
fi

# Directly echo the value with fallback
echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
echo "${cols:-100}"
}

EXIT_CODE_STOP_ON_FAILURE=4
TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
TERMINAL_WIDTH="$(env::find_terminal_width)"
FAILURES_OUTPUT_PATH=$(mktemp)
CAT="$(which cat)"
19 changes: 18 additions & 1 deletion src/main.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,20 +16,29 @@ function main::exec_tests() {
fi

# Trap SIGINT (Ctrl-C) and call the cleanup function
trap main::cleanup SIGINT
trap 'main::cleanup' SIGINT
trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT

if env::is_parallel_run_enabled && check_os::is_alpine; then
printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
printf "in a beta stage.\nThis means there may be unresolved issues, "
printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
fi

if env::is_parallel_run_enabled; then
parallel::reset
fi

console_header::print_version_with_env "$filter" "${test_files[@]}"
runner::load_test_files "$filter" "${test_files[@]}"
if env::is_parallel_run_enabled; then
wait
fi

if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
fi

console_results::print_failing_tests_and_reset
console_results::render_result
exit_code=$?
Expand All@@ -54,6 +63,14 @@ function main::cleanup() {
exit 1
}

function main::handle_stop_on_failure_sync() {
printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
console_results::print_failing_tests_and_reset
console_results::render_result
cleanup_temp_files
exit 1
}

function main::exec_assert() {
local original_assert_fn=$1
local args=("${@:2}")
Expand Down
14 changes: 14 additions & 0 deletions src/parallel.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
export _ASSERTIONS_INCOMPLETE=$total_incomplete
export _ASSERTIONS_SNAPSHOT=$total_snapshot
}

function parallel::mark_stop_on_failure() {
touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}

function parallel::must_stop_on_failure() {
[[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
}

function parallel::reset() {
# shellcheck disable=SC2153
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
}
92 changes: 41 additions & 51 deletions src/runner.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,20 @@
#!/bin/bash
# shellcheck disable=SC2155

function runner::load_test_files() {
local filter=$1
shift
local files=("${@}")
local pids=()

if env::is_parallel_run_enabled; then
rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
fi

for test_file in "${files[@]}"; do
if [[ ! -f $test_file ]]; then
continue
fi

# shellcheck source=/dev/null
source "$test_file"

runner::run_set_up_before_script
if env::is_parallel_run_enabled; then
runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
pids+=($!)
else
runner::call_test_functions "$test_file" "$filter"
fi
Expand All@@ -31,12 +24,9 @@ function runner::load_test_files() {

if env::is_parallel_run_enabled; then
wait

runner::spinner &
local spinner_pid=$!

parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"

# Kill the spinner once the aggregation finishes
disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
printf "\r " # Clear the spinner output
Expand DownExpand Up@@ -70,59 +60,56 @@ function runner::functions_for_script() {
shopt -u extdebug
}

# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
function run_test() {
runner::run_test "testing-fn" "$function_name" "$@"
}

function runner::call_test_functions() {
local script="$1"
local filter="$2"
local prefix="test"
# Use declare -F to list all function names
local all_function_names
all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions
filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")

local all_function_names=$(declare -F | awk '{print $3}')
local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
# shellcheck disable=SC2207
local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))

if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi
if [[ "${#functions_to_run[@]}" -le 0 ]]; then
return
fi

helper::check_duplicate_functions "$script" || true
if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
echo "Running $script"
fi

for function_name in "${functions_to_run[@]}"; do
local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"
helper::check_duplicate_functions "$script" || true

# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi
for function_name in "${functions_to_run[@]}"; do
if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
break
fi

local provider_data=()
while IFS=" " read -r line; do
provider_data+=("$line")
done <<< "$(helper::get_provider_data "$function_name" "$script")"

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
# No data provider found
if [[ "${#provider_data[@]}" -eq 0 ]]; then
runner::run_test "$script" "$function_name"
unset function_name
continue
fi

# Execute the test function for each line of data
for data in "${provider_data[@]}"; do
IFS=" " read -r -a args <<< "$data"
if [ "${#args[@]}" -gt 1 ]; then
runner::run_test "$script" "$function_name" "${args[@]}"
else
runner::run_test "$script" "$function_name" "$data"
fi
done
fi
unset function_name
done
}

# shellcheck disable=SC2155
function runner::run_test() {
local start_time
start_time=$(clock::now)
Expand DownExpand Up@@ -204,8 +191,13 @@ function runner::run_test() {
state::add_tests_failed
reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
runner::write_failure_result_output "$test_file" "$subshell_output"

if env::is_stop_on_failure_enabled; then
exit 1
if env::is_parallel_run_enabled; then
parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
Expand DownExpand Up@@ -264,7 +256,6 @@ function runner::parse_result() {
fi
}

# shellcheck disable=SC2155
function runner::parse_result_parallel() {
local function_name=$1
shift
Expand DownExpand Up@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
echo "$execution_result" > "$unique_test_result_file"
}

# shellcheck disable=SC2155
function runner::parse_result_sync() {
local function_name=$1
local execution_result=$2
Expand Down
14 changes: 14 additions & 0 deletions ...napshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed 
14 changes: 14 additions & 0 deletions ...shots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
✗ Failed: B error
Expected '1'
but got  '2'

Stop on failure enabled...

There was 1 failure:

|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
|✗ Failed: B error
| Expected '1'
| but got  '2'

Tests:  1 passed, 1 failed, 2 total
Assertions: 3 passed, 1 failed, 4 total

 Some tests failed