# Debugging Tests Tips
-## How to run & debug a specific test without anything else to keep the feedback loop short?
+## How to run & execute or debug a specific test without anything else to keep the feedback loop short?
There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
It will then build & run in the debugger for you.
+To just execute a test and get back a PASS or FAIL message run:
+
```bash
./scripts/debug-test.sh test-tokenizer
+```
+
+To test in GDB use the `-g` flag to enable gdb test mode.
+
+```bash
+./scripts/debug-test.sh -g test-tokenizer
# Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows:
>>> b main
```
+To speed up the testing loop, if you know your test number you can just run it similar to below:
+
+```bash
+./scripts/debug-test.sh test 23
+```
+
For further reference use `debug-test.sh -h` to print help.
make -j
```
-#### Step 3.1: Identify Test Command for Debugging
+#### Step 3: Find all tests available that matches REGEX
The output of this command will give you the command & arguments needed to run GDB.
...
```
-So for test #1 we can tell these two pieces of relevant information:
+#### Step 4: Identify Test Command for Debugging
+
+So for test #1 above we can tell these two pieces of relevant information:
* Test Binary: `~/llama.cpp/build-ci-debug/bin/test-tokenizer-0`
* Test GGUF Model: `~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf`
-#### Step 3.2: Run GDB on test command
+#### Step 5: Run GDB on test command
Based on the ctest 'test command' report above we can then run a gdb session via this command below:
#!/bin/bash
-test_suite=${1:-}
-test_number=${2:-}
PROG=${0##*/}
build_dir="build-ci-debug"
-if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
- echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
- echo "Debug specific ctest program."
- echo
- echo "Options:"
- echo " -h, --help Display this help and exit"
- echo
- echo "Arguments:"
- echo " <test_regex> (Mandatory) Supply one regex to the script to filter tests"
- echo " (test_number) (Optional) Test number to run a specific test"
- echo
- echo "Example:"
- echo " $PROG test-tokenizer"
- echo " $PROG test-tokenizer 3"
- echo
- exit 0
-fi
+# Print Color Commands
+red=$(tput setaf 1)
+green=$(tput setaf 2)
+yellow=$(tput setaf 3)
+blue=$(tput setaf 4)
+magenta=$(tput setaf 5)
+cyan=$(tput setaf 6)
+normal=$(tput sgr0)
-# Function to select and debug a test
-function select_test() {
- test_suite=${1:-test}
- test_number=${2:-}
-
- # Sanity Check If Tests Is Detected
- printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
- tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
- if [ ${#tests[@]} -eq 0 ]
- then
- echo "No tests avaliable... check your compliation process..."
- echo "Exiting."
- exit 1
- fi
- if [ -z $test_number ]
- then
- # List out avaliable tests
- printf "Which test would you like to debug?\n"
- id=0
- for s in "${tests[@]}"
- do
- echo "Test# ${id}"
- echo " $s"
- ((id++))
- done
-
- # Prompt user which test they wanted to run
- printf "\nRun test#? "
- read test_number
- else
- printf "\nUser Already Requested #${test_number}"
- fi
+# Print Help Message
+####################
- # Start GDB with the requested test binary and arguments
- printf "Debugging(GDB) test: ${tests[test_number]}\n"
- # Change IFS (Internal Field Separator)
- sIFS=$IFS
- IFS=$'\n'
+print_full_help() {
+ cat << EOF
+Usage: $PROG [OPTION]... <test_regex> (test_number)
+Debug specific ctest program.
- # Get test args
- gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
- IFS=$sIFS
- printf "Debug arguments: ${gdb_args[test_number]}\n\n"
+Options:
+ -h, --help display this help and exit
+ -g run in gdb mode
- # Expand paths if needed
- args=()
- for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\<//' -e 's/\>"//')
- do
- args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
- done
+Arguments:
+ <test_regex> (Mandatory) Supply one regex to the script to filter tests
+ (test_number) (Optional) Test number to run a specific test
- # Execute debugger
- echo "gdb args: ${args[@]}"
- gdb --args ${args[@]}
+Example:
+ $PROG test-tokenizer
+ $PROG test-tokenizer 3
+EOF
}
+abort() {
+ echo "Error: $1" >&2
+ cat << EOF >&2
+Usage: $PROG [OPTION]... <test_regex> (test_number)
+Debug specific ctest program.
+Refer to --help for full instructions.
+EOF
+ exit 1
+}
+
+
+# Dependency Sanity Check
+#########################
+
+check_dependency() {
+ command -v "$1" >/dev/null 2>&1 || {
+ abort "$1 is required but not found. Please install it and try again."
+ }
+}
+
+check_dependency ctest
+check_dependency cmake
+
+
# Step 0: Check the args
-if [ -z "$test_suite" ]
-then
- echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
- echo "Supply one regex to the script to filter tests,"
- echo "and optionally a test number to run a specific test."
- echo "Use --help flag for full instructions"
- exit 1
+########################
+
+if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
+ print_full_help >&2
+ exit 0
+fi
+
+# Parse command-line options
+gdb_mode=false
+while getopts "g" opt; do
+ case $opt in
+ g)
+ gdb_mode=true
+ echo "gdb_mode Mode Enabled"
+ ;;
+ esac
+done
+
+# Shift the option parameters
+shift $((OPTIND - 1))
+
+# Positionial Argument Processing : <test_regex>
+if [ -z "${1}" ]; then
+ abort "Test regex is required"
+else
+ test_suite=${1:-}
fi
+# Positionial Argument Processing : (test_number)
+test_number=${2:-}
+
+
# Step 1: Reset and Setup folder context
+########################################
+
## Sanity check that we are actually in a git repo
repo_root=$(git rev-parse --show-toplevel)
if [ ! -d "$repo_root" ]; then
- echo "Error: Not in a Git repository."
- exit 1
+ abort "Not in a Git repository."
fi
-## Reset folder to root context of git repo
-pushd "$repo_root" || exit 1
+## Reset folder to root context of git repo and Create and enter build directory
+pushd "$repo_root"
+rm -rf "$build_dir" && mkdir "$build_dir" || abort "Failed to make $build_dir"
-## Create and enter build directory
-rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
# Step 2: Setup Build Environment and Compile Test Binaries
-cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1
-pushd "$build_dir" && make -j || exit 1
+###########################################################
+
+# Note: test-eval-callback requires -DLLAMA_CURL
+cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_CURL=1 || abort "Failed to build enviroment"
+pushd "$build_dir"
+make -j || abort "Failed to compile"
+popd > /dev/null || exit 1
+
+
+# Step 3: Find all tests available that matches REGEX
+####################################################
+
+# Ctest Gather Tests
+# `-R test-tokenizer` : looks for all the test files named `test-tokenizer*` (R=Regex)
+# `-N` : "show-only" disables test execution & shows test commands that you can feed to GDB.
+# `-V` : Verbose Mode
+printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
+pushd "$build_dir"
+tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
+if [ ${#tests[@]} -eq 0 ]; then
+ abort "No tests avaliable... check your compliation process..."
+fi
+popd > /dev/null || exit 1
+
+
+# Step 4: Identify Test Command for Debugging
+#############################################
+
+# Select test number
+if [ -z $test_number ]; then
+ # List out avaliable tests
+ printf "Which test would you like to debug?\n"
+ id=0
+ for s in "${tests[@]}"
+ do
+ echo "Test# ${id}"
+ echo " $s"
+ ((id++))
+ done
+
+ # Prompt user which test they wanted to run
+ printf "\nRun test#? "
+ read test_number
+
+else
+ printf "\nUser Already Requested #${test_number}\n"
+
+fi
+
+# Grab all tests commands
+pushd "$build_dir"
+sIFS=$IFS # Save Initial IFS (Internal Field Separator)
+IFS=$'\n' # Change IFS (Internal Field Separator) (So we split ctest output by newline rather than by spaces)
+test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) # Get test args
+IFS=$sIFS # Reset IFS (Internal Field Separator)
+popd > /dev/null || exit 1
+
+# Grab specific test command
+single_test_name="${tests[test_number]}"
+single_test_command="${test_args[test_number]}"
+
+
+# Step 5: Execute or GDB Debug
+##############################
+
+printf "${magenta}Running Test #${test_number}: ${single_test_name}${normal}\n"
+printf "${cyan}single_test_command: ${single_test_command}${normal}\n"
-# Step 3: Debug the Test
-select_test "$test_suite" "$test_number"
+if [ "$gdb_mode" = "true" ]; then
+ # Execute debugger
+ pushd "$repo_root" || exit 1
+ eval "gdb --args ${single_test_command}"
+ popd > /dev/null || exit 1
+
+else
+ # Execute Test
+ pushd "$repo_root" || exit 1
+ eval "${single_test_command}"
+ exit_code=$?
+ popd > /dev/null || exit 1
+
+ # Print Result
+ printf "${blue}Ran Test #${test_number}: ${single_test_name}${normal}\n"
+ printf "${yellow}Command: ${single_test_command}${normal}\n"
+ if [ $exit_code -eq 0 ]; then
+ printf "${green}TEST PASS${normal}\n"
+ else
+ printf "${red}TEST FAIL${normal}\n"
+ fi
+
+fi
-# Step 4: Return to the directory from which the user ran the command.
-popd || exit 1
-popd || exit 1
-popd || exit 1
+# Return to the directory from which the user ran the command.
+popd > /dev/null || exit 1
overview / pkg / ggml / sources / llama.cpp / commitdiff