diff options
Diffstat (limited to 'test_prompts.sh')
| -rwxr-xr-x | test_prompts.sh | 40 |
1 files changed, 30 insertions, 10 deletions
diff --git a/test_prompts.sh b/test_prompts.sh index 04020c3..e22e90f 100755 --- a/test_prompts.sh +++ b/test_prompts.sh @@ -1,8 +1,27 @@ #!/bin/sh -# Default filenames -PROMPT_FILE="${1:-test_prompts.txt}" -RESULT_FILE="${2:-results.txt}" +# Usage: ./test_prompts.sh <model_path> [prompt_file] + +# Mandatory Model Argument +MODEL_PATH="$1" + +if [ -z "$MODEL_PATH" ]; then + echo "Usage: $0 <model_path> [prompt_file]" >&2 + exit 1 +fi + +if [ ! -f "$MODEL_PATH" ]; then + echo "Error: Model file '$MODEL_PATH' not found." >&2 + exit 1 +fi + +# Derive base model name and output result filename +# e.g., models/qwen2.5-7b-q4.gguf -> results_qwen2.5-7b-q4.gguf.txt +MODEL_NAME=$(basename "$MODEL_PATH") +RESULT_FILE="results_${MODEL_NAME}.txt" + +# Default prompt filename (optional 2nd argument) +PROMPT_FILE="${2:-test_prompts.txt}" EXECUTABLE="./lex" # Ensure the executable exists and is runnable @@ -28,15 +47,16 @@ fi # Create (or truncate) the result file at the start of the run > "$RESULT_FILE" -# Create a 70-character horizontal rule: " -- " + 66 "-" -RULE=" -- " +# Create horizontal rule. +RULE="" i=0 -while [ $i -lt 66 ]; do +while [ $i -lt 72 ]; do RULE="${RULE}-" i=$((i + 1)) done -echo "Starting evaluation: $total_prompts prompt(s) found." +echo "Starting evaluation with model: '$MODEL_PATH'" +echo "Evaluating $total_prompts prompt(s) from '$PROMPT_FILE'" echo "Writing results to: '$RESULT_FILE'" echo "---------------------------------------------------" @@ -66,12 +86,12 @@ while IFS= read -r prompt || [ -n "$prompt" ]; do # Log prompt header to the result file printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE" - # On OpenBSD, /usr/bin/time -lp outputs POSIX timings and rusage stats to stderr - /usr/bin/time -lp "$EXECUTABLE" "$prompt" >> "$RESULT_FILE" 2> "$TIME_LOG" + # Invoke executable with model path and prompt + /usr/bin/time -lp "$EXECUTABLE" "$MODEL_PATH" "$prompt" >> "$RESULT_FILE" 2> "$TIME_LOG" # Extract execution time and peak memory (maximum resident set size) from OpenBSD time output real_time=$(awk '/^real/ {print $2}' "$TIME_LOG") - max_rss_mb=$(awk '/maximum resident set size/ {printf "%.2f", $1/1024}' "$TIME_LOG") + max_rss_mb=$(awk '/maximum resident set size/ {printf "%.2f", $1/1024}' "$TIME_LOG") # Format metrics summary line metrics_summary="ELAPSED: ${real_time}s | MAX RSS: ${max_rss_mb} MB" |
