diff options
| -rw-r--r-- | main.c | 2 | ||||
| -rwxr-xr-x | test_prompts.sh | 77 |
2 files changed, 78 insertions, 1 deletions
@@ -181,7 +181,7 @@ static void process_request(struct llama_model *model, const char *user_prompt) printf("\n\n"); fflush(stdout); - llama_perf_context_print(ctx); + //llama_perf_context_print(ctx); llama_sampler_free(smpl); llama_free(ctx); diff --git a/test_prompts.sh b/test_prompts.sh new file mode 100755 index 0000000..3178efb --- /dev/null +++ b/test_prompts.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +# Default filenames +PROMPT_FILE="${1:-test_prompts.txt}" +RESULT_FILE="${2:-results.txt}" +EXECUTABLE="./lex" + +# Ensure the executable exists and is runnable +if [ ! -x "$EXECUTABLE" ]; then + echo "Error: Executable '$EXECUTABLE' not found or not executable." >&2 + exit 1 +fi + +# Ensure the prompt file exists +if [ ! -f "$PROMPT_FILE" ]; then + echo "Error: Prompt file '$PROMPT_FILE' not found." >&2 + exit 1 +fi + +# Count total valid prompts (ignoring empty lines and comments) +total_prompts=$(grep -cvE '^[[:space:]]*($|#)' "$PROMPT_FILE") + +if [ "$total_prompts" -eq 0 ]; then + echo "No valid prompts found in '$PROMPT_FILE'." + exit 0 +fi + +# Create (or truncate) the result file at the start of the run +> "$RESULT_FILE" + +# Create a 70-character horizontal rule: " -- " + 66 "-" +RULE=" -- " +i=0 +while [ $i -lt 66 ]; do + RULE="${RULE}-" + i=$((i + 1)) +done + +echo "Starting evaluation: $total_prompts prompt(s) found." +echo "Writing results to: '$RESULT_FILE'" +echo "---------------------------------------------------" + +current=0 +first=1 + +# Process each prompt line by line +while IFS= read -r prompt || [ -n "$prompt" ]; do + # Skip empty lines or comment lines starting with # + case "$prompt" in + ""|\#*) continue ;; + esac + + current=$((current + 1)) + + # Print real-time progress to terminal + printf "[%d/%d] Running: %s\n" "$current" "$total_prompts" "$prompt" + + # Add delimiter before every prompt except the first in the results file + if [ "$first" -eq 1 ]; then + first=0 + else + printf "%s\n\n" "$RULE" >> "$RESULT_FILE" + fi + + # Log prompt header to the result file + printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE" + + # Invoke the executable and append output to result file + "$EXECUTABLE" "$prompt" >> "$RESULT_FILE" 2>&1 + + # Ensure a trailing newline after output + printf "\n" >> "$RESULT_FILE" +done < "$PROMPT_FILE" + +echo "---------------------------------------------------" +echo "Done! Processed $total_prompts prompt(s)." + |