Add test script and disable llama metrics.

1 files changed, 77 insertions, 0 deletions

diff --git a/test_prompts.sh b/test_prompts.sh
new file mode 100755
index 0000000..3178efb
--- /dev/null
+++ b/test_prompts.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+
+# Default filenames
+PROMPT_FILE="${1:-test_prompts.txt}"
+RESULT_FILE="${2:-results.txt}"
+EXECUTABLE="./lex"
+
+# Ensure the executable exists and is runnable
+if [ ! -x "$EXECUTABLE" ]; then
+    echo "Error: Executable '$EXECUTABLE' not found or not executable." >&2
+    exit 1
+fi
+
+# Ensure the prompt file exists
+if [ ! -f "$PROMPT_FILE" ]; then
+    echo "Error: Prompt file '$PROMPT_FILE' not found." >&2
+    exit 1
+fi
+
+# Count total valid prompts (ignoring empty lines and comments)
+total_prompts=$(grep -cvE '^[[:space:]]*($|#)' "$PROMPT_FILE")
+
+if [ "$total_prompts" -eq 0 ]; then
+    echo "No valid prompts found in '$PROMPT_FILE'."
+    exit 0
+fi
+
+# Create (or truncate) the result file at the start of the run
+> "$RESULT_FILE"
+
+# Create a 70-character horizontal rule: " -- " + 66 "-"
+RULE=" -- "
+i=0
+while [ $i -lt 66 ]; do
+    RULE="${RULE}-"
+    i=$((i + 1))
+done
+
+echo "Starting evaluation: $total_prompts prompt(s) found."
+echo "Writing results to: '$RESULT_FILE'"
+echo "---------------------------------------------------"
+
+current=0
+first=1
+
+# Process each prompt line by line
+while IFS= read -r prompt || [ -n "$prompt" ]; do
+    # Skip empty lines or comment lines starting with #
+    case "$prompt" in
+        ""|\#*) continue ;;
+    esac
+
+    current=$((current + 1))
+
+    # Print real-time progress to terminal
+    printf "[%d/%d] Running: %s\n" "$current" "$total_prompts" "$prompt"
+
+    # Add delimiter before every prompt except the first in the results file
+    if [ "$first" -eq 1 ]; then
+        first=0
+    else
+        printf "%s\n\n" "$RULE" >> "$RESULT_FILE"
+    fi
+
+    # Log prompt header to the result file
+    printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE"
+
+    # Invoke the executable and append output to result file
+    "$EXECUTABLE" "$prompt" >> "$RESULT_FILE" 2>&1
+
+    # Ensure a trailing newline after output
+    printf "\n" >> "$RESULT_FILE"
+done < "$PROMPT_FILE"
+
+echo "---------------------------------------------------"
+echo "Done! Processed $total_prompts prompt(s)."
+