#!/bin/sh # Default filenames PROMPT_FILE="${1:-test_prompts.txt}" RESULT_FILE="${2:-results.txt}" EXECUTABLE="./lex" # Ensure the executable exists and is runnable if [ ! -x "$EXECUTABLE" ]; then echo "Error: Executable '$EXECUTABLE' not found or not executable." >&2 exit 1 fi # Ensure the prompt file exists if [ ! -f "$PROMPT_FILE" ]; then echo "Error: Prompt file '$PROMPT_FILE' not found." >&2 exit 1 fi # Count valid prompts using POSIX awk to avoid ksh $(...) parenthetical syntax errors total_prompts=$(awk '/^[[:space:]]*($|#)/ {next} {count++} END {print count+0}' "$PROMPT_FILE") if [ "$total_prompts" -eq 0 ]; then echo "No valid prompts found in '$PROMPT_FILE'." exit 0 fi # Create (or truncate) the result file at the start of the run > "$RESULT_FILE" # Create a 70-character horizontal rule: " -- " + 66 "-" RULE=" -- " i=0 while [ $i -lt 66 ]; do RULE="${RULE}-" i=$((i + 1)) done echo "Starting evaluation: $total_prompts prompt(s) found." echo "Writing results to: '$RESULT_FILE'" echo "---------------------------------------------------" current=0 first=1 # Temporary file to capture time / rusage output TIME_LOG=$(mktemp /tmp/lex_time.XXXXXX) trap 'rm -f "$TIME_LOG"' EXIT # Process each prompt line by line while IFS= read -r prompt || [ -n "$prompt" ]; do # Skip empty lines or comment lines starting with # case "$prompt" in ""|\#*) continue ;; esac current=$((current + 1)) # Add delimiter before every prompt except the first in the results file if [ "$first" -eq 1 ]; then first=0 else printf "%s\n\n" "$RULE" >> "$RESULT_FILE" fi # Log prompt header to the result file printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE" # On OpenBSD, /usr/bin/time -lp outputs POSIX timings and rusage stats to stderr /usr/bin/time -lp "$EXECUTABLE" "$prompt" >> "$RESULT_FILE" 2> "$TIME_LOG" # Extract execution time and peak memory (maximum resident set size) from OpenBSD time output real_time=$(awk '/^real/ {print $2}' "$TIME_LOG") max_rss_mb=$(awk '/maximum resident set size/ {printf "%.2f", $1/1024}' "$TIME_LOG") # Format metrics summary line metrics_summary="ELAPSED: ${real_time}s | MAX RSS: ${max_rss_mb} MB" # Append metrics to result file printf "METRICS: %s\n\n" "$metrics_summary" >> "$RESULT_FILE" # Print real-time progress to terminal printf "[%d/%d] Done: %s (%s)\n" "$current" "$total_prompts" "$prompt" "$metrics_summary" done < "$PROMPT_FILE" echo "---------------------------------------------------" echo "Done! Processed $total_prompts prompt(s)."