#!/bin/sh

# Usage: ./test_prompts.sh <model_path> [prompt_file]

# Mandatory Model Argument
MODEL_PATH="$1"

if [ -z "$MODEL_PATH" ]; then
    echo "Usage: $0 <model_path> [prompt_file]" >&2
    exit 1
fi

if [ ! -f "$MODEL_PATH" ]; then
    echo "Error: Model file '$MODEL_PATH' not found." >&2
    exit 1
fi

# Derive base model name and output result filename
# e.g., models/qwen2.5-7b-q4.gguf -> results_qwen2.5-7b-q4.gguf.txt
MODEL_NAME=$(basename "$MODEL_PATH")
RESULT_FILE="results_${MODEL_NAME}.txt"

# Default prompt filename (optional 2nd argument)
PROMPT_FILE="${2:-test_prompts.txt}"
EXECUTABLE="./lex"

# Ensure the executable exists and is runnable
if [ ! -x "$EXECUTABLE" ]; then
    echo "Error: Executable '$EXECUTABLE' not found or not executable." >&2
    exit 1
fi

# Ensure the prompt file exists
if [ ! -f "$PROMPT_FILE" ]; then
    echo "Error: Prompt file '$PROMPT_FILE' not found." >&2
    exit 1
fi

# Count valid prompts using POSIX awk to avoid ksh $(...) parenthetical syntax errors
total_prompts=$(awk '/^[[:space:]]*($|#)/ {next} {count++} END {print count+0}' "$PROMPT_FILE")

if [ "$total_prompts" -eq 0 ]; then
    echo "No valid prompts found in '$PROMPT_FILE'."
    exit 0
fi

# Create (or truncate) the result file at the start of the run
> "$RESULT_FILE"

# Create horizontal rule.
RULE=""
i=0
while [ $i -lt 72 ]; do
    RULE="${RULE}-"
    i=$((i + 1))
done

echo "Starting evaluation with model: '$MODEL_PATH'"
echo "Evaluating $total_prompts prompt(s) from '$PROMPT_FILE'"
echo "Writing results to: '$RESULT_FILE'"
echo "---------------------------------------------------"

current=0
first=1

# Temporary file to capture time / rusage output
TIME_LOG=$(mktemp /tmp/lex_time.XXXXXX)
trap 'rm -f "$TIME_LOG"' EXIT

# Process each prompt line by line
while IFS= read -r prompt || [ -n "$prompt" ]; do
    # Skip empty lines or comment lines starting with #
    case "$prompt" in
        ""|\#*) continue ;;
    esac

    current=$((current + 1))

    # Add delimiter before every prompt except the first in the results file
    if [ "$first" -eq 1 ]; then
        first=0
    else
        printf "%s\n\n" "$RULE" >> "$RESULT_FILE"
    fi

    # Log prompt header to the result file
    printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE"

    # Invoke executable with model path and prompt
    /usr/bin/time -lp "$EXECUTABLE" "$MODEL_PATH" "$prompt" >> "$RESULT_FILE" 2> "$TIME_LOG"

    # Extract execution time and peak memory (maximum resident set size) from OpenBSD time output
    real_time=$(awk '/^real/ {print $2}' "$TIME_LOG")
    max_rss_mb=$(awk '/maximum resident set size/ {printf "%.2f", $1/1024}' "$TIME_LOG")

    # Format metrics summary line
    metrics_summary="ELAPSED: ${real_time}s | MAX RSS: ${max_rss_mb} MB"

    # Append metrics to result file
    printf "METRICS: %s\n\n" "$metrics_summary" >> "$RESULT_FILE"

    # Print real-time progress to terminal
    printf "[%d/%d] Done: %s (%s)\n" "$current" "$total_prompts" "$prompt" "$metrics_summary"

done < "$PROMPT_FILE"

echo "---------------------------------------------------"
echo "Done! Processed $total_prompts prompt(s)."