blob: 04020c3c5ae665fc81be1f024121b0db316d44f5 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
#!/bin/sh
# Default filenames
PROMPT_FILE="${1:-test_prompts.txt}"
RESULT_FILE="${2:-results.txt}"
EXECUTABLE="./lex"
# Ensure the executable exists and is runnable
if [ ! -x "$EXECUTABLE" ]; then
echo "Error: Executable '$EXECUTABLE' not found or not executable." >&2
exit 1
fi
# Ensure the prompt file exists
if [ ! -f "$PROMPT_FILE" ]; then
echo "Error: Prompt file '$PROMPT_FILE' not found." >&2
exit 1
fi
# Count valid prompts using POSIX awk to avoid ksh $(...) parenthetical syntax errors
total_prompts=$(awk '/^[[:space:]]*($|#)/ {next} {count++} END {print count+0}' "$PROMPT_FILE")
if [ "$total_prompts" -eq 0 ]; then
echo "No valid prompts found in '$PROMPT_FILE'."
exit 0
fi
# Create (or truncate) the result file at the start of the run
> "$RESULT_FILE"
# Create a 70-character horizontal rule: " -- " + 66 "-"
RULE=" -- "
i=0
while [ $i -lt 66 ]; do
RULE="${RULE}-"
i=$((i + 1))
done
echo "Starting evaluation: $total_prompts prompt(s) found."
echo "Writing results to: '$RESULT_FILE'"
echo "---------------------------------------------------"
current=0
first=1
# Temporary file to capture time / rusage output
TIME_LOG=$(mktemp /tmp/lex_time.XXXXXX)
trap 'rm -f "$TIME_LOG"' EXIT
# Process each prompt line by line
while IFS= read -r prompt || [ -n "$prompt" ]; do
# Skip empty lines or comment lines starting with #
case "$prompt" in
""|\#*) continue ;;
esac
current=$((current + 1))
# Add delimiter before every prompt except the first in the results file
if [ "$first" -eq 1 ]; then
first=0
else
printf "%s\n\n" "$RULE" >> "$RESULT_FILE"
fi
# Log prompt header to the result file
printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE"
# On OpenBSD, /usr/bin/time -lp outputs POSIX timings and rusage stats to stderr
/usr/bin/time -lp "$EXECUTABLE" "$prompt" >> "$RESULT_FILE" 2> "$TIME_LOG"
# Extract execution time and peak memory (maximum resident set size) from OpenBSD time output
real_time=$(awk '/^real/ {print $2}' "$TIME_LOG")
max_rss_mb=$(awk '/maximum resident set size/ {printf "%.2f", $1/1024}' "$TIME_LOG")
# Format metrics summary line
metrics_summary="ELAPSED: ${real_time}s | MAX RSS: ${max_rss_mb} MB"
# Append metrics to result file
printf "METRICS: %s\n\n" "$metrics_summary" >> "$RESULT_FILE"
# Print real-time progress to terminal
printf "[%d/%d] Done: %s (%s)\n" "$current" "$total_prompts" "$prompt" "$metrics_summary"
done < "$PROMPT_FILE"
echo "---------------------------------------------------"
echo "Done! Processed $total_prompts prompt(s)."
|
