Drop CLI mode, update test script.

1 files changed, 51 insertions, 12 deletions

diff --git a/test_prompts.sh b/test_prompts.sh
index e22e90f..61fb69f 100755
--- a/test_prompts.sh
+++ b/test_prompts.sh
@@ -2,6 +2,9 @@
 
 # Usage: ./test_prompts.sh <model_path> [prompt_file]
 
+SOCK_PATH="/var/run/lex.sock"
+EXECUTABLE="./lex"
+
 # Mandatory Model Argument
 MODEL_PATH="$1"
 
@@ -22,7 +25,6 @@ RESULT_FILE="results_${MODEL_NAME}.txt"
 
 # Default prompt filename (optional 2nd argument)
 PROMPT_FILE="${2:-test_prompts.txt}"
-EXECUTABLE="./lex"
 
 # Ensure the executable exists and is runnable
 if [ ! -x "$EXECUTABLE" ]; then
@@ -55,7 +57,37 @@ while [ $i -lt 72 ]; do
     i=$((i + 1))
 done
 
-echo "Starting evaluation with model: '$MODEL_PATH'"
+# --- Start the daemon once, wait for it to come up, capture its baseline RSS ---
+
+echo "Starting daemon with model: '$MODEL_PATH'"
+"$EXECUTABLE" -m "$MODEL_PATH" &
+DAEMON_PID=$!
+
+cleanup() {
+    kill "$DAEMON_PID" 2>/dev/null
+    wait "$DAEMON_PID" 2>/dev/null
+    rm -f "$TIME_LOG"
+}
+trap cleanup EXIT INT TERM
+
+# Wait for the socket to appear (covers model load time) and for the
+# process to still be alive (covers early startup failure, e.g. bad
+# model path -> daemon exits immediately).
+waited=0
+while [ ! -S "$SOCK_PATH" ]; do
+    if ! kill -0 "$DAEMON_PID" 2>/dev/null; then
+        echo "Error: daemon exited during startup (bad model path?)" >&2
+        exit 1
+    fi
+    sleep 1
+    waited=$((waited + 1))
+    if [ "$waited" -ge 120 ]; then
+        echo "Error: timed out waiting for daemon socket at $SOCK_PATH" >&2
+        exit 1
+    fi
+done
+
+echo "Daemon ready (pid $DAEMON_PID) after ${waited}s"
 echo "Evaluating $total_prompts prompt(s) from '$PROMPT_FILE'"
 echo "Writing results to: '$RESULT_FILE'"
 echo "---------------------------------------------------"
@@ -65,7 +97,6 @@ first=1
 
 # Temporary file to capture time / rusage output
 TIME_LOG=$(mktemp /tmp/lex_time.XXXXXX)
-trap 'rm -f "$TIME_LOG"' EXIT
 
 # Process each prompt line by line
 while IFS= read -r prompt || [ -n "$prompt" ]; do
@@ -86,24 +117,32 @@ while IFS= read -r prompt || [ -n "$prompt" ]; do
     # Log prompt header to the result file
     printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE"
 
-    # Invoke executable with model path and prompt
-    /usr/bin/time -lp "$EXECUTABLE" "$MODEL_PATH" "$prompt" >> "$RESULT_FILE" 2> "$TIME_LOG"
+    # Sample the daemon's RSS right before firing the request, so we have
+    # a rough steady-state figure (won't catch a mid-request peak, but the
+    # model's footprint is dominated by the loaded weights, not per-request
+    # allocation, so this is a reasonable proxy).
+    daemon_rss_kb=$(ps -o rss= -p "$DAEMON_PID" 2>/dev/null | tr -d ' ')
+
+    # Send the prompt to the daemon over the socket, timing the round trip.
+    # This now measures request latency only (no model-load cost), which is
+    # the number that actually matters day to day with the daemon design.
+    /usr/bin/time -lp sh -c "printf '%s\n' \"\$1\" | nc -U \"\$2\"" _ "$prompt" "$SOCK_PATH" \
+        >> "$RESULT_FILE" 2> "$TIME_LOG"
 
-    # Extract execution time and peak memory (maximum resident set size) from OpenBSD time output
     real_time=$(awk '/^real/ {print $2}' "$TIME_LOG")
-    max_rss_mb=$(awk '/maximum resident set size/ {printf "%.2f", $1/1024}' "$TIME_LOG")
 
-    # Format metrics summary line
-    metrics_summary="ELAPSED: ${real_time}s | MAX RSS: ${max_rss_mb} MB"
+    if [ -n "$daemon_rss_kb" ]; then
+        daemon_rss_mb=$(awk -v kb="$daemon_rss_kb" 'BEGIN{printf "%.2f", kb/1024}')
+        metrics_summary="ELAPSED: ${real_time}s | DAEMON RSS: ${daemon_rss_mb} MB"
+    else
+        metrics_summary="ELAPSED: ${real_time}s | DAEMON RSS: unavailable"
+    fi
 
-    # Append metrics to result file
     printf "METRICS: %s\n\n" "$metrics_summary" >> "$RESULT_FILE"
 
-    # Print real-time progress to terminal
     printf "[%d/%d] Done: %s (%s)\n" "$current" "$total_prompts" "$prompt" "$metrics_summary"
 
 done < "$PROMPT_FILE"
 
 echo "---------------------------------------------------"
 echo "Done! Processed $total_prompts prompt(s)."
-