From fe4e5958572a029c6f2d2831aa7cfd8a8671ce21 Mon Sep 17 00:00:00 2001 From: Sadeep Madurange Date: Sat, 15 Aug 2026 11:21:07 +0800 Subject: Drop CLI mode, update test script. --- llm.c | 2 +- main.c | 54 +++------- results_qwen2.5-3b-instruct-q4_k_m.gguf.txt | 161 ++++++++++++++-------------- test_prompts.sh | 63 ++++++++--- 4 files changed, 145 insertions(+), 135 deletions(-) diff --git a/llm.c b/llm.c index dc3608f..e7e5853 100644 --- a/llm.c +++ b/llm.c @@ -8,7 +8,7 @@ #define SYSTEM_PROMPT \ "You are a dictionary assistant. Define the target word in the context provided.\n" \ "DO NOT repeat target word in definition. Choose one or two of Formality levels.\n\n" \ - "FOLLOW the output format:\n" \ + "Output format:\n" \ " () — \n\n" \ "DEFINITION:\n" \ "\n\n" \ diff --git a/main.c b/main.c index e89ed7a..e46c515 100644 --- a/main.c +++ b/main.c @@ -1,13 +1,9 @@ #include -#include -#include #include #include -#include #include #include #include -#include #include #include "llm.h" @@ -15,9 +11,9 @@ #define SOCK_PATH "/var/run/lex.sock" #define MODEL_PATH "qwen2.5-3b-instruct-q4_k_m.gguf" -static void usage(const char *bin) +static void usage(const char *prog) { - errx(1, "usage: %s [-m model_path] [-d] [prompt]", bin); + errx(1, "usage: %s [-m model_path]", prog); } static void loop(struct llama_model *model, int lsock) @@ -87,26 +83,19 @@ int sock_init(void) int main(int argc , char *argv[]) { - int daemon_mode, sock, opt; + int sock, opt; const char *prog; - const char *prompt; const char *model_path; struct llama_model *model; - if (pledge("stdio rpath wpath cpath unix", NULL) == -1) + if (pledge("stdio rpath wpath cpath unix unveil", NULL) == -1) err(1, "pledge failed"); - sock = -1; - daemon_mode = 0; - prompt = NULL; prog = argv[0]; model_path = MODEL_PATH; - while ((opt = getopt(argc, argv, "dm:")) != -1) { + while ((opt = getopt(argc, argv, "m:")) != -1) { switch (opt) { - case 'd': - daemon_mode = 1; - break; case 'm': model_path = optarg; break; @@ -117,42 +106,25 @@ int main(int argc , char *argv[]) argc -= optind; argv += optind; - - if (daemon_mode) { - if (argc != 0) // -d takes no positional args - usage(prog); - } else { - if (argc != 1) // normal mode requires one prompt - usage(prog); - - prompt = argv[0]; - } + if (argc != 0) + usage(prog); if (unveil(model_path, "r") == -1) err(1, "unveil %s failed", model_path); - if (daemon_mode && unveil(SOCK_PATH, "rwc") == -1) + if (unveil(SOCK_PATH, "rwc") == -1) err(1, "unveil %s failed", SOCK_PATH); if (unveil(NULL, NULL) == -1) err(1, "unveil lock failed"); - if (daemon_mode) { - sock = sock_init(); - if (pledge("stdio rpath unix", NULL) == -1) - err(1, "Failed to narrow pledge after sock_init()"); - } + sock = sock_init(); - model = llm_init(model_path); - if (!model) + if (!(model = llm_init(model_path))) errx(1, "Failed to load model"); - if (daemon_mode) - loop(model, sock); /* doesn't return */ - - if (pledge("stdio", NULL) == -1) - err(1, "Failed to narrow pledge for normal mode"); + if (pledge("stdio unix", NULL) == -1) + err(1, "secondary pledge failed"); - llm_process_request(model, prompt, stdout); - llm_free(model); + loop(model, sock); /* doesn't return */ return 0; } diff --git a/results_qwen2.5-3b-instruct-q4_k_m.gguf.txt b/results_qwen2.5-3b-instruct-q4_k_m.gguf.txt index 88e913b..d5b3043 100644 --- a/results_qwen2.5-3b-instruct-q4_k_m.gguf.txt +++ b/results_qwen2.5-3b-instruct-q4_k_m.gguf.txt @@ -3,124 +3,123 @@ PROMPT: Define imperious in 'an imperious necessity drove him forward on his que imperious (adj) — Formal DEFINITION: -Having a commanding or domineering character; asserting one's will with authority. +Having a commanding or domineering attitude; asserting authority over others. GENERAL TONE: -The necessity was so strong that it compelled him to move forward on his journey, leaving no room for resistance. +The tone is formal, emphasizing the authoritative nature of imperious necessity. SYNONYMS: -urgent, pressing, compelling +domineering, autocratic, bossy EXAMPLES: -1. The imperious necessity pushed him to finish the project. -2. His imperious nature made it hard for others to disagree with his decisions. +1. The manager's imperious attitude made the employees uncomfortable. +2. She felt an imperious necessity to finish her project before the deadline. -METRICS: ELAPSED: 19.89s | MAX RSS: 3222.74 MB +METRICS: ELAPSED: 18.79s | DAEMON RSS: 467.86 MB ------------------------------------------------------------------------ PROMPT: Define despondency in 'he fell into a state of deep, uninterrupted despondency' -despondency (noun) — Formal +despondency (noun) — Formality: Literary DEFINITION: -deep and persistent hopelessness or despair. +a state of deep and persistent depression or hopelessness. GENERAL TONE: -The tone is somber, reflecting the heavy emotional state of the person experiencing despondency. +The tone is somber, reflecting the heavy emotional state described in the sentence. SYNONYMS: -hopelessness, melancholy, gloominess +melancholy, despair, gloom EXAMPLES: 1. He fell into a state of deep, uninterrupted despondency. 2. Her friends noticed she had been in a constant state of despondency lately. -METRICS: ELAPSED: 22.21s | MAX RSS: 3223.14 MB +METRICS: ELAPSED: 18.59s | DAEMON RSS: 3160.82 MB ------------------------------------------------------------------------ PROMPT: Define arduous in 'the long and arduous journey through the frozen mountains' -arduous (adj) — Formal +arduous (adjective) — Formality: Formal DEFINITION: -Challenging or difficult, especially in a way that requires great effort. +Challenging or difficult to endure, especially physically. GENERAL TONE: -The journey through the frozen mountains was tough and required much perseverance. - +The journey through the frozen mountains was a grueling and strenuous trek. SYNONYMS: -Difficult, strenuous, laborious, taxing. +Difficult, taxing, trying, strenuous. EXAMPLES: -1. The expedition through the dense jungle was an arduous task. -2. Climbing Mount Everest is an arduous challenge for mountaineers. +1. The expedition through the icy wilderness was an arduous journey. +2. Climbing up those steep mountains was an arduous task. -METRICS: ELAPSED: 21.95s | MAX RSS: 3223.10 MB +METRICS: ELAPSED: 18.38s | DAEMON RSS: 3164.26 MB ------------------------------------------------------------------------ PROMPT: Define ephemeral in 'human history appeared as a brief and ephemeral flash in cosmic time' -ephemeral (adj) — Formal +ephemeral (adj) — Formality: Literary DEFINITION: lasting only a very short time; transient. GENERAL TONE: -The phrase suggests that human history is insignificant compared to the vastness of cosmic time. +The term evokes a sense of transience and impermanence, often used in philosophical or poetic contexts to describe fleeting moments or phenomena. SYNONYMS: -transient, fleeting, momentary, brief. +transient, fleeting, momentary. EXAMPLES: -1. The sunset was an ephemeral display in the sky. -2. His fame is just an ephemeral memory now. +1. The sunset was an ephemeral display of colors. +2. Her smile was an ephemeral light in the room. -METRICS: ELAPSED: 20.89s | MAX RSS: 3223.20 MB +METRICS: ELAPSED: 18.78s | DAEMON RSS: 3164.46 MB ------------------------------------------------------------------------ PROMPT: Define austere in 'they lived under the austere discipline of their new order' -austere (adj) — Formal +austere (adjective) — Formality: Formal DEFINITION: -strictly simple or plain in style, manner, or appearance; severe and unadorned. +Strictly simple or plain, often in a harsh or severe manner. GENERAL TONE: -The tone is formal, emphasizing the strictness and simplicity of their lifestyle under the new order. +The tone is formal and precise, suitable for historical contexts. SYNONYMS: -rigid, severe, plain, stark. +Spare, austonian, ascetic EXAMPLES: 1. The monastery was known for its austere lifestyle. -2. His writing style is quite austere and lacks ornamentation. +2. His new regime imposed a strict austere regimen on the workers. -METRICS: ELAPSED: 22.54s | MAX RSS: 3222.46 MB +METRICS: ELAPSED: 17.64s | DAEMON RSS: 3164.69 MB ------------------------------------------------------------------------ PROMPT: Define venerable in 'the venerable monuments of a long-forgotten civilization' -venerable (adj) — Formal +venerable (adjective) — Formality: Literary DEFINITION: Respected and revered due to age or historical significance. GENERAL TONE: -The monuments are ancient structures that have stood the test of time, earning them respect from modern visitors. +The tone is reverent, suggesting a sense of awe and respect for ancient structures. SYNONYMS: -Ancient, venerable, historic, revered. +Ancient, venerable, historic, esteemed EXAMPLES: -1. The venerable monuments of a long-forgotten civilization are now in ruins. -2. She felt a sense of reverence for the venerable old church. +1. The venerable monuments stood as silent witnesses to the past. +2. She felt a deep reverence for the ancient temples that were now ruins. -METRICS: ELAPSED: 22.47s | MAX RSS: 3222.97 MB +METRICS: ELAPSED: 18.57s | DAEMON RSS: 3166.86 MB ------------------------------------------------------------------------ @@ -129,103 +128,103 @@ PROMPT: Define tumultuous in 'the species endured a tumultuous period of war and tumultuous (adj) — Formality: Literary DEFINITION: -Marked by violent disturbance or disorder; stormy. +Marked by violent disturbance or disorder; tumultuous events or conditions. GENERAL TONE: -The passage conveys a chaotic and turbulent period filled with conflict and change. +The tone is formal, using precise language to describe a chaotic period of war and recovery. SYNONYMS: -chaotic, tumult, stormy, tempestuous +chaotic, turbulent, stormy, agitated EXAMPLES: -1. The city experienced a tumultuous period of unrest and protests. -2. During the tumultuous years of his reign, he faced numerous challenges. +1. The city experienced a tumultuous period of unrest and rebuilding. +2. During the tumultuous years following the revolution, society underwent significant changes. -METRICS: ELAPSED: 22.61s | MAX RSS: 3222.95 MB +METRICS: ELAPSED: 19.25s | DAEMON RSS: 3167.03 MB ------------------------------------------------------------------------ PROMPT: Define imperturbable in 'he maintained an imperturbable calm despite the surrounding chaos' -imperturbable (adj) — Formal + — Formal DEFINITION: -Unshaken or unaffected by events, circumstances, or emotions. +Unflappable; showing no sign of disturbance or emotion. GENERAL TONE: -The speaker remains composed even in chaotic situations. +The tone is formal, emphasizing the unchanging nature of calmness in a chaotic situation. SYNONYMS: -calm, composed, unruffled +Unflappable, composed, unruffled, steady. EXAMPLES: 1. Despite the storm outside, he maintained an imperturbable calm. -2. She kept her composure during the heated argument. +2. Her imperturbable composure in the face of criticism was admirable. -METRICS: ELAPSED: 21.00s | MAX RSS: 3223.41 MB +METRICS: ELAPSED: 19.17s | DAEMON RSS: 3167.21 MB ------------------------------------------------------------------------ PROMPT: Define singular in 'he was a man of a singular and unpredictable temperament' -singular (adjective) — Formal +(singular) — Formal DEFINITION: -Singular refers to something unique or individual, not part of a group. +Singular means unique or exceptional, not common. GENERAL TONE: -The term conveys uniqueness and distinctiveness without being overly formal. +The term conveys a sense of uniqueness and unpredictability in temperament. SYNONYMS: -uncommon, unique, distinctive +Uncommon, distinctive, eccentric. EXAMPLES: -1. He was a man of a singular and unpredictable temperament. -2. Her approach to problem-solving is singularly effective. +1. He was a man of singular and unpredictable temperament. +2. Her personality had a singular charm that attracted many admirers. -METRICS: ELAPSED: 22.41s | MAX RSS: 3222.57 MB +METRICS: ELAPSED: 16.05s | DAEMON RSS: 3167.36 MB ------------------------------------------------------------------------ PROMPT: Define enigmatic in 'he left behind an enigmatic smile and no further explanation' -enigmatic (adj) — Formal +enigmatic (adjective) — Formal DEFINITION: Mysterious and difficult to understand or interpret. GENERAL TONE: -The smile was a puzzle, hard to decipher. +The tone is formal, precise, and somewhat mysterious as if the smile was a puzzle waiting to be solved. SYNONYMS: -mystifying, puzzling, enigmatical +mysterious, puzzling, enigmatic, elusive EXAMPLES: -1. The riddle left me feeling enigmatic. -2. Her smile was so enigmatic it made everyone curious. +1. The riddle left behind was an enigmatic smile. +2. His words were so enigmatic that they left everyone puzzled. -METRICS: ELAPSED: 20.25s | MAX RSS: 3223.00 MB +METRICS: ELAPSED: 18.15s | DAEMON RSS: 3169.46 MB ------------------------------------------------------------------------ PROMPT: Define bestial in 'You submitted to be the slave of bestial men' -bestial (adj) — Formal +bestial (adj) — Formality: Literary DEFINITION: -Of or relating to animals; brutish. +Of or relating to animals; brutish, uncivilized. GENERAL TONE: -The phrase suggests a dehumanizing and brutal treatment of humans by other humans, emphasizing the inhumane nature of their actions. +The tone is archaic and formal, evoking a sense of medieval literature. SYNONYMS: -brutal, savage, uncivilized +Savage, brutish, uncivilized. EXAMPLES: -1. The treatment of prisoners was bestial. -2. He felt like a bestial creature in the eyes of his captors. +1. The bestial behavior of the animals in the zoo was shocking. +2. He felt like a bestial creature after being beaten by his captors. -METRICS: ELAPSED: 22.25s | MAX RSS: 3222.89 MB +METRICS: ELAPSED: 18.51s | DAEMON RSS: 3169.58 MB ------------------------------------------------------------------------ @@ -234,38 +233,38 @@ PROMPT: Define flung in 'As if plunged in a stormy water, I was flung about wild flung (verb) — Formality: Literary DEFINITION: -To throw or hurl suddenly and forcefully, often with a sense of being overwhelmed. +To throw or toss violently; to hurl. GENERAL TONE: -The tone is vividly descriptive, evoking a tumultuous experience. +The tone is vividly descriptive, evoking a tumultuous scene. SYNONYMS: -Thrust, tossed, hurled, tossed about. +Threw, tossed, hurled, tossed about. EXAMPLES: -1. As if plunged in a stormy water, I was flung about wildly, and felt myself sinking. -2. The ball was flung over the fence by the batter. +1. As if plunged in a stormy water, I was flung about wildly. +2. The ball was flung across the field by the outfielder. -METRICS: ELAPSED: 24.50s | MAX RSS: 3224.59 MB +METRICS: ELAPSED: 18.36s | DAEMON RSS: 3169.75 MB ------------------------------------------------------------------------ PROMPT: Define hewers in 'You might by this time have made the Bags hewers of wood and drawers of water' -hewers (noun) — Formal + — Formal DEFINITION: People who cut wood or perform manual labor. GENERAL TONE: -The sentence suggests a rustic, hardworking community where physical labor is valued and necessary. +The sentence suggests a rustic, possibly colonial setting where physical work is common and valued. SYNONYMS: -Cutters, woodcutters, laborers +Laborers, woodcutters, cutters of timber. EXAMPLES: -1. The hewers worked tirelessly in the forest. -2. In colonial times, many families were hewers and drawers of water. +1. The team of hewers were tasked with clearing the forest. +2. In the old days, men would often be seen as hewers of wood and drawers of water. -METRICS: ELAPSED: 21.98s | MAX RSS: 3224.26 MB +METRICS: ELAPSED: 19.62s | DAEMON RSS: 3169.91 MB diff --git a/test_prompts.sh b/test_prompts.sh index e22e90f..61fb69f 100755 --- a/test_prompts.sh +++ b/test_prompts.sh @@ -2,6 +2,9 @@ # Usage: ./test_prompts.sh [prompt_file] +SOCK_PATH="/var/run/lex.sock" +EXECUTABLE="./lex" + # Mandatory Model Argument MODEL_PATH="$1" @@ -22,7 +25,6 @@ RESULT_FILE="results_${MODEL_NAME}.txt" # Default prompt filename (optional 2nd argument) PROMPT_FILE="${2:-test_prompts.txt}" -EXECUTABLE="./lex" # Ensure the executable exists and is runnable if [ ! -x "$EXECUTABLE" ]; then @@ -55,7 +57,37 @@ while [ $i -lt 72 ]; do i=$((i + 1)) done -echo "Starting evaluation with model: '$MODEL_PATH'" +# --- Start the daemon once, wait for it to come up, capture its baseline RSS --- + +echo "Starting daemon with model: '$MODEL_PATH'" +"$EXECUTABLE" -m "$MODEL_PATH" & +DAEMON_PID=$! + +cleanup() { + kill "$DAEMON_PID" 2>/dev/null + wait "$DAEMON_PID" 2>/dev/null + rm -f "$TIME_LOG" +} +trap cleanup EXIT INT TERM + +# Wait for the socket to appear (covers model load time) and for the +# process to still be alive (covers early startup failure, e.g. bad +# model path -> daemon exits immediately). +waited=0 +while [ ! -S "$SOCK_PATH" ]; do + if ! kill -0 "$DAEMON_PID" 2>/dev/null; then + echo "Error: daemon exited during startup (bad model path?)" >&2 + exit 1 + fi + sleep 1 + waited=$((waited + 1)) + if [ "$waited" -ge 120 ]; then + echo "Error: timed out waiting for daemon socket at $SOCK_PATH" >&2 + exit 1 + fi +done + +echo "Daemon ready (pid $DAEMON_PID) after ${waited}s" echo "Evaluating $total_prompts prompt(s) from '$PROMPT_FILE'" echo "Writing results to: '$RESULT_FILE'" echo "---------------------------------------------------" @@ -65,7 +97,6 @@ first=1 # Temporary file to capture time / rusage output TIME_LOG=$(mktemp /tmp/lex_time.XXXXXX) -trap 'rm -f "$TIME_LOG"' EXIT # Process each prompt line by line while IFS= read -r prompt || [ -n "$prompt" ]; do @@ -86,24 +117,32 @@ while IFS= read -r prompt || [ -n "$prompt" ]; do # Log prompt header to the result file printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE" - # Invoke executable with model path and prompt - /usr/bin/time -lp "$EXECUTABLE" "$MODEL_PATH" "$prompt" >> "$RESULT_FILE" 2> "$TIME_LOG" + # Sample the daemon's RSS right before firing the request, so we have + # a rough steady-state figure (won't catch a mid-request peak, but the + # model's footprint is dominated by the loaded weights, not per-request + # allocation, so this is a reasonable proxy). + daemon_rss_kb=$(ps -o rss= -p "$DAEMON_PID" 2>/dev/null | tr -d ' ') + + # Send the prompt to the daemon over the socket, timing the round trip. + # This now measures request latency only (no model-load cost), which is + # the number that actually matters day to day with the daemon design. + /usr/bin/time -lp sh -c "printf '%s\n' \"\$1\" | nc -U \"\$2\"" _ "$prompt" "$SOCK_PATH" \ + >> "$RESULT_FILE" 2> "$TIME_LOG" - # Extract execution time and peak memory (maximum resident set size) from OpenBSD time output real_time=$(awk '/^real/ {print $2}' "$TIME_LOG") - max_rss_mb=$(awk '/maximum resident set size/ {printf "%.2f", $1/1024}' "$TIME_LOG") - # Format metrics summary line - metrics_summary="ELAPSED: ${real_time}s | MAX RSS: ${max_rss_mb} MB" + if [ -n "$daemon_rss_kb" ]; then + daemon_rss_mb=$(awk -v kb="$daemon_rss_kb" 'BEGIN{printf "%.2f", kb/1024}') + metrics_summary="ELAPSED: ${real_time}s | DAEMON RSS: ${daemon_rss_mb} MB" + else + metrics_summary="ELAPSED: ${real_time}s | DAEMON RSS: unavailable" + fi - # Append metrics to result file printf "METRICS: %s\n\n" "$metrics_summary" >> "$RESULT_FILE" - # Print real-time progress to terminal printf "[%d/%d] Done: %s (%s)\n" "$current" "$total_prompts" "$prompt" "$metrics_summary" done < "$PROMPT_FILE" echo "---------------------------------------------------" echo "Done! Processed $total_prompts prompt(s)." - -- cgit v1.2.3