From adaefc77ec359bec5b42711d23e7036835592e93 Mon Sep 17 00:00:00 2001 From: Sadeep Madurange Date: Fri, 28 Aug 2026 11:35:52 +0800 Subject: Remove application invocation in benchmark script + 5A5V result. --- ...ts_qwen2.5-3b-instruct-q4_k_m.gguf.5v5a+log.txt | 206 +++++++++++++++++++++ test_prompts.sh | 78 +++----- 2 files changed, 232 insertions(+), 52 deletions(-) create mode 100644 bench/results_qwen2.5-3b-instruct-q4_k_m.gguf.5v5a+log.txt mode change 100755 => 100644 test_prompts.sh diff --git a/bench/results_qwen2.5-3b-instruct-q4_k_m.gguf.5v5a+log.txt b/bench/results_qwen2.5-3b-instruct-q4_k_m.gguf.5v5a+log.txt new file mode 100644 index 0000000..770eef4 --- /dev/null +++ b/bench/results_qwen2.5-3b-instruct-q4_k_m.gguf.5v5a+log.txt @@ -0,0 +1,206 @@ +PROMPT: Define imperious in 'an imperious necessity drove him forward on his quest' + +imperious (adj) - Formal + +DEFINITION: +Having a commanding or domineering character; asserting one's will in an overbearing manner. + +GENERAL TONE: +The necessity was so strong that it compelled him to move forward on his journey, without any room for negotiation or consideration of others' opinions. +SYNONYMS: +urgent, pressing, compelling. + +EXAMPLES: +1. The imperious necessity pushed him to complete the project ahead of schedule. +2. His imperious nature made it hard for his colleagues to disagree with him on important matters. + +METRICS: ELAPSED: 42.78s | DAEMON RSS: 2114.44 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define despondency in 'he fell into a state of deep, uninterrupted despondency' + +despondency (noun) - Formal + +DEFINITION: +deep and prolonged state of depression or hopelessness. + +GENERAL TONE: +The tone is somber, reflecting the heavy emotional state described. + +SYNONYMS: +dejection, melancholy, gloominess + +EXAMPLES: +1. He fell into a state of deep, uninterrupted despondency. +2. After the loss of his job, he sank into a long period of despondency. + +METRICS: ELAPSED: 35.98s | DAEMON RSS: 2114.46 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define arduous in 'the long and arduous journey through the frozen mountains' + +arduous (adjective) - Formal + +DEFINITION: +Challenging or difficult to accomplish, especially over a long period of time. + +GENERAL TONE: +The journey through the frozen mountains was tough and took much effort. + +SYNONYMS: +Difficult, strenuous, laborious, taxing + +EXAMPLES: +1. The expedition through the dense jungle was an arduous task. +2. Climbing Mount Everest is an arduous challenge for mountaineers. + +METRICS: ELAPSED: 37.15s | DAEMON RSS: 2114.47 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define ephemeral in 'human history appeared as a brief and ephemeral flash in cosmic time' + +ephemeral (adj) - Formal + +DEFINITION: +lasting only a very short time; fleeting. + +GENERAL TONE: +The phrase suggests that human history is insignificant compared to the vastness of cosmic time. + +SYNONYMS: +transient, fleeting, momentary. + +EXAMPLES: +1. The sunset was an ephemeral display in the sky. +2. His fame is just an ephemeral glow on the internet. + +METRICS: ELAPSED: 33.48s | DAEMON RSS: 2114.47 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define austere in 'they lived under the austere discipline of their new order' + +austere (adj) - Formal + +DEFINITION: +strictly simple or plain in style, manner, or appearance; severe and unadorned. + +GENERAL TONE: +The tone is formal, emphasizing the strictness and simplicity of their lifestyle under a new order. + +SYNONYMS: +rigid, severe, plain, stark + +EXAMPLES: +1. The monastery was built in austere simplicity. +2. His speech was delivered with an austere dignity. + +METRICS: ELAPSED: 36.48s | DAEMON RSS: 2114.47 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define venerable in 'the venerable monuments of a long-forgotten civilization' + +venerable (adjective) - Formal + +DEFINITION: +Respected and held in high esteem due to age or historical significance. + +GENERAL TONE: +The monuments are ancient structures that have stood the test of time, earning them respect from modern visitors. + +SYNONYMS: +Ancient, revered, esteemed, venerable + +EXAMPLES: +1. The venerable monuments of a long-forgotten civilization stand as silent witnesses to the past. +2. Scholars hold the ancient manuscripts in high regard, considering them venerable treasures. + +METRICS: ELAPSED: 40.89s | DAEMON RSS: 2114.47 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define tumultuous in 'the species endured a tumultuous period of war and renewal' + +tumultuous (adj) - Formality: Literary + +DEFINITION: +Marked by violent disturbance or disorder; stormy. + +GENERAL TONE: +The passage conveys a sense of chaos and upheaval during the period, emphasizing the intensity and disruption caused by war and renewal. +SYNONYMS: +chaotic, stormy, turbulent, violent. + +EXAMPLES: +1. The city experienced a tumultuous period of unrest following the election. +2. During the tumultuous years of the revolution, society underwent significant changes. + +METRICS: ELAPSED: 40.50s | DAEMON RSS: 2114.47 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define imperturbable in 'he maintained an imperturbable calm despite the surrounding chaos' + +imperturbable (adjective) - Formal + +DEFINITION: +Unflappable; showing no sign of disturbance or emotion, especially in a crisis. + +GENERAL TONE: +The speaker remains composed and unruffled even amidst turmoil. + +SYNONYMS: +Unflappable, Calm, Collected, Unperturbed + +EXAMPLES: +1. Despite the stormy weather outside, he maintained an imperturbable calm. +2. Her imperturbable demeanor in the face of criticism impressed everyone present. + +METRICS: ELAPSED: 39.78s | DAEMON RSS: 2114.47 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define singular in 'he was a man of a singular and unpredictable temperament' + +singular (adjective) - Formal + +DEFINITION: +single or unique in nature; not common or typical. + +GENERAL TONE: +The term suggests a person with an unusual and unpredictable personality. +One terse sentence of standard conversational synonyms: unique, distinctive, peculiar. + +SYNONYMS: +unusual, distinctive, peculiar, eccentric. + +EXAMPLES: +1. He was a man of singular and unpredictable temperament. +2. Her approach to problem-solving is singularly effective. + +METRICS: ELAPSED: 36.01s | DAEMON RSS: 2114.47 MB | SWAP: 0.00 MB (+0.00 MB) + +------------------------------------------------------------------------ + +PROMPT: Define enigmatic in 'he left behind an enigmatic smile and no further explanation' + +enigmatic (adjective) - Formal + +DEFINITION: +Mysterious and difficult to understand or interpret. + +GENERAL TONE: +The smile was a puzzle, leaving no clues as to its meaning. +SYNONYMS: +mystifying, puzzling, enigmatical + +EXAMPLES: +1. The riddle left me with an enigmatic smile on my face. +2. His words were so enigmatic that I couldn't figure out what he meant. + +METRICS: ELAPSED: 35.39s | DAEMON RSS: 2114.47 MB | SWAP: 0.00 MB (+0.00 MB) + diff --git a/test_prompts.sh b/test_prompts.sh old mode 100755 new mode 100644 index 1f86d4a..925a26b --- a/test_prompts.sh +++ b/test_prompts.sh @@ -1,34 +1,33 @@ #!/bin/sh -# Usage: ./test_prompts.sh [prompt_file] +# Usage: ./test_prompts.sh [-i prompt_file] [-o result_file] SOCK_PATH="/var/run/lexd/sock" -EXECUTABLE="lexd" -# Mandatory Model Argument -MODEL_PATH="$1" +# Defaults +PROMPT_FILE="test_prompts.txt" +RESULT_FILE="results.txt" -if [ -z "$MODEL_PATH" ]; then - echo "Usage: $0 [prompt_file]" >&2 - exit 1 -fi +# Parse -i and -o flags +while getopts "i:o:" opt; do + case "$opt" in + i) PROMPT_FILE="$OPTARG" ;; + o) RESULT_FILE="$OPTARG" ;; + *) echo "Usage: $0 [-i prompt_file] [-o result_file]" >&2; exit 1 ;; + esac +done -if [ ! -f "$MODEL_PATH" ]; then - echo "Error: Model file '$MODEL_PATH' not found." >&2 +# Ensure socket exists (daemon must already be running) +if [ ! -S "$SOCK_PATH" ]; then + echo "Error: Socket '$SOCK_PATH' not found. Is lexd running?" >&2 exit 1 fi -# Derive base model name and output result filename -MODEL_NAME=$(basename "$MODEL_PATH") -RESULT_FILE="results_${MODEL_NAME}.txt" - -# Default prompt filename (optional 2nd argument) -PROMPT_FILE="${2:-test_prompts.txt}" +# Retrieve running daemon PID +DAEMON_PID=$(pgrep -f "lexd" | head -n 1) -# Ensure executable exists -if [ ! -x "$EXECUTABLE" ]; then - echo "Error: Executable '$EXECUTABLE' not found or not executable." >&2 - exit 1 +if [ -z "$DAEMON_PID" ]; then + echo "Warning: Could not detect DAEMON_PID. RSS metrics may be unavailable." >&2 fi # Ensure prompt file exists @@ -38,15 +37,12 @@ if [ ! -f "$PROMPT_FILE" ]; then fi # Helper function to get current swap occupancy in MB by parsing swapctl -l -# OpenBSD swapctl -l outputs 512-byte block counts by default get_swap_occupancy_mb() { swapctl -l | awk ' NR > 1 { - # Column 3 is Used blocks (512-byte blocks) used_blocks += $3 } END { - # Convert 512-byte blocks to MB (blocks * 512 / 1024 / 1024) printf "%.2f", used_blocks * 512 / 1048576 }' } @@ -70,34 +66,14 @@ while [ $i -lt 72 ]; do i=$((i + 1)) done -# --- Start daemon --- - -echo "Starting daemon with model: '$MODEL_PATH'" -"$EXECUTABLE" -m "$MODEL_PATH" & -DAEMON_PID=$! +# Temporary file to capture time / rusage output +TIME_LOG=$(mktemp /tmp/lex_time.XXXXXX) cleanup() { - kill "$DAEMON_PID" 2>/dev/null - wait "$DAEMON_PID" 2>/dev/null rm -f "$TIME_LOG" } trap cleanup EXIT INT TERM -waited=0 -while [ ! -S "$SOCK_PATH" ]; do - if ! kill -0 "$DAEMON_PID" 2>/dev/null; then - echo "Error: daemon exited during startup (bad model path?)" >&2 - exit 1 - fi - sleep 1 - waited=$((waited + 1)) - if [ "$waited" -ge 120 ]; then - echo "Error: timed out waiting for daemon socket at $SOCK_PATH" >&2 - exit 1 - fi -done - -echo "Daemon ready (pid $DAEMON_PID) after ${waited}s" echo "Evaluating $total_prompts prompt(s) from '$PROMPT_FILE'" echo "Writing results to: '$RESULT_FILE'" echo "---------------------------------------------------" @@ -105,9 +81,6 @@ echo "---------------------------------------------------" current=0 first=1 -# Temporary file to capture time / rusage output -TIME_LOG=$(mktemp /tmp/lex_time.XXXXXX) - # Process each prompt while IFS= read -r prompt || [ -n "$prompt" ]; do case "$prompt" in @@ -125,12 +98,14 @@ while IFS= read -r prompt || [ -n "$prompt" ]; do printf "PROMPT: %s\n\n" "$prompt" >> "$RESULT_FILE" # Sample daemon RSS prior to request - daemon_rss_kb=$(ps -o rss= -p "$DAEMON_PID" 2>/dev/null | tr -d ' ') + if [ -n "$DAEMON_PID" ]; then + daemon_rss_kb=$(ps -o rss= -p "$DAEMON_PID" 2>/dev/null | tr -d ' ') + fi # Capture baseline swap occupancy before sending the prompt swap_start_mb=$(get_swap_occupancy_mb) - # Send prompt to daemon over socket + # Send prompt to daemon over socket (exact original invocation) /usr/bin/time -lp sh -c "printf '%s\n' \"\$1\" | nc -U \"\$2\"" _ "$prompt" "$SOCK_PATH" \ >> "$RESULT_FILE" 2> "$TIME_LOG" @@ -162,5 +137,4 @@ while IFS= read -r prompt || [ -n "$prompt" ]; do done < "$PROMPT_FILE" echo "---------------------------------------------------" -echo "Done! Processed $total_prompts prompt(s)." - +echo "Done! Processed $total_prompts prompt(s)." \ No newline at end of file -- cgit v1.2.3