diff options
| -rw-r--r-- | main.c | 134 |
1 files changed, 50 insertions, 84 deletions
@@ -7,87 +7,72 @@ #include "mem.h" #include "llama.h" -#define MODEL_PATH "Qwen2.5-7B-Instruct-Q4_K_M.gguf" +#define MODEL_PATH "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf" #define SYSTEM_PROMPT \ - "You are a lexicographer. You are defining the EXACT target word provided.\n" \ - "Output format strictly as follows:\n" \ - "<WORD IN UPPERCASE> (<part of speech>) — <Formality: Conversational|Formal|Literary|Archaic>\n\n" \ - "DEFINITION:\n" \ - "<terse, highly accurate definition>\n\n" \ - "WHEN TO USE:\n" \ - "<1 sentence on when to use>\n\n" \ - "EXAMPLES:\n" \ - "1. <example 1>\n" \ - "2. <example 2>\n\n" \ - "Do not include intro, markdown, or extra commentary." - -#define CHATML_FMT \ - "<|im_start|>system\n%s<|im_end|>\n" \ - "<|im_start|>user\nTarget word: %s\nDefine this specific word:<|im_end|>\n" \ - "<|im_start|>assistant\n%s" - -#define MAX_TOKENS 300 - -static void upcase(char *dest, const char *src, size_t n) -{ - size_t i; - - for (i = 0; i < n - 1 && src[i] != '\0'; i++) - dest[i] = (char)toupper((unsigned char)src[i]); - - dest[i] = '\0'; -} - -static int build_prompt(char **buf, const char *word) -{ - int len; - char upper_word[128]; - - if (!buf || !word) - return -1; - - upcase(upper_word, word, sizeof(upper_word)); - - len = asprintf(buf, CHATML_FMT, SYSTEM_PROMPT, word, upper_word); - if (len < 0) { - *buf = NULL; - return -1; - } - - return len; -} - -static void process_request(struct llama_model *model, const char *word) + "You are a lexicographer. Define the target word strictly in the context provided.\n\n" \ + "Output format strictly as follows:\n" \ + "<WORD> (<part of speech>) — <Formality: Conversational|Formal|Literary|Archaic>\n\n" \ + "DEFINITION:\n" \ + "<terse, highly accurate definition>\n\n" \ + "WHEN TO USE:\n" \ + "<1 sentence on when to use>\n\n" \ + "EXAMPLES:\n" \ + "1. <example 1>\n" \ + "2. <example 2>\n\n" \ + "EXAMPLE INPUT:\n" \ + "Define torrent as in 'a torrent of rain poured down'\n\n" \ + "EXAMPLE OUTPUT:\n" \ + "TORRENT (noun) — Literary\n\n" \ + "DEFINITION:\n" \ + "A strong and fast-moving stream of water or other liquid.\n\n" \ + "WHEN TO USE:\n" \ + "Use when describing a sudden, overwhelming rush of water or emotion.\n\n" \ + "EXAMPLES:\n" \ + "1. A torrent of rain poured down on the village.\n" \ + "2. She faced a torrent of angry emails after the announcement.\n\n" \ + "CRITICAL: Do NOT echo the user's input, context sentence, or 'DEFINE' command. Start immediately with <WORD>." + +#define MAX_TOKENS 300 + +static void process_request(struct llama_model *model, const char *user_prompt) { + int n_prompt_tokens, i; char *prompt; int prompt_len; - int n_prompt_tokens, i; llama_token *prompt_tokens; llama_token new_token_id; - /* Data structure used to pass tokens into llama_decode() */ struct llama_batch batch; - struct llama_context *ctx; struct llama_context_params cparams; struct llama_sampler *smpl; struct llama_sampler_chain_params sparams; - prompt = NULL; - prompt_len = build_prompt(&prompt, word); + struct llama_chat_message messages[] = { + { "system", SYSTEM_PROMPT }, + { "user", user_prompt } + }; + prompt_len = llama_chat_apply_template(NULL, messages, 2, true, NULL, 0); if (prompt_len <= 0) { - fprintf(stderr, "Error: failed to construct prompt for '%s'\n", word); + fprintf(stderr, "Error: failed to calculate chat template size\n"); return; } - + + prompt = MALLOC((size_t)prompt_len + 1); + if (llama_chat_apply_template(NULL, messages, 2, true, prompt, prompt_len + 1) < 0) { + fprintf(stderr, "Error: failed to apply chat template\n"); + free(prompt); + return; + } + cparams = llama_context_default_params(); - cparams.n_ctx = 512; /* context size in tokens */ + cparams.n_ctx = 1024; /* context size in tokens */ cparams.n_threads = 4; cparams.n_threads_batch = 4; - + ctx = llama_init_from_model(model, cparams); if (!ctx) { fprintf(stderr, "Error: failed to create context\n"); @@ -131,7 +116,7 @@ static void process_request(struct llama_model *model, const char *word) return; } - free(prompt); /* prompt string no longer required */ + free(prompt); /* Prompt buffer is fully tokenized and no longer needed */ /* Ingest prompt tokens in one batch (parallelizes matrix * multiplications across tokens in the batch) */ @@ -146,12 +131,6 @@ static void process_request(struct llama_model *model, const char *word) free(prompt_tokens); - /* Response formatting: print the pre-filled word right as generation begins */ - char upper_word[128]; - upcase(upper_word, word, sizeof(upper_word)); - printf("%s", upper_word); - fflush(stdout); - /* Generation loop */ sparams = llama_sampler_chain_default_params(); smpl = llama_sampler_chain_init(sparams); @@ -176,18 +155,12 @@ static void process_request(struct llama_model *model, const char *word) /* Check for end-of-generation (EOG) tokens: * EOS: end-of-sequence - * EOT: end-of-turn - * Generates garbage until token limit hit or context window - * exhausted, if omitted */ + * EOT: end-of-turn */ if (llama_vocab_is_eog(vocab, new_token_id)) break; - /* LLMS process words as sub-word tokens. - * E.g.: unbelievable -> ["un", "believ", "able"] - * 128-byte buffer is sufficient */ - char buf[128]; - /* Convert numeric token id to printable text */ + char buf[128]; int n = llama_token_to_piece(vocab, new_token_id, buf, sizeof(buf), 0, false); @@ -196,16 +169,9 @@ static void process_request(struct llama_model *model, const char *word) fflush(stdout); } - /* Create batch with 1 token: - * Prompt has been evaluated. Here, we generate one token at a time. - * See autoregressive (AR), diffusion (dLLM), non-autoregressive (NAR), - * speculative/MTP for alternative frameworks. */ + /* Create batch with 1 token for the next forward pass */ batch = llama_batch_get_one(&new_token_id, 1); - /* llama_decode(): the CPU-heavy forward pass through the transformer model: - * - allocates memory and KV cache - * - runs matrix multiplications - * - generates logits (output prediction vectors) */ if (llama_decode(ctx, batch) != 0) { fprintf(stderr, "llama_decode failed!\n"); break; @@ -251,8 +217,8 @@ int main(int argc , char *argv[]) if (pledge("stdio", NULL) == -1) err(1, "secondary pledge failed"); - const char *word = argv[1]; - process_request(model, word); + const char *prompt = argv[1]; + process_request(model, prompt); llama_model_free(model); llama_backend_free(); |