main.c


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

#include <stdio.h>
#include <stdbool.h>
#include <err.h>
#include <unistd.h>

#include "llama.h"

#define MODEL_PATH "Qwen2.5-7B-Instruct-Q4_K_M.gguf"

int main(int argc , char *argv[])
{
	struct llama_model *model;
	struct llama_model_params mparams;

	if (unveil(MODEL_PATH, "r") == -1)
		err(1, "unveil %s failed", MODEL_PATH);

	if (unveil(NULL, NULL) == -1)
		err(1, "unveil lock failed");

	if (pledge("stdio rpath", NULL) == -1)
		err(1, "initial pledge failed");

	llama_backend_init();

	mparams = llama_model_default_params();
	mparams.n_gpu_layers = 0; /* force all layers onto CPU */
	mparams.load_mode = LLAMA_LOAD_MODE_MMAP;

	model = llama_model_load_from_file(MODEL_PATH, mparams);
	if (!model)
		errx(1, "failed to load model from file %s", MODEL_PATH);

	/* drop rpath after loading the model */
	if (pledge("stdio", NULL) == -1)
		err(1, "secondary pledge failed");

	llama_model_free(model);
	llama_backend_free();

	return 0;
}