From 6da102d6e0494a3eac3f05fa3b2cdcc25ba2754e Mon Sep 17 00:00:00 2001 From: Sadeep Madurange Date: Fri, 2 Jan 2026 18:39:07 +0800 Subject: Use suffix array index for search. --- .gitignore | 3 +- _log/arduino-due.md | 2 +- _site/cgi-bin/find.cgi | 250 ++++++++++++++++++++++++++++----------- _site/feed.xml | 2 +- _site/index.html | 2 +- _site/log/arduino-due/index.html | 4 +- _site/log/index.html | 2 +- _site/posts.xml | 2 +- cgi-bin/find.cgi | 192 +++++++++++++++++++++--------- cgi-bin/indexer.pl | 146 +++++++++++++++++------ 10 files changed, 439 insertions(+), 166 deletions(-) diff --git a/.gitignore b/.gitignore index 6eac243..cc4d7ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .jekyll-cache/ **/*.swp -**/search_index.dat +**/*.dat +**/*.bin diff --git a/_log/arduino-due.md b/_log/arduino-due.md index f604e89..a0bfe4a 100644 --- a/_log/arduino-due.md +++ b/_log/arduino-due.md @@ -1,5 +1,5 @@ --- -title: ATSAM3X8E bare-metal programming +title: Bare-metal ATSAM3X8E date: 2024-09-16 layout: post --- diff --git a/_site/cgi-bin/find.cgi b/_site/cgi-bin/find.cgi index fc1d4af..5f95e3a 100644 --- a/_site/cgi-bin/find.cgi +++ b/_site/cgi-bin/find.cgi @@ -1,86 +1,202 @@ #!/usr/bin/perl -use Encode qw(decode_utf8); +use strict; +use warnings; +use Storable qw(retrieve); +use Encode qw(decode_utf8 encode_utf8); +use URI::Escape qw(uri_unescape); use HTML::Escape qw(escape_html); +# --- Configuration --- +my $max_parallel = 50; # Max parallel search requests +my $lock_timeout = 30; # Seconds before dropping stale locks +my $max_results = 20; # Max search results to display +my $sa_file = 'sa.bin'; # Suffix Array index +my $cp_file = 'corpus.bin'; # Raw text corpus +my $map_file = 'file_map.dat'; # File metadata +my $lock_dir = '/tmp/search_locks'; # Semaphore directory + +# --- Concurrency Control --- +mkdir $lock_dir, 0777 unless -d $lock_dir; +my $active_count = 0; +my $now = time(); + +opendir(my $dh, $lock_dir); +while (my $file = readdir($dh)) { + next unless $file =~ /\.lock$/; + my $path = "$lock_dir/$file"; + my $mtime = (stat($path))[9] || 0; + ($now - $mtime > $lock_timeout) ? unlink($path) : $active_count++; +} +closedir($dh); + +# Template variables +my $year = (localtime)[5] + 1900; my $search_text = ''; -if ($ENV{QUERY_STRING} =~ /^q=([^&]*)/) { - $search_text = decode_utf8($1 // ""); - $search_text =~ s/\P{Print}//g; # toss any non-printable utf-8 characters +# Busy check +if ($active_count >= $max_parallel) { + print "Content-Type: text/html\n\n"; + render_html("

Server busy. Please try again in a few seconds.

", "", $year); + exit; +} + +# Create semaphore lock +my $lock_file = "$lock_dir/$$.lock"; +open(my $fh_lock, '>', $lock_file); + +# --- Query Decoding --- +if (($ENV{QUERY_STRING} || '') =~ /^q=([^&]*)/) { + my $raw_q = $1; + $raw_q =~ tr/+/ /; + $search_text = uri_unescape($raw_q); + $search_text = decode_utf8($search_text // ""); + $search_text =~ s/\P{Print}//g; $search_text = substr($search_text, 0, 64); $search_text =~ s/^\s+|\s+$//g; } -my @results; - -# Search only index.html files inside the first level of subdirectories -my $start_dir = '../log'; -my @files = glob("$start_dir/*/index.html"); - -foreach my $path (@files) { - # Skip if the path is a symlink or not a file - next if -l $path || ! -f $path; - - next unless open(my $fh, '<:utf8', $path); - my $html = do { local $/; <$fh> }; - close($fh); - - my ($text) = $html =~ m|
(.*?)
|is; - $text =~ s|<[^>]+>| |g; - $text =~ s|\s+| |g; - - next unless $text =~ /(.{0,40})(\Q$search_text\E)(.{0,40})/is; - my ($before, $actual, $after) = ($1, $2, $3); - - # Trim if we cut into the middle of a sentence - $after =~ s/\s\S*$// if length($after) > 25; - $before =~ s/^.*?\s// if length($before) > 25; - - if ($before =~ /\S/) { # If before has non-whitespace characters - $before = ucfirst($before); - } else { - $before = ""; # Clear any stray spaces - $actual = ucfirst($actual); - } +my $safe_search_text = escape_html($search_text); - my $safe_before = escape_html($before); - my $safe_actual = escape_html($actual); - my $safe_after = escape_html($after); - my $snippet = "${safe_before}${safe_actual}${safe_after}..."; +print "Content-Type: text/html\n\n"; - my ($title) = $html =~ m|(.*?)|is; - my $safe_title = escape_html($title); +if ($search_text eq '') { + final_output("

Please enter a search term above.

"); +} - $path =~ s|^\.\./||; +# --- Binary Search Logic --- +my @results; +my $query = encode_utf8(lc($search_text)); +my $query_len = length($query); + +if (-f $sa_file && -f $cp_file) { + open(my $fh_sa, '<', $sa_file) or die $!; + open(my $fh_cp, '<', $cp_file) or die $!; + binmode($fh_sa); + binmode($fh_cp); + + my $file_map = retrieve($map_file); + my $total_suffixes = (-s $sa_file) / 4; + + # Find left boundary + my ($low, $high) = (0, $total_suffixes - 1); + my $first_hit = -1; + + while ($low <= $high) { + my $mid = int(($low + $high) / 2); + seek($fh_sa, $mid * 4, 0); + read($fh_sa, my $bin_off, 4); + my $off = unpack("L", $bin_off); + seek($fh_cp, $off, 0); + read($fh_cp, my $text, $query_len); + + my $cmp = $text cmp $query; + if ($cmp >= 0) { + $first_hit = $mid if $cmp == 0; + $high = $mid - 1; + } else { + $low = $mid + 1; + } + } - push @results, { - path => $path, - title => $safe_title, - snippet => $snippet - }; + # Collect results if found + if ($first_hit != -1) { + my $last_hit = $first_hit; + ($low, $high) = ($first_hit, $total_suffixes - 1); + + # Find right boundary + while ($low <= $high) { + my $mid = int(($low + $high) / 2); + seek($fh_sa, $mid * 4, 0); + read($fh_sa, my $bin_off, 4); + my $off = unpack("L", $bin_off); + seek($fh_cp, $off, 0); + read($fh_cp, my $text, $query_len); + + if (($text cmp $query) <= 0) { + $last_hit = $mid if $text eq $query; + $low = $mid + 1; + } else { + $high = $mid - 1; + } + } + + my %seen; + for my $i ($first_hit .. $last_hit) { + seek($fh_sa, $i * 4, 0); + read($fh_sa, my $bin_off, 4); + my $offset = unpack("L", $bin_off); + + foreach my $m (@$file_map) { + if ($offset >= $m->{start} && $offset < $m->{end}) { + if (!$seen{$m->{path}}++) { + # 1. Capture slightly more than 50 chars for trimming + my $snip_start = ($offset - 30 < $m->{start}) ? $m->{start} : $offset - 30; + seek($fh_cp, $snip_start, 0); + read($fh_cp, my $raw_snip, 120); + + my $snippet = decode_utf8($raw_snip, Encode::FB_QUIET) // $raw_snip; + $snippet =~ s/\s+/ /g; # Normalize whitespace + + # 2. Trim Start: Partial word removal + if ($snip_start > $m->{start}) { + $snippet =~ s/^[^\s]*\s//; + } + + # 3. Trim End: Length limit and partial word removal + my $has_more = 0; + if (length($snippet) > 50) { + $snippet = substr($snippet, 0, 50); + $has_more = 1 if $snippet =~ s/\s+[^\s]*$//; + } + + # 4. Cleanup & Capitalize + $snippet = ucfirst($snippet); + my $clean_path = $m->{path}; + $clean_path =~ s|^\.\./_site/||; + + # 5. Build Final Snippet + my $display_snippet = escape_html($snippet) . ($has_more ? "..." : ""); + + push @results, { + path => $clean_path, + title => (split('/', $m->{path}))[-2], + snippet => $display_snippet + }; + } + last; + } + } + last if scalar @results >= $max_results; + } + } + close($fh_sa); + close($fh_cp); } -print "Content-Type: text/html\n\n"; - -my $list; -if ($search_text eq '') { - $list = "

Please enter a search term above.

"; -} elsif (@results == 0) { - $list = "

No results found for \"$search_text\".

"; +# --- Formatting & Output --- +my $list_html = ""; +if (@results == 0) { + $list_html = "

No results found for \"$safe_search_text\".

"; } else { - $list = ""; + $list_html = ""; } -my $safe_search_text = escape_html($search_text); -my $year = (localtime)[5] + 1900; +final_output($list_html); + +# --- Helpers --- +sub final_output { + my ($content) = @_; + render_html($content, $safe_search_text, $year); + if ($fh_lock) { close($fh_lock); unlink($lock_file); } + exit; +} -print <<"HTML"; +sub render_html { + my ($content, $q_val, $yr) = @_; + print <<"HTML"; @@ -105,19 +221,21 @@ print <<"HTML";

Search

- +
- $list + $content
-

© ASCIIMX - $year

+

© ASCIIMX - $yr

HTML +} + diff --git a/_site/feed.xml b/_site/feed.xml index dfe24b2..9ccb174 100644 --- a/_site/feed.xml +++ b/_site/feed.xml @@ -1 +1 @@ -Jekyll2026-01-01T16:23:09+08:00/feed.xmlASCIIMX | LogW. D. Sadeep MadurangeMatrix Rain: 2025 refactor2025-12-21T00:00:00+08:002025-12-21T00:00:00+08:00/log/matrix-digital-rainW. D. Sadeep MadurangeFingerprint door lock (LP)2025-08-18T00:00:00+08:002025-08-18T00:00:00+08:00/log/fpm-door-lock-lpW. D. Sadeep MadurangeHigh-side MOSFET switching2025-06-22T00:00:00+08:002025-06-22T00:00:00+08:00/log/mosfet-switchesW. D. Sadeep MadurangeATmega328P at 3.3V and 5V2025-06-10T00:00:00+08:002025-06-10T00:00:00+08:00/log/arduino-unoW. D. Sadeep MadurangeFingerprint door lock (RF)2025-06-05T00:00:00+08:002025-06-05T00:00:00+08:00/log/fpm-door-lock-rfW. D. Sadeep MadurangeBumblebee: browser automation2025-04-02T00:00:00+08:002025-04-02T00:00:00+08:00/log/bumblebeeW. D. Sadeep MadurangeATSAM3X8E bare-metal programming2024-09-16T00:00:00+08:002024-09-16T00:00:00+08:00/log/arduino-dueW. D. Sadeep MadurangeEtlas: e-paper dashboard2024-09-05T00:00:00+08:002024-09-05T00:00:00+08:00/log/etlasW. D. Sadeep MadurangeESP32 e-reader prototype2023-10-24T00:00:00+08:002023-10-24T00:00:00+08:00/log/e-readerW. D. Sadeep MadurangeNeo4j shortest path optimization2018-03-06T00:00:00+08:002018-03-06T00:00:00+08:00/log/neo4j-a-star-searchW. D. Sadeep Madurange \ No newline at end of file +Jekyll2026-01-02T19:06:26+08:00/feed.xmlASCIIMX | LogW. D. Sadeep MadurangeMatrix Rain: 2025 refactor2025-12-21T00:00:00+08:002025-12-21T00:00:00+08:00/log/matrix-digital-rainW. D. Sadeep MadurangeFingerprint door lock (LP)2025-08-18T00:00:00+08:002025-08-18T00:00:00+08:00/log/fpm-door-lock-lpW. D. Sadeep MadurangeHigh-side MOSFET switching2025-06-22T00:00:00+08:002025-06-22T00:00:00+08:00/log/mosfet-switchesW. D. Sadeep MadurangeATmega328P at 3.3V and 5V2025-06-10T00:00:00+08:002025-06-10T00:00:00+08:00/log/arduino-unoW. D. Sadeep MadurangeFingerprint door lock (RF)2025-06-05T00:00:00+08:002025-06-05T00:00:00+08:00/log/fpm-door-lock-rfW. D. Sadeep MadurangeBumblebee: browser automation2025-04-02T00:00:00+08:002025-04-02T00:00:00+08:00/log/bumblebeeW. D. Sadeep MadurangeBare-metal ATSAM3X8E2024-09-16T00:00:00+08:002024-09-16T00:00:00+08:00/log/arduino-dueW. D. Sadeep MadurangeEtlas: e-paper dashboard2024-09-05T00:00:00+08:002024-09-05T00:00:00+08:00/log/etlasW. D. Sadeep MadurangeESP32 e-reader prototype2023-10-24T00:00:00+08:002023-10-24T00:00:00+08:00/log/e-readerW. D. Sadeep MadurangeNeo4j shortest path optimization2018-03-06T00:00:00+08:002018-03-06T00:00:00+08:00/log/neo4j-a-star-searchW. D. Sadeep Madurange diff --git a/_site/index.html b/_site/index.html index 257c38d..90cbbfd 100644 --- a/_site/index.html +++ b/_site/index.html @@ -139,7 +139,7 @@ - ATSAM3X8E bare-metal programming + Bare-metal ATSAM3X8E diff --git a/_site/log/arduino-due/index.html b/_site/log/arduino-due/index.html index d8178c4..7bc8a0b 100644 --- a/_site/log/arduino-due/index.html +++ b/_site/log/arduino-due/index.html @@ -3,7 +3,7 @@ - ATSAM3X8E bare-metal programming + Bare-metal ATSAM3X8E @@ -40,7 +40,7 @@
-

ATSAM3X8E BARE-METAL PROGRAMMING

+

BARE-METAL ATSAM3X8E

16 SEPTEMBER 2024

Notes on programming bare-metal ATSAM3X8E chips (Arduino Due) using Serial Wire diff --git a/_site/log/index.html b/_site/log/index.html index 93d2573..32fbf8d 100644 --- a/_site/log/index.html +++ b/_site/log/index.html @@ -129,7 +129,7 @@ - ATSAM3X8E bare-metal programming + Bare-metal ATSAM3X8E diff --git a/_site/posts.xml b/_site/posts.xml index 5677165..d327a7b 100644 --- a/_site/posts.xml +++ b/_site/posts.xml @@ -1 +1 @@ -Jekyll2026-01-01T16:23:09+08:00/posts.xmlASCIIMXW. D. Sadeep Madurange \ No newline at end of file +Jekyll2026-01-02T19:06:26+08:00/posts.xmlASCIIMXW. D. Sadeep Madurange diff --git a/cgi-bin/find.cgi b/cgi-bin/find.cgi index 3e5c8a2..ab066dd 100644 --- a/cgi-bin/find.cgi +++ b/cgi-bin/find.cgi @@ -3,16 +3,18 @@ use strict; use warnings; use Storable qw(retrieve); -use Encode qw(decode_utf8); +use Encode qw(decode_utf8 encode_utf8); +use URI::Escape qw(uri_unescape); use HTML::Escape qw(escape_html); # Configuration -my $max_parallel = 50; # max no. of parallel searches -my $lock_timeout = 30; # drop stale locks after this many seconds -my $max_results = 20; # max search results -my $min_query_len = 3; # min query length to avoid matching 'a', 'e' -my $index_file = 'search_index.dat'; # index file -my $lock_dir = '/tmp/search_locks'; # lock file directory +my $max_parallel = 50; # Max parallel search requests +my $lock_timeout = 30; # Seconds before dropping stale locks +my $max_results = 20; # Max search results to display +my $sa_file = 'sa.bin'; # Suffix Array index +my $cp_file = 'corpus.bin'; # Raw text corpus +my $map_file = 'file_map.dat'; # File metadata +my $lock_dir = '/tmp/search_locks'; # Semaphore directory # Concurrency control mkdir $lock_dir, 0777 unless -d $lock_dir; @@ -24,78 +26,161 @@ while (my $file = readdir($dh)) { next unless $file =~ /\.lock$/; my $path = "$lock_dir/$file"; my $mtime = (stat($path))[9] || 0; - ( $now - $mtime > $lock_timeout ) ? unlink($path) : $active_count++; + ($now - $mtime > $lock_timeout) ? unlink($path) : $active_count++; } closedir($dh); -# Too many search requests +# Template variables +my $year = (localtime)[5] + 1900; +my $search_text = ''; + +# Busy check if ($active_count >= $max_parallel) { print "Content-Type: text/html\n\n"; - render_html("

Server busy. Please try again in a few seconds.

", "", (localtime)[5]+1900); + render_html("

Server busy. Please try again in a few seconds.

", "", $year); exit; } +# Create semaphore lock my $lock_file = "$lock_dir/$$.lock"; open(my $fh_lock, '>', $lock_file); -# Decode search text as utf-8, toss non-printable chars, trim -my $search_text = ''; +# Query decoding if (($ENV{QUERY_STRING} || '') =~ /^q=([^&]*)/) { - $search_text = decode_utf8($1 // ""); + my $raw_q = $1; + $raw_q =~ tr/+/ /; + $search_text = uri_unescape($raw_q); + $search_text = decode_utf8($search_text // ""); $search_text =~ s/\P{Print}//g; $search_text = substr($search_text, 0, 64); $search_text =~ s/^\s+|\s+$//g; } -# Pre-prepare common template variables my $safe_search_text = escape_html($search_text); -my $year = (localtime)[5] + 1900; print "Content-Type: text/html\n\n"; -# Input validation if ($search_text eq '') { final_output("

Please enter a search term above.

"); } -if (length($search_text) < $min_query_len) { - final_output("

Search term is too short. Please enter at least $min_query_len characters.

"); -} - -if (!-f $index_file) { - final_output("

Search temporarily unavailable.

"); -} - -my $index = retrieve($index_file); +# Binary search my @results; -my $found = 0; - -foreach my $url (sort keys %$index) { - last if $found >= $max_results; - my $data = $index->{$url}; - - # Grab 80 char snippet to chop at a word boundary later - next unless $data->{c} =~ /(.{0,40})(\Q$search_text\E)(.{0,40})/is; - my ($before, $actual, $after) = ($1, $2, $3); - $found++; - - # Chop at 25 or word boundary - $after =~ s/\s\S*$// if length($after) > 25; - $before =~ s/^.*?\s// if length($before) > 25; - - $before = ($before =~ /\S/) ? ucfirst($before) : ""; - $actual = ($before eq "") ? ucfirst($actual) : $actual; - - my $snippet = escape_html($before) . "" . escape_html($actual) . "" . escape_html($after) . "..."; - - push @results, { - path => $url, - title => escape_html($data->{t}), - snippet => $snippet - }; +my $query = encode_utf8(lc($search_text)); +my $query_len = length($query); + +if (-f $sa_file && -f $cp_file) { + open(my $fh_sa, '<', $sa_file) or die $!; + open(my $fh_cp, '<', $cp_file) or die $!; + binmode($fh_sa); + binmode($fh_cp); + + my $file_map = retrieve($map_file); + my $total_suffixes = (-s $sa_file) / 4; + + # Find left boundary + my ($low, $high) = (0, $total_suffixes - 1); + my $first_hit = -1; + + while ($low <= $high) { + my $mid = int(($low + $high) / 2); + seek($fh_sa, $mid * 4, 0); + read($fh_sa, my $bin_off, 4); + my $off = unpack("L", $bin_off); + seek($fh_cp, $off, 0); + read($fh_cp, my $text, $query_len); + + my $cmp = $text cmp $query; + if ($cmp >= 0) { + $first_hit = $mid if $cmp == 0; + $high = $mid - 1; + } else { + $low = $mid + 1; + } + } + + # Collect results if found + if ($first_hit != -1) { + my $last_hit = $first_hit; + ($low, $high) = ($first_hit, $total_suffixes - 1); + + # Find right boundary + while ($low <= $high) { + my $mid = int(($low + $high) / 2); + seek($fh_sa, $mid * 4, 0); + read($fh_sa, my $bin_off, 4); + my $off = unpack("L", $bin_off); + seek($fh_cp, $off, 0); + read($fh_cp, my $text, $query_len); + + if (($text cmp $query) <= 0) { + $last_hit = $mid if $text eq $query; + $low = $mid + 1; + } else { + $high = $mid - 1; + } + } + + my %seen; + for my $i ($first_hit .. $last_hit) { + seek($fh_sa, $i * 4, 0); + read($fh_sa, my $bin_off, 4); + my $offset = unpack("L", $bin_off); + + foreach my $m (@$file_map) { + if ($offset >= $m->{start} && $offset < $m->{end}) { + if (!$seen{$m->{path}}++) { + # Capture more than 50 chars for trimming + my $snip_start = ($offset - 30 < $m->{start}) ? $m->{start} : $offset - 30; + my $max_len = $m->{end} - $snip_start; + my $read_len = ($max_len > 120) ? 120 : $max_len; + seek($fh_cp, $snip_start, 0); + read($fh_cp, my $raw_snip, $read_len); + + my $snippet = decode_utf8($raw_snip, Encode::FB_QUIET) // $raw_snip; + $snippet =~ s/\s+/ /g; # Normalize whitespace + + # Trim start: Partial word removal + if ($snip_start > $m->{start}) { + $snippet =~ s/^[^\s]*\s//; + } + + # Trim end: Length limit and partial word removal + my $has_more = 0; + if (length($snippet) > 50) { + $snippet = substr($snippet, 0, 50); + $has_more = 1 if $snippet =~ s/\s+[^\s]*$//; + } + elsif ($snip_start + $read_len < $m->{end}) { + # This check handles snippets that are naturally short but + # there's still more text in the article we didn't read + $has_more = 1; + } + + # Cleanup & capitalize + $snippet = ucfirst($snippet); + $snippet = escape_html($snippet) . ($has_more ? "..." : ""); + + my $clean_path = $m->{path}; + $clean_path =~ s|^\.\./_site/||; + + push @results, { + path => $clean_path, + title => $m->{title},, + snippet => $snippet + }; + } + last; + } + } + last if scalar @results >= $max_results; + } + } + close($fh_sa); + close($fh_cp); } -# Format results list +# --- Formatting & Output --- my $list_html = ""; if (@results == 0) { $list_html = "

No results found for \"$safe_search_text\".

"; @@ -107,12 +192,11 @@ if (@results == 0) { final_output($list_html); -# Helper to ensure layout is always preserved +# --- Helpers --- sub final_output { my ($content) = @_; render_html($content, $safe_search_text, $year); - close($fh_lock) if $fh_lock; - unlink($lock_file) if -f $lock_file; + if ($fh_lock) { close($fh_lock); unlink($lock_file); } exit; } diff --git a/cgi-bin/indexer.pl b/cgi-bin/indexer.pl index 38a918e..69f6838 100644 --- a/cgi-bin/indexer.pl +++ b/cgi-bin/indexer.pl @@ -2,45 +2,115 @@ use strict; use warnings; -use Storable qw(nstore); +use File::Find; +use Storable qw(store); +use Encode qw(encode_utf8); use HTML::Entities qw(decode_entities); +use Time::HiRes qw(gettimeofday tv_interval); -# --- Configuration --- -my $built_site_dir = '../_site/log'; -my $output_file = '../_site/cgi-bin/search_index.dat'; -my %index; - -print "Building search index from $built_site_dir...\n"; - -foreach my $path (glob("$built_site_dir/*/index.html")) { - next unless open(my $fh, '<:utf8', $path); - my $html = do { local $/; <$fh> }; - close($fh); - - # Extract Title and Main Content - my ($title) = $html =~ m|(.*?)|is; - my ($main) = $html =~ m|
(.*?)
|is; - $main //= ''; - - # Strip HTML and clean prose - $main =~ s|]*>.*?| |gs; - $main =~ s|]*>.*?| |gs; - $main =~ s|<[^>]+>| |g; - $main = decode_entities($main); - $main =~ s|\s+| |g; - $main =~ s/^\s+|\s+$//g; - - # Normalize path - my $url = $path; - $url =~ s|^\.\./_site/||; # Remove local build directory - $url =~ s|^\.\./||; # Remove any leading dots - $url =~ s|^/+||; # Remove leading slashes - - $index{$url} = { - t => $title || "Untitled", - c => $main - }; +my $dir = '../_site/log'; +my $cgi_dir = '../_site/cgi-bin/'; +my $corpus_file = "${cgi_dir}corpus.bin"; +my $sa_file = "${cgi_dir}sa.bin"; +my $map_file = "${cgi_dir}file_map.dat"; + +my %excluded_files = ( + 'index.html' => 1, # /log/index.html +); + +# Start timing +my $t0 = [gettimeofday]; + +my $corpus = ""; +my @file_map; + +print "Building corpus...\n"; + +find({ + wanted => sub { + # Only index index.html files + return unless -f $_ && $_ eq 'index.html'; + + my $rel_path = $File::Find::name; + $rel_path =~ s|^\Q$dir\E/?||; + return if $excluded_files{$rel_path}; + + if (open my $fh, '<:encoding(UTF-8)', $_) { + my $content = do { local $/; <$fh> }; + close $fh; + + my ($title) = $content =~ m|(.*?)|is; + $title //= (split('/', $File::Find::name))[-2]; # Fallback to folder name + $title =~ s/^\s+|\s+$//g; + + # Extract content from
or use whole file + my ($text) = $content =~ m|
(.*?)
|is; + $text //= $content; + + # Strip tags and normalize whitespace + $text =~ s|]*>.*?| |gs; + $text =~ s|]*>.*?| |gs; + $text =~ s|<[^>]+>| |g; + $text = decode_entities($text); + $text =~ s|\s+| |g; + $text =~ s/^\s+|\s+$//g; + + # CRITICAL: Convert to lowercase and then to raw bytes + # This ensures length() and substr() work on byte offsets for seek() + my $raw_entry = encode_utf8(lc($text) . "\0"); + + my $start = length($corpus); + $corpus .= $raw_entry; + + push @file_map, { + start => $start, + end => length($corpus), + title => $title, + path => $File::Find::name + }; + } + }, + no_chdir => 0, +}, $dir); + +print "Sorting suffixes...\n"; + +# Initialize the array of indices +my @sa = 0 .. (length($corpus) - 1); + +# Use a block that forces byte-level comparison +{ + use bytes; + @sa = sort { + # First 64 bytes check (fast path) + (substr($corpus, $a, 64) cmp substr($corpus, $b, 64)) || + # Full string fallback (required for correctness) + (substr($corpus, $a) cmp substr($corpus, $b)) + } @sa; } -nstore(\%index, $output_file); -printf("Index complete: %d files (%.2f KB)\n", scalar(keys %index), (-s $output_file) / 1024); +print "Writing index files to disk...\n"; + +open my $cfh, '>', $corpus_file or die "Cannot write $corpus_file: $!"; +binmode($cfh); # Raw byte mode +print $cfh $corpus; +close $cfh; + +open my $sfh, '>', $sa_file or die "Cannot write $sa_file: $!"; +binmode($sfh); +# Pack as 32-bit unsigned integers (standard 'L') +print $sfh pack("L*", @sa); +close $sfh; + +store \@file_map, $map_file; + +my $elapsed = tv_interval($t0); +my $c_size = -s $corpus_file; +my $s_size = -s $sa_file; + +printf "\nIndexing Complete!\n"; +printf "Total Time: %.4f seconds\n", $elapsed; +printf "Corpus Size: %.2f KB\n", $c_size / 1024; +printf "Suffix Array: %.2f KB\n", $s_size / 1024; +printf "Files Processed: %d\n", scalar(@file_map); + -- cgit v1.2.3