1 files changed, 116 insertions, 0 deletions
diff --git a/cgi-bin/_site/indexer.pl b/cgi-bin/_site/indexer.pl
new file mode 100644
index 0000000..69f6838
--- /dev/null
+++ b/cgi-bin/_site/indexer.pl
@@ -0,0 +1,116 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use File::Find;
+use Storable qw(store);
+use Encode qw(encode_utf8);
+use HTML::Entities qw(decode_entities);
+use Time::HiRes qw(gettimeofday tv_interval);
+
+my $dir         = '../_site/log';
+my $cgi_dir     = '../_site/cgi-bin/';
+my $corpus_file = "${cgi_dir}corpus.bin";
+my $sa_file     = "${cgi_dir}sa.bin";
+my $map_file    = "${cgi_dir}file_map.dat";
+
+my %excluded_files = (
+    'index.html'  => 1, # /log/index.html
+);
+
+# Start timing
+my $t0 = [gettimeofday];
+
+my $corpus = "";
+my @file_map;
+
+print "Building corpus...\n";
+
+find({
+    wanted => sub {
+        # Only index index.html files
+        return unless -f $_ && $_ eq 'index.html';
+
+        my $rel_path = $File::Find::name;
+        $rel_path =~ s|^\Q$dir\E/?||; 
+        return if $excluded_files{$rel_path};
+        
+        if (open my $fh, '<:encoding(UTF-8)', $_) {
+            my $content = do { local $/; <$fh> };
+            close $fh;
+
+            my ($title) = $content =~ m|<title>(.*?)</title>|is;
+            $title //= (split('/', $File::Find::name))[-2]; # Fallback to folder name
+            $title =~ s/^\s+|\s+$//g;
+
+            # Extract content from <main> or use whole file
+            my ($text) = $content =~ m|<main>(.*?)</main>|is;
+            $text //= $content; 
+
+            # Strip tags and normalize whitespace
+            $text =~ s|<pre[^>]*>.*?</pre>| |gs;
+            $text =~ s|<code[^>]*>.*?</code>| |gs;
+            $text =~ s|<[^>]+>| |g;
+            $text = decode_entities($text);
+            $text =~ s|\s+| |g;
+            $text =~ s/^\s+|\s+$//g;
+
+            # CRITICAL: Convert to lowercase and then to raw bytes
+            # This ensures length() and substr() work on byte offsets for seek()
+            my $raw_entry = encode_utf8(lc($text) . "\0");
+
+            my $start = length($corpus);
+            $corpus .= $raw_entry;
+            
+            push @file_map, { 
+                start => $start, 
+                end   => length($corpus), 
+                title => $title, 
+                path  => $File::Find::name 
+            };
+        }
+    },
+    no_chdir => 0,
+}, $dir);
+
+print "Sorting suffixes...\n";
+
+# Initialize the array of indices
+my @sa = 0 .. (length($corpus) - 1);
+
+# Use a block that forces byte-level comparison
+{
+    use bytes; 
+    @sa = sort { 
+        # First 64 bytes check (fast path)
+        (substr($corpus, $a, 64) cmp substr($corpus, $b, 64)) || 
+        # Full string fallback (required for correctness)
+        (substr($corpus, $a) cmp substr($corpus, $b))
+    } @sa;
+}
+
+print "Writing index files to disk...\n";
+
+open my $cfh, '>', $corpus_file or die "Cannot write $corpus_file: $!";
+binmode($cfh); # Raw byte mode
+print $cfh $corpus;
+close $cfh;
+
+open my $sfh, '>', $sa_file or die "Cannot write $sa_file: $!";
+binmode($sfh);
+# Pack as 32-bit unsigned integers (standard 'L')
+print $sfh pack("L*", @sa);
+close $sfh;
+
+store \@file_map, $map_file;
+
+my $elapsed = tv_interval($t0);
+my $c_size = -s $corpus_file;
+my $s_size = -s $sa_file;
+
+printf "\nIndexing Complete!\n";
+printf "Total Time:      %.4f seconds\n", $elapsed;
+printf "Corpus Size:     %.2f KB\n", $c_size / 1024;
+printf "Suffix Array:    %.2f KB\n", $s_size / 1024;
+printf "Files Processed: %d\n", scalar(@file_map);
+