diff options
Diffstat (limited to 'cgi-bin/indexer.pl')
| -rw-r--r-- | cgi-bin/indexer.pl | 146 |
1 files changed, 108 insertions, 38 deletions
diff --git a/cgi-bin/indexer.pl b/cgi-bin/indexer.pl index 38a918e..69f6838 100644 --- a/cgi-bin/indexer.pl +++ b/cgi-bin/indexer.pl @@ -2,45 +2,115 @@ use strict; use warnings; -use Storable qw(nstore); +use File::Find; +use Storable qw(store); +use Encode qw(encode_utf8); use HTML::Entities qw(decode_entities); +use Time::HiRes qw(gettimeofday tv_interval); -# --- Configuration --- -my $built_site_dir = '../_site/log'; -my $output_file = '../_site/cgi-bin/search_index.dat'; -my %index; - -print "Building search index from $built_site_dir...\n"; - -foreach my $path (glob("$built_site_dir/*/index.html")) { - next unless open(my $fh, '<:utf8', $path); - my $html = do { local $/; <$fh> }; - close($fh); - - # Extract Title and Main Content - my ($title) = $html =~ m|<title>(.*?)</title>|is; - my ($main) = $html =~ m|<main>(.*?)</main>|is; - $main //= ''; - - # Strip HTML and clean prose - $main =~ s|<pre[^>]*>.*?</pre>| |gs; - $main =~ s|<code[^>]*>.*?</code>| |gs; - $main =~ s|<[^>]+>| |g; - $main = decode_entities($main); - $main =~ s|\s+| |g; - $main =~ s/^\s+|\s+$//g; - - # Normalize path - my $url = $path; - $url =~ s|^\.\./_site/||; # Remove local build directory - $url =~ s|^\.\./||; # Remove any leading dots - $url =~ s|^/+||; # Remove leading slashes - - $index{$url} = { - t => $title || "Untitled", - c => $main - }; +my $dir = '../_site/log'; +my $cgi_dir = '../_site/cgi-bin/'; +my $corpus_file = "${cgi_dir}corpus.bin"; +my $sa_file = "${cgi_dir}sa.bin"; +my $map_file = "${cgi_dir}file_map.dat"; + +my %excluded_files = ( + 'index.html' => 1, # /log/index.html +); + +# Start timing +my $t0 = [gettimeofday]; + +my $corpus = ""; +my @file_map; + +print "Building corpus...\n"; + +find({ + wanted => sub { + # Only index index.html files + return unless -f $_ && $_ eq 'index.html'; + + my $rel_path = $File::Find::name; + $rel_path =~ s|^\Q$dir\E/?||; + return if $excluded_files{$rel_path}; + + if (open my $fh, '<:encoding(UTF-8)', $_) { + my $content = do { local $/; <$fh> }; + close $fh; + + my ($title) = $content =~ m|<title>(.*?)</title>|is; + $title //= (split('/', $File::Find::name))[-2]; # Fallback to folder name + $title =~ s/^\s+|\s+$//g; + + # Extract content from <main> or use whole file + my ($text) = $content =~ m|<main>(.*?)</main>|is; + $text //= $content; + + # Strip tags and normalize whitespace + $text =~ s|<pre[^>]*>.*?</pre>| |gs; + $text =~ s|<code[^>]*>.*?</code>| |gs; + $text =~ s|<[^>]+>| |g; + $text = decode_entities($text); + $text =~ s|\s+| |g; + $text =~ s/^\s+|\s+$//g; + + # CRITICAL: Convert to lowercase and then to raw bytes + # This ensures length() and substr() work on byte offsets for seek() + my $raw_entry = encode_utf8(lc($text) . "\0"); + + my $start = length($corpus); + $corpus .= $raw_entry; + + push @file_map, { + start => $start, + end => length($corpus), + title => $title, + path => $File::Find::name + }; + } + }, + no_chdir => 0, +}, $dir); + +print "Sorting suffixes...\n"; + +# Initialize the array of indices +my @sa = 0 .. (length($corpus) - 1); + +# Use a block that forces byte-level comparison +{ + use bytes; + @sa = sort { + # First 64 bytes check (fast path) + (substr($corpus, $a, 64) cmp substr($corpus, $b, 64)) || + # Full string fallback (required for correctness) + (substr($corpus, $a) cmp substr($corpus, $b)) + } @sa; } -nstore(\%index, $output_file); -printf("Index complete: %d files (%.2f KB)\n", scalar(keys %index), (-s $output_file) / 1024); +print "Writing index files to disk...\n"; + +open my $cfh, '>', $corpus_file or die "Cannot write $corpus_file: $!"; +binmode($cfh); # Raw byte mode +print $cfh $corpus; +close $cfh; + +open my $sfh, '>', $sa_file or die "Cannot write $sa_file: $!"; +binmode($sfh); +# Pack as 32-bit unsigned integers (standard 'L') +print $sfh pack("L*", @sa); +close $sfh; + +store \@file_map, $map_file; + +my $elapsed = tv_interval($t0); +my $c_size = -s $corpus_file; +my $s_size = -s $sa_file; + +printf "\nIndexing Complete!\n"; +printf "Total Time: %.4f seconds\n", $elapsed; +printf "Corpus Size: %.2f KB\n", $c_size / 1024; +printf "Suffix Array: %.2f KB\n", $s_size / 1024; +printf "Files Processed: %d\n", scalar(@file_map); + |