1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/perl
use strict;
use warnings;
use File::Find;
use Storable qw(store);
use Encode qw(encode_utf8);
use HTML::Entities qw(decode_entities);
use Time::HiRes qw(gettimeofday tv_interval);
my $dir = '../_site/log';
my $cgi_dir = '../_site/cgi-bin/';
my $corpus_file = "${cgi_dir}corpus.bin";
my $sa_file = "${cgi_dir}sa.bin";
my $map_file = "${cgi_dir}file_map.dat";
my %excluded_files = (
'index.html' => 1, # /log/index.html
);
# Start timing
my $t0 = [gettimeofday];
my $corpus = "";
my @file_map;
print "Building corpus...\n";
find({
wanted => sub {
# Only index index.html files
return unless -f $_ && $_ eq 'index.html';
my $rel_path = $File::Find::name;
$rel_path =~ s|^\Q$dir\E/?||;
return if $excluded_files{$rel_path};
if (open my $fh, '<:encoding(UTF-8)', $_) {
my $content = do { local $/; <$fh> };
close $fh;
my ($title) = $content =~ m|<title>(.*?)</title>|is;
$title //= (split('/', $File::Find::name))[-2]; # Fallback to folder name
$title =~ s/^\s+|\s+$//g;
# Extract content from <main> or use whole file
my ($text) = $content =~ m|<main>(.*?)</main>|is;
$text //= $content;
# Strip tags and normalize whitespace
$text =~ s|<pre[^>]*>.*?</pre>| |gs;
$text =~ s|<code[^>]*>.*?</code>| |gs;
$text =~ s|<[^>]+>| |g;
$text = decode_entities($text);
$text =~ s|\s+| |g;
$text =~ s/^\s+|\s+$//g;
# CRITICAL: Convert to lowercase and then to raw bytes
# This ensures length() and substr() work on byte offsets for seek()
my $raw_entry = encode_utf8(lc($text) . "\0");
my $start = length($corpus);
$corpus .= $raw_entry;
push @file_map, {
start => $start,
end => length($corpus),
title => $title,
path => $File::Find::name
};
}
},
no_chdir => 0,
}, $dir);
print "Sorting suffixes...\n";
# Initialize the array of indices
my @sa = 0 .. (length($corpus) - 1);
# Use a block that forces byte-level comparison
{
use bytes;
@sa = sort {
# First 64 bytes check (fast path)
(substr($corpus, $a, 64) cmp substr($corpus, $b, 64)) ||
# Full string fallback (required for correctness)
(substr($corpus, $a) cmp substr($corpus, $b))
} @sa;
}
print "Writing index files to disk...\n";
open my $cfh, '>', $corpus_file or die "Cannot write $corpus_file: $!";
binmode($cfh); # Raw byte mode
print $cfh $corpus;
close $cfh;
open my $sfh, '>', $sa_file or die "Cannot write $sa_file: $!";
binmode($sfh);
# Pack as 32-bit unsigned integers (standard 'L')
print $sfh pack("L*", @sa);
close $sfh;
store \@file_map, $map_file;
my $elapsed = tv_interval($t0);
my $c_size = -s $corpus_file;
my $s_size = -s $sa_file;
printf "\nIndexing Complete!\n";
printf "Total Time: %.4f seconds\n", $elapsed;
printf "Corpus Size: %.2f KB\n", $c_size / 1024;
printf "Suffix Array: %.2f KB\n", $s_size / 1024;
printf "Files Processed: %d\n", scalar(@file_map);
|
