diff options
| -rw-r--r-- | README.txt | 8 | ||||
| -rw-r--r-- | _site/cgi-bin/find_sa_mmap.cgi | 212 | ||||
| -rwxr-xr-x | _site/cgi-bin/seed.sh | 2 |
3 files changed, 221 insertions, 1 deletions
diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..c0e54ea --- /dev/null +++ b/README.txt @@ -0,0 +1,8 @@ +HOW TO USE + + 1. Copy _site/cgi-bin/see.sh to _site/log/ + 2. Set the directory counta (TOTAL) + 3. Run script ./seed.sh + 4. In _site/cgi_bin/ execute indexer script (e.g., perl sa_indexer.pl) + 5. Run search query: QUERY_STRING="q=ard" perl find_sa_mmap.cgi + diff --git a/_site/cgi-bin/find_sa_mmap.cgi b/_site/cgi-bin/find_sa_mmap.cgi new file mode 100644 index 0000000..979f4d5 --- /dev/null +++ b/_site/cgi-bin/find_sa_mmap.cgi @@ -0,0 +1,212 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use Storable qw(retrieve); +use Encode qw(decode_utf8); +use HTML::Escape qw(escape_html); +use Time::HiRes qw(gettimeofday tv_interval); +use BSD::Resource; +use Sys::Mmap; + +my $sa_file = 'sa.bin'; # Suffix Array index +my $cp_file = 'corpus.bin'; # Raw text corpus +my $map_file = 'file_map.dat'; # File metadata + +# 1. Start Benchmark Timer +my $start_time = [gettimeofday]; +my $files_read = 0; # Track IO Activity + +# Decode search text +my $search_text = ''; +if (($ENV{QUERY_STRING} || '') =~ /^q=([^&]*)/) { + $search_text = decode_utf8($1 // ""); + $search_text =~ s/\P{Print}//g; + $search_text = substr($search_text, 0, 64); + $search_text =~ s/^\s+|\s+$//g; +} + +# We search using lowercase for the case-insensitive index +my $query = lc($search_text); +my $query_len = length($query); +my @results; + +if ($query_len >= 3 && -f 'sa.bin' && -f 'corpus.bin') { + open(my $fh_sa, '<', $sa_file) or die $!; + open(my $fh_cp, '<', $cp_file) or die $!; + binmode($fh_sa); + binmode($fh_cp); + + # Memory map files + my ($sa_mapped, $cp_mapped); + mmap($sa_mapped, 0, PROT_READ, MAP_SHARED, $fh_sa) or die "Could not map SA: $!"; + mmap($cp_mapped, 0, PROT_READ, MAP_SHARED, $fh_cp) or die "Could not map Corpus: $!"; + + my $file_map = retrieve($map_file); + $files_read += 3; + my $total_suffixes = (-s $sa_file) / 4; + + # Range Search: Find Left and Right boundaries + my ($low, $high) = (0, $total_suffixes - 1); + my $first_hit = -1; + while ($low <= $high) { + my $mid = int(($low + $high) / 2); + my $off = unpack("L", substr($sa_mapped, $mid * 4, 4)); + my $text = substr($cp_mapped, $off, $query_len); + my $cmp = $text cmp $query; + if ($cmp >= 0) { + $first_hit = $mid if $cmp == 0; + $high = $mid - 1; + } else { + $low = $mid + 1; + } + } + + if ($first_hit != -1) { + ($low, $high) = ($first_hit, $total_suffixes - 1); + my $last_hit = $first_hit; + while ($low <= $high) { + my $mid = int(($low + $high) / 2); + my $off = unpack("L", substr($sa_mapped, $mid * 4, 4)); + my $text = substr($cp_mapped, $off, $query_len); + if (($text cmp $query) <= 0) { + $last_hit = $mid if $text eq $query; + $low = $mid + 1; + } else { + $high = $mid - 1; + } + } + + # Collect unique file results + my %seen; + for my $i ($first_hit .. $last_hit) { + my $offset = unpack("L", substr($sa_mapped, $i * 4, 4)); + foreach my $m (@$file_map) { + if ($offset >= $m->{start} && $offset < $m->{end}) { + if (!$seen{$m->{path}}++) { + # Capture more than 50 chars for trimming + my $snip_start = ($offset - 30 < $m->{start}) ? $m->{start} : $offset - 30; + my $max_len = $m->{end} - $snip_start; + my $read_len = ($max_len > 120) ? 120 : $max_len; + my $raw_snip = substr($cp_mapped, $snip_start, $read_len); + my $snippet = decode_utf8($raw_snip, Encode::FB_QUIET) // $raw_snip; + $snippet =~ s/\s+/ /g; # Normalize whitespace + + # Trim start: Partial word removal + if ($snip_start > $m->{start}) { + $snippet =~ s/^[^\s]*\s//; + } + + # Trim end: Length limit and partial word removal + my $has_more = 0; + if (length($snippet) > 50) { + $snippet = substr($snippet, 0, 50); + $has_more = 1 if $snippet =~ s/\s+[^\s]*$//; + } + elsif ($snip_start + $read_len < $m->{end}) { + # This check handles snippets that are naturally short but + # there's still more text in the article we didn't read + $has_more = 1; + } + + # Cleanup & capitalize + $snippet = ucfirst($snippet); + $snippet = escape_html($snippet) . ($has_more ? "..." : ""); + + my $clean_path = $m->{path}; + $clean_path =~ s|^\.\./_site/||; + + push @results, { + path => $clean_path, + title => $m->{title},, + snippet => $snippet + }; + } + last; + } + } + last if scalar @results >= 1000; + } + } + close($fh_sa); + close($fh_cp); +} + +# 2. Calculate Metrics +my $end_time = [gettimeofday]; +my $elapsed = tv_interval($start_time, $end_time); + +my $rusage = getrusage(); +my $user_cpu = $rusage->utime; +my $system_cpu = $rusage->stime; +my $max_rss = $rusage->maxrss; + +# 3. Output +print "Content-Type: text/html\n\n"; + +my $list; +if ($search_text eq '') { + $list = "<p>Please enter a search term above.</p>"; +} elsif (@results == 0) { + $list = "<p>No results found for \"<b>" . escape_html($search_text) . "</b>\".</p>"; +} else { + $list = "<ul>"; + foreach my $res (@results) { + $list .= "<li><a href=\"/$res->{path}\">$res->{title}</a><br><small>$res->{snippet}</small></li>"; + } + $list .= "</ul>"; +} + +my $safe_search_text = escape_html($search_text); +my $year = (localtime)[5] + 1900; + +print <<"HTML"; +<!DOCTYPE html> +<html lang="en-us"> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>Search</title> + <link rel="stylesheet" href="/assets/css/main.css"> + <link rel="stylesheet" href="/assets/css/skeleton.css"> +</head> +<body> + <div id="nav-container" class="container"> + <ul id="navlist" class="left"> + <li><a href="/" class="link-decor-none">hme</a></li> + <li><a href="/log/" class="link-decor-none">log</a></li> + <li><a href="/projects/" class="link-decor-none">poc</a></li> + <li><a href="/about/" class="link-decor-none">abt</a></li> + <li class="active"><a href="/cgi-bin/find.cgi" class="link-decor-none">sws</a></li> + <li><a href="/feed.xml" class="link-decor-none">rss</a></li> + </ul> + </div> + <main class="container" id="main"> + <div class="container"> + <h2>Search</h2> + <form action="" method="GET"> + <input id="search-box" type="text" name="q" value="$safe_search_text"> + <input id="search-btn" type="submit" value="Search"> + </form> + $list + + <div style="background: #f4f4f4; padding: 10px; border-radius: 5px; font-family: monospace; font-size: 0.85em; margin-top: 20px; border: 1px solid #ddd;"> + <strong>Performance Metrics:</strong><br> + Total Time: @{[ sprintf("%.4f", $elapsed) ]} seconds<br> + User CPU: $user_cpu s<br> + System CPU: $system_cpu s<br> + Peak RAM: $max_rss KB<br> + Files Read: $files_read (IO Activity) + </div> + </div> + </main> + <div class="footer"> + <div class="container"> + <div class="twelve columns right container-2"> + <p id="footer-text">© ASCIIMX - $year</p> + </div> + </div> + </div> +</body> +</html> +HTML diff --git a/_site/cgi-bin/seed.sh b/_site/cgi-bin/seed.sh index 9d85108..5ae14df 100755 --- a/_site/cgi-bin/seed.sh +++ b/_site/cgi-bin/seed.sh @@ -1,7 +1,7 @@ #!/bin/ksh # Set the number of files/dirs -TOTAL=10000 +TOTAL=500 # Approximate size in blocks BLOCK_SIZE=16 COUNT=1 |