Refactor benchmark output filenames, discard Bloom filter query, and restore Diff link in footer

ftosoni · ftosoni · commit 2a8f66fbe64b · 2026-06-04T14:23:36.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -13,7 +13,6 @@ backend/snippets.db
 backend/raw_functions.json
 backend/raw_snippets.json
 backend/test_repo/
-backend/CoCo-trie/
 backend/*.npy
 backend/*.db
 
diff --git a/backend/build_index.py b/backend/build_index.py
@@ -51,8 +51,8 @@ def build_index():
     # A rule of thumb is 4 * sqrt(N), but for 2,484+ repos we can use a higher fixed value
     nlist = min(100, num_vectors) 
     quantizer = faiss.IndexFlatL2(EMBEDDING_DIM)
-    # 32: Number of sub-quantizers, 8: Number of bits per sub-vector
-    index = faiss.IndexIVFPQ(quantizer, EMBEDDING_DIM, nlist, 32, 8)
+    # 128: Number of sub-quantizers (was 32), 8: Number of bits per sub-vector
+    index = faiss.IndexIVFPQ(quantizer, EMBEDDING_DIM, nlist, 128, 8)
     
     print("Training quantizer...")
     index.train(embeddings.astype('float32'))
@@ -61,7 +61,7 @@ def build_index():
     
     faiss.write_index(index, FAISS_INDEX_PATH)
     print(f"Index saved to {FAISS_INDEX_PATH}")
-    print("✅ Index building complete.")
+    print("[OK] Index building complete.")
 
 if __name__ == "__main__":
     build_index()
diff --git a/frontend/i18n/as.json b/frontend/i18n/as.json
@@ -52,5 +52,6 @@
     "footer_section_status": "অনুজ্ঞাপত্ৰ আৰু স্থিতি",
     "footer_release_date": "প্ৰকাশৰ তাৰিখ:",
     "example_queries_label": "মূল্যায়ন কোয়েরি উদাহরণ চেষ্টা করুন (Try an evaluation query example)",
-    "example_queries_placeholder": "-- এটা মূল্যায়ন কোয়েরি বাছক --"
+    "example_queries_placeholder": "-- এটা মূল্যায়ন কোয়েরি বাছক --",
+    "diff_post": "ৱিকিমিডিয়া ডিফ পোষ্ট (Wikimedia Diff post)"
 }
diff --git a/frontend/i18n/bn.json b/frontend/i18n/bn.json
@@ -52,5 +52,6 @@
     "footer_section_status": "লাইসেন্স এবং স্থিতি",
     "footer_release_date": "প্রকাশের তারিখ:",
     "example_queries_label": "মূল্যায়ন কোয়েরি উদাহরণ চেষ্টা করুন",
-    "example_queries_placeholder": "-- একটি মূল্যায়ন কোয়েরি নির্বাচন করুন --"
+    "example_queries_placeholder": "-- একটি মূল্যায়ন কোয়েরি নির্বাচন করুন --",
+    "diff_post": "উইকিমিডিয়া ডিফ পোস্ট"
 }
diff --git a/frontend/i18n/en.json b/frontend/i18n/en.json
@@ -45,6 +45,7 @@
     "source": "Source",
     "wikidata": "Wikidata Q-item",
     "toolhub": "Toolhub",
+    "diff_post": "Wikimedia Diff post",
     "docs": "API",
     "presentation": "Presentation",
     "group_label": "Repository groups",
diff --git a/frontend/i18n/fr.json b/frontend/i18n/fr.json
@@ -54,5 +54,6 @@
     "footer_section_status": "Licence et Statut",
     "footer_release_date": "Date de sortie :",
     "example_queries_label": "Essayer un exemple de requête d'évaluation",
-    "example_queries_placeholder": "-- Sélectionner une requête d'évaluation --"
+    "example_queries_placeholder": "-- Sélectionner une requête d'évaluation --",
+    "diff_post": "Article sur Wikimedia Diff"
 }
diff --git a/frontend/i18n/gu.json b/frontend/i18n/gu.json
@@ -52,5 +52,6 @@
     "footer_section_status": "લાયસન્સ અને સ્થિતિ",
     "footer_release_date": "પ્રકાશન તારીખ:",
     "example_queries_label": "મૂલ્યાંકন ક્વેરી ઉદાહરણ અજમાવો",
-    "example_queries_placeholder": "-- મૂલ્યાંકન ક્વેરી પસંદ કરો --"
+    "example_queries_placeholder": "-- મૂલ્યાંકન ક્વેરી પસંદ કરો --",
+    "diff_post": "વિકિમીડિયા ડિફ પોસ્ટ"
 }
diff --git a/frontend/i18n/hi.json b/frontend/i18n/hi.json
@@ -52,5 +52,6 @@
     "footer_section_status": "लाइसेंस और स्थिति",
     "footer_release_date": "रिलीज़ की तारीख:",
     "example_queries_label": "मूल्यांकन क्वेरी उदाहरण आज़माएं",
-    "example_queries_placeholder": "-- मूल्यांकन क्वेरी चुनें --"
+    "example_queries_placeholder": "-- मूल्यांकन क्वेरी चुनें --",
+    "diff_post": "विकिमीडिया डिफ़ पोस्ट"
 }
diff --git a/frontend/i18n/it.json b/frontend/i18n/it.json
@@ -35,6 +35,7 @@
     "source": "Sorgente",
     "wikidata": "Elemento Q di Wikidata",
     "toolhub": "Toolhub",
+    "diff_post": "Post su Wikimedia Diff",
     "docs": "API",
     "presentation": "Presentazione",
     "group_label": "Gruppi di repository",
diff --git a/frontend/i18n/kn.json b/frontend/i18n/kn.json
@@ -52,5 +52,6 @@
     "footer_section_status": "ಪರವಾನಗಿ ಮತ್ತು ಸ್ಥಿತಿ",
     "footer_release_date": "ಬಿಡುగಡೆ ದಿನಾಂಕ:",
     "example_queries_label": "ಮೌಲ್ಯಮಾಪನ ಪ್ರಶ್ನೆಯ ಉದಾಹರಣೆಯನ್ನು ప్రయత్ಿಸಿ",
-    "example_queries_placeholder": "-- ಮೌಲ್ಯಮಾపನ ಪ್ರಶ್ನೆಯನ್ನು ఆರಿಸಿ --"
+    "example_queries_placeholder": "-- ಮೌಲ್ಯಮಾపನ ಪ್ರಶ್ನೆಯನ್ನು ఆರಿಸಿ --",
+    "diff_post": "ವಿಕಿಮೀಡಿಯา ಡಿಫ್ ಪೋಸ್ಟ್"
 }
diff --git a/frontend/i18n/mai.json b/frontend/i18n/mai.json
@@ -52,5 +52,6 @@
     "footer_section_status": "लाइसेंस आ स्थिति",
     "footer_release_date": "रिलीज़क तारीख:",
     "example_queries_label": "मूल्यांकन क्वेरी उदाहरण आज़माएं",
-    "example_queries_placeholder": "-- मूल्यांकन क्वेरी चुनु --"
+    "example_queries_placeholder": "-- मूल्यांकन क्वेरी चुनु --",
+    "diff_post": "विकिमीडिया डिफ़ पोस्ट"
 }
diff --git a/frontend/i18n/ml.json b/frontend/i18n/ml.json
@@ -52,5 +52,6 @@
     "footer_section_status": "ലൈസൻസും നിലയും",
     "footer_release_date": "റിലീസ് തീയതി:",
     "example_queries_label": "ഒരു മൂല്യനിർണ്ണയ അന്വേഷണ ഉദാഹരണം പരീക്ഷിക്കുക",
-    "example_queries_placeholder": "-- ഒരു മൂല്യനിർണ്ണയ അന്വേഷണം തിരഞ്ഞെടുക്കുക --"
+    "example_queries_placeholder": "-- ഒരു മൂല്യനിർണ്ണയ അന്വേഷണം തിരഞ്ഞെടുക്കുക --",
+    "diff_post": "വിക്കിമീഡിയ ഡിഫ് പോസ്റ്റ്"
 }
diff --git a/frontend/i18n/mr.json b/frontend/i18n/mr.json
@@ -52,5 +52,6 @@
     "footer_section_status": "परवाना आणि स्थिती",
     "footer_release_date": "प्रकाशन तारीख:",
     "example_queries_label": "मूल्यांकन क्वेरीचे उदाहरण पहा",
-    "example_queries_placeholder": "-- मूल्यांकन क्वेरी निवडा --"
+    "example_queries_placeholder": "-- मूल्यांकन क्वेरी निवडा --",
+    "diff_post": "विकिमीडिया डिफ पोस्ट"
 }
diff --git a/frontend/i18n/or.json b/frontend/i18n/or.json
@@ -52,5 +52,6 @@
     "footer_section_status": "ଲାଇସେନ୍ସ ଏବଂ ସ୍ଥିତି",
     "footer_release_date": "ରିଲିଜ୍ ତାରିଖ:",
     "example_queries_label": "ମୂଲ୍ୟାଙ୍କନ ପ୍ରଶ୍ନର ଏକ ଉଦାহରଣ ଚେଷ୍ଟာ କରନ୍ତু",
-    "example_queries_placeholder": "-- ମୂଲ୍ୟାଙ୍କନ ପ୍ରଶ୍ନ ଚୟନ କରନ୍ତୁ --"
+    "example_queries_placeholder": "-- ମୂଲ୍ୟାଙ୍କନ ପ୍ରଶ୍ନ ଚୟନ କରନ୍ତୁ --",
+    "diff_post": "ଉଇକିମିଡ଼ିଆ ଡିଫ୍ ପୋଷ୍ଟ"
 }
diff --git a/frontend/i18n/pa.json b/frontend/i18n/pa.json
@@ -52,5 +52,6 @@
     "footer_section_status": "ਲਾਇਸੈਂਸ ਅਤੇ ਸਥਿਤੀ",
     "footer_release_date": "ਜਾਰੀ ਕਰਨ ਦੀ ਮਿਤੀ:",
     "example_queries_label": "ਮੁਲਾਂਕਣ ਪੁੱਛਗਿੱਛ ਦੀ ਉਦਾਹরਣ ਵੇਖੋ",
-    "example_queries_placeholder": "-- ਮੁਲਾਂਕਣ ਪੁੱਛਗਿੱਛ ਦੀ ਚੋਣ ਕਰੋ --"
+    "example_queries_placeholder": "-- ਮੁਲਾਂਕਣ ਪੁੱਛਗਿੱਛ ਦੀ ਚੋਣ ਕਰੋ --",
+    "diff_post": "ਵਿਕੀਮੀਡੀਆ ਡਿਫ ਪੋਸਟ"
 }
diff --git a/frontend/i18n/qqq.json b/frontend/i18n/qqq.json
@@ -45,6 +45,7 @@
     "source": "Source",
     "wikidata": "Wikidata Q-item",
     "toolhub": "Toolhub",
+    "diff_post": "Wikimedia Diff post",
     "docs": "API Documentation",
     "presentation": "Presentation",
     "group_label": "Repository groups",
diff --git a/frontend/i18n/sat.json b/frontend/i18n/sat.json
@@ -52,5 +52,6 @@
     "footer_section_status": "ᱞᱟᱭᱥᱮᱱᱥ ᱟᱨ ᱦᱟᱞᱟᱛ",
     "footer_release_date": "ᱨᱤᱞᱤᱡᱽ ᱢᱟᱹᱦིତ:",
     "example_queries_label": "ᱢᱩᱞᱟᱹᱭᱟᱹᱱ ᱠᱩᱠᱞᱤ ᱨᱮᱱᱟᱜ ᱩᱫᱟᱹᱦᱚᱨᱚᱱ ᱧᱮᱞ ᱢᱮ",
-    "example_queries_placeholder": "-- ᱢᱩᱞᱟᱹᱭᱟᱹᱱ ᱠᱩᱠᱞᱤ ᱵᱟᱪᱷᱟᱣ ᱢᱮ --"
+    "example_queries_placeholder": "-- ᱢᱩᱞᱟᱹᱭᱟᱹᱱ ᱠᱩᱠᱞᱤ ᱵᱟᱪᱷᱟᱣ ᱢᱮ --",
+    "diff_post": "ᱣᱤᱠᱤᱢᱤᱰᱤᱭา ᱰᱤᱯᱷ ᱯᱳᱥᱴ"
 }
diff --git a/frontend/i18n/ta.json b/frontend/i18n/ta.json
@@ -52,5 +52,6 @@
     "footer_section_status": "உரிமம் & நிலை",
     "footer_release_date": "வெளியீட்டு தேதி:",
     "example_queries_label": "மதிப்பீட்டுக் குறியீடு உதாரணத்தை முயற்சிக்கவும்",
-    "example_queries_placeholder": "-- மதிப்பீட்டுக் குறியீட்டைத் தேர்ந்தெடுக்கவும் --"
+    "example_queries_placeholder": "-- மதிப்பீட்டுக் குறியீட்டைத் தேர்ந்தெடுக்கவும் --",
+    "diff_post": "விக்கிமீடியா டிஃப் இடுகை"
 }
diff --git a/frontend/i18n/te.json b/frontend/i18n/te.json
@@ -52,5 +52,6 @@
     "footer_section_status": "లైసెన్స్ & స్థితి",
     "footer_release_date": "ವಿಡುదల తేదీ:",
     "example_queries_label": "మూల్యాంకన క్వెరీ ఉదాహరణను ప్రయత్నించండి",
-    "example_queries_placeholder": "-- మూల్యాంకన క్వెరీని ఎంచుకోండి --"
+    "example_queries_placeholder": "-- మూల్యాంకన క్వెరీని ఎంచుకోండి --",
+    "diff_post": "వికీమీడియా డిఫ్ పోస్ట్"
 }
diff --git a/frontend/i18n/ur.json b/frontend/i18n/ur.json
@@ -52,5 +52,6 @@
     "footer_section_status": "لائسنس اور حیثیت",
     "footer_release_date": "تاریخ اشاعت:",
     "example_queries_label": "تشخیص سوال کی مثال آزمائیں",
-    "example_queries_placeholder": "-- تشخیص سوال منتخب کریں --"
+    "example_queries_placeholder": "-- تشخیص سوال منتخب کریں --",
+    "diff_post": "ویکی میڈیا ڈف پوسٹ"
 }
diff --git a/frontend/index.html b/frontend/index.html
@@ -199,7 +199,6 @@ <h1 class="page-hero__heading">Search source code</h1>
                             <option value="D2">D2 - Git-compatible SHA-1 / SWHID computation (Python)</option>
                             <option value="D3">D3 - IRC message line parser (Python)</option>
                             <option value="D4">D4 - MediaWiki hook dispatcher (PHP)</option>
-                            <option value="D5">D5 - Bloom filter membership check (Go)</option>
                         </optgroup>
                     </select>
                 </div>
@@ -369,6 +368,11 @@ <h3 class="filter-label">Result type</h3>
                         target="_blank" rel="noopener noreferrer"
                         style="font-size: 0.8125rem; color: #3366cc; text-decoration: none;">Toolhub</a>
                 </div>
+                <div>
+                    <a id="footer-diff" href="https://diff.wikimedia.org/2026/04/14/introducing-mediawiki-code2code-search-semantic-search-to-find-code-by-under-the-surface-similarity/"
+                        target="_blank" rel="noopener noreferrer"
+                        style="font-size: 0.8125rem; color: #3366cc; text-decoration: none;">Wikimedia Diff post</a>
+                </div>
             </div>
 
             <!-- Column 3: Source & Support -->
diff --git a/frontend/js/main.js b/frontend/js/main.js
@@ -54,8 +54,7 @@ const EVAL_QUERIES = {
     D1: { code: "public function extractWikiLinks(string $wikitext): array {\n    $links = [];\n    if (preg_match_all(\n            '/\\[\\[([^|\\]]+)(?:\\|[^\\]]+)?\\]\\]/', $wikitext, $m)) {\n        foreach ($m[1] as $target) {\n            $links[] = Title::newFromText(trim($target));\n        }\n    }\n    return array_filter($links);\n}", lang: "PHP" },
     D2: { code: "def compute_swhid(content: bytes) -> str:\n    sha1 = hashlib.new('sha1')\n    header = f\"blob {len(content)}\\0\".encode()\n    sha1.update(header + content)\n    return f\"swh:1:cnt:{sha1.hexdigest()}\"", lang: "Python" },
     D3: { code: "def parse_irc_message(raw: str) -> dict:\n    prefix, command, params = None, None, []\n    if raw.startswith(':'):\n        prefix, raw = raw[1:].split(' ', 1)\n    parts = raw.split(' ', 1)\n    command = parts[0]\n    if len(parts) > 1:\n        ti = parts[1].find(' :')\n        if ti >= 0:\n            params = parts[1][:ti].split()\n            params.append(parts[1][ti+2:])\n        else:\n            params = parts[1].split()\n    return {'prefix': prefix, 'command': command, 'params': params}", lang: "Python" },
-    D4: { code: "public function run(string $hook, array $args = []): bool {\n    foreach ($this->getHandlers($hook) as $handler) {\n        $ret = $handler(...$args);\n        if ($ret === false) {\n            return false;\n        }\n    }\n    return true;\n}", lang: "PHP" },
-    D5: { code: "func (f *BloomFilter) Contains(item []byte) bool {\n    for _, h := range f.hashFunctions(item) {\n        idx := h % uint64(len(f.bits))\n        if f.bits[idx/8]&(1<<(idx%8)) == 0 {\n            return false\n        }\n    }\n    return true\n}", lang: "Go" }
+    D4: { code: "public function run(string $hook, array $args = []): bool {\n    foreach ($this->getHandlers($hook) as $handler) {\n        $ret = $handler(...$args);\n        if ($ret === false) {\n            return false;\n        }\n    }\n    return true;\n}", lang: "PHP" }
 };
 
 let currentLang = localStorage.getItem('code2code_lang') || CONFIG.defaultLang;
@@ -106,26 +105,20 @@ function setupEventListeners() {
             const queryData = EVAL_QUERIES[qid];
             document.getElementById('search-query').value = queryData.code;
 
-            // Update language filter chip to match query language
-            const lang = queryData.lang;
-            const langContainer = document.getElementById('filter-langs');
-            const chips = langContainer.querySelectorAll('.cdx-button');
-            chips.forEach(c => c.classList.remove('active'));
-
-            const supportedLangs = ['Python', 'C++', 'C', 'PHP', 'JavaScript', 'TypeScript', 'Lua', 'Go', 'Java', 'Rust', 'Ruby', 'Perl'];
-            if (supportedLangs.includes(lang)) {
-                const targetChip = langContainer.querySelector(`[data-value="${lang}"]`);
-                if (targetChip) {
-                    targetChip.classList.add('active');
-                    activeFilters.langs = [lang];
+            // Reset filters to "All" to match the benchmarking settings
+            const containers = ['filter-repos', 'filter-langs', 'filter-types'];
+            containers.forEach(containerId => {
+                const container = document.getElementById(containerId);
+                if (container) {
+                    const chips = container.querySelectorAll('.cdx-button');
+                    chips.forEach(c => c.classList.remove('active'));
+                    const allChip = container.querySelector('[data-value="all"]');
+                    if (allChip) allChip.classList.add('active');
                 }
-            } else {
-                const allChip = langContainer.querySelector(`[data-value="all"]`);
-                if (allChip) {
-                    allChip.classList.add('active');
-                    activeFilters.langs = ['all'];
-                }
-            }
+            });
+            activeFilters.repos = ['all'];
+            activeFilters.langs = ['all'];
+            activeFilters.types = ['all'];
             updateTemplateStatus();
         }
     });
@@ -293,6 +286,9 @@ function applyI18n() {
     const footerToolhub = document.getElementById('footer-toolhub');
     if (footerToolhub) footerToolhub.textContent = i18nData.toolhub || 'Toolhub';
 
+    const footerDiff = document.getElementById('footer-diff');
+    if (footerDiff) footerDiff.textContent = i18nData.diff_post || 'Wikimedia Diff post';
+
     const footerSource = document.getElementById('footer-source');
     if (footerSource) {
         footerSource.innerHTML = `${i18nData.source || 'Source'}:&nbsp;<a href="https://github.com/ftosoni/mediawiki-code2code-search" target="_blank">GitHub</a>&nbsp;·&nbsp;<a href="https://archive.softwareheritage.org/browse/origin/directory/?origin_url=https://github.com/ftosoni/mediawiki-code2code-search" target="_blank">SWH</a>`;
diff --git a/scripts/evaluation/benchmark_bm25.py b/scripts/evaluation/benchmark_bm25.py
@@ -89,6 +89,8 @@ def save_queries_to_json(queries, json_path):
 
 def main():
     parser = argparse.ArgumentParser(description="BM25 Search Baseline Evaluation")
+    parser.add_argument("--url", default="https://code2codesearch.toolforge.org/search", help="Target API search endpoint for filename alignment")
+    parser.add_argument("--runs", type=int, default=7, help="Number of repetitions for filename alignment")
     parser.add_argument("--queries-json", default=None, help="Path to evaluation queries JSON file")
     parser.add_argument("--tex-path", default=None, help="Path to evaluation queries LaTeX file")
     parser.add_argument("--save-results", default=None, help="Path to save BM25 evaluation results JSON")
@@ -99,11 +101,17 @@ def main():
     script_dir = os.path.dirname(os.path.abspath(__file__))
     project_root = os.path.dirname(os.path.dirname(script_dir))
 
+    # Sanitize URL for filename
+    url_part = re.sub(r'^https?://', '', args.url)
+    url_part = re.sub(r'[^a-zA-Z0-9]', '_', url_part)
+    url_part = re.sub(r'_+', '_', url_part).strip('_')
+    suffix = f"{url_part}_{args.runs}runs"
+
     # Determine paths
     db_path = args.db_path or os.path.join(project_root, "backend", "snippets.db")
     tex_path = args.tex_path or os.path.join(project_root, "manuscript", "evaluation_queries.tex")
     queries_json_path = args.queries_json or os.path.join(script_dir, "evaluation_queries.json")
-    save_results_path = args.save_results or os.path.join(script_dir, "bm25_results.json")
+    save_results_path = args.save_results or os.path.join(script_dir, f"bm25_results_{suffix}.json")
     index_path = args.index_path or os.path.join(project_root, "backend", "bm25_index.pkl")
 
     # Load or parse queries
diff --git a/scripts/evaluation/benchmark_latency.py b/scripts/evaluation/benchmark_latency.py
diff --git a/scripts/evaluation/bm25_results.json b/scripts/evaluation/bm25_results.json
diff --git a/scripts/evaluation/dump_for_judging.py b/scripts/evaluation/dump_for_judging.py
diff --git a/scripts/evaluation/duplication_stats.py b/scripts/evaluation/duplication_stats.py
diff --git a/scripts/evaluation/evaluation_queries.json b/scripts/evaluation/evaluation_queries.json
diff --git a/scripts/evaluation/evaluation_results.json b/scripts/evaluation/evaluation_results.json
diff --git a/scripts/evaluation/judge_annotations.json b/scripts/evaluation/judge_annotations.json
diff --git a/scripts/evaluation/judge_scores.py b/scripts/evaluation/judge_scores.py
diff --git a/scripts/evaluation/latency_boxplot.png b/scripts/evaluation/latency_boxplot.png
diff --git a/scripts/evaluation/latency_boxplot.tex b/scripts/evaluation/latency_boxplot.tex

Original file line number	Diff line number	Diff line change
`@@ -52,5 +52,6 @@`
`52`	`52`	`"footer_section_status": "অনুজ্ঞাপত্ৰ আৰু স্থিতি",`
`53`	`53`	`"footer_release_date": "প্ৰকাশৰ তাৰিখ:",`
`54`	`54`	`"example_queries_label": "মূল্যায়ন কোয়েরি উদাহরণ চেষ্টা করুন (Try an evaluation query example)",`
`55`		`- "example_queries_placeholder": "-- এটা মূল্যায়ন কোয়েরি বাছক --"`
	`55`	`+ "example_queries_placeholder": "-- এটা মূল্যায়ন কোয়েরি বাছক --",`
	`56`	`+ "diff_post": "ৱিকিমিডিয়া ডিফ পোষ্ট (Wikimedia Diff post)"`
`56`	`57`	`}`
Original file line number	Diff line number	Diff line change
`@@ -54,5 +54,6 @@`
`54`	`54`	`"footer_section_status": "Licence et Statut",`
`55`	`55`	`"footer_release_date": "Date de sortie :",`
`56`	`56`	`"example_queries_label": "Essayer un exemple de requête d'évaluation",`
`57`		`- "example_queries_placeholder": "-- Sélectionner une requête d'évaluation --"`
	`57`	`+ "example_queries_placeholder": "-- Sélectionner une requête d'évaluation --",`
	`58`	`+ "diff_post": "Article sur Wikimedia Diff"`
`58`	`59`	`}`