speed optimizations

Author: Zulko

src/App.svelte

@@ -100,18 +100,24 @@
     };
 
     const entries = await getGeoEntriesInBounds(bounds);
-    const hashlevel = Math.max(1, Math.min(8, mapZoom / 2));
-    const uniqueEntries = getUniqueByGeoHash({
+    let hashlevel = Math.max(1, Math.min(8, mapZoom / 2));
+    let uniqueEntries = getUniqueByGeoHash({
       entries,
       hashLength: hashlevel,
       scoreField: "page_len",
     });
+    if (uniqueEntries.length > 400) {
+      console.log("uniqueEntries", uniqueEntries.length, "so reducing");
+      uniqueEntries.sort((a, b) => b.page_len - a.page_len);
+      uniqueEntries = uniqueEntries.slice(0, 400);
+    }
     if (
       selectedMarker &&
       !uniqueEntries.some((entry) => entry.id === selectedMarker.id)
     ) {
       uniqueEntries.push(selectedMarker);
     }
+    console.log("nMarkers", uniqueEntries.length);
 
     addMarkerClasses(uniqueEntries, hashlevel);
     markers = uniqueEntries;

src/lib/Map.svelte

@@ -66,9 +66,6 @@
         label = fullLabel;
       }
     }
-    if (marker.displayClass === "selected") {
-      console.log("Found selected in icon generation", marker);
-    }
     return `
     <div class="map-marker marker-display-${marker.displayClass}" >
         <div class="marker-icon-circle">
@@ -167,15 +164,13 @@
   }
 
   $: if (markers && map) {
-    console.log("markers changed");
     updateMarkers();
   }
 
   // Change from window.boundsChangeTimeout to a local variable
   let boundsChangeTimeout = null;
 
-  function handleBoundsChange(evt) {
-    console.log("handleBoundsChange", evt);
+  function handleBoundsChange() {
     if (!map || isFlying) return; // Skip if map is flying
 
     const bounds = map.getBounds();
@@ -213,7 +208,6 @@
 
     // Track which markers we've processed to identify removals
     const processedIds = new Set();
-    console.log("marker update");
 
     // Update or add markers
     markers.forEach((marker) => {
@@ -265,7 +259,6 @@
         mapMarker.on("mouseover", () => {
           if (hoveredMarkerId !== marker.id) {
             hoveredMarkerId = marker.id;
-            console.log("mouseover", marker);
             updateMarkers();
           }
         });
@@ -282,9 +275,9 @@
     });
 
     // Remove markers that are no longer in the data
-    for (const [markerId, mapMarker] of existingMapMarkers.entries()) {
+    for (const [markerId, entry] of existingMapMarkers.entries()) {
       if (!processedIds.has(markerId)) {
-        markerLayer.removeLayer(mapMarker);
+        markerLayer.removeLayer(entry.existingMarker);
         existingMapMarkers.delete(markerId);
       }
     }

src/lib/geodata.js

@@ -13,15 +13,15 @@ const db = new Loki('geodata.db', {
 
 // Create a collection for geo data with optimized configuration
 const geoCollection = db.addCollection('geodata', {
-  indices: ['geo2', 'category'],
+  indices: ['geo3', 'category'],
   adaptiveBinaryIndices: false,  // Disable adaptive indices for bulk operations
   transactional: false,  // Disable transactions for better performance
   clone: false,  // Disable object cloning for better performance
   disableMeta: true  // Disable meta properties for better performance
 });
 
 const tinyGeoCollection = db.addCollection('tinygeodata', {
-  indices: ['geo2', 'category'],
+  indices: ['geo3', 'category'],
   adaptiveBinaryIndices: false,
   transactional: false,
   clone: false,
@@ -30,7 +30,7 @@ const tinyGeoCollection = db.addCollection('tinygeodata', {
 
 /**
  * @typedef {Object} GeoDocument
- * @property {string} geo2 - Geohash
+ * @property {string} geo3 - Geohash
  * @property {string} category - Category
  * @property {number} lat - Latitude
  * @property {number} lon - Longitude
@@ -51,7 +51,7 @@ function addLatLonToRows(rows) {
       const latLon = ngeohash.decode(row.geohash);
       row.lat = latLon.latitude;
       row.lon = latLon.longitude;
-      row.geo2 = row.geohash.substring(0, 2);
+      row.geo3 = row.geohash.substring(0, 3);
     }
   }
 }
@@ -162,14 +162,17 @@ async function downloadMissingData(urls) {
 
 
 function queryGeoTable(table, minLat, maxLat, minLon, maxLon) {
-  const geohashes_2 = ngeohash.bboxes(minLat, minLon, maxLat, maxLon, 2);
-    return table.find({geo2: { '$in': geohashes_2 }}).filter(doc => 
-      doc.lat > minLat && 
-      doc.lat < maxLat && 
-      doc.lon > minLon && 
-      doc.lon < maxLon
-    );
-  }
+  const geohashes_3 = ngeohash.bboxes(minLat, minLon, maxLat, maxLon, 3);
+  console.log("geohashes_3", geohashes_3);
+  // Use LokiJS chaining to filter by geo3 and lat first
+  // const data = table.find()
+  // console.log("data", data.length);
+  // console.log("geo3 only", data.filter(doc => geohashes_3.includes(doc.geo3)).length);
+  return table.chain()
+  .find({ geo3: { '$in': geohashes_3 } })
+  .where(obj => obj.lat >= minLat && obj.lat <= maxLat && obj.lon >= minLon && obj.lon <= maxLon)
+  .data();
+}
 
 /**
  * Get geo entries within bounds - optimized version