fix: route on dirt trails, mountain bike paths, and park service roads

The OSM fetch and routing graph only allowed standard road highway
types, so trails never entered the graph or the Strava ridden-road
matching. Added a shared highway-type filter (lib/highwayFilter.ts):
path/track/bridleway always routable, footway routable unless tagged
as a sidewalk/crossing or closed to bikes, service routable unless
it's a driveway/alley/parking_aisle/emergency_access.

Also fixes two Overpass reliability bugs surfaced while debugging:
a 509 rate-limit response was misclassified as a hard failure (5min
circuit-open) instead of transient (1min), and the ridden-roads
background recompute had no pacing between tile fetches, letting it
trip the self-hosted instance's rate limit and starve interactive
routing requests.

Author: chadbeaudin

app/api/ridden-roads/route.ts

@@ -6,7 +6,8 @@ import { roadsFromOSM } from '@/lib/roadsFromOSM';
 import { prisma } from '@/lib/prisma';
 
 const ts = () => `[${new Date().toTimeString().slice(0, 8)}]`;
-const RIDDEN_VERSION = 1;
+const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+const RIDDEN_VERSION = 3; // bumped: highway=service (park maintenance/multi-use paths) now included in the road fetch
 const FRESH_TTL_MS = 24 * 60 * 60 * 1000;
 const TILE = 0.02; // ~2.2km tiles to gather OSM roads over the riding footprint
 // Guard against a runaway precompute. Self-hosted Overpass (OVERPASS_URL) has
@@ -39,6 +40,11 @@ async function compute(riddenRoads: [number, number][][]): Promise<[number, numb
             console.warn(`${ts()} RiddenRoads: tile ${t} failed: ${e.message}`);
         }
         if (++i % 25 === 0) console.log(`${ts()} RiddenRoads: fetched ${i}/${tiles.size} tiles (${failed} failed)`);
+        // Small pacing delay — a full recompute (e.g. after a cache-version bump) can hit
+        // hundreds/thousands of tiles back-to-back and trip the Overpass instance's own
+        // rate limit (509), which cascades into blocking interactive routing for everyone
+        // via the shared circuit breaker. This keeps the background job well under that.
+        await delay(75);
     }
 
     // Don't cache a badly incomplete overlay (e.g. Overpass down) — throw so the

lib/graph.ts

@@ -4,6 +4,7 @@ import createGraph, { Graph } from 'ngraph.graph';
 import path from 'ngraph.path';
 
 import { OSMWay, OSMNode, OverpassResponse } from './types';
+import { isRoutableHighway } from './highwayFilter';
 
 interface NodeData {
     lat: number;
@@ -223,12 +224,8 @@ export class StreetGraph {
 
                 const highway = way.tags?.highway;
 
-                // Reject non-routable highway types regardless of data source (cache may contain
-                // footways/paths from an earlier fetch before the OSM API filter was added).
-                const ROUTABLE_HIGHWAYS = new Set(['motorway','trunk','primary','secondary','tertiary',
-                    'unclassified','residential','living_street','motorway_link','trunk_link',
-                    'primary_link','secondary_link','tertiary_link','track','cycleway']);
-                if (!highway || !ROUTABLE_HIGHWAYS.has(highway)) continue;
+                // Reject non-routable highway types regardless of data source.
+                if (!isRoutableHighway(highway, way.tags)) continue;
 
                 // SAFETY: Exclude actual motorway lanes — cyclists cannot ride on them.
                 // trunk is kept (with isAvoided=true) so divided-highway crossings remain connected.
@@ -247,7 +244,7 @@ export class StreetGraph {
                 if (options?.avoidHighways && majorHighways.includes(highway || '')) {
                     isAvoided = true;
                 }
-                if (!isAvoided && options?.avoidTrails && ['path', 'track', 'footway', 'cycleway'].includes(highway || '')) {
+                if (!isAvoided && options?.avoidTrails && ['path', 'track', 'footway', 'cycleway', 'bridleway'].includes(highway || '')) {
                     isAvoided = true;
                 }
                 if (!isAvoided && options?.avoidGravel) {

lib/highwayFilter.test.ts

@@ -0,0 +1,48 @@
+import { isRoutableHighway } from './highwayFilter';
+
+describe('isRoutableHighway', () => {
+    it('allows standard road types', () => {
+        expect(isRoutableHighway('residential')).toBe(true);
+        expect(isRoutableHighway('primary')).toBe(true);
+        expect(isRoutableHighway('cycleway')).toBe(true);
+        expect(isRoutableHighway('track')).toBe(true);
+    });
+
+    it('allows dirt trails, singletrack, and bridleways unconditionally', () => {
+        expect(isRoutableHighway('path')).toBe(true);
+        expect(isRoutableHighway('bridleway')).toBe(true);
+        expect(isRoutableHighway('path', { surface: 'dirt' })).toBe(true);
+        expect(isRoutableHighway('path', { mtb_scale: '2' })).toBe(true);
+    });
+
+    it('allows plain footway (standalone park/trail paths) but excludes sidewalks and crossings', () => {
+        expect(isRoutableHighway('footway')).toBe(true);
+        expect(isRoutableHighway('footway', {})).toBe(true);
+        expect(isRoutableHighway('footway', { surface: 'dirt' })).toBe(true);
+        expect(isRoutableHighway('footway', { footway: 'sidewalk' })).toBe(false);
+        expect(isRoutableHighway('footway', { footway: 'crossing' })).toBe(false);
+    });
+
+    it('excludes footway explicitly closed to bikes', () => {
+        expect(isRoutableHighway('footway', { bicycle: 'no' })).toBe(false);
+        expect(isRoutableHighway('footway', { bicycle: 'private' })).toBe(false);
+        expect(isRoutableHighway('footway', { bicycle: 'yes' })).toBe(true);
+        expect(isRoutableHighway('footway', { bicycle: 'designated' })).toBe(true);
+    });
+
+    it('allows plain service ways (park maintenance/multi-use paths) but excludes driveways, alleys, and parking aisles', () => {
+        expect(isRoutableHighway('service')).toBe(true);
+        expect(isRoutableHighway('service', {})).toBe(true);
+        expect(isRoutableHighway('service', { surface: 'compacted' })).toBe(true);
+        expect(isRoutableHighway('service', { service: 'alley' })).toBe(false);
+        expect(isRoutableHighway('service', { service: 'driveway' })).toBe(false);
+        expect(isRoutableHighway('service', { service: 'parking_aisle' })).toBe(false);
+        expect(isRoutableHighway('service', { service: 'emergency_access' })).toBe(false);
+    });
+
+    it('rejects unknown or missing highway types', () => {
+        expect(isRoutableHighway(undefined)).toBe(false);
+        expect(isRoutableHighway('steps')).toBe(false);
+        expect(isRoutableHighway('pedestrian')).toBe(false);
+    });
+});

lib/highwayFilter.ts

@@ -0,0 +1,37 @@
+// Highway types considered part of the routable street network. `path`,
+// `track`, and `bridleway` are always included — OSM convention doesn't use
+// them for sidewalks, so this is how dirt trails, singletrack, forest roads,
+// and rail-trails get in.
+const BASE_ROUTABLE_HIGHWAYS = new Set([
+    'motorway', 'trunk', 'primary', 'secondary', 'tertiary',
+    'unclassified', 'residential', 'living_street', 'motorway_link', 'trunk_link',
+    'primary_link', 'secondary_link', 'tertiary_link', 'track', 'cycleway', 'path', 'bridleway',
+]);
+
+// `footway` is ambiguous: it covers both standalone park/trail paths (almost
+// never tagged with `bicycle`) and sidewalks running alongside a road. The
+// `footway=sidewalk`/`crossing` subtag is the reliable signal for "this is a
+// sidewalk" — plain `footway` with no subtag is treated as a trail.
+const SIDEWALK_SUBTYPES = new Set(['sidewalk', 'crossing']);
+const BIKE_FORBIDDEN_TAGS = new Set(['no', 'private']);
+
+// `service` is similarly ambiguous: park maintenance/multi-use paths are
+// commonly tagged plain `highway=service` (no subtype) instead of `path` —
+// this is how paved/compacted paths inside parks (e.g. High Bridge Park's
+// interior network) get in. The `service` subtag is the reliable signal for
+// "this is functional property access, not a path" — driveways, alleys, and
+// parking aisles stay excluded.
+const SERVICE_EXCLUDED_SUBTYPES = new Set(['alley', 'driveway', 'parking_aisle', 'emergency_access']);
+
+export function isRoutableHighway(highway: string | undefined, tags?: Record<string, string>): boolean {
+    if (!highway) return false;
+    if (highway === 'footway') {
+        if (SIDEWALK_SUBTYPES.has(tags?.footway ?? '')) return false;
+        if (BIKE_FORBIDDEN_TAGS.has(tags?.bicycle ?? '')) return false;
+        return true;
+    }
+    if (highway === 'service') {
+        return !SERVICE_EXCLUDED_SUBTYPES.has(tags?.service ?? '');
+    }
+    return BASE_ROUTABLE_HIGHWAYS.has(highway);
+}

lib/osmDbCache.test.ts

@@ -60,7 +60,7 @@ describe('OSM DB cache', () => {
 
     it('returns DB-cached data without hitting the network', async () => {
         mockFindUnique.mockResolvedValue({
-            key: 'v4_47.6000,-117.5000,47.7000,-117.3000',
+            key: 'v6_47.6000,-117.5000,47.7000,-117.3000',
             data: mockOsmData,
             fetchedAt: new Date()
         });
@@ -87,7 +87,7 @@ describe('OSM DB cache', () => {
     it('falls through to network when DB cache is expired', async () => {
         const thirtyOneDaysAgo = new Date(Date.now() - 31 * 24 * 60 * 60 * 1000);
         mockFindUnique.mockResolvedValue({
-            key: 'v4_47.6000,-117.5000,47.7000,-117.3000',
+            key: 'v6_47.6000,-117.5000,47.7000,-117.3000',
             data: mockOsmData,
             fetchedAt: thirtyOneDaysAgo
         });
@@ -114,14 +114,14 @@ describe('OSM DB cache', () => {
         // Allow fire-and-forget upsert to resolve
         await new Promise(resolve => setTimeout(resolve, 10));
         expect(mockUpsert).toHaveBeenCalledWith(expect.objectContaining({
-            where: { key: expect.stringContaining('v4_') },
+            where: { key: expect.stringContaining('v6_') },
             create: expect.objectContaining({ data: mockOsmData })
         }));
     });
 
     it('warms memory cache from DB so second call skips both DB and network', async () => {
         mockFindUnique.mockResolvedValueOnce({
-            key: 'v4_47.6000,-117.5000,47.7000,-117.3000',
+            key: 'v6_47.6000,-117.5000,47.7000,-117.3000',
             data: mockOsmData,
             fetchedAt: new Date()
         });

lib/overpass.ts

@@ -1,4 +1,5 @@
 import { BoundingBox, OverpassResponse, OSMElement } from './types';
+import { isRoutableHighway } from './highwayFilter';
 import { prisma } from './prisma';
 import { readDiskCache, writeDiskCache } from './osmDiskCache';
 
@@ -81,11 +82,7 @@ function parseOSMXML(xml: string): OverpassResponse {
     }
   }
 
-  // Parse ways — must match the highway types in the Overpass query
-  const ALLOWED_HIGHWAYS = new Set(['motorway','trunk','primary','secondary','tertiary',
-    'unclassified','residential','living_street','motorway_link','trunk_link',
-    'primary_link','secondary_link','tertiary_link','track','cycleway']);
-
+  // Parse ways — must match the highway types allowed by the Overpass query
   const wayRe = /<way\b([^>]*)>([\s\S]*?)<\/way>/g;
   while ((m = wayRe.exec(xml)) !== null) {
     const attrs = m[1];
@@ -98,7 +95,7 @@ function parseOSMXML(xml: string): OverpassResponse {
     let t: RegExpExecArray | null;
     while ((t = tagRe.exec(body)) !== null) tags[t[1]] = t[2];
 
-    if (!tags['highway'] || !ALLOWED_HIGHWAYS.has(tags['highway'])) continue;
+    if (!isRoutableHighway(tags['highway'], tags)) continue;
     if (tags['access'] === 'private' || tags['access'] === 'no') continue;
 
     const nodes: number[] = [];
@@ -347,7 +344,7 @@ export async function fetchOSMData(requestedBbox: BoundingBox): Promise<Overpass
   // Tile coords are already multiples of TILE_DEG; format to fixed precision for a stable key.
   const k = (n: number) => n.toFixed(4);
   // v4: fine-grained tile-aligned cache keys (0.005° tiles, ~500m). Old v3 entries ignored.
-  const cacheKey = `v4_${k(bbox.south)},${k(bbox.west)},${k(bbox.north)},${k(bbox.east)}`;
+  const cacheKey = `v6_${k(bbox.south)},${k(bbox.west)},${k(bbox.north)},${k(bbox.east)}`;
   const now = Date.now();
 
   // For medium/large areas, reject cached responses that are clearly incomplete.
@@ -410,9 +407,17 @@ export async function fetchOSMData(requestedBbox: BoundingBox): Promise<Overpass
   const requestPromise = (async () => {
     const bikeQuery = `
       [out:json][timeout:90];
-      way["highway"~"motorway|trunk|primary|secondary|tertiary|unclassified|residential|living_street|motorway_link|trunk_link|primary_link|secondary_link|tertiary_link|track|cycleway"]
-         ["access"!~"private|no"]
-         (${bbox.south},${bbox.west},${bbox.north},${bbox.east});
+      (
+        way["highway"~"motorway|trunk|primary|secondary|tertiary|unclassified|residential|living_street|motorway_link|trunk_link|primary_link|secondary_link|tertiary_link|track|cycleway|path|bridleway"]
+           ["access"!~"private|no"]
+           (${bbox.south},${bbox.west},${bbox.north},${bbox.east});
+        way["highway"="footway"]["footway"!~"sidewalk|crossing"]["bicycle"!~"no|private"]
+           ["access"!~"private|no"]
+           (${bbox.south},${bbox.west},${bbox.north},${bbox.east});
+        way["highway"="service"]["service"!~"alley|driveway|parking_aisle|emergency_access"]
+           ["access"!~"private|no"]
+           (${bbox.south},${bbox.west},${bbox.north},${bbox.east});
+      );
       out geom;
     `;
 
@@ -472,7 +477,7 @@ export async function fetchOSMData(requestedBbox: BoundingBox): Promise<Overpass
               return data;
             }
 
-            if (response.status === 504 || response.status === 429) {
+            if (response.status === 504 || response.status === 429 || response.status === 509) {
               console.warn(`${ts()} Endpoint ${endpoint} failed with ${response.status}. Trying next...`);
               lastError = new Error(`Overpass API error: ${response.status}`);
               recordFailure(endpoint, true); // transient — short circuit TTL

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "street-sweep",
-  "version": "0.6.45",
+  "version": "0.6.46",
   "private": true,
   "scripts": {
     "dev": "next dev -p 3888",