Self hosted maptile generation with tippecanoe (#1263)

Author: jboolean

backend/serverless.yml

@@ -4,6 +4,7 @@ package:
   patterns:
     - 'certs/**'
     - '!chromium/**'
+    - '!tippecanoe-layer.zip'
 
 provider:
   name: aws
@@ -42,6 +43,13 @@ plugins:
   - serverless-domain-manager
   - serverless-offline
 
+layers:
+  tippecanoe:
+    package:
+      artifact: tippecanoe-layer.zip
+    name: tippecanoe-layer
+    description: felt/tippecanoe
+
 functions:
   app:
     handler: appHandler.handler
@@ -93,6 +101,16 @@ functions:
     events:
       - schedule: cron(0 12 * * ? *)
     timeout: 900
+  syncMapSelfHosted:
+    stages:
+      - production
+      - staging
+    handler: syncMapSelfHosted.handler
+    events:
+      - schedule: cron(10 12 * * ? *)
+    timeout: 900
+    layers:
+      - Ref: TippecanoeLambdaLayer
   generateStoryTitles:
     stages:
       - production

backend/src/business/utils/TippecanoeTypes.ts

@@ -0,0 +1,35 @@
+/**
+ * Tippecanoe options based on the official documentation
+ * @see https://github.com/felt/tippecanoe
+ */
+export interface TippecanoeOptions {
+  /** Minimum zoom level (default: 0) */
+  minZoom?: number;
+  /** Maximum zoom level (default: 14) */
+  maxZoom?: number;
+  /** Base zoom level (default: 14) */
+  baseZoom?: number;
+  /** Drop rate (default: 2) */
+  dropRate?: number;
+  /** Buffer size in pixels (default: 5) */
+  buffer?: number;
+  /** Simplification tolerance in pixels (default: 0) */
+  tolerance?: number;
+  /** Layer name for the output tiles */
+  layer?: string;
+  /** Read input as newline-delimited GeoJSON */
+  newlineDelimited?: boolean;
+  /** Drop densest features as needed when tiles become too large */
+  dropDensestAsNeeded?: boolean;
+  /** Extend zoom levels if features are still being dropped */
+  extendZoomsIfStillDropping?: boolean;
+  /** Additional tippecanoe arguments as key-value pairs */
+  additionalArgs?: Record<string, string | number | boolean>;
+}
+
+export interface TippecanoeResult {
+  /** Path to the generated PMTiles file */
+  outputPath: string;
+  /** Symbol for automatic disposal */
+  [Symbol.dispose](): void;
+}

backend/src/business/utils/tippecanoe.ts

@@ -0,0 +1,151 @@
+import { spawn } from 'child_process';
+import { randomUUID } from 'crypto';
+import { unlink } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { TippecanoeOptions, TippecanoeResult } from './TippecanoeTypes';
+
+// Map of option names to their CLI argument names
+const OPTION_MAP: Record<string, string> = {
+  minZoom: '--minimum-zoom',
+  maxZoom: '--maximum-zoom',
+  baseZoom: '--base-zoom',
+  dropRate: '--drop-rate',
+  buffer: '--buffer',
+  tolerance: '--tolerance',
+  layer: '--layer',
+  dropDensestAsNeeded: '--drop-densest-as-needed',
+  extendZoomsIfStillDropping: '--extend-zooms-if-still-dropping',
+};
+
+/**
+ * Process a GeoJSON stream into a PMTiles file using tippecanoe CLI
+ *
+ * Usage examples:
+ *
+ * 1. Basic usage with automatic cleanup (recommended):
+ * ```typescript
+ * await using result = await tippecanoe(geojsonStream);
+ * // Use result.outputPath to access the PMTiles file
+ * // File is automatically cleaned up when result goes out of scope
+ * ```
+ *
+ * 2. Process with options:
+ * ```typescript
+ * await using result = await tippecanoe(geojsonStream, {
+ *   minZoom: 0,
+ *   maxZoom: 16
+ * });
+ * ```
+ *
+ * 3. Process newline-delimited GeoJSON:
+ * ```typescript
+ * await using result = await tippecanoe(geojsonStream, {
+ *   newlineDelimited: true,
+ *   minZoom: 5,
+ *   maxZoom: 14
+ * });
+ * ```
+ *
+ * 4. Error handling with automatic cleanup:
+ * ```typescript
+ * try {
+ *   await using result = await tippecanoe(geojsonStream);
+ *   console.log('PMTiles generation completed successfully');
+ *   // Cleanup happens automatically, even if an error is thrown
+ * } catch (error) {
+ *   console.error('Tippecanoe process error:', error);
+ * }
+ * ```
+ *
+ * @param inputStream - NodeJS.ReadableStream containing GeoJSON data
+ * @param options - Tippecanoe configuration options
+ * @returns Promise that resolves to result object with output path and automatic cleanup
+ */
+export default async function tippecanoe(
+  inputStream: NodeJS.ReadableStream,
+  options: TippecanoeOptions = {}
+): Promise<TippecanoeResult> {
+  const { newlineDelimited, additionalArgs = {} } = options;
+
+  // Create temporary output file
+  const outputPath = join(tmpdir(), `tippecanoe-${randomUUID()}.pmtiles`);
+
+  // Build tippecanoe command arguments - only add if explicitly provided
+  const args: string[] = ['--output', outputPath];
+
+  // Add numeric options
+  Object.entries(OPTION_MAP).forEach(([optionName, cliArg]) => {
+    const value = options[optionName as keyof TippecanoeOptions];
+    if (value !== undefined) {
+      if (typeof value === 'boolean') {
+        if (value) {
+          args.push(cliArg);
+        }
+      } else {
+        args.push(cliArg, value.toString());
+      }
+    }
+  });
+
+  if (newlineDelimited) {
+    args.push('--newline-delimited');
+  }
+
+  // Add additional arguments with validation
+  for (const [key, value] of Object.entries(additionalArgs)) {
+    if (value === undefined || value === null) {
+      continue; // Skip undefined/null values
+    }
+
+    if (typeof value === 'boolean') {
+      if (value) {
+        args.push(`--${key}`);
+      }
+    } else if (typeof value === 'string' || typeof value === 'number') {
+      args.push(`--${key}`, value.toString());
+    }
+  }
+
+  // Spawn tippecanoe process
+  const tippecanoeProcess = spawn('tippecanoe', args, {
+    stdio: ['pipe', 'pipe', 'pipe'],
+  });
+
+  // Set up progress monitoring (stderr is where tippecanoe writes progress)
+  tippecanoeProcess.stderr?.on('data', (data: Buffer) => {
+    // Forward tippecanoe's stderr to the parent process stderr
+    process.stderr.write(data);
+  });
+
+  // Pipe input stream to tippecanoe stdin
+  inputStream.pipe(tippecanoeProcess.stdin);
+
+  // Handle process completion
+  const processPromise = new Promise<void>((resolve, reject) => {
+    tippecanoeProcess.on('error', (error) => {
+      reject(new Error(`Tippecanoe process error: ${error.message}`));
+    });
+
+    tippecanoeProcess.on('exit', (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`Tippecanoe exited with code ${code ?? 'unknown'}`));
+      }
+    });
+  });
+
+  // Wait for process to complete
+  await processPromise;
+
+  return {
+    outputPath,
+    [Symbol.dispose]() {
+      // Automatic cleanup when using 'await using'
+      unlink(outputPath).catch((error) => {
+        console.warn(`Failed to cleanup temporary file ${outputPath}:`, error);
+      });
+    },
+  };
+}

backend/src/cron/syncMapSelfHosted.ts

@@ -0,0 +1,50 @@
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { createReadStream } from 'fs';
+import { getRepository } from 'typeorm';
+import GeojsonEncoder from '../business/geodata/GeojsonEncoder';
+import tippecanoe from '../business/utils/tippecanoe';
+import EffectiveAddress from '../entities/EffectiveAddress';
+import EffectiveGeocode from '../entities/EffectiveGeocode';
+
+const s3 = new S3Client();
+
+export default async function syncMap(): Promise<void> {
+  console.log('Refreshing effective addresses...');
+  await getRepository(EffectiveAddress).query(
+    'REFRESH MATERIALIZED VIEW effective_addresses_view WITH DATA'
+  );
+
+  console.log('Refreshing effective geocodes...');
+  await getRepository(EffectiveGeocode).query(
+    'REFRESH MATERIALIZED VIEW effective_geocodes_view WITH DATA'
+  );
+
+  const encoder = new GeojsonEncoder('newline-delimited-geojson');
+
+  console.log('Beginning sync of map data...');
+
+  const dataStream = await encoder.createGeojson('1940');
+
+  console.log('Generating PMTiles file...');
+  // eslint-disable-next-line prettier/prettier
+  await using result = await tippecanoe(dataStream, {
+    layer: 'photos-1940s',
+    dropDensestAsNeeded: true,
+    extendZoomsIfStillDropping: true,
+  });
+
+  console.log('Uploading PMTiles to S3...');
+  const pmtilesStream = createReadStream(result.outputPath);
+  
+  await s3.send(
+    new PutObjectCommand({
+      Bucket: 'fourties-maps',
+      Key: 'photos-1940s.pmtiles',
+      Body: pmtilesStream,
+      ContentType: 'application/vnd.pmtiles',
+      CacheControl: 'max-age=86400',
+    })
+  );
+
+  console.log('Sync of map data complete.');
+}

backend/syncMapSelfHosted.ts

@@ -0,0 +1,7 @@
+import syncMapImpl from './src/cron/syncMapSelfHosted';
+
+// Importing @sentry/tracing patches the global hub for tracing to work.
+import '@sentry/tracing';
+import withSetup from './withSetup';
+
+export const handler = withSetup(syncMapImpl);

backend/tippecanoe-layer.zip

@@ -15,6 +15,7 @@ module.exports = {
     checkStaleStories: './checkStaleStories.ts',
     checkMerchQueue: './checkMerchQueue.ts',
     syncMap: './syncMap.ts',
+    syncMapSelfHosted: './syncMapSelfHosted.ts',
     generateStoryTitles: './generateStoryTitles.ts',
     sendEmailCampaigns: './sendEmailCampaigns.ts',
     renderMerchPrintfiles: './renderMerchPrintfiles.ts',

backend/webpack.config.js

@@ -1,4 +1,5 @@
 import 'reflect-metadata';
+
 import 'source-map-support/register';
 import createConnection from './src/createConnection';