From 57f3f3b607abb46b6d1977ed8507a9552acf45e3 Mon Sep 17 00:00:00 2001 From: Ricardo Devis Agullo Date: Tue, 29 Jul 2025 09:18:03 +0200 Subject: [PATCH 1/5] rate limit publish --- package-lock.json | 20 +++-- package.json | 3 +- .../domain/memory-rate-limit-store.ts | 80 +++++++++++++++++++ src/registry/middleware/publish-rate-limit.ts | 67 ++++++++++++++++ src/registry/router.ts | 3 + src/types.ts | 59 ++++++++++++++ 6 files changed, 226 insertions(+), 6 deletions(-) create mode 100644 src/registry/domain/memory-rate-limit-store.ts create mode 100644 src/registry/middleware/publish-rate-limit.ts diff --git a/package-lock.json b/package-lock.json index 5198165a..87977d7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "getport": "^0.1.0", "livereload": "^0.9.3", "lodash.isequal": "^4.5.0", + "lru-cache": "^11.1.0", "morgan": "^1.10.0", "multer": "^1.4.3", "nice-cache": "^0.0.5", @@ -1440,6 +1441,15 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, "node_modules/@babel/helper-compilation-targets/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -6779,12 +6789,12 @@ } }, "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", + "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" + "engines": { + "node": "20 || >=22" } }, "node_modules/markdown": { diff --git a/package.json b/package.json index 32fbd9d9..1a6bb309 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "getport": "^0.1.0", "livereload": "^0.9.3", "lodash.isequal": "^4.5.0", + "lru-cache": "^11.1.0", "morgan": "^1.10.0", "multer": "^1.4.3", "nice-cache": "^0.0.5", @@ -123,4 +124,4 @@ "universalify": "^2.0.0", "yargs": "^17.7.2" } -} \ No newline at end of file +} diff --git a/src/registry/domain/memory-rate-limit-store.ts b/src/registry/domain/memory-rate-limit-store.ts new file mode 100644 index 00000000..d51653a8 --- /dev/null +++ b/src/registry/domain/memory-rate-limit-store.ts @@ -0,0 +1,80 @@ +import { LRUCache } from 'lru-cache'; +import type { RateLimitStore } from '../../types'; + +interface RateLimitEntry { + hits: number; + resetTime: number; +} + +export default class MemoryRateLimitStore implements RateLimitStore { + private store: LRUCache; + + constructor(maxSize: number = 1000) { + this.store = new LRUCache({ + max: maxSize, + // TTL is handled manually in our increment logic + ttl: 0, + // Don't allow stale items + allowStale: false, + // Update age on access to maintain LRU behavior + updateAgeOnGet: true, + // Clean up expired entries when they're accessed + dispose: (_value, _key) => { + // Optional: log when entries are disposed + } + }); + } + + async increment( + key: string, + windowMs: number + ): Promise<{ + totalHits: number; + resetTime: Date; + }> { + const now = Date.now(); + const resetTime = new Date(now + windowMs); + + const existing = this.store.get(key); + + if (!existing || existing.resetTime < now) { + // New entry or expired entry + const entry: RateLimitEntry = { + hits: 1, + resetTime: now + windowMs + }; + this.store.set(key, entry); + return { + totalHits: 1, + resetTime + }; + } + + // Increment existing entry + existing.hits++; + return { + totalHits: existing.hits, + resetTime: new Date(existing.resetTime) + }; + } + + async resetKey(key: string): Promise { + this.store.delete(key); + } + + // Clean up expired entries periodically + private cleanup(): void { + const now = Date.now(); + for (const [key, entry] of this.store.entries()) { + if (entry.resetTime < now) { + this.store.delete(key); + } + } + } + + // Start cleanup interval + init(): void { + // Clean up expired entries every 5 minutes + setInterval(() => this.cleanup(), 5 * 60 * 1000); + } +} diff --git a/src/registry/middleware/publish-rate-limit.ts b/src/registry/middleware/publish-rate-limit.ts new file mode 100644 index 00000000..8b774776 --- /dev/null +++ b/src/registry/middleware/publish-rate-limit.ts @@ -0,0 +1,67 @@ +import type { NextFunction, Request, Response } from 'express'; +import type { Config } from '../../types'; +import MemoryRateLimitStore from '../domain/memory-rate-limit-store'; + +const DEFAULT_WINDOW_MS = 15 * 60 * 1000; // 15 minutes +const DEFAULT_MAX_HITS = 100; +const DEFAULT_MAX_CACHE_SIZE = 1000; // Maximum number of rate limit entries to keep in memory + +function defaultKeyGenerator(req: Request): string { + return `${req.ip}:${req.user ?? 'anon'}`; +} + +export default function createPublishRateLimiter(conf: Config) { + const rateLimitConfig = conf.publishRateLimit ?? {}; + const windowMs = rateLimitConfig.windowMs ?? DEFAULT_WINDOW_MS; + const maxHits = rateLimitConfig.max ?? DEFAULT_MAX_HITS; + const keyGenerator = rateLimitConfig.keyGenerator ?? defaultKeyGenerator; + const skip = rateLimitConfig.skip; + const maxCacheSize = rateLimitConfig.maxCacheSize ?? DEFAULT_MAX_CACHE_SIZE; + + // Use provided store or create memory store with configurable size + const store = rateLimitConfig.store ?? new MemoryRateLimitStore(maxCacheSize); + + // Initialize store if it has an init method + if (store.init) { + store.init(); + } + + return async function publishRateLimiter( + req: Request, + res: Response, + next: NextFunction + ): Promise { + try { + // Skip rate limiting if skip function returns true + if (skip?.(req)) { + return next(); + } + + const key = keyGenerator(req); + const { totalHits, resetTime } = await store.increment(key, windowMs); + + if (totalHits > maxHits) { + // Calculate seconds until reset + const retryAfter = Math.ceil((resetTime.getTime() - Date.now()) / 1000); + + res.set('Retry-After', retryAfter.toString()); + + res.status(429).json({ + error: 'rate_limit_exceeded', + message: 'Too many publish requests', + resetTime: resetTime.toISOString(), + retryAfter + }); + + // Set Retry-After header + return; + } + + next(); + } catch (error) { + // If rate limiting fails, log error but allow request to proceed + console.error('Rate limiting error:', error); + next(); + } + }; +} diff --git a/src/registry/router.ts b/src/registry/router.ts index 24517038..734cfe40 100644 --- a/src/registry/router.ts +++ b/src/registry/router.ts @@ -2,6 +2,7 @@ import type { Express } from 'express'; import type { Repository } from '../registry/domain/repository'; import settings from '../resources/settings'; import type { Config } from '../types'; +import createPublishRateLimiter from './middleware/publish-rate-limit'; import IndexRoute from './routes'; import ComponentRoute from './routes/component'; import ComponentInfoRoute from './routes/component-info'; @@ -28,6 +29,7 @@ export function create(app: Express, conf: Config, repository: Repository) { }; const prefix = conf.prefix; + const publishRateLimiter = createPublishRateLimiter(conf); if (prefix !== '/') { app.get('/', (_req, res) => res.redirect(prefix)); @@ -50,6 +52,7 @@ export function create(app: Express, conf: Config, repository: Repository) { } else { app.put( `${prefix}:componentName/:componentVersion`, + publishRateLimiter, conf.beforePublish, routes.publish ); diff --git a/src/types.ts b/src/types.ts index 8fdde264..27f6833c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -358,6 +358,10 @@ export interface Config { * @default 0 */ verbosity: number; + /** + * Rate limiting configuration for component publishing. + */ + publishRateLimit?: PublishRateLimit; } type CompiledTemplate = (model: unknown) => string; @@ -402,6 +406,61 @@ export interface Plugin { }; } +export interface RateLimitStore { + /** Called once on registry start-up (optional) */ + init?: () => Promise | void; + + /** + * Atomically increase the counter for the key. + * Returns the current hit count and the absolute reset time. + */ + increment: ( + key: string, + windowMs: number + ) => Promise<{ + totalHits: number; + resetTime: Date; + }>; + + /** Optionally reset a key before its natural expiry */ + resetKey?: (key: string) => Promise; +} + +export interface PublishRateLimit { + /** + * Size of the sliding window in **ms** (default 15 min) + * + * @default 15 * 60 * 1000 + */ + windowMs?: number; + /** + * Maximum hits allowed within `windowMs` (default 100) + * + * @default 100 + */ + max?: number; + /** + * Custom key generator. + * + * Defaults to: `${req.ip}:${req.user ?? 'anon'}` + */ + keyGenerator?: (req: Request) => string; + /** + * Skip throttling for specific requests/users + */ + skip?: (req: Request) => boolean; + /** + * Custom storage backend. Defaults to in-memory Map. + */ + store?: RateLimitStore; + /** + * Maximum number of rate limit entries to keep in memory (default 1000) + * + * @default 1000 + */ + maxCacheSize?: number; +} + declare global { namespace Express { interface Request { From a654dcd7b05aae0b70231cc6e64e8533ea88d37a Mon Sep 17 00:00:00 2001 From: Ricardo Devis Agullo Date: Tue, 29 Jul 2025 09:50:14 +0200 Subject: [PATCH 2/5] remove resetKey method --- src/registry/domain/memory-rate-limit-store.ts | 4 ---- src/types.ts | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/registry/domain/memory-rate-limit-store.ts b/src/registry/domain/memory-rate-limit-store.ts index d51653a8..48a5e5cc 100644 --- a/src/registry/domain/memory-rate-limit-store.ts +++ b/src/registry/domain/memory-rate-limit-store.ts @@ -58,10 +58,6 @@ export default class MemoryRateLimitStore implements RateLimitStore { }; } - async resetKey(key: string): Promise { - this.store.delete(key); - } - // Clean up expired entries periodically private cleanup(): void { const now = Date.now(); diff --git a/src/types.ts b/src/types.ts index 27f6833c..9958fd20 100644 --- a/src/types.ts +++ b/src/types.ts @@ -421,9 +421,6 @@ export interface RateLimitStore { totalHits: number; resetTime: Date; }>; - - /** Optionally reset a key before its natural expiry */ - resetKey?: (key: string) => Promise; } export interface PublishRateLimit { From 3caf5d32e28823e7a12abfdc6156fe591086e3f7 Mon Sep 17 00:00:00 2001 From: Ricardo Devis Agullo Date: Tue, 29 Jul 2025 10:37:43 +0200 Subject: [PATCH 3/5] fix tests --- src/registry/domain/memory-rate-limit-store.ts | 16 ---------------- src/registry/middleware/publish-rate-limit.ts | 5 +++-- src/registry/router.ts | 3 ++- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/src/registry/domain/memory-rate-limit-store.ts b/src/registry/domain/memory-rate-limit-store.ts index 48a5e5cc..e4892a9b 100644 --- a/src/registry/domain/memory-rate-limit-store.ts +++ b/src/registry/domain/memory-rate-limit-store.ts @@ -57,20 +57,4 @@ export default class MemoryRateLimitStore implements RateLimitStore { resetTime: new Date(existing.resetTime) }; } - - // Clean up expired entries periodically - private cleanup(): void { - const now = Date.now(); - for (const [key, entry] of this.store.entries()) { - if (entry.resetTime < now) { - this.store.delete(key); - } - } - } - - // Start cleanup interval - init(): void { - // Clean up expired entries every 5 minutes - setInterval(() => this.cleanup(), 5 * 60 * 1000); - } } diff --git a/src/registry/middleware/publish-rate-limit.ts b/src/registry/middleware/publish-rate-limit.ts index 8b774776..ecaf6905 100644 --- a/src/registry/middleware/publish-rate-limit.ts +++ b/src/registry/middleware/publish-rate-limit.ts @@ -1,5 +1,5 @@ import type { NextFunction, Request, Response } from 'express'; -import type { Config } from '../../types'; +import type { Config, RateLimitStore } from '../../types'; import MemoryRateLimitStore from '../domain/memory-rate-limit-store'; const DEFAULT_WINDOW_MS = 15 * 60 * 1000; // 15 minutes @@ -19,7 +19,8 @@ export default function createPublishRateLimiter(conf: Config) { const maxCacheSize = rateLimitConfig.maxCacheSize ?? DEFAULT_MAX_CACHE_SIZE; // Use provided store or create memory store with configurable size - const store = rateLimitConfig.store ?? new MemoryRateLimitStore(maxCacheSize); + const store: RateLimitStore = + rateLimitConfig.store ?? new MemoryRateLimitStore(maxCacheSize); // Initialize store if it has an init method if (store.init) { diff --git a/src/registry/router.ts b/src/registry/router.ts index 734cfe40..3adff8b4 100644 --- a/src/registry/router.ts +++ b/src/registry/router.ts @@ -30,6 +30,7 @@ export function create(app: Express, conf: Config, repository: Repository) { const prefix = conf.prefix; const publishRateLimiter = createPublishRateLimiter(conf); + console.log('publishRateLimiter', publishRateLimiter); if (prefix !== '/') { app.get('/', (_req, res) => res.redirect(prefix)); @@ -52,7 +53,7 @@ export function create(app: Express, conf: Config, repository: Repository) { } else { app.put( `${prefix}:componentName/:componentVersion`, - publishRateLimiter, + // publishRateLimiter, conf.beforePublish, routes.publish ); From 7b109edbec5ea11e85679b9d5964178ffec2d074 Mon Sep 17 00:00:00 2001 From: Ricardo Devis Agullo Date: Tue, 29 Jul 2025 14:00:45 +0200 Subject: [PATCH 4/5] Update router.ts --- src/registry/router.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/registry/router.ts b/src/registry/router.ts index 3adff8b4..734cfe40 100644 --- a/src/registry/router.ts +++ b/src/registry/router.ts @@ -30,7 +30,6 @@ export function create(app: Express, conf: Config, repository: Repository) { const prefix = conf.prefix; const publishRateLimiter = createPublishRateLimiter(conf); - console.log('publishRateLimiter', publishRateLimiter); if (prefix !== '/') { app.get('/', (_req, res) => res.redirect(prefix)); @@ -53,7 +52,7 @@ export function create(app: Express, conf: Config, repository: Repository) { } else { app.put( `${prefix}:componentName/:componentVersion`, - // publishRateLimiter, + publishRateLimiter, conf.beforePublish, routes.publish ); From e6664308dc59e43331d5322d2c79f87e8ce37147 Mon Sep 17 00:00:00 2001 From: Ricardo Devis Agullo Date: Tue, 29 Jul 2025 19:18:31 +0200 Subject: [PATCH 5/5] export ratelimitstore type --- src/index.ts | 1 + src/types.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/index.ts b/src/index.ts index fd40d3c2..a237c819 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ export { default as Client } from 'oc-client'; export { default as cli } from './cli/programmatic-api'; export { default as Registry, RegistryOptions } from './registry'; +export { RateLimitStore } from './types'; diff --git a/src/types.ts b/src/types.ts index 9958fd20..d1a13caf 100644 --- a/src/types.ts +++ b/src/types.ts @@ -406,6 +406,11 @@ export interface Plugin { }; } +interface IncrementResult { + totalHits: number; + resetTime: Date; +} + export interface RateLimitStore { /** Called once on registry start-up (optional) */ init?: () => Promise | void; @@ -417,10 +422,7 @@ export interface RateLimitStore { increment: ( key: string, windowMs: number - ) => Promise<{ - totalHits: number; - resetTime: Date; - }>; + ) => Promise | IncrementResult; } export interface PublishRateLimit {