Skip to content

ci: optimize directory metadata generation #8404

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@
"eslint-plugin-next": "^0.0.0",
"eslint-plugin-prettier": "^5.0.1",
"fs-extra": "^9.0.1",
"git-jiggy": "1.1.1",
"husky": "^8.0.1",
"jest": "^29.7.0",
"jest-cli": "^29.7.0",
Expand Down
67 changes: 66 additions & 1 deletion src/directory/generateDirectory.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,80 @@ import { promises as fs } from 'fs';
import { fileURLToPath } from 'url';
import { cwd } from 'node:process';
import path from 'path';
import { execFile } from 'child_process';
import { promisify } from 'util';
import JSON5 from 'json5';
import { directory } from './directory.mjs';
import { writeFile } from 'fs/promises';
import { getLastModifiedDate } from 'git-jiggy';
import { API_CATEGORIES, API_SUB_CATEGORIES } from '../data/api-categories.mjs';

const execFileAsync = promisify(execFile);

// Set up the root path so that we can get the correct path from the current working directory
const rootPath = path.resolve(cwd(), 'src/pages');

let gitDatesCache;

/**
* Initializes and caches Git last-modified dates for files under `rootPath`.
* Executes:
* git log --follow --name-only --pretty=format:%ct -- .
* The output is a sequence of lines like:
* 1625030400
* src/pages/index.js
* src/pages/about.js
*
* Parses lines of Unix timestamps and file paths, mapping each file to the ISO date
* of its most recent commit. Caches the result to avoid repeated Git invocations.
*/
async function initializeGitCache() {
if (gitDatesCache) return gitDatesCache;

try {
const { stdout } = await execFileAsync(
'git',
['log', '--follow', '--name-only', '--pretty=format:%ct', '--', '.'],
{ cwd: rootPath, maxBuffer: 50 * 1024 * 1024, timeout: 30000 }
);

const cache = new Map();
let currentDate;

for (const line of stdout.trim().split('\n')) {
if (/^\d+$/.test(line)) {
currentDate = new Date(Number(line) * 1000).toISOString();
} else if (line && !cache.has(line)) {
cache.set(line, currentDate);
}
}

gitDatesCache = cache;
} catch (err) {
console.warn(`Git operation failed: ${err.message}`);
gitDatesCache = new Map();
}

return gitDatesCache;
}

/**
* Retrieves the last modified date for a given file by looking it up
* in the cached Git dates map. Initializes the cache on first invocation.
*
* @param {string} filePath - Absolute path to the target file.
* @returns {Promise<string>} ISO 8601 date string of the last Git commit touching that file.
* @throws {Error} If no Git date is found for the given file.
*/
async function getLastModifiedDate(filePath) {
const cache = await initializeGitCache();

if (!cache.has(filePath)) {
throw new Error(`Git date not found for: ${filePath}`);
}

return cache.get(filePath);
}

/**
* Helper function to use RegEx to grab the "meta" object
* @param {string} filePath
Expand Down
81 changes: 1 addition & 80 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3598,13 +3598,6 @@ __metadata:
languageName: node
linkType: hard

"@sindresorhus/is@npm:0.15.0":
version: 0.15.0
resolution: "@sindresorhus/is@npm:0.15.0"
checksum: 10c0/1d7dba430c13dd2e1a977d9191e0137a131dd83e7166fb1b69378e6627b5baef267d09ed11e5753b24c8e852ad5cdbcab9573b1c9a4bdc2992a1b06aa872517e
languageName: node
linkType: hard

"@sinonjs/commons@npm:^3.0.0":
version: 3.0.1
resolution: "@sinonjs/commons@npm:3.0.1"
Expand Down Expand Up @@ -6859,7 +6852,6 @@ __metadata:
eslint-plugin-next: "npm:^0.0.0"
eslint-plugin-prettier: "npm:^5.0.1"
fs-extra: "npm:^9.0.1"
git-jiggy: "npm:1.1.1"
husky: "npm:^8.0.1"
jest: "npm:^29.7.0"
jest-cli: "npm:^29.7.0"
Expand Down Expand Up @@ -7738,21 +7730,6 @@ __metadata:
languageName: node
linkType: hard

"execa@npm:1.0.0":
version: 1.0.0
resolution: "execa@npm:1.0.0"
dependencies:
cross-spawn: "npm:^6.0.0"
get-stream: "npm:^4.0.0"
is-stream: "npm:^1.1.0"
npm-run-path: "npm:^2.0.0"
p-finally: "npm:^1.0.0"
signal-exit: "npm:^3.0.0"
strip-eof: "npm:^1.0.0"
checksum: 10c0/cc71707c9aa4a2552346893ee63198bf70a04b5a1bc4f8a0ef40f1d03c319eae80932c59191f037990d7d102193e83a38ec72115fff814ec2fb3099f3661a590
languageName: node
linkType: hard

"execa@npm:^5.0.0, execa@npm:^5.1.1":
version: 5.1.1
resolution: "execa@npm:5.1.1"
Expand Down Expand Up @@ -8336,15 +8313,6 @@ __metadata:
languageName: node
linkType: hard

"get-stream@npm:^4.0.0":
version: 4.1.0
resolution: "get-stream@npm:4.1.0"
dependencies:
pump: "npm:^3.0.0"
checksum: 10c0/294d876f667694a5ca23f0ca2156de67da950433b6fb53024833733975d32582896dbc7f257842d331809979efccf04d5e0b6b75ad4d45744c45f193fd497539
languageName: node
linkType: hard

"get-stream@npm:^5.1.0":
version: 5.2.0
resolution: "get-stream@npm:5.2.0"
Expand Down Expand Up @@ -8400,16 +8368,6 @@ __metadata:
languageName: node
linkType: hard

"git-jiggy@npm:1.1.1":
version: 1.1.1
resolution: "git-jiggy@npm:1.1.1"
dependencies:
"@sindresorhus/is": "npm:0.15.0"
execa: "npm:1.0.0"
checksum: 10c0/1b5e963555967548bc73c0c9bdfb0d204a7ce926865eca69b69115a28a2b0a1a959b1ea21eb32519ee79c7e71c71ab1edd4dfb70322068882cecbfd2cac9d6f7
languageName: node
linkType: hard

"github-slugger@npm:^2.0.0":
version: 2.0.0
resolution: "github-slugger@npm:2.0.0"
Expand Down Expand Up @@ -9456,13 +9414,6 @@ __metadata:
languageName: node
linkType: hard

"is-stream@npm:^1.1.0":
version: 1.1.0
resolution: "is-stream@npm:1.1.0"
checksum: 10c0/b8ae7971e78d2e8488d15f804229c6eed7ed36a28f8807a1815938771f4adff0e705218b7dab968270433f67103e4fef98062a0beea55d64835f705ee72c7002
languageName: node
linkType: hard

"is-stream@npm:^2.0.0":
version: 2.0.1
resolution: "is-stream@npm:2.0.1"
Expand Down Expand Up @@ -12159,15 +12110,6 @@ __metadata:
languageName: node
linkType: hard

"npm-run-path@npm:^2.0.0":
version: 2.0.2
resolution: "npm-run-path@npm:2.0.2"
dependencies:
path-key: "npm:^2.0.0"
checksum: 10c0/95549a477886f48346568c97b08c4fda9cdbf7ce8a4fbc2213f36896d0d19249e32d68d7451bdcbca8041b5fba04a6b2c4a618beaf19849505c05b700740f1de
languageName: node
linkType: hard

"npm-run-path@npm:^4.0.1":
version: 4.0.1
resolution: "npm-run-path@npm:4.0.1"
Expand Down Expand Up @@ -12408,13 +12350,6 @@ __metadata:
languageName: node
linkType: hard

"p-finally@npm:^1.0.0":
version: 1.0.0
resolution: "p-finally@npm:1.0.0"
checksum: 10c0/6b8552339a71fe7bd424d01d8451eea92d379a711fc62f6b2fe64cad8a472c7259a236c9a22b4733abca0b5666ad503cb497792a0478c5af31ded793d00937e7
languageName: node
linkType: hard

"p-limit@npm:^2.2.0":
version: 2.3.0
resolution: "p-limit@npm:2.3.0"
Expand Down Expand Up @@ -12617,13 +12552,6 @@ __metadata:
languageName: node
linkType: hard

"path-key@npm:^2.0.0":
version: 2.0.1
resolution: "path-key@npm:2.0.1"
checksum: 10c0/dd2044f029a8e58ac31d2bf34c34b93c3095c1481942960e84dd2faa95bbb71b9b762a106aead0646695330936414b31ca0bd862bf488a937ad17c8c5d73b32b
languageName: node
linkType: hard

"path-key@npm:^3.0.0, path-key@npm:^3.1.0":
version: 3.1.1
resolution: "path-key@npm:3.1.1"
Expand Down Expand Up @@ -13848,7 +13776,7 @@ __metadata:
languageName: node
linkType: hard

"signal-exit@npm:^3.0.0, signal-exit@npm:^3.0.2, signal-exit@npm:^3.0.3, signal-exit@npm:^3.0.7":
"signal-exit@npm:^3.0.2, signal-exit@npm:^3.0.3, signal-exit@npm:^3.0.7":
version: 3.0.7
resolution: "signal-exit@npm:3.0.7"
checksum: 10c0/25d272fa73e146048565e08f3309d5b942c1979a6f4a58a8c59d5fa299728e9c2fcd1a759ec870863b1fd38653670240cd420dad2ad9330c71f36608a6a1c912
Expand Down Expand Up @@ -14300,13 +14228,6 @@ __metadata:
languageName: node
linkType: hard

"strip-eof@npm:^1.0.0":
version: 1.0.0
resolution: "strip-eof@npm:1.0.0"
checksum: 10c0/f336beed8622f7c1dd02f2cbd8422da9208fae81daf184f73656332899978919d5c0ca84dc6cfc49ad1fc4dd7badcde5412a063cf4e0d7f8ed95a13a63f68f45
languageName: node
linkType: hard

"strip-final-newline@npm:^2.0.0":
version: 2.0.0
resolution: "strip-final-newline@npm:2.0.0"
Expand Down