Skip to content

Commit de2d9c7

Browse files
JakeSCahillJakeSCahill
authored
fix: Add ACTIONS_BOT_TOKEN and VBOT_GITHUB_API_TOKEN to token check (#167)
* fix: Add ACTIONS_BOT_TOKEN and VBOT_GITHUB_API_TOKEN to token check The getGitHubToken() helper was missing ACTIONS_BOT_TOKEN (used in CI) and VBOT_GITHUB_API_TOKEN (used in legacy code), causing authentication to fail and hitting unauthenticated rate limits (60 req/hr instead of 5000 req/hr). This was causing 403/429 errors when checking cloud-docs for 471 connectors. Updated token priority order: 1. REDPANDA_GITHUB_TOKEN 2. ACTIONS_BOT_TOKEN (CI) 3. GITHUB_TOKEN (GitHub Actions default) 4. VBOT_GITHUB_API_TOKEN (legacy) 5. GH_TOKEN (GitHub CLI) * chore: Bump version to 4.13.6 * perf: Optimize cloud-docs checking from 471 API calls to 1 Use GitHub Tree API to fetch entire directory structure in a single API call instead of checking each connector individually. This reduces API calls from 471 to 1, making the check ~108x faster (from 30-60s to ~280ms). Changes: - Fetch recursive tree of modules/develop/pages/connect/components/ - Build in-memory set of existing files for O(1) lookup - Check connectors against this set (no additional API calls) - Graceful fallback to individual checks if tree API fails Performance: - OLD: 471 individual API calls (~30-60 seconds) - NEW: 1 tree API call (~280ms) - Speedup: 108x faster * refactor: Remove emojis from cloud-docs checking output Replace emoji characters with standard text labels (INFO, WARNING) for better compatibility and cleaner log output.
1 parent 8e286a7 commit de2d9c7

File tree

4 files changed

+69
-65
lines changed

4 files changed

+69
-65
lines changed

‎cli-utils/github-token.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,18 @@
99
* Get GitHub token from environment variables
1010
* Checks multiple common variable names in priority order:
1111
* 1. REDPANDA_GITHUB_TOKEN - Custom Redpanda token
12-
* 2. GITHUB_TOKEN - GitHub Actions default
13-
* 3. GH_TOKEN - GitHub CLI default
12+
* 2. ACTIONS_BOT_TOKEN - GitHub Actions bot token
13+
* 3. GITHUB_TOKEN - GitHub Actions default
14+
* 4. VBOT_GITHUB_API_TOKEN - Legacy bot token
15+
* 5. GH_TOKEN - GitHub CLI default
1416
*
1517
* @returns {string|null} GitHub token or null if not found
1618
*/
1719
function getGitHubToken() {
1820
return process.env.REDPANDA_GITHUB_TOKEN ||
21+
process.env.ACTIONS_BOT_TOKEN ||
1922
process.env.GITHUB_TOKEN ||
23+
process.env.VBOT_GITHUB_API_TOKEN ||
2024
process.env.GH_TOKEN ||
2125
null;
2226
}

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@redpanda-data/docs-extensions-and-macros",
3-
"version": "4.13.5",
3+
"version": "4.13.6",
44
"description": "Antora extensions and macros developed for Redpanda documentation.",
55
"keywords": [
66
"antora",

‎tools/redpanda-connect/rpcn-connector-docs-handler.js‎

Lines changed: 60 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1172,12 +1172,40 @@ async function handleRpcnConnectorDocs (options) {
11721172
const missingFromCloudDocs = []
11731173
const cloudDocsErrors = []
11741174
if (cloudSupportedSet.size > 0 && options.checkCloudDocs !== false) {
1175-
console.log('\n ℹ️ Checking cloud-docs repository for missing connector pages...')
1175+
console.log('\n INFO: Checking cloud-docs repository for missing connector pages...')
11761176

11771177
// Use shared Octokit instance
11781178
const octokit = require('../../cli-utils/octokit-client')
11791179

11801180
try {
1181+
// Optimization: Fetch entire directory tree in 1 API call instead of 471 individual calls
1182+
console.log(' Fetching cloud-docs directory tree (1 API call)...')
1183+
1184+
let existingFiles = new Set()
1185+
1186+
try {
1187+
// Get the tree for the components directory
1188+
const { data: tree } = await octokit.git.getTree({
1189+
owner: 'redpanda-data',
1190+
repo: 'cloud-docs',
1191+
tree_sha: 'main:modules/develop/pages/connect/components',
1192+
recursive: true
1193+
})
1194+
1195+
// Build a set of existing file paths for O(1) lookup
1196+
tree.tree.forEach(item => {
1197+
if (item.type === 'blob' && item.path.endsWith('.adoc')) {
1198+
existingFiles.add(item.path)
1199+
}
1200+
})
1201+
1202+
console.log(` Loaded ${existingFiles.size} existing connector pages from cloud-docs`)
1203+
} catch (treeError) {
1204+
console.log(` WARNING: Could not fetch tree (${treeError.status}), falling back to individual checks`)
1205+
// If tree API fails, fall back to individual checks (old behavior)
1206+
existingFiles = null
1207+
}
1208+
11811209
// Check each cloud-supported connector
11821210
// Filter to only check actual connector/component types that need individual pages
11831211
const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'buffers', 'scanners', 'metrics', 'tracers']
@@ -1198,95 +1226,67 @@ async function handleRpcnConnectorDocs (options) {
11981226
}
11991227
}
12001228

1201-
const cloudDocsPath = `modules/develop/pages/connect/components/${type}/${name}.adoc`
1229+
const relativePath = `${type}/${name}.adoc`
1230+
const fullPath = `modules/develop/pages/connect/components/${relativePath}`
1231+
1232+
// Fast path: Check against tree if we have it
1233+
if (existingFiles !== null) {
1234+
if (!existingFiles.has(relativePath)) {
1235+
missingFromCloudDocs.push({ type, name, path: fullPath })
1236+
}
1237+
continue
1238+
}
12021239

1240+
// Fallback path: Individual API calls (only if tree fetch failed)
12031241
try {
12041242
await octokit.repos.getContent({
12051243
owner: 'redpanda-data',
12061244
repo: 'cloud-docs',
1207-
path: cloudDocsPath,
1245+
path: fullPath,
12081246
ref: 'main'
12091247
})
12101248
// File exists, no action needed
12111249
} catch (error) {
12121250
if (error.status === 404) {
12131251
// File doesn't exist in cloud-docs
1214-
missingFromCloudDocs.push({ type, name, path: cloudDocsPath })
1252+
missingFromCloudDocs.push({ type, name, path: fullPath })
12151253
} else {
1216-
// Non-404 error (auth, rate-limit, network, etc.)
1217-
// Try fallback: check raw URL without authentication
1218-
const rawUrl = `https://raw.githubusercontent.com/redpanda-data/cloud-docs/main/${cloudDocsPath}`
1219-
try {
1220-
const https = require('https')
1221-
const { URL } = require('url')
1222-
const parsedUrl = new URL(rawUrl)
1223-
1224-
await new Promise((resolve, reject) => {
1225-
const req = https.request({
1226-
hostname: parsedUrl.hostname,
1227-
path: parsedUrl.pathname,
1228-
method: 'HEAD',
1229-
timeout: 5000
1230-
}, (res) => {
1231-
if (res.statusCode === 200) {
1232-
resolve() // File exists
1233-
} else if (res.statusCode === 404) {
1234-
reject(new Error('404'))
1235-
} else {
1236-
reject(new Error(`Status ${res.statusCode}`))
1237-
}
1238-
})
1239-
req.on('error', reject)
1240-
req.on('timeout', () => {
1241-
req.destroy()
1242-
reject(new Error('Timeout'))
1243-
})
1244-
req.end()
1245-
})
1246-
// Fallback succeeded - file exists, no action needed
1247-
} catch (fallbackError) {
1248-
if (fallbackError.message === '404') {
1249-
// Confirmed missing via fallback
1250-
missingFromCloudDocs.push({ type, name, path: cloudDocsPath })
1251-
} else {
1252-
// Both API and fallback failed
1253-
cloudDocsErrors.push({
1254-
type,
1255-
name,
1256-
path: cloudDocsPath,
1257-
status: error.status || 'unknown',
1258-
message: `API: ${error.message}; Fallback: ${fallbackError.message}`
1259-
})
1260-
}
1261-
}
1254+
// Non-404 error - record as error
1255+
cloudDocsErrors.push({
1256+
type,
1257+
name,
1258+
path: fullPath,
1259+
status: error.status || 'unknown',
1260+
message: error.message
1261+
})
12621262
}
12631263
}
12641264
}
12651265

12661266
// Report results
12671267
if (cloudDocsErrors.length > 0) {
1268-
console.log(` ⚠️ Encountered ${cloudDocsErrors.length} error(s) while checking cloud-docs (check inconclusive):`)
1268+
console.log(` WARNING: Encountered ${cloudDocsErrors.length} error(s) while checking cloud-docs (check inconclusive):`)
12691269
cloudDocsErrors.forEach(({ type, name, status, message }) => {
1270-
console.log(` • ${type}/${name} - Status ${status}: ${message}`)
1270+
console.log(` - ${type}/${name} - Status ${status}: ${message}`)
12711271
})
1272-
console.log(` ℹ️ Please resolve these errors (e.g., check GITHUB_TOKEN or VBOT_GITHUB_API_TOKEN, API rate limits, network connectivity)`)
1272+
console.log(` INFO: Please resolve these errors (e.g., check GITHUB_TOKEN or VBOT_GITHUB_API_TOKEN, API rate limits, network connectivity)`)
12731273
if (missingFromCloudDocs.length > 0) {
1274-
console.log(` ℹ️ Additionally, ${missingFromCloudDocs.length} connector(s) confirmed missing from cloud-docs:`)
1274+
console.log(` INFO: Additionally, ${missingFromCloudDocs.length} connector(s) confirmed missing from cloud-docs:`)
12751275
missingFromCloudDocs.forEach(({ type, name }) => {
1276-
console.log(` • ${type}/${name}`)
1276+
console.log(` - ${type}/${name}`)
12771277
})
12781278
}
12791279
} else if (missingFromCloudDocs.length > 0) {
1280-
console.log(` ⚠️ Found ${missingFromCloudDocs.length} cloud-supported connector(s) missing from cloud-docs:`)
1280+
console.log(` WARNING: Found ${missingFromCloudDocs.length} cloud-supported connector(s) missing from cloud-docs:`)
12811281
missingFromCloudDocs.forEach(({ type, name }) => {
1282-
console.log(` • ${type}/${name}`)
1282+
console.log(` - ${type}/${name}`)
12831283
})
1284-
console.log(` ℹ️ These connectors need pages added to https://github.com/redpanda-data/cloud-docs`)
1284+
console.log(` INFO: These connectors need pages added to https://github.com/redpanda-data/cloud-docs`)
12851285
} else {
1286-
console.log(` ✓ All cloud-supported connectors have pages in cloud-docs`)
1286+
console.log(` All cloud-supported connectors have pages in cloud-docs`)
12871287
}
12881288
} catch (error) {
1289-
console.log(` ⚠️ Could not check cloud-docs: ${error.message}`)
1289+
console.log(` WARNING: Could not check cloud-docs: ${error.message}`)
12901290
}
12911291
}
12921292

0 commit comments

Comments
 (0)