Skip to content

chore: enable ssl verification for fetch-supporters #7607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 19, 2025
Merged

chore: enable ssl verification for fetch-supporters #7607

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 74 additions & 96 deletions src/utilities/fetch-supporters.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,107 +92,85 @@ const nodeToSupporter = (node) => ({
});

const getAllNodes = async (graphqlQuery, getNodes) => {
// Store original value
const originalTlsRejectUnauthorized =
process.env.NODE_TLS_REJECT_UNAUTHORIZED;
const isCI = process.env.CI === 'true' || (process.env.CI && process.env.VERCEL);

try {
// Only disable SSL verification in local development
if (!isCI) {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
console.log('Running locally - SSL verification disabled');
const body = {
query: graphqlQuery,
variables: {
limit: graphqlPageSize,
offset: 0,
dateFrom: new Date(
new Date().setFullYear(new Date().getFullYear() - 1)
).toISOString(), // data from last year
},
};

let allNodes = [];

let limit = 10,
remaining = 10,
reset;
if (process.env.OPENCOLLECTIVE_API_KEY) {
limit = 100;
remaining = 100;
}
// Handling pagination if necessary

while (true) {
if (remaining === 0) {
console.log(`Rate limit exceeded. Sleeping until ${new Date(reset)}.`);
await new Promise((resolve) =>
setTimeout(resolve, reset - Date.now() + 100)
);
}

const body = {
query: graphqlQuery,
variables: {
limit: graphqlPageSize,
offset: 0,
dateFrom: new Date(
new Date().setFullYear(new Date().getFullYear() - 1)
).toISOString(), // data from last year
const result = await fetch(graphqlEndpoint, {
method: 'POST',
body: JSON.stringify(body),
headers: {
'Content-Type': 'application/json',
},
};

let allNodes = [];

let limit = 10,
remaining = 10,
reset;
if (process.env.OPENCOLLECTIVE_API_KEY) {
limit = 100;
remaining = 100;
}
// Handling pagination if necessary

while (true) {
if (remaining === 0) {
console.log(`Rate limit exceeded. Sleeping until ${new Date(reset)}.`);
await new Promise((resolve) =>
setTimeout(resolve, reset - Date.now() + 100)
);
}
const fetchOptions = {
method: 'POST',
body: JSON.stringify(body),
headers: {
'Content-Type': 'application/json',
},
};

const result = await fetch(graphqlEndpoint, fetchOptions).then(
async (response) => {
if (response.headers.get('content-type').includes('json')) {
const json = await response.json();
console.log('json', json);
if (json.error) {
// when rate limit exceeded, api won't return headers data like x-ratelimit-limit, etc.
remaining = 0;
reset = Date.now() + 1000 * 60; // 1 minute
} else {
limit = response.headers.get('x-ratelimit-limit') * 1;
remaining = response.headers.get('x-ratelimit-remaining') * 1;
reset = response.headers.get('x-ratelimit-reset') * 1000;
console.log(
`Rate limit: ${remaining}/${limit} remaining. Reset in ${new Date(
reset
)}`
);
}
return json;
} else {
// utilities/fetch-supporters: SyntaxError: Unexpected token < in JSON at position 0
console.log('something wrong when fetching supporters');
return {
error: {
message: await response.text(),
},
};
}
}
);
// when rate limit exceeded, api will return {error: {message: ''}}
// but we could hopefully avoid rate limit by sleeping in the beginning of the loop
// however, when there're multiple task running simultaneously, it's still possible to hit the rate limit
if (result.error) {
console.log('error', result.error);
// let the loop continue
} else {
const nodes = getNodes(result.data);
allNodes = [...allNodes, ...nodes];
body.variables.offset += graphqlPageSize;
if (nodes.length < graphqlPageSize) {
return allNodes;
}).then(async (response) => {
if (response.headers.get('content-type').includes('json')) {
const json = await response.json();
console.log('json', json);
if (json.error) {
// when rate limit exceeded, api won't return headers data like x-ratelimit-limit, etc.
remaining = 0;
reset = Date.now() + 1000 * 60; // 1 minute
} else {
// more nodes to fetch
limit = response.headers.get('x-ratelimit-limit') * 1;
remaining = response.headers.get('x-ratelimit-remaining') * 1;
reset = response.headers.get('x-ratelimit-reset') * 1000;
console.log(
`Rate limit: ${remaining}/${limit} remaining. Reset in ${new Date(
reset
)}`
);
}
return json;
} else {
// utilities/fetch-supporters: SyntaxError: Unexpected token < in JSON at position 0
console.log('something wrong when fetching supporters');
return {
error: {
message: await response.text(),
},
};
}
});
// when rate limit exceeded, api will return {error: {message: ''}}
// but we could hopefully avoid rate limit by sleeping in the beginning of the loop
// however, when there're multiple task running simultaneously, it's still possible to hit the rate limit
if (result.error) {
console.log('error', result.error);
// let the loop continue
} else {
const nodes = getNodes(result.data);
allNodes = [...allNodes, ...nodes];
body.variables.offset += graphqlPageSize;
if (nodes.length < graphqlPageSize) {
return allNodes;
} else {
// more nodes to fetch
}
}
} finally {
// Only restore if we modified it
if (!isCI) {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = originalTlsRejectUnauthorized;
}
}
};
Expand Down
Loading