diff --git a/content/copilot/concepts/about-mcp.md b/content/copilot/concepts/about-mcp.md index 17309bfd656e..71aa1dbf3ad9 100644 --- a/content/copilot/concepts/about-mcp.md +++ b/content/copilot/concepts/about-mcp.md @@ -10,8 +10,6 @@ topics: contentType: concepts --- -{% data reusables.copilot.mcp-availability-and-preview-note %} - ## Overview of Model Context Protocol (MCP) The Model Context Protocol (MCP) is an open standard that defines how applications share context with large language models (LLMs). MCP provides a standardized way to connect AI models to different data sources and tools, enabling them to work together more effectively. diff --git a/content/copilot/how-tos/provide-context/use-mcp/extend-copilot-chat-with-mcp.md b/content/copilot/how-tos/provide-context/use-mcp/extend-copilot-chat-with-mcp.md index 128ffceeb254..e26defea2a09 100644 --- a/content/copilot/how-tos/provide-context/use-mcp/extend-copilot-chat-with-mcp.md +++ b/content/copilot/how-tos/provide-context/use-mcp/extend-copilot-chat-with-mcp.md @@ -17,8 +17,6 @@ redirect_from: contentType: how-tos --- -{% data reusables.copilot.mcp-availability-and-preview-note %} - ## Introduction The Model Context Protocol (MCP) is an open standard that defines how applications share context with large language models (LLMs). For an overview of MCP, see [AUTOTITLE](/copilot/concepts/about-mcp). @@ -121,6 +119,77 @@ If you already have an MCP configuration in Claude Desktop, you can use that con {% endvscode %} +{% visualstudio %} + +## Prerequisites + +* **Access to {% data variables.product.prodname_copilot_short %}**. {% data reusables.copilot.subscription-prerequisite %} +* **{% data variables.product.prodname_vs %} version 17.14 or later**. For more information on installing {% data variables.product.prodname_vs %}, see the [{% data variables.product.prodname_vs %} downloads page](https://visualstudio.microsoft.com/downloads/). +* **Sign in to {% data variables.product.company_short %} from {% data variables.product.prodname_vs %}**. +* {% data reusables.copilot.mcp-policy-requirement %} + +## Configuring MCP servers in {% data variables.product.prodname_vs %} + +1. In the {% data variables.product.prodname_vs %} menu bar, click **View**, then click **{% data variables.copilot.copilot_chat %}**. +1. At the bottom of the chat panel, select **Agent** from the mode dropdown. +1. In the {% data variables.copilot.copilot_chat_short %} window, click the tools icon, then click the plus icon in the tool picker window. +1. In the "Configure MCP server" pop-up window, fill out the fields, including server ID, type, and any additional fields required for the specific MCP server configuration. + + {% data variables.product.prodname_vs %} supports both remote and local servers. Remote servers, defined with a URL and credentials, are hosted externally for easier setup and sharing, while local servers, defined with command-line invocation, run on your local machine and can access local resources. See example configurations below, using the {% data variables.product.github %} MCP server as an example. + +1. Click **Save**. +1. If you are using a remote server with OAuth authentication, in the `mcp.json` file, click **Auth** from the CodeLens above the server to authenticate to the server. A pop-up or new window will appear, allowing you to authenticate with your account. The server will only be able to access the scopes you approve, and that your organization policies allow. +1. In the {% data variables.copilot.copilot_chat_short %} window, click the tools icon. You should now see additional tools from the MCP server that you configured. + +### Remote server configuration example with OAuth + + 1. For "Server ID", type `github`. + 1. For "Type", select "HTTP/SSE" from the dropdown. + 1. For "URL", type `https://api.githubcopilot.com/mcp/`. + 1. After clicking **Save**, the configuration in the `mcp.json` file should look like this: + + ```json copy + { + "servers": { + "github": { + "url": "https://api.githubcopilot.com/mcp/" + } + } + } + ``` + + 1. In the `mcp.json` file, click **Auth** from the CodeLens above the server to authenticate to the server. A pop-up will come up allowing you to authenticate with your {% data variables.product.github %} account. + +### Local server configuration example + + 1. For "Server ID", type `github`. + 1. For "Type", select "stdio" from the dropdown. + 1. For "Command (with optional arguments)", type `docker "run", "-i", "--rm", "-e", "GITHUB_PERSONAL_ACCESS_TOKEN", "ghcr.io/github/github-mcp-server"` + 1. Add an environment variable "GITHUB_PERSONAL_ACCESS_TOKEN" set to your {% data variables.product.pat_generic %}. + 1. After clicking **Save**, the configuration in the `mcp.json` file should look like this: + + ```json copy + { + "servers": { + "github": { + "type": "stdio", + "command": "docker", + "args": [ + "run", "-i", "--rm", "-e", "GITHUB_PERSONAL_ACCESS_TOKEN", + "ghcr.io/github/github-mcp-server" + ], + "env": { + "GITHUB_PERSONAL_ACCESS_TOKEN": "YOUR_GITHUB_PAT" + } + } + } + } + ``` + +For more information on configuring MCP servers in {% data variables.product.prodname_vs %}, see [Use MCP servers in {% data variables.product.prodname_vs %} (Preview)](https://learn.microsoft.com/en-us/visualstudio/ide/mcp-servers?view=vs-2022) in the {% data variables.product.prodname_vs %} documentation. + +{% endvisualstudio %} + {% jetbrains %} ## Prerequisites diff --git a/content/copilot/how-tos/provide-context/use-mcp/use-the-github-mcp-server.md b/content/copilot/how-tos/provide-context/use-mcp/use-the-github-mcp-server.md index 6198c0cc851d..d7762cf9d35a 100644 --- a/content/copilot/how-tos/provide-context/use-mcp/use-the-github-mcp-server.md +++ b/content/copilot/how-tos/provide-context/use-mcp/use-the-github-mcp-server.md @@ -17,8 +17,8 @@ contentType: how-tos >[!NOTE] > * The remote {% data variables.product.github %} MCP server is currently in {% data variables.release-phases.public_preview %} and subject to change; use of the {% data variables.product.github %} MCP server locally is generally available (GA). -> * MCP support is generally available (GA) in {% data variables.product.prodname_copilot_short %} for {% data variables.product.prodname_vscode %}, JetBrains, Eclipse, and Xcode. -> * The **MCP servers in {% data variables.product.prodname_copilot_short %}** policy for enterprises and organizations, disabled by default, controls use of MCP where MCP server support is generally available (GA). +> * MCP support is generally available (GA) in {% data variables.product.prodname_copilot_short %} for {% data variables.product.prodname_vscode %}, {% data variables.product.prodname_vs %}, JetBrains, Eclipse, and Xcode. +> * The **MCP servers in {% data variables.product.prodname_copilot_short %}** policy for enterprises and organizations, disabled by default, controls the use of MCP. > * While in {% data variables.release-phases.public_preview %}, access to the remote {% data variables.product.github %} MCP server through OAuth in {% data variables.product.prodname_copilot_short %} is governed by the {% data variables.product.prodname_copilot_short %} **Editor preview features** policy at the organization or enterprise level. PAT access to the server is managed by PAT policies. {% vscode %} @@ -41,10 +41,10 @@ You can choose to set up the {% data variables.product.github %} MCP server eith {% data reusables.copilot.mcp.mcp-configuration-location %} -The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server will have the same access as your personal account. If you use a PAT, the MCP server will have access to the scopes granted by the PAT. +The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server can only access the scopes you approve during sign-in. In organization-owned contexts, access may also be limited by admin policies that control which scopes and apps are permitted. If you use a PAT, the MCP server will have access to the scopes granted by the PAT, which is also subject to any PAT restrictions configured by the organization. > [!NOTE] -> If you are an {% data variables.product.prodname_emu %} with PAT restrictions, you won't be able to use PAT authentication. If you have OAuth access policy restrictions, you will need the OAuth apps for each client to be enabled (except {% data variables.product.prodname_vscode %} and {% data variables.product.prodname_vs %}). +> If you are an {% data variables.product.prodname_emu %}, then PAT is disabled by default, unless enabled by an enterprise administrator. If PAT is disabled, you won't be able to use PAT authentication. If you have OAuth access policy restrictions, you will need the OAuth App for each client (MCP host application) to be enabled (except {% data variables.product.prodname_vscode %} and {% data variables.product.prodname_vs %}). * [Remote MCP server configuration with OAuth](#remote-mcp-server-configuration-with-oauth) * [Remote MCP server configuration with PAT](#remote-mcp-server-configuration-with-pat) @@ -215,6 +215,100 @@ The {% data variables.product.github %} MCP server enables you to perform a wide {% endvscode %} +{% visualstudio %} + +{% data reusables.copilot.mcp.about-github-mcp-server %} + +## Prerequisites + +* **Access to {% data variables.product.prodname_copilot_short %}**. {% data reusables.copilot.subscription-prerequisite %} +* **{% data variables.product.prodname_vs %} version 17.14 or later**. For more information on installing {% data variables.product.prodname_vs %}, see the [{% data variables.product.prodname_vs %} downloads page](https://visualstudio.microsoft.com/downloads/). +* **Sign in to {% data variables.product.company_short %} from {% data variables.product.prodname_vs %}**. +* {% data reusables.copilot.mcp-policy-requirement %} + +## Setting up the {% data variables.product.github %} MCP server in {% data variables.product.prodname_vs %} + +The instructions below guide you through setting up the {% data variables.product.github %} MCP server in {% data variables.product.prodname_vs %}. Other MCP-compatible editors may have similar steps, but the exact process may vary. + +The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server can only access the scopes you approve during sign-in. In organization-owned contexts, access may also be limited by admin policies that control which scopes and apps are permitted. If you use a PAT, the MCP server will have access to the scopes granted by the PAT, which is also subject to any PAT restrictions configured by the organization. + +> [!NOTE] +> If you are an {% data variables.product.prodname_emu %}, then PAT is disabled by default, unless enabled by an enterprise administrator. If PAT is disabled, you won't be able to use PAT authentication. If you have OAuth access policy restrictions, you will need the OAuth App for each client (MCP host application) to be enabled (except {% data variables.product.prodname_vscode %} and {% data variables.product.prodname_vs %}). + +For information on setting up the {% data variables.product.github %} MCP server locally, see the [GitHub MCP server repository](https://github.com/github/github-mcp-server#usage-in-other-mcp-hosts-1). + +### Remote MCP server configuration with OAuth + +You do not need to create a PAT or install any additional software to use the remote {% data variables.product.github %} MCP server with OAuth. You can set it up directly in {% data variables.product.prodname_vs %}. + +1. In the {% data variables.product.prodname_vs %} menu bar, click **View**, then click **{% data variables.copilot.copilot_chat %}**. +1. At the bottom of the chat panel, select **Agent** from the mode dropdown. +1. In the {% data variables.copilot.copilot_chat_short %} window, click the tools icon, then click the plus icon in the tool picker window. +1. In the "Configure MCP server" pop-up window, fill out the fields. + 1. For "Server ID", type `github`. + 1. For "Type", select "HTTP/SSE" from the dropdown. + 1. For "URL", type `https://api.githubcopilot.com/mcp/`. +1. Click **Save**. The configuration in the `mcp.json` file should look like this: + + ```json copy + { + "servers": { + "github": { + "url": "https://api.githubcopilot.com/mcp/" + } + } + } + ``` + +1. In the `mcp.json` file, click **Auth** from the CodeLens above the server to authenticate to the server. A pop-up will come up allowing you to authenticate with your {% data variables.product.github %} account. + +### Remote MCP server configuration with PAT + +To configure the remote {% data variables.product.github %} MCP server with a PAT, ensure you have created a PAT with the necessary scopes for the access you want to grant to the MCP server. For more information, see [AUTOTITLE](/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). + +1. In the {% data variables.product.prodname_vs %} menu bar, click **View**, then click **{% data variables.copilot.copilot_chat %}**. +1. At the bottom of the chat panel, select **Agent** from the mode dropdown. +1. In the {% data variables.copilot.copilot_chat_short %} window, click the tools icon, then click the plus icon in the tool picker window. +1. In the "Configure MCP server" pop-up window, fill out the fields. + 1. For "Server ID", type `github`. + 1. For "Type", select "HTTP/SSE" from the dropdown. + 1. For "URL", type `https://api.githubcopilot.com/mcp/`. + 1. Add a new header under "Headers", called "Authorization" and set to the value `Bearer YOUR_GITHUB_PAT`, replacing "YOUR_GITHUB_PAT" with your PAT. +1. Click **Save**. The configuration in the `mcp.json` file should look like this: + + ```json copy + { + "servers": { + "github": { + "url": "https://api.githubcopilot.com/mcp/", + "requestInit": { + "headers": { + "Authorization": "Bearer YOUR_GITHUB_PAT" + } + } + } + } + } + ``` + +For more information on configuring MCP servers in {% data variables.product.prodname_vs %}, see [Use MCP servers in {% data variables.product.prodname_vs %} (Preview)](https://learn.microsoft.com/en-us/visualstudio/ide/mcp-servers?view=vs-2022) in the {% data variables.product.prodname_vs %} documentation. + +## Using the {% data variables.product.github %} MCP server in {% data variables.product.prodname_vs %} + +The {% data variables.product.github %} MCP server enables you to perform a wide range of actions on {% data variables.product.github %}, via {% data variables.copilot.copilot_chat_short %} in {% data variables.product.prodname_vs %}. + +1. In the {% data variables.product.prodname_vs %} menu bar, click **View**, then click **{% data variables.copilot.copilot_chat %}**. +1. At the bottom of the chat panel, select **Agent** from the mode dropdown. +1. In the {% data variables.copilot.copilot_chat_short %} window, click the tools icon. + * Under **{% data variables.product.github %}**, you will see a list of available tools. +1. In the {% data variables.copilot.copilot_chat_short %} box, type a command or question related to the action you want to perform, and press **Enter**. + * For example, you can ask the {% data variables.product.github %} MCP server to create a new issue, list pull requests, or retrieve repository information. +1. The {% data variables.product.github %} MCP server will process your request and provide a response in the chat interface. + * In the {% data variables.copilot.copilot_chat_short %} box, you may be asked to give additional permissions or provide more information to complete the action. +1. Follow the prompts to complete the action. + +{% endvisualstudio %} + {% jetbrains %} {% data reusables.copilot.mcp.mcp-ide-preview-note %} @@ -234,10 +328,10 @@ The {% data variables.product.github %} MCP server enables you to perform a wide The instructions below guide you through setting up the {% data variables.product.github %} MCP server in JetBrains IDEs. Other MCP-compatible editors may have similar steps, but the exact process may vary. -We recommend setting up the {% data variables.product.github %} MCP server remotely. The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server will have the same access as your {% data variables.product.github %} account. If you use a PAT, the MCP server will have access to the scopes granted by the PAT. +The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server can only access the scopes you approve during sign-in. In organization-owned contexts, access may also be limited by admin policies that control which scopes and apps are permitted. If you use a PAT, the MCP server will have access to the scopes granted by the PAT, which is also subject to any PAT restrictions configured by the organization. > [!NOTE] -> If you are an {% data variables.product.prodname_emu %} with PAT restrictions, you won't be able to use PAT authentication. +> If you are an {% data variables.product.prodname_emu %}, then PAT is disabled by default, unless enabled by an enterprise administrator. If PAT is disabled, you won't be able to use PAT authentication. If you have OAuth access policy restrictions, you will need the OAuth App for each client (MCP host application) to be enabled (except {% data variables.product.prodname_vscode %} and {% data variables.product.prodname_vs %}). For information on setting up the {% data variables.product.github %} MCP server locally, see the [GitHub MCP server repository](https://github.com/github/github-mcp-server#usage-in-other-mcp-hosts-1). @@ -311,10 +405,10 @@ The {% data variables.product.github %} MCP server enables you to perform a wide The instructions below guide you through setting up the {% data variables.product.github %} MCP server in Xcode. Other MCP-compatible editors may have similar steps, but the exact process may vary. -We recommend setting up the {% data variables.product.github %} MCP server remotely. The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server will have the same access as your {% data variables.product.github %} account. If you use a PAT, the MCP server will have access to the scopes granted by the PAT. +The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server can only access the scopes you approve during sign-in. In organization-owned contexts, access may also be limited by admin policies that control which scopes and apps are permitted. If you use a PAT, the MCP server will have access to the scopes granted by the PAT, which is also subject to any PAT restrictions configured by the organization. > [!NOTE] -> If you are an {% data variables.product.prodname_emu %} with PAT restrictions, you won't be able to use PAT authentication. +> If you are an {% data variables.product.prodname_emu %}, then PAT is disabled by default, unless enabled by an enterprise administrator. If PAT is disabled, you won't be able to use PAT authentication. If you have OAuth access policy restrictions, you will need the OAuth App for each client (MCP host application) to be enabled (except {% data variables.product.prodname_vscode %} and {% data variables.product.prodname_vs %}). For information on setting up the {% data variables.product.github %} MCP server locally, see the [GitHub MCP server repository](https://github.com/github/github-mcp-server#usage-in-other-mcp-hosts-1). @@ -386,10 +480,10 @@ The {% data variables.product.github %} MCP server enables you to perform a wide The instructions below guide you through setting up the {% data variables.product.github %} MCP server in Eclipse. Other MCP-compatible editors may have similar steps, but the exact process may vary. -We recommend setting up the {% data variables.product.github %} MCP server remotely. The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server will have the same access as your {% data variables.product.github %} account. If you use a PAT, the MCP server will have access to the scopes granted by the PAT. +The remote {% data variables.product.github %} MCP server uses one-click OAuth authentication by default, but you can also manually configure it to use a {% data variables.product.pat_generic %} (PAT) for authentication. If you use OAuth, the MCP server can only access the scopes you approve during sign-in. In organization-owned contexts, access may also be limited by admin policies that control which scopes and apps are permitted. If you use a PAT, the MCP server will have access to the scopes granted by the PAT, which is also subject to any PAT restrictions configured by the organization. > [!NOTE] -> If you are an {% data variables.product.prodname_emu %} with PAT restrictions, you won't be able to use PAT authentication. +> If you are an {% data variables.product.prodname_emu %}, then PAT is disabled by default, unless enabled by an enterprise administrator. If PAT is disabled, you won't be able to use PAT authentication. If you have OAuth access policy restrictions, you will need the OAuth App for each client (MCP host application) to be enabled (except {% data variables.product.prodname_vscode %} and {% data variables.product.prodname_vs %}). For information on setting up the {% data variables.product.github %} MCP server locally, see the [GitHub MCP server repository](https://github.com/github/github-mcp-server#usage-in-other-mcp-hosts-1). diff --git a/data/reusables/copilot/mcp-availability-and-preview-note.md b/data/reusables/copilot/mcp-availability-and-preview-note.md index 2e54d1204a4f..2b9e1e006483 100644 --- a/data/reusables/copilot/mcp-availability-and-preview-note.md +++ b/data/reusables/copilot/mcp-availability-and-preview-note.md @@ -1,5 +1,3 @@ >[!NOTE] > -> * MCP support is generally available (GA) in {% data variables.product.prodname_copilot_short %} for {% data variables.product.prodname_vscode %}, JetBrains, Eclipse, and Xcode. -> * MCP support for {% data variables.product.prodname_copilot_short %} in {% data variables.product.prodname_vs %} is in {% data variables.release-phases.public_preview %} and is subject to change. -> * The [AUTOTITLE](/free-pro-team@latest/site-policy/github-terms/github-pre-release-license-terms) apply only to {% data variables.product.prodname_copilot_short %} in IDEs where MCP support is still in preview. GA terms apply when using MCP for {% data variables.product.prodname_copilot_short %} in {% data variables.product.prodname_vscode %}, JetBrains, Eclipse, and Xcode. +> **MCP servers in {% data variables.product.prodname_copilot_short %}** policy for enterprises and organizations, disabled by default, controls the use of MCP. diff --git a/data/reusables/copilot/mcp/mcp-policy.md b/data/reusables/copilot/mcp/mcp-policy.md index abd5da414041..ea63528fe29d 100644 --- a/data/reusables/copilot/mcp/mcp-policy.md +++ b/data/reusables/copilot/mcp/mcp-policy.md @@ -1,3 +1 @@ -Enterprises and organizations can choose to enable or disable use of MCP for members of their organization or enterprise. The policy is disabled by default. See [AUTOTITLE](/copilot/how-tos/administer/enterprises/managing-policies-and-features-for-copilot-in-your-enterprise) and [AUTOTITLE](/copilot/how-tos/administer-copilot/manage-for-organization/manage-policies). The MCP policy **only** applies to users who have a {% data variables.copilot.copilot_business_short %} or {% data variables.copilot.copilot_enterprise_short %} subscription from an organization or enterprise that configures the policy. {% data variables.copilot.copilot_free_short %}, {% data variables.copilot.copilot_pro_short %}, or {% data variables.copilot.copilot_pro_plus_short %} **do not** have their MCP access governed by this policy. - -{% data reusables.copilot.mcp-servers-policy-note %} +Enterprises and organizations can choose to enable or disable use of MCP for members of their organization or enterprise with the **MCP servers in {% data variables.product.prodname_copilot_short %}** policy. The policy is disabled by default. See [AUTOTITLE](/copilot/how-tos/administer/enterprises/managing-policies-and-features-for-copilot-in-your-enterprise) and [AUTOTITLE](/copilot/how-tos/administer-copilot/manage-for-organization/manage-policies). The MCP policy **only** applies to users who have a {% data variables.copilot.copilot_business_short %} or {% data variables.copilot.copilot_enterprise_short %} subscription from an organization or enterprise that configures the policy. {% data variables.copilot.copilot_free_short %}, {% data variables.copilot.copilot_pro_short %}, or {% data variables.copilot.copilot_pro_plus_short %} **do not** have their MCP access governed by this policy. diff --git a/src/events/lib/hydro.ts b/src/events/lib/hydro.ts index 34242d41c8e8..3fb1e91e8b61 100644 --- a/src/events/lib/hydro.ts +++ b/src/events/lib/hydro.ts @@ -1,6 +1,5 @@ import { createHmac } from 'crypto' -import { Agent } from 'node:https' -import got from 'got' +import { fetchWithRetry } from '@/frame/lib/fetch-utils' import { isNil } from 'lodash-es' import statsd from '@/observability/lib/statsd' import { report } from '@/observability/lib/failbot' @@ -15,7 +14,6 @@ const X_HYDRO_APP = 'docs-production' const CLUSTER = 'potomac' // We only have ability to publish externally to potomac cluster const TIMEOUT = MAX_REQUEST_TIMEOUT - 1000 // Limit because Express will terminate at MAX_REQUEST_TIMEOUT const RETRIES = 0 // We care about aggregate statistics; a few dropped events isn't a big deal -const httpsAgent = new Agent({ keepAlive: true, maxSockets: 32 }) // keepAlive: https://gh.io/AAk2qio -- 32: https://bit.ly/3Tywd1U const { NODE_ENV, HYDRO_SECRET, HYDRO_ENDPOINT } = process.env const inProd = NODE_ENV === 'production' @@ -48,19 +46,27 @@ async function _publish( }) const token = createHmac('sha256', secret).update(requestBody).digest('hex') - const response = await got.post(endpoint, { - body: requestBody, - agent: { https: httpsAgent }, - headers: { - Authorization: `Hydro ${token}`, - 'Content-Type': 'application/json', - 'X-Hydro-App': X_HYDRO_APP, + // Note: Custom HTTPS agent (keepAlive, maxSockets) not supported with native fetch + // Consider using undici.fetch() if custom agent behavior is critical + const response = await fetchWithRetry( + endpoint, + { + method: 'POST', + body: requestBody, + headers: { + Authorization: `Hydro ${token}`, + 'Content-Type': 'application/json', + 'X-Hydro-App': X_HYDRO_APP, + }, }, - throwHttpErrors: false, - retry: { limit: RETRIES }, - timeout: { request: TIMEOUT }, - }) - const { statusCode, body } = response + { + retries: RETRIES, + timeout: TIMEOUT, + throwHttpErrors: false, + }, + ) + const statusCode = response.status + const body = await response.text() statsd.increment('hydro.response_code.all', 1, [`response_code:${statusCode}`]) diff --git a/src/frame/lib/fetch-utils.ts b/src/frame/lib/fetch-utils.ts index c6e6017e7e93..f90bc3f6c61f 100644 --- a/src/frame/lib/fetch-utils.ts +++ b/src/frame/lib/fetch-utils.ts @@ -2,12 +2,13 @@ * Utility functions for fetch with retry and timeout functionality * to replace got library functionality */ - export interface FetchWithRetryOptions { retries?: number retryDelay?: number timeout?: number throwHttpErrors?: boolean + // Note: Custom HTTPS agents are not supported in native fetch + // Consider using undici or node-fetch if custom agent support is critical } /** diff --git a/src/observability/lib/failbot.ts b/src/observability/lib/failbot.ts index 9ca9b21d573f..226def0ae62d 100644 --- a/src/observability/lib/failbot.ts +++ b/src/observability/lib/failbot.ts @@ -1,49 +1,31 @@ -import got, { type OptionsOfTextResponseBody, type Method } from 'got' +import { fetchWithRetry } from '@/frame/lib/fetch-utils' import { Failbot, HTTPBackend } from '@github/failbot' import { getLoggerContext } from '@/observability/logger/lib/logger-context' const HAYSTACK_APP = 'docs' -async function retryingGot(input: RequestInfo | URL, init?: RequestInit): Promise { +async function retryingFetch(input: RequestInfo | URL, init?: RequestInit): Promise { const url = typeof input === 'string' ? input : input.toString() - // Extract body from fetch init for got options - const gotOptions: OptionsOfTextResponseBody = { - method: (init?.method as Method) || 'GET', - body: typeof init?.body === 'string' ? init.body : undefined, - headers: init?.headers as Record | undefined, - // With the timeout at 3000 (milliseconds) and the retry.limit - // at 4 (times), the total worst-case is: - // 3000 * 4 + 1000 + 2000 + 3000 + 4000 + 8000 = 30 seconds - timeout: { - response: 3000, + // Use fetchWithRetry with retry configuration matching got's behavior + // With the timeout at 3000 (milliseconds) and the retry.limit + // at 4 (times), the total worst-case is: + // 3000 * 4 + 1000 + 2000 + 3000 + 4000 + 8000 = 30 seconds + const response = await fetchWithRetry( + url, + { + method: init?.method || 'GET', + body: init?.body, + headers: init?.headers, }, - retry: { - // This means it will wait... - // 1. 1000ms - // 2. 2000ms - // 3. 4000ms - // 4. 8000ms - // 5. give up! - // - // From the documentation: - // - // Delays between retries counts with function - // 1000 * Math.pow(2, retry - 1) + Math.random() * 100, - // where retry is attempt number (starts from 1). - // - limit: 4, + { + timeout: 3000, + retries: 4, + throwHttpErrors: false, // Let failbot handle HTTP errors }, - } - - const gotResponse = await got(url, gotOptions) + ) - // Convert got response to fetch-compatible Response - return new Response(gotResponse.body, { - status: gotResponse.statusCode, - statusText: gotResponse.statusMessage, - headers: gotResponse.headers as HeadersInit, - }) + return response } export function report(error: Error, metadata?: Record) { @@ -55,7 +37,7 @@ export function report(error: Error, metadata?: Record) { const backends = [ new HTTPBackend({ haystackURL: process.env.HAYSTACK_URL, - fetchFn: retryingGot, + fetchFn: retryingFetch, }), ] const failbot = new Failbot({ diff --git a/src/search/middleware/general-search-middleware.ts b/src/search/middleware/general-search-middleware.ts index 3eda5b6cebbc..c9533fcae763 100644 --- a/src/search/middleware/general-search-middleware.ts +++ b/src/search/middleware/general-search-middleware.ts @@ -6,7 +6,7 @@ This file & middleware is for when a user requests our /search page e.g. 'docs.g When a user directly hits our API e.g. /api/search/v1?query=foo, they will hit the routes in ./search-routes.ts */ -import got from 'got' +import { fetchWithRetry } from '@/frame/lib/fetch-utils' import { Request, Response, NextFunction } from 'express' import { errors } from '@elastic/elasticsearch' import statsd from '@/observability/lib/statsd' @@ -172,5 +172,10 @@ async function getProxySearch( // Add client_name for external API requests url.searchParams.set('client_name', 'docs.github.com-client') console.log(`Proxying search to ${url}`) - return got(url).json() + + const response = await fetchWithRetry(url.toString()) + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`) + } + return response.json() as Promise } diff --git a/src/search/scripts/scrape/lib/build-records.ts b/src/search/scripts/scrape/lib/build-records.ts index 9b7e1643e897..479d3a06b050 100644 --- a/src/search/scripts/scrape/lib/build-records.ts +++ b/src/search/scripts/scrape/lib/build-records.ts @@ -2,7 +2,6 @@ import eventToPromise from 'event-to-promise' import chalk from 'chalk' import dotenv from 'dotenv' import boxen from 'boxen' -import { HTTPError } from 'got' import languages from '@/languages/lib/languages' import parsePageSectionsIntoRecords from '@/search/scripts/scrape/lib/parse-page-sections-into-records' @@ -12,6 +11,23 @@ import { getAllVersionsKeyFromIndexVersion } from '@/search/lib/elasticsearch-ve import type { Page, Permalink, Record, Config, Redirects } from '@/search/scripts/scrape/types' +// Custom error class to replace got's HTTPError +class HTTPError extends Error { + response: { ok: boolean; statusCode?: number } + request: { requestUrl?: { pathname?: string } } + + constructor( + message: string, + response: { ok: boolean; statusCode?: number }, + request: { requestUrl?: { pathname?: string } }, + ) { + super(message) + this.name = 'HTTPError' + this.response = response + this.request = request + } +} + const pageMarker = chalk.green('|') const recordMarker = chalk.grey('.') const port = 4002 diff --git a/src/search/scripts/scrape/lib/domwaiter.ts b/src/search/scripts/scrape/lib/domwaiter.ts index fe70a1d9fedd..ac8cbe199ab7 100644 --- a/src/search/scripts/scrape/lib/domwaiter.ts +++ b/src/search/scripts/scrape/lib/domwaiter.ts @@ -1,10 +1,27 @@ import { EventEmitter } from 'events' import Bottleneck from 'bottleneck' -import got from 'got' +import { fetchWithRetry } from '@/frame/lib/fetch-utils' import cheerio from 'cheerio' import type { Permalink } from '@/search/scripts/scrape/types' +// Custom error class to match got's HTTPError interface +class HTTPError extends Error { + response: { ok: boolean; statusCode?: number } + request: { requestUrl?: { pathname?: string } } + + constructor( + message: string, + response: { ok: boolean; statusCode?: number }, + request: { requestUrl?: { pathname?: string } }, + ) { + super(message) + this.name = 'HTTPError' + this.response = response + this.request = request + } +} + interface DomWaiterOptions { parseDOM?: boolean json?: boolean @@ -45,7 +62,15 @@ async function getPage(page: Permalink, emitter: EventEmitter, opts: DomWaiterOp if (opts.json) { try { - const json = await got(page.url!).json() + const response = await fetchWithRetry(page.url!) + if (!response.ok) { + throw new HTTPError( + `HTTP ${response.status}: ${response.statusText}`, + { ok: response.ok, statusCode: response.status }, + { requestUrl: { pathname: page.url } }, + ) + } + const json = await response.json() const pageCopy = Object.assign({}, page, { json }) emitter.emit('page', pageCopy) } catch (err) { @@ -53,7 +78,15 @@ async function getPage(page: Permalink, emitter: EventEmitter, opts: DomWaiterOp } } else { try { - const body = (await got(page.url!)).body + const response = await fetchWithRetry(page.url!) + if (!response.ok) { + throw new HTTPError( + `HTTP ${response.status}: ${response.statusText}`, + { ok: response.ok, statusCode: response.status }, + { requestUrl: { pathname: page.url } }, + ) + } + const body = await response.text() const pageCopy = Object.assign({}, page, { body }) if (opts.parseDOM) (pageCopy as any).$ = cheerio.load(body) emitter.emit('page', pageCopy) diff --git a/src/secret-scanning/data/public-docs.yml b/src/secret-scanning/data/public-docs.yml index b31dd502cb83..f2bb510e23cf 100644 --- a/src/secret-scanning/data/public-docs.yml +++ b/src/secret-scanning/data/public-docs.yml @@ -2690,6 +2690,17 @@ hasPushProtection: true hasValidityCheck: '{% ifversion fpt or ghes %}false{% else %}true{% endif %}' isduplicate: false +- provider: hCaptcha + supportedSecret: hCaptcha Siteverify Secret + secretType: hcaptcha_siteverify_secret + versions: + fpt: '*' + ghec: '*' + isPublic: false + isPrivateWithGhas: true + hasPushProtection: false + hasValidityCheck: false + isduplicate: false - provider: Heroku supportedSecret: Heroku Platform API OAuth2 Token secretType: heroku_platform_api_oauth2_token @@ -4925,6 +4936,17 @@ hasPushProtection: false hasValidityCheck: false isduplicate: false +- provider: Tencent + supportedSecret: Tencent WeChat Pay Token + secretType: tencent_wechat_pay_token + versions: + fpt: '*' + ghec: '*' + isPublic: false + isPrivateWithGhas: true + hasPushProtection: false + hasValidityCheck: false + isduplicate: false - provider: Thunderstore supportedSecret: Thunderstore IO API Token secretType: thunderstore_io_api_token diff --git a/src/secret-scanning/lib/config.json b/src/secret-scanning/lib/config.json index 414e12d870ec..5242dd7a5533 100644 --- a/src/secret-scanning/lib/config.json +++ b/src/secret-scanning/lib/config.json @@ -1,5 +1,5 @@ { - "sha": "31555964edb2fad67e84a11330cf6a5fd1b63e28", - "blob-sha": "e681fbbcc1832b62a7de3864e5fe4dc4eaf8cc5b", + "sha": "7daab1873298e18f96c5efef95ffecf8e5d37f8c", + "blob-sha": "ad16586dfcf7e5bdfe7f0c7071f30dbf006c7f59", "targetFilename": "code-security/secret-scanning/introduction/supported-secret-scanning-patterns" } \ No newline at end of file diff --git a/src/workflows/experimental/readability-report.ts b/src/workflows/experimental/readability-report.ts index 8052748c26dc..0ca6d7232415 100644 --- a/src/workflows/experimental/readability-report.ts +++ b/src/workflows/experimental/readability-report.ts @@ -39,7 +39,7 @@ import fs from 'fs' import path from 'path' import cheerio from 'cheerio' -import got from 'got' +import { fetchWithRetry } from '@/frame/lib/fetch-utils' interface ReadabilityMetrics { fleschReadingEase: number @@ -174,7 +174,12 @@ async function waitForServer(): Promise { for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { - await got(makeURL('/'), { timeout: { request: 5000 } }) + const response = await fetchWithRetry(makeURL('/'), undefined, { + timeout: 5000, + }) + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`) + } console.log('Server is ready!') return } catch (error) { @@ -202,18 +207,19 @@ async function analyzeFile(filePath: string): Promise { try { // Fetch the rendered page - const response = await got(makeURL(urlPath), { - timeout: { request: 30000 }, + const response = await fetchWithRetry(makeURL(urlPath), undefined, { + timeout: 30000, throwHttpErrors: false, }) - if (response.statusCode !== 200) { - console.warn(`Skipping ${urlPath}: HTTP ${response.statusCode}`) + if (response.status !== 200) { + console.warn(`Skipping ${urlPath}: HTTP ${response.status}`) return null } // Parse HTML and extract content - const $ = cheerio.load(response.body) + const body = await response.text() + const $ = cheerio.load(body) // Get page title const title = $('h1').first().text().trim() || $('title').text().trim() || 'Untitled'