diff --git a/bin/clover/src/pipelines/gcp/funcs/actions/create.ts b/bin/clover/src/pipelines/gcp/funcs/actions/create.ts index 17164c78b8..8bfea02b03 100644 --- a/bin/clover/src/pipelines/gcp/funcs/actions/create.ts +++ b/bin/clover/src/pipelines/gcp/funcs/actions/create.ts @@ -19,12 +19,7 @@ async function main(component: Input): Promise { } // Get API path metadata from domain.extra - const insertApiPathJson = _.get( - component.properties, - ["domain", "extra", "insertApiPath"], - "", - ); - + const insertApiPathJson = _.get(component.properties, ["domain", "extra", "insertApiPath"], ""); if (!insertApiPathJson) { return { status: "error", @@ -35,15 +30,8 @@ async function main(component: Input): Promise { const insertApiPath = JSON.parse(insertApiPathJson); const baseUrl = _.get(component.properties, ["domain", "extra", "baseUrl"], ""); - // Get the get API path to determine how to extract resourceId later - // APIs using {+name} need the full path; APIs using {name} need short name - const getApiPathJson = _.get( - component.properties, - ["domain", "extra", "getApiPath"], - "", - ); - const getApiPath = getApiPathJson ? JSON.parse(getApiPathJson) : null; - const usesFullResourcePath = getApiPath?.path?.includes("{+"); + // Get resourceIdStyle to determine how to extract resourceId from response + const resourceIdStyle = _.get(component.properties, ["domain", "extra", "resourceIdStyle"], "simpleName"); // Get authentication token const serviceAccountJson = requestStorage.getEnv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); @@ -53,61 +41,8 @@ async function main(component: Input): Promise { const { token, projectId } = await getAccessToken(serviceAccountJson); - // Build the URL by replacing path parameters - let url = `${baseUrl}${insertApiPath.path}`; - const queryParams: string[] = []; - - // Replace path parameters with values from resource_value or domain - // Parameters not found in the path template are added as query parameters - if (insertApiPath.parameterOrder) { - for (const paramName of insertApiPath.parameterOrder) { - let paramValue; - - // Use extracted project_id for project/projectId parameter - if (paramName === "project" || paramName === "projectId") { - paramValue = projectId; - } else if (paramName === "parent") { - paramValue = _.get(component.properties, ["domain", "parent"]); - if (!paramValue && projectId) { - // auto-construct for project-only resources - const availableScopesJson = _.get(component.properties, ["domain", "extra", "availableScopes"]); - const availableScopes = availableScopesJson ? JSON.parse(availableScopesJson) : []; - const isProjectOnly = availableScopes.length === 1 && availableScopes[0] === "projects"; - - if (isProjectOnly) { - const location = _.get(component.properties, ["domain", "location"]) || - _.get(component.properties, ["domain", "zone"]) || - _.get(component.properties, ["domain", "region"]); - if (location) { - paramValue = `projects/${projectId}/locations/${location}`; - } - } - } - } else { - paramValue = _.get(component.properties, ["domain", paramName]); - } - - if (paramValue) { - // Handle {+param} (reserved expansion - don't encode, allows slashes) - if (url.includes(`{+${paramName}}`)) { - url = url.replace(`{+${paramName}}`, paramValue); - } else if (url.includes(`{${paramName}}`)) { - // Handle {param} (simple expansion - encode) - url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); - } else { - // Parameter not in path template - add as query parameter - // This handles APIs like GCS where project is a query param, not a path param - queryParams.push(`${paramName}=${encodeURIComponent(paramValue)}`); - } - } - } - } - - // Append query parameters to URL - if (queryParams.length > 0) { - url += (url.includes("?") ? "&" : "?") + queryParams.join("&"); - } - + // Build the URL + const url = buildUrlWithParams(baseUrl, insertApiPath, component, projectId); const httpMethod = insertApiPath.httpMethod || "POST"; // Make the API request with retry logic @@ -135,161 +70,212 @@ ${errorText}` return resp; }, { - isRateLimitedFn: (error) => error.status === 429 - }).then((r) => r.result); + isRateLimitedFn: (error: any) => error.status === 429 + }).then((r: any) => r.result); const responseJson = await response.json(); - // Handle Google Cloud Long-Running Operations (LRO) - // Check if this is an operation response: - // - Compute Engine uses "kind" containing "operation" - // - GKE/Container API uses "operationType" field - // - Other APIs (API Keys, etc.) use "name" starting with "operations/" - const isLRO = (responseJson.kind && responseJson.kind.includes("operation")) || + // Check if this resource uses Long-Running Operations based on metadata + const lroStyle = _.get(component.properties, ["domain", "extra", "lroStyle"], "none"); + + // Detect LRO response - only check if lroStyle indicates LRO support + const isLRO = lroStyle !== "none" && ( + (responseJson.kind && responseJson.kind.includes("operation")) || responseJson.operationType || - (responseJson.name && responseJson.name.startsWith("operations/")); + (responseJson.name && responseJson.name.startsWith("operations/")) + ); + if (isLRO) { console.log(`[CREATE] LRO detected, polling for completion...`); - // Use selfLink or construct URL from operation name - // For APIs that don't provide selfLink, we need to construct the URL - // The API version prefix (v1, v2, etc.) comes from the API paths - let pollingUrl = responseJson.selfLink; - if (!pollingUrl && responseJson.name) { - // Extract version from one of the API paths (e.g., "v2/{+parent}/keys" -> "v2") - const insertApiPathJson = _.get(component.properties, ["domain", "extra", "insertApiPath"], ""); - const getApiPathJson = _.get(component.properties, ["domain", "extra", "getApiPath"], ""); - const pathJson = insertApiPathJson || getApiPathJson; - let apiVersion = ""; - if (pathJson) { - const apiPath = JSON.parse(pathJson); - const versionMatch = apiPath.path?.match(/^(v\d+)\//); - if (versionMatch) { - apiVersion = versionMatch[1] + "/"; - } - } - pollingUrl = `${baseUrl}${apiVersion}${responseJson.name}`; - } + const pollingUrl = buildLroPollingUrl(baseUrl, responseJson, component); - // Poll the operation until it completes using new siExec.pollLRO - const finalResource = await siExec.pollLRO({ + // Poll until complete, then extract resourceId from the operation result + const operationResult = await siExec.pollLRO({ url: pollingUrl, headers: { "Authorization": `Bearer ${token}` }, maxAttempts: 20, baseDelay: 2000, maxDelay: 30000, - isCompleteFn: (response, body) => body.status === "DONE" || body.done === true, - isErrorFn: (response, body) => !!body.error, - extractResultFn: async (response, body) => { - // If operation has error, throw it + isCompleteFn: (_response: any, body: any) => body.status === "DONE" || body.done === true, + isErrorFn: (_response: any, body: any) => !!body.error, + extractResultFn: async (_response: any, body: any) => { if (body.error) { - throw new Error(`Operation failed: ${JSON.stringify(body.error)}`); + throw new Error(`Create operation failed: ${JSON.stringify(body.error)}`); } - - // For create operations, get the final resource from the operation response - // Some operations include the created resource in the response field - if (body.response) { - return body.response; - } - - // GCP pattern: fetch the final resource from targetLink - if (body.targetLink) { - const resourceResponse = await fetch(body.targetLink, { - method: "GET", - headers: { "Authorization": `Bearer ${token}` }, - }); - - if (!resourceResponse.ok) { - throw new Error(`Failed to fetch final resource: ${resourceResponse.status}`); - } - - return await resourceResponse.json(); - } - - // Fallback: Try to extract resource name from operation metadata and fetch using getApiPath - // This handles APIs that don't provide targetLink or response in the operation - const operationMetadata = body.metadata; - if (operationMetadata?.target) { - const getApiPathJson = _.get(component.properties, ["domain", "extra", "getApiPath"], ""); - if (getApiPathJson) { - const getApiPath = JSON.parse(getApiPathJson); - let getUrl = `${baseUrl}${getApiPath.path}`; - - // Replace {+name} or {name} with the target resource name - if (getUrl.includes("{+name}")) { - getUrl = getUrl.replace("{+name}", operationMetadata.target); - } else if (getUrl.includes("{name}")) { - getUrl = getUrl.replace("{name}", encodeURIComponent(operationMetadata.target)); - } - - const resourceResponse = await fetch(getUrl, { - method: "GET", - headers: { "Authorization": `Bearer ${token}` }, - }); - - if (resourceResponse.ok) { - return await resourceResponse.json(); - } - } - } - - console.warn("[GCP] Operation completed but couldn't fetch final resource"); return body; } }); - // Extract resource ID from the final resource - // For GKE and similar APIs using {+name}, we need the full resource path - // For Compute Engine style APIs using {name}, we need just the short name - const resourceId = extractResourceId(finalResource, usesFullResourcePath); - + // Extract resourceId from the operation result + const resourceId = extractResourceIdFromOperation(operationResult, resourceIdStyle); console.log(`[CREATE] Operation complete, resourceId: ${resourceId}`); + return { resourceId: resourceId ? resourceId.toString() : undefined, status: "ok", - payload: normalizeGcpResourceValues(finalResource), }; } // Handle synchronous response - const resourceId = extractResourceId(responseJson, usesFullResourcePath); + const resourceId = extractResourceId(responseJson, resourceIdStyle === "fullPath"); - if (resourceId) { - return { - resourceId: resourceId.toString(), - status: "ok", - payload: normalizeGcpResourceValues(responseJson), - }; - } else { - return { - status: "ok", - payload: normalizeGcpResourceValues(responseJson), - }; + return { + resourceId: resourceId ? resourceId.toString() : undefined, + status: "ok", + }; +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// Get location from component (domain only for create - no resource yet) +function getLocation(component: Input): string | undefined { + return _.get(component.properties, ["domain", "location"]) || + _.get(component.properties, ["domain", "zone"]) || + _.get(component.properties, ["domain", "region"]); +} + +// Resolve a parameter value from component properties (create uses domain only) +function resolveParamValue( + component: Input, + paramName: string, + projectId: string | undefined +): string | undefined { + if (paramName === "project" || paramName === "projectId") { + return projectId; } + + if (paramName === "parent") { + let parentValue = _.get(component.properties, ["domain", "parent"]); + if (!parentValue && projectId) { + const location = getLocation(component); + const supportsAutoConstruct = _.get(component.properties, ["domain", "extra", "supportsParentAutoConstruct"]) === "true"; + + if (supportsAutoConstruct && location) { + parentValue = `projects/${projectId}/locations/${location}`; + } + } + return parentValue; + } + + return _.get(component.properties, ["domain", paramName]); } -// Extract the resource ID from a GCP resource response -// For APIs using {+name} (like GKE), we need the full resource path from selfLink -// For APIs using {name} (like Compute Engine), we use the simple name/id -function extractResourceId(resource: any, useFullPath: boolean): string | undefined { - // For APIs using {+name}, extract the full path from selfLink - if (useFullPath && resource.selfLink && typeof resource.selfLink === "string") { - try { - const url = new URL(resource.selfLink); - const pathParts = url.pathname.split("/").filter(Boolean); - // Find "projects" and take everything from there +// Build URL by replacing path parameters using RFC 6570 URI templates +// For create, parameters not in path are added as query params (e.g., GCS project) +function buildUrlWithParams( + baseUrl: string, + apiPath: { path: string; parameterOrder?: string[] }, + component: Input, + projectId: string | undefined +): string { + let url = `${baseUrl}${apiPath.path}`; + const queryParams: string[] = []; + + if (apiPath.parameterOrder) { + for (const paramName of apiPath.parameterOrder) { + const paramValue = resolveParamValue(component, paramName, projectId); + + if (paramValue) { + if (url.includes(`{+${paramName}}`)) { + url = url.replace(`{+${paramName}}`, paramValue); + } else if (url.includes(`{${paramName}}`)) { + url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); + } else { + // Parameter not in path template - add as query parameter (e.g., GCS project) + queryParams.push(`${paramName}=${encodeURIComponent(paramValue)}`); + } + } + } + } + + if (queryParams.length > 0) { + url += (url.includes("?") ? "&" : "?") + queryParams.join("&"); + } + + return url; +} + +// Build LRO polling URL from operation response +function buildLroPollingUrl(baseUrl: string, responseJson: any, component: Input): string { + if (responseJson.selfLink) { + return responseJson.selfLink; + } + + // Extract API version from paths to construct polling URL + const insertApiPathJson = _.get(component.properties, ["domain", "extra", "insertApiPath"], ""); + const getApiPathJson = _.get(component.properties, ["domain", "extra", "getApiPath"], ""); + const pathJson = insertApiPathJson || getApiPathJson; + + let apiVersion = ""; + if (pathJson) { + const apiPath = JSON.parse(pathJson); + const versionMatch = apiPath.path?.match(/^(v\d+)\//); + if (versionMatch) { + apiVersion = versionMatch[1] + "/"; + } + } + + return `${baseUrl}${apiVersion}${responseJson.name}`; +} + +// Extract resourceId from LRO operation result +function extractResourceIdFromOperation(operation: any, resourceIdStyle: string): string | undefined { + // Check common locations for the created resource info + // 1. response field (modern APIs) + if (operation.response) { + return extractResourceId(operation.response, resourceIdStyle === "fullPath"); + } + + // 2. targetLink field (Compute Engine) + if (operation.targetLink) { + return extractResourceIdFromUrl(operation.targetLink, resourceIdStyle === "fullPath"); + } + + // 3. metadata.target (some APIs) + if (operation.metadata?.target) { + return operation.metadata.target; + } + + // 4. targetId (Compute Engine) + if (operation.targetId) { + return operation.targetId; + } + + return undefined; +} + +// Extract resourceId from a full URL +function extractResourceIdFromUrl(urlString: string, useFullPath: boolean): string | undefined { + try { + const url = new URL(urlString); + const pathParts = url.pathname.split("/").filter(Boolean); + + if (useFullPath) { const projectsIdx = pathParts.indexOf("projects"); if (projectsIdx !== -1) { return pathParts.slice(projectsIdx).join("/"); } - // Fallback: skip the version (v1, v1beta1, etc.) and return the rest const versionIdx = pathParts.findIndex(p => /^v\d/.test(p)); if (versionIdx !== -1 && versionIdx + 1 < pathParts.length) { return pathParts.slice(versionIdx + 1).join("/"); } - } catch { - // If URL parsing fails, fall through to name/id } + + return pathParts[pathParts.length - 1]; + } catch { + return undefined; + } +} + +// Extract resourceId from a GCP resource response +function extractResourceId(resource: any, useFullPath: boolean): string | undefined { + // For APIs using {+name}, extract the full path from selfLink + if (useFullPath && resource.selfLink && typeof resource.selfLink === "string") { + const extracted = extractResourceIdFromUrl(resource.selfLink, true); + if (extracted) return extracted; } // For Compute Engine style APIs or fallback, use simple name/id @@ -297,77 +283,26 @@ function extractResourceId(resource: any, useFullPath: boolean): string | undefi } async function getAccessToken(serviceAccountJson: string): Promise<{ token: string; projectId: string | undefined }> { - // Parse service account JSON to extract project_id (optional) let projectId: string | undefined; try { const serviceAccount = JSON.parse(serviceAccountJson); projectId = serviceAccount.project_id; } catch { - // If parsing fails or project_id is missing, continue without it projectId = undefined; } const activateResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "activate-service-account", - "--key-file=-", - "--quiet" - ], { - input: serviceAccountJson - }); + "auth", "activate-service-account", "--key-file=-", "--quiet" + ], { input: serviceAccountJson }); if (activateResult.exitCode !== 0) { throw new Error(`Failed to activate service account: ${activateResult.stderr}`); } - const tokenResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "print-access-token" - ]); - + const tokenResult = await siExec.waitUntilEnd("gcloud", ["auth", "print-access-token"]); if (tokenResult.exitCode !== 0) { throw new Error(`Failed to get access token: ${tokenResult.stderr}`); } - return { - token: tokenResult.stdout.trim(), - projectId, - }; -} - -// URL normalization for GCP resource values -const GCP_URL_PATTERN = /^https:\/\/[^/]*\.?googleapis\.com\//; -const LOCATION_SEGMENTS = new Set(["regions", "zones", "locations"]); - -function normalizeGcpResourceValues(obj: T): T { - if (obj === null || obj === undefined) return obj; - if (Array.isArray(obj)) return obj.map(item => normalizeGcpResourceValues(item)) as T; - if (typeof obj === "object") { - const normalized: Record = {}; - for (const [key, value] of Object.entries(obj)) { - if (typeof value === "string" && GCP_URL_PATTERN.test(value)) { - const pathParts = new URL(value).pathname.split("/").filter(Boolean); - if (pathParts.length >= 2 && LOCATION_SEGMENTS.has(pathParts[pathParts.length - 2])) { - normalized[key] = pathParts[pathParts.length - 1]; - } else { - const projectsIdx = pathParts.indexOf("projects"); - if (projectsIdx !== -1) { - normalized[key] = pathParts.slice(projectsIdx).join("/"); - } else { - // For non-project APIs (e.g., Storage), extract after API version (v1, v2, etc.) - const versionIdx = pathParts.findIndex(p => /^v\d+/.test(p)); - normalized[key] = versionIdx !== -1 && versionIdx + 1 < pathParts.length - ? pathParts.slice(versionIdx + 1).join("/") - : pathParts[pathParts.length - 1] || value; - } - } - } else if (typeof value === "object" && value !== null) { - normalized[key] = normalizeGcpResourceValues(value); - } else { - normalized[key] = value; - } - } - return normalized as T; - } - return obj; + return { token: tokenResult.stdout.trim(), projectId }; } diff --git a/bin/clover/src/pipelines/gcp/funcs/actions/delete.ts b/bin/clover/src/pipelines/gcp/funcs/actions/delete.ts index 203a668709..cb6c460fea 100644 --- a/bin/clover/src/pipelines/gcp/funcs/actions/delete.ts +++ b/bin/clover/src/pipelines/gcp/funcs/actions/delete.ts @@ -36,81 +36,13 @@ async function main(component: Input): Promise { const { token, projectId } = await getAccessToken(serviceAccountJson); - // Build the URL by replacing path parameters - let url: string; - - // If resourceId is already a full path matching the API structure, use it directly - // This handles cases where resourceId is "projects/xxx/datasets/yyy/tables/zzz" - if (isFullResourcePath(resourceId, deleteApiPath.path)) { - url = `${baseUrl}${resourceId}`; - } else { - url = `${baseUrl}${deleteApiPath.path}`; - - // Replace path parameters with values from resource_value or domain - // GCP APIs use RFC 6570 URI templates: {param} and {+param} (reserved expansion) - if (deleteApiPath.parameterOrder) { - for (const paramName of deleteApiPath.parameterOrder) { - let paramValue; - - // For the resource identifier, use resourceId - if (paramName === deleteApiPath.parameterOrder[deleteApiPath.parameterOrder.length - 1]) { - paramValue = resourceId; - } else if (paramName === "project" || paramName === "projectId") { - // Use extracted project_id for project/projectId parameter - paramValue = projectId; - } else if (paramName === "parent") { - // "parent" is a common GCP pattern: projects/{project}/locations/{location} - paramValue = _.get(component.properties, ["resource", "payload", "parent"]) || - _.get(component.properties, ["domain", "parent"]); - if (!paramValue && projectId) { - // Only auto-construct for project-only resources - // Multi-scope resources require explicit parent - const availableScopesJson = _.get(component.properties, ["domain", "extra", "availableScopes"]); - const availableScopes = availableScopesJson ? JSON.parse(availableScopesJson) : []; - const isProjectOnly = availableScopes.length === 1 && availableScopes[0] === "projects"; - - if (isProjectOnly) { - const location = _.get(component.properties, ["resource", "payload", "location"]) || - _.get(component.properties, ["domain", "location"]) || - _.get(component.properties, ["domain", "zone"]) || - _.get(component.properties, ["domain", "region"]); - if (location) { - paramValue = `projects/${projectId}/locations/${location}`; - } - } - } - } else { - paramValue = _.get(component.properties, ["resource", "payload", paramName]) || - _.get(component.properties, ["domain", paramName]); - - // GCP often returns full URLs for reference fields e.g. - // region: //www.googleapis.com/compute/v1/projects/myproject/regions/us-central1 - // network: //www.googleapis.com/compute/v1/projects/myproject/networks/my-network - - // Extract just the resource name from the URL - if (paramValue && typeof paramValue === "string" && paramValue.startsWith("https://")) { - const urlParts = paramValue.split("/"); - paramValue = urlParts[urlParts.length - 1]; - } - } - - if (paramValue) { - // Handle {+param} (reserved expansion - don't encode, allows slashes) - if (url.includes(`{+${paramName}}`)) { - url = url.replace(`{+${paramName}}`, paramValue); - } else { - // Handle {param} (simple expansion - encode) - url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); - } - } - } - } - } + // Build the URL + const url = buildUrlWithParams(baseUrl, deleteApiPath, component, projectId, { resourceId }); // Make the API request with retry logic const response = await siExec.withRetry(async () => { const resp = await fetch(url, { - method: httpMethod, // Usually DELETE, but some APIs use POST (e.g., deleteConnection) + method: httpMethod, headers: { "Authorization": `Bearer ${token}`, }, @@ -123,7 +55,6 @@ async function main(component: Input): Promise { } const errorText = await resp.text(); - const error = new Error(`Unable to delete resource; Called "${url}" API returned ${resp.status} ${resp.statusText}: @@ -137,39 +68,37 @@ ${errorText}` return resp; }, { - isRateLimitedFn: (error) => error.status === 429 - }).then((r) => r.result); + isRateLimitedFn: (error: any) => error.status === 429 + }).then((r: any) => r.result); // Handle 404 as success for delete operations if (response.status === 404) { - return { - status: "ok", - }; + return { status: "ok" }; } // Handle 204 No Content (common for successful deletes like GCS) if (response.status === 204) { - return { - status: "ok", - }; + return { status: "ok" }; } // Try to parse response body - some APIs return empty body on success const responseText = await response.text(); if (!responseText) { - return { - status: "ok", - }; + return { status: "ok" }; } const responseJson = JSON.parse(responseText); - // Handle Google Cloud Long-Running Operations (LRO) - // Check if this is an operation response: - // - Compute Engine uses "kind" containing "operation" - // - GKE/Container API uses "operationType" field - const isLRO = (responseJson.kind && responseJson.kind.includes("operation")) || - responseJson.operationType; + // Check if this resource uses Long-Running Operations based on metadata + const lroStyle = _.get(component.properties, ["domain", "extra", "lroStyle"], "none"); + + // Detect LRO response - only check if lroStyle indicates LRO support + const isLRO = lroStyle !== "none" && ( + (responseJson.kind && responseJson.kind.includes("operation")) || + responseJson.operationType || + (responseJson.name && responseJson.name.startsWith("operations/")) + ); + if (isLRO) { console.log(`[DELETE] LRO detected, polling for completion...`); @@ -183,10 +112,9 @@ ${errorText}` maxAttempts: 20, baseDelay: 2000, maxDelay: 30000, - isCompleteFn: (response, body) => body.status === "DONE", - isErrorFn: (response, body) => !!body.error, - extractResultFn: async (response, body) => { - // If operation has error, throw it + isCompleteFn: (_response: any, body: any) => body.status === "DONE" || body.done === true, + isErrorFn: (_response: any, body: any) => !!body.error, + extractResultFn: async (_response: any, body: any) => { if (body.error) { throw new Error(`Delete operation failed: ${JSON.stringify(body.error)}`); } @@ -197,64 +125,139 @@ ${errorText}` console.log(`[DELETE] Operation complete`); } - return { - status: "ok", - }; + return { status: "ok" }; +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// Get location from component, checking resource payload first then domain +function getLocation(component: Input): string | undefined { + return _.get(component.properties, ["resource", "payload", "location"]) || + _.get(component.properties, ["domain", "location"]) || + _.get(component.properties, ["domain", "zone"]) || + _.get(component.properties, ["domain", "region"]); +} + +// Resolve a parameter value from component properties +function resolveParamValue( + component: Input, + paramName: string, + projectId: string | undefined +): string | undefined { + if (paramName === "project" || paramName === "projectId") { + return projectId; + } + + if (paramName === "parent") { + let parentValue = _.get(component.properties, ["resource", "payload", "parent"]) || + _.get(component.properties, ["domain", "parent"]); + if (!parentValue && projectId) { + const location = getLocation(component); + const supportsAutoConstruct = _.get(component.properties, ["domain", "extra", "supportsParentAutoConstruct"]) === "true"; + + if (supportsAutoConstruct && location) { + parentValue = `projects/${projectId}/locations/${location}`; + } + } + return parentValue; + } + + let paramValue = _.get(component.properties, ["resource", "payload", paramName]) || + _.get(component.properties, ["domain", paramName]); + + // GCP often returns full URLs for reference fields - extract just the resource name + if (paramValue && typeof paramValue === "string" && paramValue.startsWith("https://")) { + const urlParts = paramValue.split("/"); + paramValue = urlParts[urlParts.length - 1]; + } + + return paramValue; +} + +// Check if resourceId is already a full path matching the API path structure +// Uses proper segment matching (not substring) to avoid false positives +function isFullResourcePath(resourceId: string, pathTemplate: string): boolean { + if (!resourceId.includes('/')) return false; + + const templateSegments = pathTemplate.split('/').filter(s => !s.startsWith('{')); + const resourceSegments = resourceId.split('/'); + let templateIdx = 0; + + for (const seg of resourceSegments) { + if (templateIdx < templateSegments.length && seg === templateSegments[templateIdx]) { + templateIdx++; + } + } + + return templateIdx === templateSegments.length; +} + +// Build URL by replacing path parameters using RFC 6570 URI templates +function buildUrlWithParams( + baseUrl: string, + apiPath: { path: string; parameterOrder?: string[] }, + component: Input, + projectId: string | undefined, + options: { resourceId?: string } = {} +): string { + // If resourceId is already a full path matching the API structure, use it directly + if (options.resourceId && isFullResourcePath(options.resourceId, apiPath.path)) { + return `${baseUrl}${options.resourceId}`; + } + + let url = `${baseUrl}${apiPath.path}`; + + if (apiPath.parameterOrder) { + const lastParam = apiPath.parameterOrder[apiPath.parameterOrder.length - 1]; + + for (const paramName of apiPath.parameterOrder) { + let paramValue: string | undefined; + + // For the resource identifier, use resourceId + if (options.resourceId && paramName === lastParam) { + paramValue = options.resourceId; + } else { + paramValue = resolveParamValue(component, paramName, projectId); + } + + if (paramValue) { + // Handle {+param} (reserved expansion - don't encode, allows slashes) + if (url.includes(`{+${paramName}}`)) { + url = url.replace(`{+${paramName}}`, paramValue); + } else if (url.includes(`{${paramName}}`)) { + // Handle {param} (simple expansion - encode) + url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); + } + } + } + } + + return url; } async function getAccessToken(serviceAccountJson: string): Promise<{ token: string; projectId: string | undefined }> { - // Parse service account JSON to extract project_id (optional) let projectId: string | undefined; try { const serviceAccount = JSON.parse(serviceAccountJson); projectId = serviceAccount.project_id; } catch { - // If parsing fails or project_id is missing, continue without it projectId = undefined; } const activateResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "activate-service-account", - "--key-file=-", - "--quiet" - ], { - input: serviceAccountJson - }); + "auth", "activate-service-account", "--key-file=-", "--quiet" + ], { input: serviceAccountJson }); if (activateResult.exitCode !== 0) { throw new Error(`Failed to activate service account: ${activateResult.stderr}`); } - const tokenResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "print-access-token" - ]); - + const tokenResult = await siExec.waitUntilEnd("gcloud", ["auth", "print-access-token"]); if (tokenResult.exitCode !== 0) { throw new Error(`Failed to get access token: ${tokenResult.stderr}`); } - return { - token: tokenResult.stdout.trim(), - projectId, - }; -} - -// Check if resourceId is already a full path matching the API path structure -function isFullResourcePath(resourceId: string, pathTemplate: string): boolean { - if (!resourceId.includes('/')) return false; - - // Extract static segments from template (non-parameter parts) - // e.g., "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}" -> ["projects", "datasets", "tables"] - const templateSegments = pathTemplate.split('/').filter(s => !s.startsWith('{')); - - // Check if resourceId contains these segments in order - let lastIdx = -1; - for (const seg of templateSegments) { - const idx = resourceId.indexOf(seg, lastIdx + 1); - if (idx === -1) return false; - lastIdx = idx; - } - return true; + return { token: tokenResult.stdout.trim(), projectId }; } diff --git a/bin/clover/src/pipelines/gcp/funcs/actions/refresh.ts b/bin/clover/src/pipelines/gcp/funcs/actions/refresh.ts index 4cdc23643d..08331809ef 100644 --- a/bin/clover/src/pipelines/gcp/funcs/actions/refresh.ts +++ b/bin/clover/src/pipelines/gcp/funcs/actions/refresh.ts @@ -23,7 +23,7 @@ async function main(component: Input): Promise { // Resolve a parameter value from component properties // forList: controls parent auto-construction behavior // - true (list operations): always auto-construct parent, fallback to projects/${projectId} -// - false (get operations): only auto-construct for project-only resources, require location +// - false (get operations): only auto-construct if supportsParentAutoConstruct is true function resolveParamValue( component: Input, paramName: string, @@ -38,23 +38,15 @@ function resolveParamValue( let parentValue = _.get(component.properties, ["resource", "payload", "parent"]) || _.get(component.properties, ["domain", "parent"]); if (!parentValue && projectId) { - const location = _.get(component.properties, ["resource", "payload", "location"]) || - _.get(component.properties, ["domain", "location"]) || - _.get(component.properties, ["domain", "zone"]) || - _.get(component.properties, ["domain", "region"]); + const location = getLocation(component); + const supportsAutoConstruct = _.get(component.properties, ["domain", "extra", "supportsParentAutoConstruct"]) === "true"; if (forList) { // List operations: always auto-construct, fallback to project-only parentValue = location ? `projects/${projectId}/locations/${location}` : `projects/${projectId}`; - } else { - // Get operations: only auto-construct for project-only resources with location - const availableScopesJson = _.get(component.properties, ["domain", "extra", "availableScopes"]); - const availableScopes = availableScopesJson ? JSON.parse(availableScopesJson) : []; - const isProjectOnly = availableScopes.length === 1 && availableScopes[0] === "projects"; - - if (isProjectOnly && location) { - parentValue = `projects/${projectId}/locations/${location}`; - } + } else if (supportsAutoConstruct && location) { + // Get/update/delete operations: only auto-construct if metadata says we can + parentValue = `projects/${projectId}/locations/${location}`; } } return parentValue; @@ -72,7 +64,16 @@ function resolveParamValue( return paramValue; } +// Get location from component, checking resource payload first then domain +function getLocation(component: Input): string | undefined { + return _.get(component.properties, ["resource", "payload", "location"]) || + _.get(component.properties, ["domain", "location"]) || + _.get(component.properties, ["domain", "zone"]) || + _.get(component.properties, ["domain", "region"]); +} + // Check if resourceId is already a full path matching the API path structure +// Uses proper segment matching (not substring) to avoid false positives function isFullResourcePath(resourceId: string, pathTemplate: string): boolean { if (!resourceId.includes('/')) return false; @@ -80,14 +81,18 @@ function isFullResourcePath(resourceId: string, pathTemplate: string): boolean { // e.g., "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}" -> ["projects", "datasets", "tables"] const templateSegments = pathTemplate.split('/').filter(s => !s.startsWith('{')); - // Check if resourceId contains these segments in order - let lastIdx = -1; - for (const seg of templateSegments) { - const idx = resourceId.indexOf(seg, lastIdx + 1); - if (idx === -1) return false; - lastIdx = idx; + // Check if resourceId contains these as actual path segments (bounded by /) + // This prevents false positives like "my-projects-datasets" matching "projects/datasets" + const resourceSegments = resourceId.split('/'); + let templateIdx = 0; + + for (const seg of resourceSegments) { + if (templateIdx < templateSegments.length && seg === templateSegments[templateIdx]) { + templateIdx++; + } } - return true; + + return templateIdx === templateSegments.length; } // Build URL by replacing path parameters using RFC 6570 URI templates diff --git a/bin/clover/src/pipelines/gcp/funcs/actions/update.ts b/bin/clover/src/pipelines/gcp/funcs/actions/update.ts index 69085323c4..6cbaf1eba6 100644 --- a/bin/clover/src/pipelines/gcp/funcs/actions/update.ts +++ b/bin/clover/src/pipelines/gcp/funcs/actions/update.ts @@ -14,14 +14,11 @@ async function main(component: Input): Promise { const currentResource = component.properties?.resource?.payload; // Filter to only changed fields (GCP PATCH requires this for some resources) - // Include fields if they're different from current resource OR if they're being set for the first time if (currentResource) { const changedFields: Record = {}; for (const [key, value] of Object.entries(updatePayload)) { - // Include field if: - // 1. It doesn't exist in current resource (new field being set) - // 2. OR it exists but has a different value + // Include field if it doesn't exist in current resource or has different value if (!(key in currentResource) || !_.isEqual(value, currentResource[key])) { changedFields[key] = value; } @@ -30,25 +27,15 @@ async function main(component: Input): Promise { updatePayload = changedFields; // GCP requires fingerprint for updates to prevent concurrent modifications - // Always include the fingerprint from the current resource if it exists if (currentResource.fingerprint) { updatePayload.fingerprint = currentResource.fingerprint; } } // Try to get update API path first, fall back to patch - let updateApiPathJson = _.get( - component.properties, - ["domain", "extra", "updateApiPath"], - "", - ); - + let updateApiPathJson = _.get(component.properties, ["domain", "extra", "updateApiPath"], ""); if (!updateApiPathJson) { - updateApiPathJson = _.get( - component.properties, - ["domain", "extra", "patchApiPath"], - "", - ); + updateApiPathJson = _.get(component.properties, ["domain", "extra", "patchApiPath"], ""); } if (!updateApiPathJson) { @@ -78,76 +65,10 @@ async function main(component: Input): Promise { const { token, projectId } = await getAccessToken(serviceAccountJson); - // Build the URL by replacing path parameters - let url: string; - - // If resourceId is already a full path matching the API structure, use it directly - // This handles cases where resourceId is "projects/xxx/datasets/yyy/tables/zzz" - if (isFullResourcePath(resourceId, updateApiPath.path)) { - url = `${baseUrl}${resourceId}`; - } else { - url = `${baseUrl}${updateApiPath.path}`; - - // Replace path parameters with values from resource_value or domain - // GCP APIs use RFC 6570 URI templates: {param} and {+param} (reserved expansion) - if (updateApiPath.parameterOrder) { - for (const paramName of updateApiPath.parameterOrder) { - let paramValue; - - // For the resource identifier, use resourceId - if (paramName === updateApiPath.parameterOrder[updateApiPath.parameterOrder.length - 1]) { - paramValue = resourceId; - } else if (paramName === "project" || paramName === "projectId") { - // Use extracted project_id for project/projectId parameter - paramValue = projectId; - } else if (paramName === "parent") { - // "parent" is a common GCP pattern: projects/{project}/locations/{location} - paramValue = _.get(component.properties, ["resource", "payload", "parent"]) || - _.get(component.properties, ["domain", "parent"]); - if (!paramValue && projectId) { - // Only auto-construct for project-only resources - // Multi-scope resources require explicit parent - const availableScopesJson = _.get(component.properties, ["domain", "extra", "availableScopes"]); - const availableScopes = availableScopesJson ? JSON.parse(availableScopesJson) : []; - const isProjectOnly = availableScopes.length === 1 && availableScopes[0] === "projects"; - - if (isProjectOnly) { - const location = _.get(component.properties, ["resource", "payload", "location"]) || - _.get(component.properties, ["domain", "location"]) || - _.get(component.properties, ["domain", "zone"]) || - _.get(component.properties, ["domain", "region"]); - if (location) { - paramValue = `projects/${projectId}/locations/${location}`; - } - } - } - } else { - paramValue = _.get(component.properties, ["resource", "payload", paramName]) || - _.get(component.properties, ["domain", paramName]); - - // GCP often returns full URLs for reference fields (e.g., region, zone, network) - // Extract just the resource name from the URL - if (paramValue && typeof paramValue === "string" && paramValue.startsWith("https://")) { - const urlParts = paramValue.split("/"); - paramValue = urlParts[urlParts.length - 1]; - } - } + // Build the URL + let url = buildUrlWithParams(baseUrl, updateApiPath, component, projectId, { resourceId }); - if (paramValue) { - // Handle {+param} (reserved expansion - don't encode, allows slashes) - if (url.includes(`{+${paramName}}`)) { - url = url.replace(`{+${paramName}}`, paramValue); - } else { - // Handle {param} (simple expansion - encode) - url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); - } - } - } - } - } - - // Many GCP APIs require or benefit from an updateMask query parameter - // that specifies which fields are being updated + // Add updateMask query parameter (GCP APIs require/benefit from specifying which fields are being updated) const updateFields = Object.keys(updatePayload).filter(k => k !== 'fingerprint'); if (updateFields.length > 0) { const updateMask = updateFields.join(','); @@ -167,7 +88,6 @@ async function main(component: Input): Promise { if (!resp.ok) { const errorText = await resp.text(); - const error = new Error(`Unable to update resource; Called "${url}" API returned ${resp.status} ${resp.statusText}: @@ -181,209 +101,198 @@ ${errorText}` return resp; }, { - isRateLimitedFn: (error) => error.status === 429 - }).then((r) => r.result); + isRateLimitedFn: (error: any) => error.status === 429 + }).then((r: any) => r.result); const responseJson = await response.json(); - // Handle Google Cloud Long-Running Operations (LRO) - // Check if this is an operation response: - // - Compute Engine uses "kind" containing "operation" - // - GKE/Container API uses "operationType" field - // - Other APIs (API Keys, etc.) use "name" starting with "operations/" - const isLRO = (responseJson.kind && responseJson.kind.includes("operation")) || + // Check if this resource uses Long-Running Operations based on metadata + const lroStyle = _.get(component.properties, ["domain", "extra", "lroStyle"], "none"); + + // Detect LRO response - only check if lroStyle indicates LRO support + const isLRO = lroStyle !== "none" && ( + (responseJson.kind && responseJson.kind.includes("operation")) || responseJson.operationType || - (responseJson.name && responseJson.name.startsWith("operations/")); + (responseJson.name && responseJson.name.startsWith("operations/")) + ); + if (isLRO) { console.log(`[UPDATE] LRO detected, polling for completion...`); - // Use selfLink or construct URL from operation name - // For APIs that don't provide selfLink, we need to construct the URL - // The API version prefix (v1, v2, etc.) comes from the API paths - let pollingUrl = responseJson.selfLink; - if (!pollingUrl && responseJson.name) { - // Extract version from one of the API paths (e.g., "v2/{+parent}/keys" -> "v2") - const insertApiPathJson = _.get(component.properties, ["domain", "extra", "insertApiPath"], ""); - const getApiPathJson = _.get(component.properties, ["domain", "extra", "getApiPath"], ""); - const pathJson = insertApiPathJson || getApiPathJson; - let apiVersion = ""; - if (pathJson) { - const apiPath = JSON.parse(pathJson); - const versionMatch = apiPath.path?.match(/^(v\d+)\//); - if (versionMatch) { - apiVersion = versionMatch[1] + "/"; - } - } - pollingUrl = `${baseUrl}${apiVersion}${responseJson.name}`; - } + const pollingUrl = buildLroPollingUrl(baseUrl, responseJson, component); - // Poll the operation until it completes using new siExec.pollLRO - const finalResource = await siExec.pollLRO({ + // Poll until complete - don't fetch final resource, let refresh handle that + await siExec.pollLRO({ url: pollingUrl, headers: { "Authorization": `Bearer ${token}` }, maxAttempts: 20, baseDelay: 2000, maxDelay: 30000, - isCompleteFn: (response: any, body: any) => body.status === "DONE" || body.done === true, - isErrorFn: (response: any, body: any) => !!body.error, - extractResultFn: async (response: any, body: any) => { - // If operation has error, throw it + isCompleteFn: (_response: any, body: any) => body.status === "DONE" || body.done === true, + isErrorFn: (_response: any, body: any) => !!body.error, + extractResultFn: async (_response: any, body: any) => { if (body.error) { - throw new Error(`Operation failed: ${JSON.stringify(body.error)}`); + throw new Error(`Update operation failed: ${JSON.stringify(body.error)}`); } + return body; + } + }); - // Some operations include the updated resource in the response field - if (body.response) { - return body.response; - } + console.log(`[UPDATE] Operation complete`); + } - // For update operations, fetch the final resource from targetLink - if (body.targetLink) { - const resourceResponse = await fetch(body.targetLink, { - method: "GET", - headers: { "Authorization": `Bearer ${token}` }, - }); + // Return success - refresh will fetch the final resource state + return { status: "ok" }; +} - if (!resourceResponse.ok) { - throw new Error(`Failed to fetch final resource: ${resourceResponse.status}`); - } +// ============================================================================ +// Helper Functions +// ============================================================================ - return await resourceResponse.json(); - } +// Get location from component, checking resource payload first then domain +function getLocation(component: Input): string | undefined { + return _.get(component.properties, ["resource", "payload", "location"]) || + _.get(component.properties, ["domain", "location"]) || + _.get(component.properties, ["domain", "zone"]) || + _.get(component.properties, ["domain", "region"]); +} - // Fallback: Use resourceId with getApiPath to fetch final resource - // This handles APIs like API Keys that don't provide targetLink or response - const getApiPathJson = _.get(component.properties, ["domain", "extra", "getApiPath"], ""); - if (getApiPathJson && resourceId) { - const getApiPath = JSON.parse(getApiPathJson); - let getUrl = `${baseUrl}${getApiPath.path}`; - - // Replace {+name} or {name} with resourceId - if (getUrl.includes("{+name}")) { - getUrl = getUrl.replace("{+name}", resourceId); - } else if (getUrl.includes("{name}")) { - getUrl = getUrl.replace("{name}", encodeURIComponent(resourceId)); - } - - const resourceResponse = await fetch(getUrl, { - method: "GET", - headers: { "Authorization": `Bearer ${token}` }, - }); - - if (resourceResponse.ok) { - return await resourceResponse.json(); - } - } +// Resolve a parameter value from component properties +function resolveParamValue( + component: Input, + paramName: string, + projectId: string | undefined +): string | undefined { + if (paramName === "project" || paramName === "projectId") { + return projectId; + } - console.warn("[GCP] Operation completed but couldn't fetch final resource"); - return body; + if (paramName === "parent") { + let parentValue = _.get(component.properties, ["resource", "payload", "parent"]) || + _.get(component.properties, ["domain", "parent"]); + if (!parentValue && projectId) { + const location = getLocation(component); + const supportsAutoConstruct = _.get(component.properties, ["domain", "extra", "supportsParentAutoConstruct"]) === "true"; + + if (supportsAutoConstruct && location) { + parentValue = `projects/${projectId}/locations/${location}`; } - }); + } + return parentValue; + } - console.log(`[UPDATE] Operation complete`); - return { - payload: normalizeGcpResourceValues(finalResource), - status: "ok", - }; + let paramValue = _.get(component.properties, ["resource", "payload", paramName]) || + _.get(component.properties, ["domain", paramName]); + + // GCP often returns full URLs for reference fields - extract just the resource name + if (paramValue && typeof paramValue === "string" && paramValue.startsWith("https://")) { + const urlParts = paramValue.split("/"); + paramValue = urlParts[urlParts.length - 1]; } - // Handle synchronous response - return { - payload: normalizeGcpResourceValues(responseJson), - status: "ok", - }; + return paramValue; +} + +// Check if resourceId is already a full path matching the API path structure +function isFullResourcePath(resourceId: string, pathTemplate: string): boolean { + if (!resourceId.includes('/')) return false; + + const templateSegments = pathTemplate.split('/').filter(s => !s.startsWith('{')); + const resourceSegments = resourceId.split('/'); + let templateIdx = 0; + + for (const seg of resourceSegments) { + if (templateIdx < templateSegments.length && seg === templateSegments[templateIdx]) { + templateIdx++; + } + } + + return templateIdx === templateSegments.length; +} + +// Build URL by replacing path parameters using RFC 6570 URI templates +function buildUrlWithParams( + baseUrl: string, + apiPath: { path: string; parameterOrder?: string[] }, + component: Input, + projectId: string | undefined, + options: { resourceId?: string } = {} +): string { + if (options.resourceId && isFullResourcePath(options.resourceId, apiPath.path)) { + return `${baseUrl}${options.resourceId}`; + } + + let url = `${baseUrl}${apiPath.path}`; + + if (apiPath.parameterOrder) { + const lastParam = apiPath.parameterOrder[apiPath.parameterOrder.length - 1]; + + for (const paramName of apiPath.parameterOrder) { + let paramValue: string | undefined; + + if (options.resourceId && paramName === lastParam) { + paramValue = options.resourceId; + } else { + paramValue = resolveParamValue(component, paramName, projectId); + } + + if (paramValue) { + if (url.includes(`{+${paramName}}`)) { + url = url.replace(`{+${paramName}}`, paramValue); + } else if (url.includes(`{${paramName}}`)) { + url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); + } + } + } + } + + return url; +} + +// Build LRO polling URL from operation response +function buildLroPollingUrl(baseUrl: string, responseJson: any, component: Input): string { + if (responseJson.selfLink) { + return responseJson.selfLink; + } + + // Extract API version from paths to construct polling URL + const insertApiPathJson = _.get(component.properties, ["domain", "extra", "insertApiPath"], ""); + const getApiPathJson = _.get(component.properties, ["domain", "extra", "getApiPath"], ""); + const pathJson = insertApiPathJson || getApiPathJson; + + let apiVersion = ""; + if (pathJson) { + const apiPath = JSON.parse(pathJson); + const versionMatch = apiPath.path?.match(/^(v\d+)\//); + if (versionMatch) { + apiVersion = versionMatch[1] + "/"; + } + } + + return `${baseUrl}${apiVersion}${responseJson.name}`; } async function getAccessToken(serviceAccountJson: string): Promise<{ token: string; projectId: string | undefined }> { - // Parse service account JSON to extract project_id (optional) let projectId: string | undefined; try { const serviceAccount = JSON.parse(serviceAccountJson); projectId = serviceAccount.project_id; } catch { - // If parsing fails or project_id is missing, continue without it projectId = undefined; } const activateResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "activate-service-account", - "--key-file=-", - "--quiet" - ], { - input: serviceAccountJson - }); + "auth", "activate-service-account", "--key-file=-", "--quiet" + ], { input: serviceAccountJson }); if (activateResult.exitCode !== 0) { throw new Error(`Failed to activate service account: ${activateResult.stderr}`); } - const tokenResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "print-access-token" - ]); - + const tokenResult = await siExec.waitUntilEnd("gcloud", ["auth", "print-access-token"]); if (tokenResult.exitCode !== 0) { throw new Error(`Failed to get access token: ${tokenResult.stderr}`); } - return { - token: tokenResult.stdout.trim(), - projectId, - }; -} - -// URL normalization for GCP resource values -const GCP_URL_PATTERN = /^https:\/\/[^/]*\.?googleapis\.com\//; -const LOCATION_SEGMENTS = new Set(["regions", "zones", "locations"]); - -function normalizeGcpResourceValues(obj: T): T { - if (obj === null || obj === undefined) return obj; - if (Array.isArray(obj)) return obj.map(item => normalizeGcpResourceValues(item)) as T; - if (typeof obj === "object") { - const normalized: Record = {}; - for (const [key, value] of Object.entries(obj)) { - if (typeof value === "string" && GCP_URL_PATTERN.test(value)) { - const pathParts = new URL(value).pathname.split("/").filter(Boolean); - if (pathParts.length >= 2 && LOCATION_SEGMENTS.has(pathParts[pathParts.length - 2])) { - normalized[key] = pathParts[pathParts.length - 1]; - } else { - const projectsIdx = pathParts.indexOf("projects"); - if (projectsIdx !== -1) { - normalized[key] = pathParts.slice(projectsIdx).join("/"); - } else { - // For non-project APIs (e.g., Storage), extract after API version (v1, v2, etc.) - const versionIdx = pathParts.findIndex(p => /^v\d+/.test(p)); - normalized[key] = versionIdx !== -1 && versionIdx + 1 < pathParts.length - ? pathParts.slice(versionIdx + 1).join("/") - : pathParts[pathParts.length - 1] || value; - } - } - } else if (typeof value === "object" && value !== null) { - normalized[key] = normalizeGcpResourceValues(value); - } else { - normalized[key] = value; - } - } - return normalized as T; - } - return obj; -} - -// Check if resourceId is already a full path matching the API path structure -function isFullResourcePath(resourceId: string, pathTemplate: string): boolean { - if (!resourceId.includes('/')) return false; - - // Extract static segments from template (non-parameter parts) - // e.g., "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}" -> ["projects", "datasets", "tables"] - const templateSegments = pathTemplate.split('/').filter(s => !s.startsWith('{')); - - // Check if resourceId contains these segments in order - let lastIdx = -1; - for (const seg of templateSegments) { - const idx = resourceId.indexOf(seg, lastIdx + 1); - if (idx === -1) return false; - lastIdx = idx; - } - return true; + return { token: tokenResult.stdout.trim(), projectId }; } diff --git a/bin/clover/src/pipelines/gcp/funcs/management/discover.ts b/bin/clover/src/pipelines/gcp/funcs/management/discover.ts index 9f6b8620bd..6c44244292 100644 --- a/bin/clover/src/pipelines/gcp/funcs/management/discover.ts +++ b/bin/clover/src/pipelines/gcp/funcs/management/discover.ts @@ -2,12 +2,7 @@ async function main({ thisComponent }: Input): Promise { const component = thisComponent; // Get API path metadata from domain.extra - const listApiPathJson = _.get( - component.properties, - ["domain", "extra", "listApiPath"], - "", - ); - + const listApiPathJson = _.get(component.properties, ["domain", "extra", "listApiPath"], ""); if (!listApiPathJson) { return { status: "error", @@ -15,13 +10,7 @@ async function main({ thisComponent }: Input): Promise { }; } - const listApiPath = JSON.parse(listApiPathJson); - const getApiPathJson = _.get( - component.properties, - ["domain", "extra", "getApiPath"], - "", - ); - + const getApiPathJson = _.get(component.properties, ["domain", "extra", "getApiPath"], ""); if (!getApiPathJson) { return { status: "error", @@ -29,13 +18,10 @@ async function main({ thisComponent }: Input): Promise { }; } + const listApiPath = JSON.parse(listApiPathJson); const getApiPath = JSON.parse(getApiPathJson); const baseUrl = _.get(component.properties, ["domain", "extra", "baseUrl"], ""); - const gcpResourceType = _.get( - component.properties, - ["domain", "extra", "GcpResourceType"], - "", - ); + const gcpResourceType = _.get(component.properties, ["domain", "extra", "GcpResourceType"], ""); // Get authentication token const serviceAccountJson = requestStorage.getEnv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); @@ -48,15 +34,88 @@ async function main({ thisComponent }: Input): Promise { console.log(`Discovering ${gcpResourceType} resources...`); // Build refinement filter from domain properties - const refinement = _.cloneDeep(thisComponent.properties.domain); + const refinement = buildRefinementFilter(thisComponent.properties.domain); + + // Build list URL and fetch all resources with pagination + const listUrl = buildListUrl(baseUrl, listApiPath, component, projectId); + const resources = await fetchAllResources(listUrl, token); + + console.log(`Found ${resources.length} resources`); + + // Process each resource + const create: Output["ops"]["create"] = {}; + const actions: Record = {}; + let importCount = 0; + + for (const resource of resources) { + const resourceId = resource.name || resource.id || resource.selfLink; + if (!resourceId) { + console.log(`Skipping resource without ID`); + continue; + } + + console.log(`Importing ${resourceId}`); + + // Fetch full resource details + const fullResource = await fetchResourceDetails( + baseUrl, getApiPath, component, projectId, token, resource, resourceId + ); + + if (!fullResource) { + console.log(`Failed to fetch ${resourceId}, skipping`); + continue; + } + + const properties = { + si: { resourceId }, + domain: { + ...component.properties?.domain || {}, + ...fullResource, + }, + resource: fullResource, + }; + + // Apply refinement filter + if (_.isEmpty(refinement) || _.isMatch(properties.domain, refinement)) { + const newAttributes: Output["ops"]["create"][string]["attributes"] = {}; + for (const [skey, svalue] of Object.entries(component.sources || {})) { + newAttributes[skey] = { $source: svalue }; + } + + create[resourceId] = { + kind: gcpResourceType || component.properties?.domain?.extra?.GcpResourceType, + properties, + attributes: newAttributes, + }; + actions[resourceId] = { remove: ["create"] }; + importCount++; + } else { + console.log(`Skipping import of ${resourceId}; it did not match refinements`); + } + } + + return { + status: "ok", + message: `Discovered ${importCount} ${gcpResourceType} resources`, + ops: { create, actions }, + }; +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// Build refinement filter from domain properties +function buildRefinementFilter(domain: any): Record { + const refinement = _.cloneDeep(domain); delete refinement["extra"]; - // Remove any empty values, as they are never refinements + for (const [key, value] of Object.entries(refinement)) { if (_.isEmpty(value)) { delete refinement[key]; } else if (_.isPlainObject(value)) { refinement[key] = _.pickBy( - value, + value as Record, (v) => !_.isEmpty(v) || _.isNumber(v) || _.isBoolean(v), ); if (_.isEmpty(refinement[key])) { @@ -65,80 +124,95 @@ async function main({ thisComponent }: Input): Promise { } } - // Build list URL by replacing path parameters - let listUrl = `${baseUrl}${listApiPath.path}`; + return refinement; +} + +// Get location from component +function getLocation(component: any): string | undefined { + return _.get(component.properties, ["domain", "location"]) || + _.get(component.properties, ["domain", "zone"]) || + _.get(component.properties, ["domain", "region"]); +} + +// Resolve parameter value for list operations (discovery) +function resolveListParamValue( + component: any, + paramName: string, + projectId: string | undefined +): string | undefined { + if (paramName === "project" || paramName === "projectId") { + return projectId; + } + + if (paramName === "parent") { + let parentValue = _.get(component.properties, ["domain", "parent"]); + if (!parentValue && projectId) { + const location = getLocation(component); + // For discovery, always auto-construct parent (fallback to project-only) + parentValue = location + ? `projects/${projectId}/locations/${location}` + : `projects/${projectId}`; + } + return parentValue; + } + + return _.get(component.properties, ["domain", paramName]); +} + +// Build list URL by replacing path parameters +function buildListUrl( + baseUrl: string, + listApiPath: { path: string; parameterOrder?: string[] }, + component: any, + projectId: string | undefined +): string { + let url = `${baseUrl}${listApiPath.path}`; const queryParams: string[] = []; if (listApiPath.parameterOrder) { for (const paramName of listApiPath.parameterOrder) { - let paramValue; - - if (paramName === "project") { - paramValue = projectId; - } else if (paramName === "parent") { - // Try explicit parent first, otherwise auto-construct from projectId - paramValue = _.get(component.properties, ["domain", "parent"]); - if (!paramValue && projectId) { - // Check if resource requires location in the parent path - const location = _.get(component.properties, ["domain", "location"]) || - _.get(component.properties, ["domain", "zone"]) || - _.get(component.properties, ["domain", "region"]); - - if (location) { - // Resource uses location: projects/{project}/locations/{location} - paramValue = `projects/${projectId}/locations/${location}`; - } else { - // Resource doesn't use location: projects/{project} - // This works for both project-only and multi-scope resources - // (multi-scope resources default to project scope for discovery) - paramValue = `projects/${projectId}`; - } - } - } else { - paramValue = _.get(component.properties, ["domain", paramName]); - } + const paramValue = resolveListParamValue(component, paramName, projectId); if (paramValue) { - if (listUrl.includes(`{+${paramName}}`)) { - listUrl = listUrl.replace(`{+${paramName}}`, paramValue); - } else if (listUrl.includes(`{${paramName}}`)) { - listUrl = listUrl.replace(`{${paramName}}`, encodeURIComponent(paramValue)); + if (url.includes(`{+${paramName}}`)) { + url = url.replace(`{+${paramName}}`, paramValue); + } else if (url.includes(`{${paramName}}`)) { + url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); } } } } - // Handle parent as query parameter for APIs like Resource Manager Folders - // that don't use parent in the path but require it as a query parameter - if (!listUrl.includes("parent=") && !listApiPath.path.includes("{parent}") && !listApiPath.path.includes("{+parent}")) { + // Handle parent as query parameter for some APIs + if (!url.includes("parent=") && !listApiPath.path.includes("{parent}") && !listApiPath.path.includes("{+parent}")) { const parentValue = _.get(component.properties, ["domain", "parent"]); if (parentValue) { queryParams.push(`parent=${encodeURIComponent(parentValue)}`); } } - // Append query parameters if (queryParams.length > 0) { - listUrl += (listUrl.includes("?") ? "&" : "?") + queryParams.join("&"); + url += (url.includes("?") ? "&" : "?") + queryParams.join("&"); } - // Handle pagination with pageToken + return url; +} + +// Fetch all resources with pagination +async function fetchAllResources(listUrl: string, token: string): Promise { let resources: any[] = []; let nextPageToken: string | null = null; do { let currentUrl = listUrl; if (nextPageToken) { - const separator = listUrl.includes("?") ? "&" : "?"; - currentUrl = `${listUrl}${separator}pageToken=${encodeURIComponent(nextPageToken)}`; + currentUrl += (listUrl.includes("?") ? "&" : "?") + `pageToken=${encodeURIComponent(nextPageToken)}`; } - const listResponse = await siExec.withRetry(async () => { + const response = await siExec.withRetry(async () => { const resp = await fetch(currentUrl, { method: "GET", - headers: { - "Authorization": `Bearer ${token}`, - }, + headers: { "Authorization": `Bearer ${token}` }, }); if (!resp.ok) { @@ -151,18 +225,14 @@ async function main({ thisComponent }: Input): Promise { return resp; }, { - isRateLimitedFn: (error) => error.status === 429 - }).then((r) => r.result); + isRateLimitedFn: (error: any) => error.status === 429 + }).then((r: any) => r.result); - const listData = await listResponse.json(); + const listData = await response.json(); - // GCP list responses vary in structure: - // - Compute Engine uses "items" array - // - Other APIs use the plural resource name (e.g., "contacts", "clusters", "buckets") - // Try "items" first, then look for any array property that isn't metadata + // GCP list responses vary - find the array containing resources let items = listData.items; if (!items) { - // Find the first array property that likely contains resources for (const [key, value] of Object.entries(listData)) { if (Array.isArray(value) && key !== "unreachable" && key !== "warnings") { items = value; @@ -170,8 +240,8 @@ async function main({ thisComponent }: Input): Promise { } } } - items = items || []; - resources = resources.concat(items); + + resources = resources.concat(items || []); nextPageToken = listData.nextPageToken || null; if (nextPageToken) { @@ -179,161 +249,91 @@ async function main({ thisComponent }: Input): Promise { } } while (nextPageToken); - console.log(`Found ${resources.length} resources`); - - const create: Output["ops"]["create"] = {}; - const actions = {}; - let importCount = 0; - - for (const resource of resources) { - // The resource ID is typically in the "name" or "id" field - const resourceId = resource.name || resource.id || resource.selfLink; - - if (!resourceId) { - console.log(`Skipping resource without ID`); - continue; - } - - console.log(`Importing ${resourceId}`); - - // Build the get URL to fetch full resource details - let getUrl = `${baseUrl}${getApiPath.path}`; - - if (getApiPath.parameterOrder) { - for (const paramName of getApiPath.parameterOrder) { - let paramValue; - - // For the resource identifier, use resourceId - if (paramName === getApiPath.parameterOrder[getApiPath.parameterOrder.length - 1]) { - paramValue = resourceId; - } else if (paramName === "project") { - paramValue = projectId; - } else { - paramValue = _.get(resource, [paramName]) || - _.get(component.properties, ["domain", paramName]); - } + return resources; +} - if (paramValue) { - // {+param} = reserved expansion (no encoding, allows slashes) - // {param} = simple expansion (URL encoded) - if (getUrl.includes(`{+${paramName}}`)) { - getUrl = getUrl.replace(`{+${paramName}}`, paramValue); - } else if (getUrl.includes(`{${paramName}}`)) { - getUrl = getUrl.replace(`{${paramName}}`, encodeURIComponent(paramValue)); - } - } +// Fetch full resource details +async function fetchResourceDetails( + baseUrl: string, + getApiPath: { path: string; parameterOrder?: string[] }, + component: any, + projectId: string | undefined, + token: string, + resource: any, + resourceId: string +): Promise { + let url = `${baseUrl}${getApiPath.path}`; + + if (getApiPath.parameterOrder) { + const lastParam = getApiPath.parameterOrder[getApiPath.parameterOrder.length - 1]; + + for (const paramName of getApiPath.parameterOrder) { + let paramValue: string | undefined; + + if (paramName === lastParam) { + paramValue = resourceId; + } else if (paramName === "project" || paramName === "projectId") { + paramValue = projectId; + } else { + paramValue = _.get(resource, [paramName]) || + _.get(component.properties, ["domain", paramName]); } - } - // Fetch the full resource details with retry - let resourceResponse; - try { - resourceResponse = await siExec.withRetry(async () => { - const resp = await fetch(getUrl, { - method: "GET", - headers: { - "Authorization": `Bearer ${token}`, - }, - }); - - if (!resp.ok) { - const error = new Error(`Failed to fetch ${resourceId}`) as any; - error.status = resp.status; - throw error; + if (paramValue) { + if (url.includes(`{+${paramName}}`)) { + url = url.replace(`{+${paramName}}`, paramValue); + } else if (url.includes(`{${paramName}}`)) { + url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); } - - return resp; - }, { - isRateLimitedFn: (error) => error.status === 429 - }).then((r) => r.result); - } catch (error) { - console.log(`Failed to fetch ${resourceId} after retries, skipping`); - continue; + } } + } - const fullResource = await resourceResponse.json(); - - const properties = { - si: { - resourceId, - }, - domain: { - ...component.properties?.domain || {}, - ...fullResource, - }, - resource: fullResource, - }; + try { + const response = await siExec.withRetry(async () => { + const resp = await fetch(url, { + method: "GET", + headers: { "Authorization": `Bearer ${token}` }, + }); - // Apply refinement filter - if (_.isEmpty(refinement) || _.isMatch(properties.domain, refinement)) { - const newAttributes: Output["ops"]["create"][string]["attributes"] = {}; - for (const [skey, svalue] of Object.entries(component.sources || {})) { - newAttributes[skey] = { - $source: svalue, - }; + if (!resp.ok) { + const error = new Error(`Failed to fetch ${resourceId}`) as any; + error.status = resp.status; + throw error; } - create[resourceId] = { - kind: gcpResourceType || component.properties?.domain?.extra?.GcpResourceType, - properties, - attributes: newAttributes, - }; - actions[resourceId] = { - remove: ["create"], - }; - importCount++; - } else { - console.log( - `Skipping import of ${resourceId}; it did not match refinements`, - ); - } - } + return resp; + }, { + isRateLimitedFn: (error: any) => error.status === 429 + }).then((r: any) => r.result); - return { - status: "ok", - message: `Discovered ${importCount} ${gcpResourceType} resources`, - ops: { - create, - actions, - }, - }; + return await response.json(); + } catch { + return null; + } } async function getAccessToken(serviceAccountJson: string): Promise<{ token: string; projectId: string | undefined }> { - // Parse service account JSON to extract project_id (optional) let projectId: string | undefined; try { const serviceAccount = JSON.parse(serviceAccountJson); projectId = serviceAccount.project_id; } catch { - // If parsing fails or project_id is missing, continue without it projectId = undefined; } const activateResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "activate-service-account", - "--key-file=-", - "--quiet" - ], { - input: serviceAccountJson - }); + "auth", "activate-service-account", "--key-file=-", "--quiet" + ], { input: serviceAccountJson }); if (activateResult.exitCode !== 0) { throw new Error(`Failed to activate service account: ${activateResult.stderr}`); } - const tokenResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "print-access-token" - ]); - + const tokenResult = await siExec.waitUntilEnd("gcloud", ["auth", "print-access-token"]); if (tokenResult.exitCode !== 0) { throw new Error(`Failed to get access token: ${tokenResult.stderr}`); } - return { - token: tokenResult.stdout.trim(), - projectId, - }; + return { token: tokenResult.stdout.trim(), projectId }; } diff --git a/bin/clover/src/pipelines/gcp/funcs/management/import.ts b/bin/clover/src/pipelines/gcp/funcs/management/import.ts index aef560ba7f..b09371b697 100644 --- a/bin/clover/src/pipelines/gcp/funcs/management/import.ts +++ b/bin/clover/src/pipelines/gcp/funcs/management/import.ts @@ -1,7 +1,7 @@ async function main({ thisComponent }: Input): Promise { const component = thisComponent.properties; const resourcePayload = _.get(component, ["resource", "payload"], ""); - let resourceId = _.get(component, ["si", "resourceId"]); + const resourceId = _.get(component, ["si", "resourceId"]); if (!resourceId) { return { @@ -11,12 +11,7 @@ async function main({ thisComponent }: Input): Promise { } // Get API path metadata from domain.extra - const getApiPathJson = _.get( - component, - ["domain", "extra", "getApiPath"], - "", - ); - + const getApiPathJson = _.get(component, ["domain", "extra", "getApiPath"], ""); if (!getApiPathJson) { return { status: "error", @@ -35,59 +30,14 @@ async function main({ thisComponent }: Input): Promise { const { token, projectId } = await getAccessToken(serviceAccountJson); - // Build the URL by replacing path parameters - let url = `${baseUrl}${getApiPath.path}`; - - // Replace path parameters with values from resource_value or domain - if (getApiPath.parameterOrder) { - for (const paramName of getApiPath.parameterOrder) { - let paramValue; - - // For the resource identifier, use resourceId - if (paramName === getApiPath.parameterOrder[getApiPath.parameterOrder.length - 1]) { - paramValue = resourceId; - } else if (paramName === "project") { - paramValue = projectId; - } else if (paramName === "parent") { - // Try explicit parent first, otherwise auto-construct for project-only resources - paramValue = _.get(component, ["domain", "parent"]); - if (!paramValue && projectId) { - const availableScopesJson = _.get(component, ["domain", "extra", "availableScopes"]); - const availableScopes = availableScopesJson ? JSON.parse(availableScopesJson) : []; - const isProjectOnly = availableScopes.length === 1 && availableScopes[0] === "projects"; - - if (isProjectOnly) { - const location = _.get(component, ["domain", "location"]) || - _.get(component, ["domain", "zone"]) || - _.get(component, ["domain", "region"]); - if (location) { - paramValue = `projects/${projectId}/locations/${location}`; - } - } - } - } else { - paramValue = _.get(component, ["domain", paramName]); - } - - if (paramValue) { - // {+param} = reserved expansion (no encoding, allows slashes) - // {param} = simple expansion (URL encoded) - if (url.includes(`{+${paramName}}`)) { - url = url.replace(`{+${paramName}}`, paramValue); - } else if (url.includes(`{${paramName}}`)) { - url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); - } - } - } - } + // Build the URL + const url = buildUrlWithParams(baseUrl, getApiPath, component, projectId, resourceId); // Make the API request with retry logic const response = await siExec.withRetry(async () => { const resp = await fetch(url, { method: "GET", - headers: { - "Authorization": `Bearer ${token}`, - }, + headers: { "Authorization": `Bearer ${token}` }, }); if (!resp.ok) { @@ -102,8 +52,8 @@ async function main({ thisComponent }: Input): Promise { return resp; }, { - isRateLimitedFn: (error) => error.status === 429 - }).then((r) => r.result); + isRateLimitedFn: (error: any) => error.status === 429 + }).then((r: any) => r.result); const resourceProperties = await response.json(); console.log(resourceProperties); @@ -127,9 +77,7 @@ async function main({ thisComponent }: Input): Promise { const ops = { update: { - self: { - properties, - }, + self: { properties }, }, actions: { self: { @@ -152,43 +100,101 @@ async function main({ thisComponent }: Input): Promise { }; } +// ============================================================================ +// Helper Functions +// ============================================================================ + +// Get location from component +function getLocation(component: any): string | undefined { + return _.get(component, ["domain", "location"]) || + _.get(component, ["domain", "zone"]) || + _.get(component, ["domain", "region"]); +} + +// Resolve a parameter value from component properties +function resolveParamValue( + component: any, + paramName: string, + projectId: string | undefined +): string | undefined { + if (paramName === "project" || paramName === "projectId") { + return projectId; + } + + if (paramName === "parent") { + let parentValue = _.get(component, ["domain", "parent"]); + if (!parentValue && projectId) { + const location = getLocation(component); + const supportsAutoConstruct = _.get(component, ["domain", "extra", "supportsParentAutoConstruct"]) === "true"; + + if (supportsAutoConstruct && location) { + parentValue = `projects/${projectId}/locations/${location}`; + } + } + return parentValue; + } + + return _.get(component, ["domain", paramName]); +} + +// Build URL by replacing path parameters +function buildUrlWithParams( + baseUrl: string, + apiPath: { path: string; parameterOrder?: string[] }, + component: any, + projectId: string | undefined, + resourceId: string +): string { + let url = `${baseUrl}${apiPath.path}`; + + if (apiPath.parameterOrder) { + const lastParam = apiPath.parameterOrder[apiPath.parameterOrder.length - 1]; + + for (const paramName of apiPath.parameterOrder) { + let paramValue: string | undefined; + + if (paramName === lastParam) { + paramValue = resourceId; + } else { + paramValue = resolveParamValue(component, paramName, projectId); + } + + if (paramValue) { + if (url.includes(`{+${paramName}}`)) { + url = url.replace(`{+${paramName}}`, paramValue); + } else if (url.includes(`{${paramName}}`)) { + url = url.replace(`{${paramName}}`, encodeURIComponent(paramValue)); + } + } + } + } + + return url; +} + async function getAccessToken(serviceAccountJson: string): Promise<{ token: string; projectId: string | undefined }> { - // Parse service account JSON to extract project_id (optional) let projectId: string | undefined; try { const serviceAccount = JSON.parse(serviceAccountJson); projectId = serviceAccount.project_id; } catch { - // If parsing fails or project_id is missing, continue without it projectId = undefined; } const activateResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "activate-service-account", - "--key-file=-", - "--quiet" - ], { - input: serviceAccountJson - }); + "auth", "activate-service-account", "--key-file=-", "--quiet" + ], { input: serviceAccountJson }); if (activateResult.exitCode !== 0) { throw new Error(`Failed to activate service account: ${activateResult.stderr}`); } - const tokenResult = await siExec.waitUntilEnd("gcloud", [ - "auth", - "print-access-token" - ]); - + const tokenResult = await siExec.waitUntilEnd("gcloud", ["auth", "print-access-token"]); if (tokenResult.exitCode !== 0) { throw new Error(`Failed to get access token: ${tokenResult.stderr}`); } - return { - token: tokenResult.stdout.trim(), - projectId, - }; + return { token: tokenResult.stdout.trim(), projectId }; } // URL normalization for GCP resource values @@ -210,7 +216,6 @@ function normalizeGcpResourceValues(obj: T): T { if (projectsIdx !== -1) { normalized[key] = pathParts.slice(projectsIdx).join("/"); } else { - // For non-project APIs (e.g., Storage), extract after API version (v1, v2, etc.) const versionIdx = pathParts.findIndex(p => /^v\d+/.test(p)); normalized[key] = versionIdx !== -1 && versionIdx + 1 < pathParts.length ? pathParts.slice(versionIdx + 1).join("/") diff --git a/bin/clover/src/pipelines/gcp/pipeline-steps/addDefaultProps.ts b/bin/clover/src/pipelines/gcp/pipeline-steps/addDefaultProps.ts index 50488a587d..c6f2dd7504 100644 --- a/bin/clover/src/pipelines/gcp/pipeline-steps/addDefaultProps.ts +++ b/bin/clover/src/pipelines/gcp/pipeline-steps/addDefaultProps.ts @@ -188,6 +188,65 @@ export function addDefaultProps(specs: ExpandedPkgSpec[]): ExpandedPkgSpec[] { extraProp.entries.push(listOnlyProp); } + // resourceIdStyle: determines how to interpret resourceId values + // "fullPath" = APIs using {+name} need full resource path (e.g., "projects/x/instances/y") + // "simpleName" = APIs using {name} need just the resource name (e.g., "my-instance") + { + const usesFullPath = gcpSchema.methods.get?.path?.includes("{+"); + const resourceIdStyleProp = createScalarProp( + "resourceIdStyle", + "string", + extraProp.metadata.propPath, + false, + ); + resourceIdStyleProp.data.hidden = true; + resourceIdStyleProp.data.defaultValue = usesFullPath + ? "fullPath" + : "simpleName"; + extraProp.entries.push(resourceIdStyleProp); + } + + // supportsParentAutoConstruct: whether parent can be auto-built from projectId + location + // Only project-only resources support this (multi-scope resources need explicit parent) + { + const isProjectOnly = gcpSchema.availableScopes?.length === 1 && + gcpSchema.availableScopes[0] === "projects"; + const supportsParentAutoConstructProp = createScalarProp( + "supportsParentAutoConstruct", + "string", + extraProp.metadata.propPath, + false, + ); + supportsParentAutoConstructProp.data.hidden = true; + supportsParentAutoConstructProp.data.defaultValue = isProjectOnly + ? "true" + : "false"; + extraProp.entries.push(supportsParentAutoConstructProp); + } + + // lroStyle: how to detect and handle Long Running Operations + // "compute" = Compute Engine style (kind contains "operation", status: "DONE") + // "modern" = Modern APIs (name starts with "operations/", done: true) + // "none" = Synchronous operations (no LRO handling needed) + { + const lroStyleProp = createScalarProp( + "lroStyle", + "string", + extraProp.metadata.propPath, + false, + ); + lroStyleProp.data.hidden = true; + // Default to "compute" for resources with insert/update/delete methods + // as most GCP APIs that need LRO handling use Compute Engine style + // The "modern" style is less common and harder to detect at build time + const hasModifyingMethods = gcpSchema.methods.insert || + gcpSchema.methods.update || + gcpSchema.methods.patch || + gcpSchema.methods.delete; + lroStyleProp.data.defaultValue = hasModifyingMethods ? "compute" : "none"; + extraProp.entries.push(lroStyleProp); + } + // For global-only resources, set the location prop to default "global" // This allows auto-construction of parent path without user input if (gcpSchema.isGlobalOnly) { diff --git a/bin/clover/src/pipelines/gcp/spec.ts b/bin/clover/src/pipelines/gcp/spec.ts index 5586404bb1..c6bae6cb01 100644 --- a/bin/clover/src/pipelines/gcp/spec.ts +++ b/bin/clover/src/pipelines/gcp/spec.ts @@ -314,7 +314,7 @@ function buildGcpResourceSpec( if (listResponse.properties) { // Find the array property that contains the resource items for ( - const [propName, propDef] of Object.entries(listResponse.properties) + const [_, propDef] of Object.entries(listResponse.properties) ) { if (propDef.type === "array" && propDef.items) { resourceSchema = propDef.items;