diff --git a/tools/postman/Makefile b/tools/postman/Makefile index c9635bf112..3987886940 100644 --- a/tools/postman/Makefile +++ b/tools/postman/Makefile @@ -1,6 +1,7 @@ export OPENAPI_FOLDER=./openapi export TMP_FOLDER=./tmp export FULL_OPENAPI_FOLDER=../../openapi/v2/ +ENV_FILE = ./local.env default: build @@ -44,3 +45,16 @@ compare_and_send_forks: compare_forks send_forks .PHONY: clean clean: rm ./openapi/*; rm ./tmp/* + +# Command to load local environment variables from file +load-env: + @if [ -f $(ENV_FILE) ]; then \ + echo "Loading environment variables from $(ENV_FILE)..."; \ + for var in $$(grep -v '^#' $(ENV_FILE) | cut -d= -f1); do \ + echo "Loaded: $$var"; \ + done; \ + export $$(grep -v '^#' $(ENV_FILE) | xargs); \ + else \ + echo "Error: $(ENV_FILE) file not found."; \ + exit 1; \ + fi diff --git a/tools/postman/README.md b/tools/postman/README.md index 88a3d4e8a7..2e78671ab7 100644 --- a/tools/postman/README.md +++ b/tools/postman/README.md @@ -47,3 +47,41 @@ flowchart TD for upload. 5. **Upload Collection to Postman**: Use the Postman API to upload the Collection to Postman. + +## Local setup and validation + +### Prerequisites + +To push to your own workspace via Postman API, [create a private workspace](https://learning.postman.com/docs/collaborating-in-postman/using-workspaces/use-workspaces/) and retrieve the workspace id + +Then, [create a Postman API Key](https://learning.postman.com/docs/developer/postman-api/authentication/) that has access to this workspace + +### Env var setup + +To test the scripts locally, you should set up the required environment variables in `tools/postman/scripts` by creating a `local.env` file populated with the following (at the minimum): + +``` +BASE_URL="" +WORKSPACE_ID="" +POSTMAN_API_KEY="" +``` + +Run `make load-env` to set your required and overriding env vars. + +The OpenAPI path for Postman generation and configured feature flags can also be overriden based on provided env vars. + +### Running locally + +Once env vars are configured, the setup scripts can be run locally using the Make following commands: +- `make fetch_openapi` +- `make convert_to_collection` +- `make transform_collection` +- `make upload_collection` + + +### Limitations + +Our Postman collection generation has several limitations, meaning some manual user actions may be necessary during setup: + +- Only a single content-type header is supported generation, meaning users may have to manually update this header if the API supports more than one; see docs links on endpoint level +- Only a single auth scheme is supported during generation, meaning users may have to update their auth on the collection level or endpoint level if not selecting the default \ No newline at end of file diff --git a/tools/postman/scripts/convert-to-collection.sh b/tools/postman/scripts/convert-to-collection.sh index dec4db894f..e739532358 100755 --- a/tools/postman/scripts/convert-to-collection.sh +++ b/tools/postman/scripts/convert-to-collection.sh @@ -21,7 +21,7 @@ jq 'del(.. | select(type == "object") | .pattern?)' "$OPENAPI_FOLDER"/"$OPENAPI_ echo "Installing openapi-to-postmanv2" npm install -echo "Converting from OpenAPI to PostmanV2" +echo "Converting $OPENAPI_FOLDER/$OPENAPI_FILE_NAME from OpenAPI to PostmanV2" ./node_modules/.bin/openapi2postmanv2 -s "$TMP_FOLDER"/tmp.json -o "$TMP_FOLDER"/"$COLLECTION_FILE_NAME" -O folderStrategy=Tags rm "$TMP_FOLDER"/tmp.json diff --git a/tools/postman/scripts/transform-for-api.sh b/tools/postman/scripts/transform-for-api.sh index 3b96404e5b..74e062fe31 100755 --- a/tools/postman/scripts/transform-for-api.sh +++ b/tools/postman/scripts/transform-for-api.sh @@ -9,10 +9,11 @@ set -euo pipefail # OPENAPI_FILE_NAME - name of the openapi specification file # OPENAPI_FOLDER - folder where openapi file is saved # TMP_FOLDER - folder for temporary files during transformations -# TOGGLE_USE_ENVIRONMENT_AUTH - bool for if auth variables are stored at the environment or collection level -# TOGGLE_INCLUDE_BODY - bool for if generated bodies should be removed or kept # VERSION_FILE_NAME - name of the file where the current version is stored # DESCRIPTION_FILE - name for the markdown description file +# TOGGLE_INCLUDE_BODY - bool for if generated bodies should be removed or kept +# TOGGLE_ADD_DOCS_LINKS - updates requests with corresponding docs links +# TOKEN_URL_ENV - client credentials auth path to set at the environment level, will not be set if unpopulated # BASE_URL - the default base url the Postman Collection will use ######################################################### @@ -22,17 +23,17 @@ OPENAPI_FILE_NAME=${OPENAPI_FILE_NAME:-"atlas-api.json"} OPENAPI_FOLDER=${OPENAPI_FOLDER:-"../openapi"} TMP_FOLDER=${TMP_FOLDER:-"../tmp"} VERSION_FILE_NAME=${VERSION_FILE_NAME:-"version.txt"} - DESCRIPTION_FILE=${DESCRIPTION_FILE:-"../collection-description.md"} -TOGGLE_USE_ENVIRONMENT_AUTH=${TOGGLE_USE_ENVIRONMENT_AUTH:-true} TOGGLE_INCLUDE_BODY=${TOGGLE_INCLUDE_BODY:-true} +TOGGLE_ADD_DOCS_LINKS=${TOGGLE_ADD_DOCS_LINKS:-true} +TOKEN_URL_ENV=${TOKEN_URL_ENV:-""} current_api_revision=$(<"$OPENAPI_FOLDER/$VERSION_FILE_NAME") pushd "${TMP_FOLDER}" -echo "Wrapping Collection in \"collection\" tag" +echo "Wrapping Collection $COLLECTION_FILE_NAME in \"collection\" tag" jq '{"collection": .}' "$COLLECTION_FILE_NAME" > intermediateCollectionWrapped.json echo "Disabling query params by default" @@ -47,48 +48,52 @@ jq 'del(.collection.info._postman_id)' \ echo "Removing circular references" sed 's/\\"value\\": \\""]* detected>\\"//g' intermediateCollectionNoPostmanID.json > intermediateCollectionNoCircular.json -echo "Updating name with version" +echo "Updating name with version $current_api_revision" jq --arg api_version "$current_api_revision" \ '.collection.info.name = ("MongoDB Atlas Administration API " + $api_version)' \ intermediateCollectionNoCircular.json > intermediateCollectionWithName.json -echo "Adding Collection description" +echo "Adding Collection description to $DESCRIPTION_FILE" description=$(<"$DESCRIPTION_FILE") jq --arg desc "$description" \ '.collection.info.description.content = $desc' \ intermediateCollectionWithName.json > intermediateCollectionWithDescription.json -echo "Updating baseUrl" +echo "Adding baseUrl env $BASE_URL" jq --arg base_url "$BASE_URL" \ '.collection.variable[0].value = $base_url' \ intermediateCollectionWithDescription.json > intermediateCollectionWithBaseURL.json -echo "Adding links to docs" -cp intermediateCollectionWithBaseURL.json intermediateCollectionWithLinks.json +if [ "$TOGGLE_ADD_DOCS_LINKS" = "true" ]; then + echo "Adding links to docs for each request" + cp intermediateCollectionWithBaseURL.json intermediateCollectionWithLinks.json -# Store all paths to requests. The summary field is the same as the title in the collection -paths=$(jq 'path(.. | objects | select(has("summary"))) | @sh' "$OLDPWD"/"$OPENAPI_FOLDER"/"$OPENAPI_FILE_NAME") -declare -a paths_array="($paths)" + # Store all paths to requests. The summary field is the same as the title in the collection + paths=$(jq 'path(.. | objects | select(has("summary"))) | @sh' "$OLDPWD"/"$OPENAPI_FOLDER"/"$OPENAPI_FILE_NAME") + declare -a paths_array="($paths)" -for path in "${paths_array[@]}"; do - declare -a single_path_array="($path)" - path_json=$(jq -n '$ARGS.positional' --args "${single_path_array[@]}") + for path in "${paths_array[@]}"; do + declare -a single_path_array="($path)" + path_json=$(jq -n '$ARGS.positional' --args "${single_path_array[@]}") - # Use the path to get all the information about this request without searching - requestInfo=$(jq --argjson path "$path_json" 'getpath($path)' "$OLDPWD"/"$OPENAPI_FOLDER"/"$OPENAPI_FILE_NAME") + # Use the path to get all the information about this request without searching + requestInfo=$(jq --argjson path "$path_json" 'getpath($path)' "$OLDPWD"/"$OPENAPI_FOLDER"/"$OPENAPI_FILE_NAME") - title=$(echo "$requestInfo" | jq -r '.summary') - operationId=$(echo "$requestInfo" | jq -r '.operationId') - tag=$(echo "$requestInfo" | jq -r '.tags[0]' | tr " " "-") + title=$(echo "$requestInfo" | jq -r '.summary') + operationId=$(echo "$requestInfo" | jq -r '.operationId') + tag=$(echo "$requestInfo" | jq -r '.tags[0]' | tr " " "-") - url="https://mongodb.com/docs/atlas/reference/api-resources-spec/v2/#tag/${tag}/operation/$operationId" + url="https://mongodb.com/docs/atlas/reference/api-resources-spec/v2/#tag/${tag}/operation/$operationId" - # Search the collection for the request with the matching name. Add the link to its description - jq --arg title "$title" --arg url "$url" \ - 'first(.collection.item[].item[].request | select(.name == $title).description.content) += "\n\nFind out more at " + $url' \ - intermediateCollectionWithLinks.json > tmp.json && mv tmp.json intermediateCollectionWithLinks.json + # Search the collection for the request with the matching name. Add the link to its description + jq --arg title "$title" --arg url "$url" \ + 'first(.collection.item[].item[].request | select(.name == $title).description.content) += "\n\nFind out more at " + $url' \ + intermediateCollectionWithLinks.json > tmp.json && mv tmp.json intermediateCollectionWithLinks.json -done + done +else + cp intermediateCollectionWithBaseURL.json intermediateCollectionWithLinks.json +fi # Togglable features if [ "$TOGGLE_INCLUDE_BODY" = "false" ]; then @@ -107,11 +112,10 @@ else cp intermediateCollectionWithLinks.json intermediateCollectionPostBody.json fi -if [ "$TOGGLE_USE_ENVIRONMENT_AUTH" = "false" ]; then - echo "Adding auth variables" - jq '.collection.variable += [{"key": "digestAuthUsername", "value": ""}, - {"key": "digestAuthPassword", "value": ""}, - {"key": "realm", "value": ""}]' intermediateCollectionPostBody.json > "$COLLECTION_TRANSFORMED_FILE_NAME" +if [ "$TOKEN_URL_ENV" != "" ]; then + echo "Adding client credentials auth url variable $TOKEN_URL_ENV" + jq --arg token_url "$TOKEN_URL_ENV" '.collection.variable += [{"key": "clientCredentialsTokenUrl", "value": $token_url}]' \ + intermediateCollectionPostBody.json > "$COLLECTION_TRANSFORMED_FILE_NAME" else cp intermediateCollectionPostBody.json "$COLLECTION_TRANSFORMED_FILE_NAME" fi diff --git a/tools/postman/scripts/upload-collection.sh b/tools/postman/scripts/upload-collection.sh index 7ba3c4dfe8..bad672eecd 100755 --- a/tools/postman/scripts/upload-collection.sh +++ b/tools/postman/scripts/upload-collection.sh @@ -28,6 +28,9 @@ pushd "${OPENAPI_FOLDER}" current_collection_name="MongoDB Atlas Administration API ${current_api_revision}" echo "Fetching list of current collections" +echo "curl -o ${COLLECTIONS_LIST_FILE} + --location 'https://api.getpostman.com/collections?workspace=${WORKSPACE_ID}' + --header 'X-API-Key: **********'" curl --show-error --fail --silent -o "${COLLECTIONS_LIST_FILE}" \ --location "https://api.getpostman.com/collections?workspace=${WORKSPACE_ID}" \ --header "X-API-Key: ${POSTMAN_API_KEY}" @@ -37,15 +40,25 @@ collection_exists=$(jq '.collections | any(.name=="'"${current_collection_name}" if [ "$collection_exists" = "false" ]; then # Create new collection echo "Creating new remote collection ${current_collection_name}" + echo "curl -o ${COLLECTIONS_LIST_FILE} + --location 'https://api.getpostman.com/collections?workspace=${WORKSPACE_ID}' + --header 'Content-Type: application/json' + --header 'X-API-Key: **********' + --data ${collection_transformed_path}" curl --show-error --fail --retry 5 --retry-all-errors --silent \ --location "https://api.getpostman.com/collections?workspace=${WORKSPACE_ID}" \ --header "Content-Type: application/json" \ --header "X-API-Key: ${POSTMAN_API_KEY}" \ - --data "@${collection_transformed_path}" + --data "@${collection_transformed_path}" \ else # Find collection ID and update collection echo "Updating remote collection ${current_collection_name}" + echo "curl --request PUT + --location 'https://api.getpostman.com/collections/${collection_id}' + --header 'Content-Type: application/json' + --header 'X-API-Key: **********' + --data ${collection_transformed_path}" collection_id=$(jq -r '.collections | map(select(.name=="'"${current_collection_name}"'").id)[0]' "${COLLECTIONS_LIST_FILE}") curl --show-error --fail --retry 5 --retry-all-errors --silent --request PUT \ --location "https://api.getpostman.com/collections/${collection_id}" \