diff --git a/docs/api/post-v1-determineversion.md b/docs/api/post-v1-determineversion.md index 38df4bcebcd..b88e56c237d 100644 --- a/docs/api/post-v1-determineversion.md +++ b/docs/api/post-v1-determineversion.md @@ -37,7 +37,7 @@ To confirm if the package you are interested in can be versioned by the determin 1. All available package information can be found [here](https://storage.googleapis.com/osv-indexer-configs). 2. You can look up your specific package using a url in the form `https://storage.googleapis.com/osv-indexer-configs/generated/{your-package}.yaml` For example, if you are interested in the library `protobuf`, you can find information for it at [`https://storage.googleapis.com/osv-indexer-configs/generated/protobuf.yaml`](https://storage.googleapis.com/osv-indexer-configs/generated/protobuf.yaml). -3. You can use [gsutil](https://cloud.google.com/storage/docs/gsutil) to copy everything: `gsutil -m cp -r gs://osv-indexer-configs/ .` +3. You can use [gcloud storage](https://cloud.google.com/sdk/gcloud/reference/storage) to copy everything: `gcloud storage cp -r gs://osv-indexer-configs/ .` ## Try the API with our tool diff --git a/docs/data.md b/docs/data.md index 6159e4f5bbd..3865e437b8b 100644 --- a/docs/data.md +++ b/docs/data.md @@ -124,7 +124,7 @@ E.g. for PyPI vulnerabilities: ```bash # Or download over HTTP via https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip -gsutil cp gs://osv-vulnerabilities/PyPI/all.zip . +gcloud storage cp gs://osv-vulnerabilities/PyPI/all.zip . ``` ### Downloading recent changes diff --git a/gcp/workers/cron/generate_sitemap/generate_and_upload.sh b/gcp/workers/cron/generate_sitemap/generate_and_upload.sh index 03f48bb1534..e14e1c5d368 100755 --- a/gcp/workers/cron/generate_sitemap/generate_and_upload.sh +++ b/gcp/workers/cron/generate_sitemap/generate_and_upload.sh @@ -14,4 +14,4 @@ echo "Begin sitemap generation for $BASE_URL_PATH" echo "Begin Syncing with cloud to $OUTPUT_BUCKET" -gsutil -m rsync -c -d $SITEMAP_OUTPUT "gs://$OUTPUT_BUCKET/" +gcloud storage rsync $SITEMAP_OUTPUT "gs://$OUTPUT_BUCKET/" --delete-unmatched-destination-objects -c diff --git a/vulnfeeds/cmd/alpine/run_alpine_convert.sh b/vulnfeeds/cmd/alpine/run_alpine_convert.sh index 106e7e86112..ddc4f293925 100755 --- a/vulnfeeds/cmd/alpine/run_alpine_convert.sh +++ b/vulnfeeds/cmd/alpine/run_alpine_convert.sh @@ -16,5 +16,5 @@ rm -rf $OSV_PARTS_OUTPUT && mkdir -p $OSV_PARTS_OUTPUT ./alpine-osv echo "Begin Syncing with cloud" -gsutil -q -m rsync -c -d $OSV_PARTS_OUTPUT "gs://$OUTPUT_BUCKET/$OSV_PARTS_OUTPUT" +gcloud storage rsync $OSV_PARTS_OUTPUT "gs://$OUTPUT_BUCKET/$OSV_PARTS_OUTPUT" --delete-unmatched-destination-objects -q -c echo "Successfully synced with cloud" diff --git a/vulnfeeds/cmd/combine-to-osv/README.md b/vulnfeeds/cmd/combine-to-osv/README.md index 79829b823d4..ab08d852dbb 100644 --- a/vulnfeeds/cmd/combine-to-osv/README.md +++ b/vulnfeeds/cmd/combine-to-osv/README.md @@ -43,6 +43,6 @@ This statically overrides the record generated, meaning any and all of the input #### Procedure -1. `gsutil cp gs://cve-osv-conversion/osv-output/CVE-YYYY-NNNN.json` +1. `gcloud storage cp gs://cve-osv-conversion/osv-output/CVE-YYYY-NNNN.json` 2. manually edit the file -3. `gsutil cp gs://cve-osv-conversion/osv-output-overrides/CVE-YYYY-NNNN.json` +3. `gcloud storage cp gs://cve-osv-conversion/osv-output-overrides/CVE-YYYY-NNNN.json` diff --git a/vulnfeeds/cmd/combine-to-osv/run_combine_to_osv_convert.sh b/vulnfeeds/cmd/combine-to-osv/run_combine_to_osv_convert.sh index c2ed9274efc..8363ae6ccdf 100755 --- a/vulnfeeds/cmd/combine-to-osv/run_combine_to_osv_convert.sh +++ b/vulnfeeds/cmd/combine-to-osv/run_combine_to_osv_convert.sh @@ -49,5 +49,5 @@ echo "Override" gcloud --no-user-output-enabled storage rsync "gs://${INPUT_BUCKET}/osv-output-overrides/" $OSV_OUTPUT echo "Begin syncing output to GCS bucket ${OUTPUT_BUCKET}" -gsutil -q -m rsync -c -d "${OSV_OUTPUT}" "gs://${OUTPUT_BUCKET}/osv-output/" +gcloud storage rsync "${OSV_OUTPUT}" "gs://${OUTPUT_BUCKET}/osv-output/" --delete-unmatched-destination-objects -q -c echo "Successfully synced to GCS bucket" diff --git a/vulnfeeds/cmd/cpe-repo-gen/cpe-repo-gen_map.sh b/vulnfeeds/cmd/cpe-repo-gen/cpe-repo-gen_map.sh index 056de54c86c..97d256ad24a 100755 --- a/vulnfeeds/cmd/cpe-repo-gen/cpe-repo-gen_map.sh +++ b/vulnfeeds/cmd/cpe-repo-gen/cpe-repo-gen_map.sh @@ -38,7 +38,7 @@ unzip -o "${WORK_DIR}/cpe.zip" -d "${CPE_JSON_DIR}" MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="" if [[ -n "${DEBIAN_COPYRIGHT_GCS_PATH}" ]]; then - gsutil ${BE_VERBOSE="-q"} cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}" + gcloud storage cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}" ${BE_VERBOSE="-q"} tar -C "${WORK_DIR}" -xf "${WORK_DIR}/$(basename ${DEBIAN_COPYRIGHT_GCS_PATH})" MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="--debian_metadata_path ${WORK_DIR}/metadata.ftp-master.debian.org" fi @@ -49,4 +49,4 @@ fi --output_dir="${WORK_DIR}" -gsutil ${BE_VERBOSE="-q"} cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}" +gcloud storage cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}" ${BE_VERBOSE="-q"} diff --git a/vulnfeeds/cmd/cve-bulk-converter/run-cvelist-converter.sh b/vulnfeeds/cmd/cve-bulk-converter/run-cvelist-converter.sh index 66dbb58287a..2875c336ea7 100755 --- a/vulnfeeds/cmd/cve-bulk-converter/run-cvelist-converter.sh +++ b/vulnfeeds/cmd/cve-bulk-converter/run-cvelist-converter.sh @@ -64,19 +64,19 @@ find "${LOCAL_OUT_DIR}" -type f -name \*.json \ # Copy (and remove any missing) results to GCS bucket, with some sanity # checking. -objs_present=$(gsutil ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) -objs_deleted=$(gsutil -m rsync -c -n -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) +objs_present=$(gcloud storage ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) +objs_deleted=$(gcloud storage rsync --dry-run --delete-unmatched-destination-objects -c "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) threshold=$(echo "scale=2; ${objs_present} * (${SAFETY_THRESHOLD_PCT:-2} / 100)" | bc) # # Bash can't deal with floats if (( $(echo "${objs_deleted} > ${threshold}" | bc -l) )); then echo "Aborting. Unexpectedly high (${objs_deleted}) number of CVE records would be deleted!" >> /dev/stderr - gsutil -m rsync -c -n -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr + gcloud storage rsync --dry-run --delete-unmatched-destination-objects -c "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr exit 1 fi echo "Copying CVEList records successfully converted to GCS bucket" -gsutil -q -m rsync -c -d "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" +gcloud storage rsync -q --delete-unmatched-destination-objects -c "${LOCAL_OUT_DIR}/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" echo "Conversion run complete" diff --git a/vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror.sh index bbc6d0c8fc3..dfd3ddbba78 100755 --- a/vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror.sh +++ b/vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror.sh @@ -33,4 +33,4 @@ python debian-copyright-mirror.py "${WORK_DIR}/metadata.ftp-master.debian.org/ch tar -C "${WORK_DIR}" -cf "${WORK_DIR}/$(basename ${GCS_PATH})" . -gsutil ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" +gcloud storage cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}" ${BE_VERBOSE="-q"} diff --git a/vulnfeeds/cmd/debian/run_debian_convert.sh b/vulnfeeds/cmd/debian/run_debian_convert.sh index 72b1b8942d4..95bb89ceda9 100755 --- a/vulnfeeds/cmd/debian/run_debian_convert.sh +++ b/vulnfeeds/cmd/debian/run_debian_convert.sh @@ -16,5 +16,5 @@ rm -rf $OSV_PARTS_OUTPUT && mkdir -p $OSV_PARTS_OUTPUT ./debian-osv echo "Begin Syncing with cloud, GCS bucket: ${OUTPUT_BUCKET}" -gsutil -q -m rsync -c -d $OSV_PARTS_OUTPUT "gs://$OUTPUT_BUCKET/$OSV_PARTS_OUTPUT" +gcloud storage rsync $OSV_PARTS_OUTPUT "gs://$OUTPUT_BUCKET/$OSV_PARTS_OUTPUT" --delete-unmatched-destination-objects -q -c echo "Successfully synced with cloud" diff --git a/vulnfeeds/cmd/download-cves/mirror_nvd.sh b/vulnfeeds/cmd/download-cves/mirror_nvd.sh index 7fa814315ff..d271e2b61d3 100755 --- a/vulnfeeds/cmd/download-cves/mirror_nvd.sh +++ b/vulnfeeds/cmd/download-cves/mirror_nvd.sh @@ -35,4 +35,4 @@ mkdir -p "${WORK_DIR}/nvd" echo "Copying files to GCS bucket" gcloud config set storage/parallel_composite_upload_enabled True -gcloud --no-user-output-enabled storage rsync "${WORK_DIR}/nvd/" "gs://${BUCKET}/nvd/" --checksums-only -c --delete-unmatched-destination-objects -q +gcloud --no-user-output-enabled storage rsync "${WORK_DIR}/nvd/" "gs://${BUCKET}/nvd/" --checksums-only -c --delete-unmatched-destination-objects -q -J diff --git a/vulnfeeds/cmd/nvd-cve-osv/run_cve_to_osv_generation.sh b/vulnfeeds/cmd/nvd-cve-osv/run_cve_to_osv_generation.sh index 58b43e51201..74e58969f7d 100755 --- a/vulnfeeds/cmd/nvd-cve-osv/run_cve_to_osv_generation.sh +++ b/vulnfeeds/cmd/nvd-cve-osv/run_cve_to_osv_generation.sh @@ -68,19 +68,19 @@ done # Copy (and remove any missing) results to GCS bucket, with some sanity # checking. -objs_present=$(gsutil ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) -objs_deleted=$(gsutil -m rsync -c -n -d "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) +objs_present=$(gcloud storage ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) +objs_deleted=$(gcloud storage rsync --dry-run --delete-unmatched-destination-objects -c "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) threshold=$(echo "scale=2; ${objs_present} * (${SAFETY_THRESHOLD_PCT:-2} / 100)" | bc) # Bash can't deal with floats if (( $(echo "${objs_deleted} > ${threshold}" | bc -l) )); then echo "Warning. Unexpectedly high (${objs_deleted}) number of CVE records would be deleted!" >> /dev/stderr - gsutil -m rsync -c -n -d "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr - # TODO: add back in once nvd-mirror issue fixed: exit 1 + gcloud storage rsync --dry-run --delete-unmatched-destination-objects -c "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr + exit 1 fi echo "Copying NVD CVE records successfully converted to GCS bucket" -gsutil -q -m rsync -c "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" +gcloud storage rsync -q --delete-unmatched-destination-objects -c "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" echo "Conversion run complete" diff --git a/vulnfeeds/pypi/cloudbuild.yaml b/vulnfeeds/pypi/cloudbuild.yaml index e07547058d2..ef4eb12fdac 100644 --- a/vulnfeeds/pypi/cloudbuild.yaml +++ b/vulnfeeds/pypi/cloudbuild.yaml @@ -25,5 +25,6 @@ steps: args: - -c - bq query --max_rows=10000000 --format=json --nouse_legacy_sql < vulnfeeds/pypi/pypi_versions.sql > pypi_versions.json -- name: gcr.io/cloud-builders/gsutil - args: ['cp', 'pypi_links.json', 'pypi_versions.json', 'gs://pypa-advisory-db/triage/'] +- name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' + entrypoint: 'gcloud' + args: ['storage', 'cp', 'pypi_links.json', 'pypi_versions.json', 'gs://pypa-advisory-db/triage/'] diff --git a/vulnfeeds/tools/debian/run_convert_debian.sh b/vulnfeeds/tools/debian/run_convert_debian.sh index 8a44235a31d..8ba9d7448b7 100755 --- a/vulnfeeds/tools/debian/run_convert_debian.sh +++ b/vulnfeeds/tools/debian/run_convert_debian.sh @@ -44,7 +44,7 @@ poetry run python3 convert_debian.py --adv_type=DTSA -o $OSV_DTSA_OUT $WEBWML_PA popd echo "Begin Syncing with cloud" -gsutil -m rsync -c -d $OSV_DSA_OUT gs://$OUTPUT_BUCKET/dsa-osv -gsutil -m rsync -c -d "$OSV_DLA_OUT" "gs://${OUTPUT_BUCKET}/dla-osv" -gsutil -m rsync -c -d "$OSV_DTSA_OUT" "gs://${OUTPUT_BUCKET}/dtsa-osv" +gcloud storage rsync $OSV_DSA_OUT gs://$OUTPUT_BUCKET/dsa-osv --delete-unmatched-destination-objects -c +gcloud storage rsync "$OSV_DLA_OUT" "gs://${OUTPUT_BUCKET}/dla-osv" --delete-unmatched-destination-objects -c +gcloud storage rsync "$OSV_DTSA_OUT" "gs://${OUTPUT_BUCKET}/dtsa-osv" --delete-unmatched-destination-objects -c echo "Successfully synced with cloud" \ No newline at end of file diff --git a/vulnfeeds/tools/debian/run_first_package_finder.sh b/vulnfeeds/tools/debian/run_first_package_finder.sh index 0dfebc5747d..2b4e106f9ae 100755 --- a/vulnfeeds/tools/debian/run_first_package_finder.sh +++ b/vulnfeeds/tools/debian/run_first_package_finder.sh @@ -25,7 +25,7 @@ echo "Finding first packages" poetry run python3 first_package_finder.py echo "Syncing with cloud first_package_output ${OUTPUT_BUCKET}" -gsutil -q -m rsync -c -d 'first_package_output' "gs://${OUTPUT_BUCKET}/first_package_output" +gcloud storage rsync 'first_package_output' "gs://${OUTPUT_BUCKET}/first_package_output" --delete-unmatched-destination-objects -q -c echo "Successfully synced with cloud" popd \ No newline at end of file