Merge pull request #519 from GoogleCloudPlatform/all_components_staging

add an image with all existing components and push it to the scanning repo…

Author: anindyatahsin

all_components/Dockerfile

@@ -0,0 +1,30 @@
+FROM marketplace.gcr.io/google/debian12:latest
+ARG CLOUD_SDK_VERSION
+ENV CLOUD_SDK_VERSION=$CLOUD_SDK_VERSION
+ENV PATH /google-cloud-sdk/bin:$PATH
+RUN groupadd -r -g 1000 cloudsdk && \
+    useradd -r -u 1000 -m -s /bin/bash -g cloudsdk cloudsdk
+RUN apt-get update -qqy && apt-get -qqy upgrade && apt-get install -qqy \
+        curl \
+        gcc \
+        python3-dev \
+        python3-pip \
+        python3-crcmod \
+        apt-transport-https \
+        lsb-release \
+        openssh-client \
+        git \
+        make \
+        gnupg \	
+        openjdk-17-jre-headless
+RUN if [ `uname -m` = 'x86_64' ]; then echo -n "x86_64" > /tmp/arch; else echo -n "arm" > /tmp/arch; fi;
+RUN ARCH=`cat /tmp/arch` && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${CLOUD_SDK_VERSION}-linux-${ARCH}.tar.gz && \
+    tar xzf google-cloud-cli-${CLOUD_SDK_VERSION}-linux-${ARCH}.tar.gz && \
+    rm google-cloud-cli-${CLOUD_SDK_VERSION}-linux-${ARCH}.tar.gz
+RUN echo -n "app-engine-java app-engine-python alpha beta pubsub-emulator cloud-datastore-emulator app-engine-go bigtable cbt app-engine-python-extras kubectl gke-gcloud-auth-plugin kustomize minikube skaffold kpt local-extract appctl cloud-sql-proxy docker-credential-gcr package-go-module cloud-firestore-emulator cloud-run-proxy cloud-spanner-emulator harbourbridge log-streaming managed-flink-client spanner-migration-tool terraform-tools config-connector enterprise-certificate-proxy istioctl kubectl-oidc pkg" > /tmp/additional_components
+# These components are not available on ARM right now.
+RUN if [ `uname -m` = 'x86_64' ]; then echo -n " appctl nomos anthos-auth" >> /tmp/additional_components; fi;
+RUN /google-cloud-sdk/install.sh --bash-completion=false --path-update=true --usage-reporting=false \
+	--additional-components `cat /tmp/additional_components` && rm -rf /google-cloud-sdk/.install/.backup
+RUN git config --system credential.'https://source.developers.google.com'.helper gcloud.sh
+VOLUME ["/root/.config", "/root/.kube"]

cloudbuild-hotfix.yaml

@@ -32,6 +32,10 @@ steps:
   - 'inspect'
   - '--bootstrap'
   waitFor: ['multi_arch_step2']
+- name: 'gcr.io/cloud-builders/docker'
+  id: scanning_all_components
+  args: ['build', '--build-arg', 'CLOUD_SDK_VERSION=$_CLI_VERSION', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/scanning/google-cloud-cli:all_components', 'all_components/']
+  waitFor: ['-']
 - name: 'gcr.io/cloud-builders/docker'
   id: alpine
   args: ['build', '--build-arg', 'CLOUD_SDK_VERSION=$_CLI_VERSION', '-t', 'google/cloud-sdk:alpine', '-t', 'google/cloud-sdk:$TAG_NAME-alpine-$_DATE', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/cloud-sdk:alpine', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:alpine', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE', '-t', 'europe-docker.pkg.dev/google.com/cloudsdktool/eu.gcr.io/cloud-sdk:alpine', '-t', 'europe-docker.pkg.dev/google.com/cloudsdktool/eu.gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE', '-t', 'asia-docker.pkg.dev/google.com/cloudsdktool/asia.gcr.io/cloud-sdk:alpine', '-t', 'asia-docker.pkg.dev/google.com/cloudsdktool/asia.gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE', 'alpine/']
@@ -161,6 +165,7 @@ images:
 - 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/cloud-sdk:stable'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/google-cloud-cli:$TAG_NAME-$_DATE'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/google-cloud-cli:latest'
+- 'us-docker.pkg.dev/google.com/cloudsdktool/scanning/google-cloud-cli:all_components'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME-$_DATE'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME-debian_component_based-$_DATE'

cloudbuild.yaml

@@ -31,6 +31,10 @@ steps:
   - 'inspect'
   - '--bootstrap'
   waitFor: ['multi_arch_step2']
+- name: 'gcr.io/cloud-builders/docker'
+  id: scanning_all_components
+  args: ['build', '--build-arg', 'CLOUD_SDK_VERSION=$_CLI_VERSION', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/scanning/google-cloud-cli:all_components', 'all_components/']
+  waitFor: ['-']
 - name: 'gcr.io/cloud-builders/docker'
   id: alpine
   args: ['build', '--build-arg', 'CLOUD_SDK_VERSION=$_CLI_VERSION', '-t', 'google/cloud-sdk:alpine', '-t', 'google/cloud-sdk:$TAG_NAME-alpine', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/cloud-sdk:alpine', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/cloud-sdk:$TAG_NAME-alpine', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:alpine', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME-alpine', '-t', 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE', '-t', 'europe-docker.pkg.dev/google.com/cloudsdktool/eu.gcr.io/cloud-sdk:alpine', '-t', 'europe-docker.pkg.dev/google.com/cloudsdktool/eu.gcr.io/cloud-sdk:$TAG_NAME-alpine', '-t', 'europe-docker.pkg.dev/google.com/cloudsdktool/eu.gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE', '-t', 'asia-docker.pkg.dev/google.com/cloudsdktool/asia.gcr.io/cloud-sdk:alpine', '-t', 'asia-docker.pkg.dev/google.com/cloudsdktool/asia.gcr.io/cloud-sdk:$TAG_NAME-alpine', '-t', 'asia-docker.pkg.dev/google.com/cloudsdktool/asia.gcr.io/cloud-sdk:$TAG_NAME-alpine-$_DATE', 'alpine/']
@@ -181,6 +185,7 @@ images:
 - 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/google-cloud-cli:$TAG_NAME'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/google-cloud-cli:$TAG_NAME-$_DATE'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/gcr.io/google-cloud-cli:latest'
+- 'us-docker.pkg.dev/google.com/cloudsdktool/scanning/google-cloud-cli:all_components'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME-$_DATE'
 - 'us-docker.pkg.dev/google.com/cloudsdktool/us.gcr.io/cloud-sdk:$TAG_NAME-alpine'

generate_cloudbuild.py

@@ -38,6 +38,7 @@
   - 'inspect'
   - '--bootstrap'
   waitFor: ['multi_arch_step2']
+{SCANNINGSTEPS}
 {BUILDSTEPS}
 # END OF PROD BUILDING STEPS
 {MULTIARCH_BUILDSTEPS}
@@ -59,20 +60,35 @@
 
 GCRIO_PROJECT='google.com/cloudsdktool'
 GCR_PREFIXES = [('us-docker.pkg.dev', 'gcr.io'), ('us-docker.pkg.dev', 'us.gcr.io'), ('europe-docker.pkg.dev','eu.gcr.io'), ('asia-docker.pkg.dev', 'asia.gcr.io')]
+SCANNING_PREFIXES = [('us-docker.pkg.dev', 'scanning')]
 DOCKERHUB_PREFIX='google'
 OLD_NAME='cloud-sdk'
 REBRAND_NAME='google-cloud-cli'
 IMAGES=['alpine', 'debian_slim', 'default', 'debian_component_based', 'emulators', 'stable']
 MULTI_ARCH=['debian_slim', 'debian_component_based', 'alpine', 'emulators', 'stable']
+SCANNING_IMAGES=['all_components']
 LABEL_FOR_IMAGE={
     'alpine': 'alpine',
     'debian_slim': 'slim',
     'default': '',
     'debian_component_based': 'debian_component_based',
     'emulators': 'emulators',
-    'stable': 'stable'  # change it to stable when the image is ready to release.
+    'stable': 'stable',
+    'all_components': 'all_components'
     }
 
+def MakeScanningTags (label):
+    t = []
+    for gcr_prefix, gcr_suffix in SCANNING_PREFIXES:
+        t.append(
+                '\'{gcrprefix}/{gcrio_project}/{gcrio_suffix}/{rebrand_name}:{label}\''
+                .format(gcrprefix=gcr_prefix,
+                        gcrio_project=GCRIO_PROJECT,
+                        gcrio_suffix=gcr_suffix,
+                        rebrand_name=REBRAND_NAME,
+                        label=label))
+    return t
+
 def MakeGcrTags(label_without_tag,
                 label_with_tag,
                 maybe_hypen,
@@ -178,6 +194,32 @@ def MakeGcrTags(label_without_tag,
                                            label_with_tag,
                                            maybe_hypen,
                                            include_old_name=False))
+# Make scanning tags and save them
+scanning_tags={}
+for i in SCANNING_IMAGES:
+    scanning_tags[i]=[]
+    label_name = LABEL_FOR_IMAGE[i]
+    if i == 'default':
+        label_name = 'latest'
+    scanning_tags[i].extend(MakeScanningTags(label_name))
+
+scanning_steps=''
+for i in SCANNING_IMAGES:
+    image_directory = '{}/'.format(i)
+    if i == 'default':
+        image_directory = '.'
+
+    scanning_step = """- name: 'gcr.io/cloud-builders/docker'
+  id: scanning_{image_name}
+  args: ['build', '--build-arg', 'CLOUD_SDK_VERSION=$_CLI_VERSION', {scanning_tags}, '{image_directory}']
+  waitFor: ['-']"""
+    output_scanning_step = scanning_step.format(
+        image_name=i,
+        scanning_tags=', '.join(['\'-t\', {}'.format(t) for t in scanning_tags[i]]),
+        image_directory=image_directory)
+    if len(scanning_steps) > 0:
+        scanning_steps+='\n'
+    scanning_steps+=output_scanning_step
 
 build_steps=''
 for i in IMAGES:
@@ -230,12 +272,15 @@ def MakeGcrTags(label_without_tag,
 all_images_tags=[]
 for i in IMAGES:
     all_images_tags.extend([t for t in tags[i] if not t.startswith('\'google/cloud-sdk')])
+for i in SCANNING_IMAGES:
+    all_images_tags.extend(t for t in scanning_tags[i])
 for tag in sorted(all_images_tags):
     if len(all_gcr_io_tags_for_images) > 0:
         all_gcr_io_tags_for_images+='\n'
     all_gcr_io_tags_for_images+='- {}'.format(tag)
 
 print(MAIN_TEMPLATE.format(
+    SCANNINGSTEPS=scanning_steps,
     BUILDSTEPS=build_steps,
     MULTIARCH_BUILDSTEPS=multi_arch_build_steps,
     DOCKER_PUSHSTEPS=docker_push_steps,