feat: add ability to set PYTHON_VERSION via variables #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-single-product-part | ||
|
Check failure on line 1 in .github/workflows/build-single-product-part.yml
|
||
| # will hopefully possible very soon with the new linter | ||
| # run-name: build for component ${{ inputs.component }} | ||
| on: | ||
| workflow_call: | ||
| secrets: | ||
| DATAVISYN_BOT_REPO_TOKEN: | ||
| required: false | ||
| DV_BOT_USER: | ||
| required: false | ||
| NODE_VERSION: | ||
| required: false | ||
| PYTHON_VERSION: | ||
| required: false | ||
| GITLAB_TOKEN: | ||
| required: false | ||
| GITLAB_HOST: | ||
| required: false | ||
| CHECKOUT_TOKEN: | ||
| required: false | ||
| description: "token to use for the checkout actions to access private repositories" | ||
| inputs: | ||
| component: | ||
| description: "component that should be built" | ||
| required: true | ||
| type: string | ||
| image_tag1: | ||
| description: "image tag 1 to push the image" | ||
| required: true | ||
| type: string | ||
| image_tag2: | ||
| description: "image tag 2 for labeling" | ||
| required: true | ||
| type: string | ||
| build_time: | ||
| description: "actually build time (in RFC 3339)" | ||
| required: true | ||
| type: string | ||
| stage: | ||
| description: "stage for the image (develop or production) depending on the branch name" | ||
| required: true | ||
| type: string | ||
| timeout: | ||
| description: "Timeout for each job in minutes." | ||
| type: number | ||
| required: false | ||
| default: 60 | ||
| runs_on: | ||
| type: string | ||
| required: false | ||
| default: "ubuntu-22.04" | ||
| env: | ||
| TIME_ZONE: "Europe/Vienna" | ||
| NODE_VERSION: "20.9" | ||
| PYTHON_VERSION: ${{ vars.PYTHON_VERSION || "3.10" }} | ||
| WORKFLOW_BRANCH: "main" | ||
| PYTHON_BASE_IMAGE: "python:3.10.18-slim-bullseye" | ||
| DATAVISYN_PYTHON_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/python:main" | ||
| DATAVISYN_NGINX_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/nginx:main" | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| jobs: | ||
| build-components: | ||
| timeout-minutes: ${{ fromJSON(inputs.timeout) }} | ||
| runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }} | ||
| steps: | ||
| - name: Remove unnecessary files | ||
| run: | | ||
| sudo rm -rf /usr/share/dotnet | ||
| sudo rm -rf /usr/local/lib/android | ||
| sudo rm -rf /opt/ghc | ||
| # checkout specific repository | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| # checkout this workflow repository to get actions | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| repository: datavisyn/github-workflows | ||
| ref: ${{ env.WORKFLOW_BRANCH }} | ||
| path: ./tmp/github-workflows | ||
| - name: get parameters | ||
| id: get-parameters | ||
| run: | | ||
| type=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.type' ./visyn_product.json) | ||
| directory=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.directory' ./visyn_product.json) | ||
| ecr_repo=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.ecr_repo' ./visyn_product.json) | ||
| skip_image_check=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.skip_image_check' ./visyn_product.json) | ||
| dockerfile_in_app=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.dockerfile_in_app' ./visyn_product.json) | ||
| if [ "$dockerfile_in_app" == "null" ]; then | ||
| dockerfile_in_app=docker/Dockerfile | ||
| fi | ||
| app=$(jq -rc '.app' ./visyn_product.json) | ||
| repo=$(jq -rc '.repo' ./visyn_product.json) | ||
| branch=$(jq -rc '.branch' ./visyn_product.json) | ||
| if [[ ${type} == web ]]; then | ||
| image_base_label=$DATAVISYN_NGINX_BASE_IMAGE | ||
| elif [[ ${type} == api ]]; then | ||
| image_base_label=$DATAVISYN_PYTHON_BASE_IMAGE | ||
| fi | ||
| image_url="$REPOSITORY_HTML/$directory/Dockerfile" | ||
| image_url="${image_url/\/\.\//\/}" | ||
| echo "type=$type" | ||
| echo "directory=$directory" | ||
| echo "ecr_repo=$ecr_repo" | ||
| echo "dockerfile_in_app=$dockerfile_in_app" | ||
| echo "skip_image_check=$skip_image_check" | ||
| echo "app=$app" | ||
| echo "repo=$repo" | ||
| echo "branch=$branch" | ||
| echo "image_base_label=$image_base_label" | ||
| echo "image_url=$image_url" | ||
| # echo to output | ||
| # shellcheck disable=SC2129 | ||
| echo "type=$type" >> "$GITHUB_OUTPUT" | ||
| echo "directory=$directory" >> "$GITHUB_OUTPUT" | ||
| echo "ecr_repo=$ecr_repo" >> "$GITHUB_OUTPUT" | ||
| echo "dockerfile_in_app=$dockerfile_in_app" >> "$GITHUB_OUTPUT" | ||
| echo "skip_image_check=$skip_image_check" >> "$GITHUB_OUTPUT" | ||
| echo "app=$app" >> "$GITHUB_OUTPUT" | ||
| echo "repo=$repo" >> "$GITHUB_OUTPUT" | ||
| echo "branch=$branch" >> "$GITHUB_OUTPUT" | ||
| echo "image_base_label=$image_base_label" >> "$GITHUB_OUTPUT" | ||
| echo "image_url=$image_url" >> "$GITHUB_OUTPUT" | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| DATAVISYN_NGINX_BASE_IMAGE: ${{ env.DATAVISYN_NGINX_BASE_IMAGE }} | ||
| DATAVISYN_PYTHON_BASE_IMAGE: ${{ env.DATAVISYN_PYTHON_BASE_IMAGE }} | ||
| REPOSITORY_HTML: ${{ github.event.repository.html_url }} | ||
| - name: setup node | ||
| if: ${{ steps.get-parameters.outputs.type == 'web' }} | ||
| uses: actions/setup-node@v4 | ||
| with: | ||
| node-version: ${{ secrets.NODE_VERSION || env.NODE_VERSION }} | ||
| registry-url: ${{ env.NPM_REGISTRY }} | ||
| - name: Git config | ||
| run: | | ||
| git config --global url."https://${_GITHUB_TOKEN}@github".insteadOf "https://github" | ||
| git config --add --global url."https://[email protected]/".insteadOf ssh://[email protected]/ | ||
| git config --add --global url."https://[email protected]/".insteadOf [email protected]: | ||
| if [[ -n $GITLAB_TOKEN && -n $GITLAB_HOST ]] ; then | ||
| echo "set gitlab config" | ||
| git config --global url."https://$GITLAB_TOKEN@$GITLAB_HOST/".insteadOf "git@$GITLAB_HOST:" | ||
| git config --global url."git+https://$GITLAB_TOKEN@$GITLAB_HOST/".insteadOf "git+ssh://git@$GITLAB_HOST:" | ||
| fi | ||
| env: | ||
| _GITHUB_TOKEN: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | ||
| GITLAB_HOST: ${{ secrets.GITLAB_HOST }} | ||
| - name: Git clone app | ||
| run: | | ||
| mkdir "./tmp/$COMPONENT" | ||
| cd "./tmp/$COMPONENT" | ||
| repository="$REPO" | ||
| if [[ $repository == *"gitlab"* && -n $GITLAB_TOKEN ]] ; then | ||
| repository=${repository/"https://"/"https://$GITLAB_TOKEN@"} | ||
| else | ||
| repository=https://[email protected]/${repository}.git | ||
| fi | ||
| echo "$repository" | ||
| git clone -b "$BRANCH" --depth 1 "$repository" "$APP" | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| REPO: ${{ steps.get-parameters.outputs.repo }} | ||
| BRANCH: ${{ steps.get-parameters.outputs.branch }} | ||
| APP: ${{ steps.get-parameters.outputs.app }} | ||
| GH_TOKEN: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | ||
| GITLAB_HOST: ${{ secrets.GITLAB_HOST }} | ||
| - name: Copy docker sources | ||
| run: | | ||
| if [[ -d "$COMPONENT_DIRECTORY/docker" ]]; then | ||
| echo "copy docker directory" | ||
| cp -r "$COMPONENT_DIRECTORY/docker" "./tmp/$COMPONENT/$APP/docker" | ||
| ls -lah "./tmp/$COMPONENT/$APP/docker" | ||
| fi | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }} | ||
| APP: ${{ steps.get-parameters.outputs.app }} | ||
| # Enable yarn download cache, @see https://github.com/actions/setup-node/issues/325 | ||
| - name: Restore yarn cache | ||
| if: ${{ steps.get-parameters.outputs.type == 'web' }} | ||
| uses: actions/cache@v4 | ||
| with: | ||
| # This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder | ||
| path: ~/.yarn/berry/cache/ | ||
| # TODO: Add any postfix to make it unique? i.e. -${{ hashFiles('package.json') }} | ||
| key: yarn-download-cache | ||
| restore-keys: | | ||
| yarn-download-cache | ||
| - name: Build web | ||
| if: ${{ steps.get-parameters.outputs.type == 'web' }} | ||
| run: | | ||
| stage_str=".dev" | ||
| if [[ $STAGE == "production" ]]; then | ||
| stage_str=".prod" | ||
| elif [[ $STAGE == "insight" ]]; then | ||
| stage_str=".insight" | ||
| fi | ||
| echo "$stage_str" | ||
| if [[ -f "$COMPONENT_DIRECTORY/.env" ]]; then | ||
| cp "$COMPONENT_DIRECTORY/.env" "./tmp/$COMPONENT/$APP" | ||
| cat "./tmp/$COMPONENT/$APP/.env" | ||
| fi | ||
| if [[ -f "${COMPONENT_DIRECTORY}/${stage_str}.env" ]]; then | ||
| cat "${COMPONENT_DIRECTORY}/${stage_str}.env" >> "./tmp/$COMPONENT/$APP/.env" | ||
| cat "./tmp/$COMPONENT/$APP/.env" | ||
| fi | ||
| cd "./tmp/$COMPONENT/$APP" | ||
| ls -lah | ||
| touch ./yarn.lock | ||
| yarn install --no-immutable | ||
| yarn info --name-only | ||
| yarn run bundle:prod || yarn run webpack:prod | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| APP: ${{ steps.get-parameters.outputs.app }} | ||
| COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }} | ||
| STAGE: ${{ inputs.stage }} | ||
| - name: setup python | ||
| if: ${{ steps.get-parameters.outputs.type == 'api' }} | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }} | ||
| # cache: 'pip' Disable cache as uv is probably faster anyways: https://github.com/actions/setup-python/issues/822 | ||
| - name: Build api | ||
| if: ${{ steps.get-parameters.outputs.type == 'api' }} | ||
| run: | | ||
| cd "./tmp/$COMPONENT/$APP" | ||
| ls -lah | ||
| python -m pip install --upgrade pip uv | ||
| uv pip install setuptools wheel --system | ||
| make install | ||
| make build | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| APP: ${{ steps.get-parameters.outputs.app }} | ||
| COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }} | ||
| # checkout this workflow repository to get actions | ||
| - uses: ./tmp/github-workflows/.github/actions/build-push-image | ||
| with: | ||
| aws_role: ${{ vars.DV_AWS_ECR_ROLE }} | ||
| aws_region: ${{ vars.DV_AWS_REGION }} | ||
| ecr_registry: ${{ vars.DV_AWS_ECR_REGISTRY }} | ||
| ecr_repository: ${{ steps.get-parameters.outputs.ecr_repo }} | ||
| docker_file: ./tmp/${{ inputs.component }}/${{ steps.get-parameters.outputs.app }}/${{ steps.get-parameters.outputs.dockerfile_in_app }} | ||
| current_directory: ./tmp/${{ inputs.component }}/${{ steps.get-parameters.outputs.app }} | ||
| image_tag: ${{ inputs.image_tag1 }} | ||
| build_args: | | ||
| PYTHON_BASE_IMAGE=${{ env.PYTHON_BASE_IMAGE }} | ||
| DATAVISYN_PYTHON_BASE_IMAGE=${{ env.DATAVISYN_PYTHON_BASE_IMAGE }} | ||
| GIT_ACCESS_TOKEN=${{ secrets.DATAVISYN_BOT_REPO_TOKEN }} | ||
| DATAVISYN_NGINX_BASE_IMAGE=${{ env.DATAVISYN_NGINX_BASE_IMAGE }} | ||
| labels: | | ||
| name=${{ steps.get-parameters.outputs.ecr_repo }} | ||
| version=${{ inputs.image_tag2 }} | ||
| org.opencontainers.image.description=${{ inputs.component }} image for ${{ steps.get-parameters.outputs.app }} | ||
| org.opencontainers.image.base.name=${{ steps.get-parameters.outputs.image_base_label }} | ||
| org.opencontainers.image.source=${{ steps.get-parameters.outputs.image_url }} | ||
| org.opencontainers.image.url=${{ steps.get-parameters.outputs.image_url }} | ||
| org.opencontainers.image.title=${{ steps.get-parameters.outputs.ecr_repo }} | ||
| org.opencontainers.image.version=${{ inputs.image_tag2 }} | ||
| org.opencontainers.image.created=${{ inputs.build_time }} | ||
| org.opencontainers.image.revision=${{ github.sha }} | ||
| - name: scan image | ||
| if: ${{ fromJson(vars.SKIP_IMAGE_SCAN || 'false') != true && steps.get-parameters.outputs.skip_image_check != 'true' }} | ||
| id: get-ecr-scan-result | ||
| uses: ./tmp/github-workflows/.github/actions/get-ecr-scan-result | ||
| with: | ||
| aws_role: ${{ vars.DV_AWS_ECR_ROLE }} | ||
| aws_region: ${{ vars.DV_AWS_REGION }} | ||
| ecr_registry: ${{ vars.DV_AWS_ECR_REGISTRY }} | ||
| ecr_repository: ${{ steps.get-parameters.outputs.ecr_repo }} | ||
| image_tag: ${{ inputs.image_tag1 }} | ||
| - name: check scan results | ||
| if: ${{ steps.get-parameters.outputs.skip_image_check != 'true' }} | ||
| run: | | ||
| if [ "${{ steps.get-ecr-scan-result.outputs.critical }}" != "null" ] || [ "${{ steps.get-ecr-scan-result.outputs.high }}" != "null" ]; then | ||
| echo "Docker image contains vulnerabilities at critical or high level" | ||
| exit 1 #exit execution due to docker image vulnerabilities | ||
| fi | ||
