feat: add ability to set PYTHON_VERSION via variables #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-workspace-product-part | ||
|
Check failure on line 1 in .github/workflows/build-workspace-product-part.yml
|
||
| # will hopefully possible very soon with the new linter | ||
| # run-name: build for component ${{ inputs.component }} | ||
| on: | ||
| workflow_call: | ||
| secrets: | ||
| DATAVISYN_BOT_REPO_TOKEN: | ||
| required: false | ||
| DV_BOT_USER: | ||
| required: false | ||
| NODE_VERSION: | ||
| required: false | ||
| PYTHON_VERSION: | ||
| required: false | ||
| GITLAB_TOKEN: | ||
| required: false | ||
| GITLAB_HOST: | ||
| required: false | ||
| inputs: | ||
| component: | ||
| description: "component that should be built" | ||
| required: true | ||
| type: string | ||
| image_tag1: | ||
| description: "image tag 1 to push the image" | ||
| required: true | ||
| type: string | ||
| image_tag2: | ||
| description: "image tag 2 for labeling" | ||
| required: true | ||
| type: string | ||
| build_time: | ||
| description: "actually build time (in RFC 3339)" | ||
| required: true | ||
| type: string | ||
| stage: | ||
| description: "stage for the image (develop or production) depending on the branch name" | ||
| required: true | ||
| type: string | ||
| timeout: | ||
| description: "Timeout for each job in minutes." | ||
| type: number | ||
| required: false | ||
| default: 60 | ||
| env: | ||
| VISYN_SCRIPTS_VERSION: "v7" # visyn_scripts@v7 is the last version with workspace support | ||
| TIME_ZONE: "Europe/Vienna" | ||
| NODE_VERSION: "20.9" | ||
| PYTHON_VERSION: ${{ vars.PYTHON_VERSION || "3.10" }} | ||
| WORKFLOW_BRANCH: "main" | ||
| PYTHON_BASE_IMAGE: "python:3.10.18-slim-bullseye" | ||
| DATAVISYN_PYTHON_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/python:main" | ||
| DATAVISYN_NGINX_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/nginx:main" | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| jobs: | ||
| build-components: | ||
| timeout-minutes: ${{ fromJSON(inputs.timeout) }} | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| # checkout specific repository | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| # checkout this workflow repository to get actions | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| repository: datavisyn/github-workflows | ||
| ref: ${{ env.WORKFLOW_BRANCH }} | ||
| path: ./tmp/github-workflows | ||
| - name: get parameters | ||
| id: get-parameters | ||
| run: | | ||
| type=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.type' ./visyn_product.json) | ||
| directory=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.directory' ./visyn_product.json) | ||
| ecr_repo=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.ecr_repo' ./visyn_product.json) | ||
| app=$(jq -rc '.app' ./visyn_product.json) | ||
| default_app=$(jq -rc '.defaultApp' ./visyn_product.json) | ||
| repo=$(jq -rc '.repo' ./visyn_product.json) | ||
| branch=$(jq -rc '.branch' ./visyn_product.json) | ||
| if [[ ${type} == web ]]; then | ||
| image_base_label=$DATAVISYN_NGINX_BASE_IMAGE | ||
| elif [[ ${type} == api ]]; then | ||
| image_base_label=$DATAVISYN_PYTHON_BASE_IMAGE | ||
| fi | ||
| image_url="$REPOSITORY_HTML/$directory/Dockerfile" | ||
| image_url="${image_url/\/\.\//\/}" | ||
| echo "type=$type" | ||
| echo "directory=$directory" | ||
| echo "ecr_repo=$ecr_repo" | ||
| echo "app=$app" | ||
| echo "default_app=$default_app" | ||
| echo "repo=$repo" | ||
| echo "branch=$branch" | ||
| echo "image_base_label=$image_base_label" | ||
| echo "image_url=$image_url" | ||
| # echo to output | ||
| # shellcheck disable=SC2129 | ||
| echo "type=$type" >> "$GITHUB_OUTPUT" | ||
| echo "directory=$directory" >> "$GITHUB_OUTPUT" | ||
| echo "ecr_repo=$ecr_repo" >> "$GITHUB_OUTPUT" | ||
| echo "app=$app" >> "$GITHUB_OUTPUT" | ||
| echo "default_app=$default_app" >> "$GITHUB_OUTPUT" | ||
| echo "repo=$repo" >> "$GITHUB_OUTPUT" | ||
| echo "branch=$branch" >> "$GITHUB_OUTPUT" | ||
| echo "image_base_label=$image_base_label" >> "$GITHUB_OUTPUT" | ||
| echo "image_url=$image_url" >> "$GITHUB_OUTPUT" | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| DATAVISYN_NGINX_BASE_IMAGE: ${{ env.DATAVISYN_NGINX_BASE_IMAGE }} | ||
| DATAVISYN_PYTHON_BASE_IMAGE: ${{ env.DATAVISYN_PYTHON_BASE_IMAGE }} | ||
| REPOSITORY_HTML: ${{ github.event.repository.html_url }} | ||
| - name: setup node | ||
| if: ${{ steps.get-parameters.outputs.type == 'web' }} | ||
| uses: actions/setup-node@v4 | ||
| with: | ||
| node-version: ${{ secrets.NODE_VERSION || env.NODE_VERSION }} | ||
| registry-url: ${{ env.NPM_REGISTRY }} | ||
| - name: setup python | ||
| if: ${{ steps.get-parameters.outputs.type == 'api' }} | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }} | ||
| # TODO: This fails with Error: No file in ..._product matched to [**/requirements.txt] | ||
| # cache: 'pip' | ||
| - name: Git config | ||
| run: | | ||
| git config --global url."https://${_GITHUB_TOKEN}@github".insteadOf "https://github" | ||
| git config --add --global url."https://[email protected]/".insteadOf ssh://[email protected]/ | ||
| git config --add --global url."https://[email protected]/".insteadOf [email protected]: | ||
| if [[ -n $GITLAB_TOKEN && -n $GITLAB_HOST ]] ; then | ||
| echo "set gitlab config" | ||
| git config --global url."https://$GITLAB_TOKEN@$GITLAB_HOST/".insteadOf "git@$GITLAB_HOST:" | ||
| git config --global url."git+https://$GITLAB_TOKEN@$GITLAB_HOST/".insteadOf "git+ssh://git@$GITLAB_HOST:" | ||
| fi | ||
| env: | ||
| _GITHUB_TOKEN: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | ||
| GITLAB_HOST: ${{ secrets.GITLAB_HOST }} | ||
| - name: Install visyn_scripts | ||
| run: | | ||
| if [[ "$VISYN_SCRIPTS_VERSION" == v[0-9]* ]] ; then | ||
| npm install -g "visyn_scripts@$VISYN_SCRIPTS_VERSION" | ||
| else | ||
| npm install -g "https://github.com/datavisyn/visyn_scripts.git#$VISYN_SCRIPTS_VERSION" | ||
| fi | ||
| env: | ||
| VISYN_SCRIPTS_VERSION: ${{ env.VISYN_SCRIPTS_VERSION }} | ||
| - name: create workspace | ||
| run: | | ||
| mkdir "./tmp/$COMPONENT" | ||
| cd "./tmp/$COMPONENT" | ||
| repository="$REPO" | ||
| if [[ $repository == *"gitlab"* && -n $GITLAB_TOKEN ]] ; then | ||
| repository=${repository/"https://"/"https://$GITLAB_TOKEN@"} | ||
| else | ||
| repository=https://[email protected]/${repository}.git | ||
| fi | ||
| echo "$repository" | ||
| git clone -b "$BRANCH_NAME" --depth 1 "$repository" | ||
| readarray -t my_array < <(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.additionals[]?' ../../visyn_product.json) | ||
| for additional in "${my_array[@]}"; do | ||
| echo "$additional" | ||
| name=$(echo "$additional" | jq -rc '.name') | ||
| repository=$(echo "$additional" | jq -rc '.repo') | ||
| if [[ $repository == *"gitlab"* && -n $GITLAB_TOKEN ]] ; then | ||
| repository=${repository/"https://"/"https://$GITLAB_TOKEN@"} | ||
| else | ||
| repository=https://[email protected]/${repository}.git | ||
| fi | ||
| branch=$(echo "$additional" | jq -rc '.branch') | ||
| git clone -b "$branch" --depth 1 "$repository" "$name" | ||
| done | ||
| rm -f ./../../.yo-rc.json | ||
| visyn_scripts workspace-update --defaultApp="$DEFAULT_APP" --noAdditionals=true --addWorkspaceRepos=false | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| REPO: ${{ steps.get-parameters.outputs.repo }} | ||
| BRANCH_NAME: ${{ steps.get-parameters.outputs.branch }} | ||
| APP: ${{ steps.get-parameters.outputs.app }} | ||
| DEFAULT_APP: ${{ steps.get-parameters.outputs.default_app }} | ||
| GH_TOKEN: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | ||
| GITLAB_HOST: ${{ secrets.GITLAB_HOST }} | ||
| - name: Inject version from product package.json into workspace package.json | ||
| run: | | ||
| product_version=$(jq -rc '.version' ./package.json) | ||
| echo "product_version=$product_version" | ||
| if [[ $product_version == *"SNAPSHOT"* ]]; then | ||
| echo "replace SNAPSHOT in version with timestamp" | ||
| product_version="${product_version//SNAPSHOT/$(date +%Y%m%d-%H%M%S)}" | ||
| echo "product_version=$product_version" | ||
| fi | ||
| workspace_version=$(jq -rc '.version' "./tmp/$COMPONENT/package.json") | ||
| echo "workspace_version=$workspace_version" | ||
| if [[ "$product_version" != "$workspace_version" ]]; then | ||
| echo "update workspace version" | ||
| jq --arg version "$product_version" '.version = $version' "./tmp/$COMPONENT/package.json" > "./tmp/$COMPONENT/package.json.tmp" | ||
| mv "./tmp/$COMPONENT/package.json.tmp" "./tmp/$COMPONENT/package.json" | ||
| echo "workspace version updated to $(jq -rc '.version' './tmp/$COMPONENT/package.json')" | ||
| fi | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| - name: Copy docker sources | ||
| run: | | ||
| ls -lah . | ||
| ls -lah "$COMPONENT_DIRECTORY" | ||
| if [[ -d "$COMPONENT_DIRECTORY/docker" ]]; then | ||
| echo "copy docker directory" | ||
| cp -r "$COMPONENT_DIRECTORY/docker" "./tmp/$COMPONENT/docker" | ||
| ls -lah "./tmp/$COMPONENT/docker" | ||
| fi | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }} | ||
| APP: ${{ steps.get-parameters.outputs.app }} | ||
| - name: Build web | ||
| if: ${{ steps.get-parameters.outputs.type == 'web' }} | ||
| run: | | ||
| stage_str=".dev" | ||
| if [[ "$STAGE" == "production" ]]; then | ||
| stage_str=".prod" | ||
| elif [[ "$STAGE" == "insight" ]]; then | ||
| stage_str=".insight" | ||
| fi | ||
| echo "$stage_str" | ||
| if [[ -f "$COMPONENT_DIRECTORY/.env" ]]; then | ||
| echo "copy .env" | ||
| cat "$COMPONENT_DIRECTORY/.env" | ||
| cp "$COMPONENT_DIRECTORY/.env" "./tmp/$COMPONENT" | ||
| fi | ||
| if [[ -f "${COMPONENT_DIRECTORY}/${stage_str}.env" ]]; then | ||
| cat "${COMPONENT_DIRECTORY}/${stage_str}.env" >> "./tmp/$COMPONENT/.env" | ||
| fi | ||
| if [[ -f "$COMPONENT_DIRECTORY/workspace.scss" ]]; then | ||
| echo "copy workspace.scss" | ||
| cp "$COMPONENT_DIRECTORY/workspace.scss" "./tmp/$COMPONENT" | ||
| fi | ||
| cd "./tmp/$COMPONENT" | ||
| ls -lah | ||
| touch ./yarn.lock | ||
| yarn install --no-immutable | ||
| yarn info --name-only | ||
| yarn run dist | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }} | ||
| STAGE: ${{ inputs.stage }} | ||
| - name: Build api | ||
| if: ${{ steps.get-parameters.outputs.type == 'api' }} | ||
| run: | | ||
| cd "./tmp/$COMPONENT" | ||
| ls -lah | ||
| python -m pip install --upgrade pip uv | ||
| uv pip install setuptools wheel --system | ||
| mkdir -p ./build/source | ||
| mkdir -p ./dist_python/ | ||
| cd "$DEFAULT_APP" | ||
| if test -f "Makefile"; then | ||
| echo "Makefile exists in $DEFAULT_APP, trying to build" | ||
| make build | ||
| cp -r ./build/lib/* ../build/source/ | ||
| cp -r "./${DEFAULT_APP,,}.egg-info" "../build/source/${DEFAULT_APP,,}.egg-info" | ||
| cp -r ./dist_python/* ../dist_python/ | ||
| else | ||
| echo "$APP has no Makefile, therefore no backend can be built" | ||
| fi | ||
| cd .. | ||
| readarray -t my_array < <(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.additionals[]?' ../../visyn_product.json) | ||
| for additional in "${my_array[@]}"; do | ||
| echo "$additional" | ||
| name=$(echo "$additional" | jq -rc '.name') | ||
| # shellcheck disable=SC2001 | ||
| name=$(echo "$name" | sed -e 's|.*/||') | ||
| cd "$name" | ||
| make build | ||
| cp -r ./build/lib/* ../build/source/ | ||
| cp -r "./${name,,}.egg-info" "../build/source/${name,,}.egg-info" | ||
| cp -r ./dist_python/* ../dist_python/ | ||
| cd .. | ||
| done | ||
| env: | ||
| COMPONENT: ${{ inputs.component }} | ||
| APP: ${{ steps.get-parameters.outputs.app }} | ||
| DEFAULT_APP: ${{ steps.get-parameters.outputs.default_app }} | ||
| COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }} | ||
| # checkout this workflow repository to get actions | ||
| - uses: ./tmp/github-workflows/.github/actions/build-push-image | ||
| with: | ||
| aws_role: ${{ vars.DV_AWS_ECR_ROLE }} | ||
| aws_region: ${{ vars.DV_AWS_REGION }} | ||
| ecr_registry: ${{ vars.DV_AWS_ECR_REGISTRY }} | ||
| ecr_repository: ${{ steps.get-parameters.outputs.ecr_repo }} | ||
| docker_file: ./tmp/${{ inputs.component }}/docker/Dockerfile | ||
| current_directory: ./tmp/${{ inputs.component }} | ||
| image_tag: ${{ inputs.image_tag1 }} | ||
| build_args: | | ||
| PYTHON_BASE_IMAGE=${{ env.PYTHON_BASE_IMAGE }} | ||
| DATAVISYN_PYTHON_BASE_IMAGE=${{ env.DATAVISYN_PYTHON_BASE_IMAGE }} | ||
| GIT_ACCESS_TOKEN=${{ secrets.DATAVISYN_BOT_REPO_TOKEN }} | ||
| DATAVISYN_NGINX_BASE_IMAGE=${{ env.DATAVISYN_NGINX_BASE_IMAGE }} | ||
| labels: | | ||
| name=${{ steps.get-parameters.outputs.ecr_repo }} | ||
| version=${{ inputs.image_tag2 }} | ||
| org.opencontainers.image.description=${{ inputs.component }} image for ${{ steps.get-parameters.outputs.app }} | ||
| org.opencontainers.image.base.name=${{ steps.get-parameters.outputs.image_base_label }} | ||
| org.opencontainers.image.source=${{ steps.get-parameters.outputs.image_url }} | ||
| org.opencontainers.image.url=${{ steps.get-parameters.outputs.image_url }} | ||
| org.opencontainers.image.title=${{ steps.get-parameters.outputs.ecr_repo }} | ||
| org.opencontainers.image.version=${{ inputs.image_tag2 }} | ||
| org.opencontainers.image.created=${{ inputs.build_time }} | ||
| org.opencontainers.image.revision=${{ github.sha }} | ||
| - name: scan image | ||
| id: get-ecr-scan-result | ||
| uses: ./tmp/github-workflows/.github/actions/get-ecr-scan-result | ||
| with: | ||
| aws_role: ${{ vars.DV_AWS_ECR_ROLE }} | ||
| aws_region: ${{ vars.DV_AWS_REGION }} | ||
| ecr_registry: ${{ vars.DV_AWS_ECR_REGISTRY }} | ||
| ecr_repository: ${{ steps.get-parameters.outputs.ecr_repo }} | ||
| image_tag: ${{ inputs.image_tag1 }} | ||
| - name: check scan results | ||
| run: | | ||
| if [ "${{ steps.get-ecr-scan-result.outputs.critical }}" != "null" ] || [ "${{ steps.get-ecr-scan-result.outputs.high }}" != "null" ]; then | ||
| echo "Docker image contains vulnerabilities at critical or high level" | ||
| exit 1 #exit execution due to docker image vulnerabilities | ||
| fi | ||
