diff --git a/.github/actions/deploy-ecs/action.yaml b/.github/actions/deploy-ecs/action.yaml new file mode 100644 index 0000000000..39097023e0 --- /dev/null +++ b/.github/actions/deploy-ecs/action.yaml @@ -0,0 +1,60 @@ +name: "Deploy to ECS" +description: "Deploy new image to given ECS service by updating task definition file" +inputs: + aws-role: + required: true + description: "AWS ROLE" + aws-region: + required: true + description: "AWS REGION" + task-definition: + required: true + description: "TASK DEFINITION" + container-name: + required: true + description: "CONTAINER NAME" + ecs-service: + required: true + description: "ECS SERVICE" + ecs-cluster: + required: true + description: "ECS CLUSTER" + image: + required: true + description: "APP IMAGE" +runs: + using: "composite" + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ inputs.aws-region }} + + - name: Download task definition + run: | + aws ecs describe-task-definition --task-definition ${{ inputs.task-definition }} \ + --query taskDefinition > task-definition.json + shell: bash + + - name: Fill in the new image ID in the Amazon ECS task definition + id: task_def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition.json + container-name: ${{ inputs.container-name }} + image: ${{ inputs.image }} + + - name: Remove unwanted fields from task definition + id: task_def_cleanup + shell: bash + run: | + jq 'del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy)' ${{ steps.task_def.outputs.task-definition }} > updated-task-definition.json + + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: updated-task-definition.json + service: ${{ inputs.ecs-service }} + cluster: ${{ inputs.ecs-cluster }} + wait-for-service-stability: true diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000000..f592f390e4 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,34 @@ +# Conductor CI/CD Workflow + +This repository contains workflow files for implementing Continuous Integration (CI) and Continuous Deployment (CD) processes separately for Conductor UI and server components. The workflow is designed to deploy to both development (dev) and production (prd) environments on AWS (ECS). + +## Workflow Overview + +The CI/CD workflow is triggered manually & comprises two main components: + +1. **Conductor UI CI/CD:** + - Workflow file: `.github/workflows/ci-ui.yaml` + `.github/workflows/cd-ui.yaml` + - These workflows handle the CI & CD process for Conductor UI. + +2. **Conductor Server CI/CD:** + - Workflow file: `.github/workflows/ci-server.yaml` + `.github/workflows/cd-server.yaml` + - These workflows handle the CI & CD process for Conductor server. + +## Deployment Strategy + +- **Branches:** + - The `production` branch is considered the master branch for all deployments. + - All deployments to both development and production environments are triggered from the `production` branch. + +- **Input Variables:** + - The workflow takes the following input variables: + 1. **Branch:** Specifies the branch to be deployed (e.g., `production`). + 2. **Environment:** Specifies the deployment environment (e.g., `dev` or `prd`). + 3. **Tag:** Specifies the version to be deployed. This version is used for tagging the Docker image. + +## Versioning and Docker Image Tagging + +The version provided as an input variable is crucial for versioning and tagging Docker images. The workflow utilizes this version to tag the Docker image before deploying to the AWS Elastic Container Registry (ECR). During ECS deployment, this tagged image is fetched from ECR. + diff --git a/.github/workflows/cd-server.yaml b/.github/workflows/cd-server.yaml new file mode 100644 index 0000000000..14895943f4 --- /dev/null +++ b/.github/workflows/cd-server.yaml @@ -0,0 +1,162 @@ +name: Deploy Conductor Server + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - stg + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) +permissions: + id-token: write + contents: write + packages: read + actions: read +env: + SERVICE_NAME: conductor + AWS_REGION: "ap-south-1" + HELM_CHART_NAME: "application-helm-chart" + +jobs: + prepare-env: + name: Prepare Env + runs-on: "ubuntu-latest" + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + K8S_CLUSTER: ${{ steps.vars.outputs.K8S_CLUSTER }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + AWS_ACCOUNT_ID: ${{ steps.vars.outputs.AWS_ACCOUNT_ID}} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + IMAGE_TAG=${{ github.event.inputs.tag }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-prd-mb-prime" >> $GITHUB_OUTPUT + elif [ $ENV == 'stg' ] + then + echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-dev-mb-prime" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-dev-mb-prime" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + echo ":label: Image Tag - $IMAGE_TAG " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-conductor-server" >> $GITHUB_OUTPUT + + # Deploy Conductor UI Image to ECS + deploy-to-k8s: + name: Deploy to k8s + runs-on: ubuntu-latest + container: + image: public.ecr.aws/kvsiren-dev/pipeline/helm-deploy:latest + timeout-minutes: 15 + permissions: + id-token: write + pull-requests: write + contents: read + needs: + - prepare-env + env: + AWS_ACCOUNT_ID: ${{ needs.prepare-env.outputs.AWS_ACCOUNT_ID }} + ECR_REPOSITORY: ${{ needs.prepare-env.outputs.ECR_REPOSITORY }} + steps: + - name: Checkout code from action + uses: actions/checkout@v2 + + - name: Checkout values.yaml from siren-infra + uses: actions/checkout@v4 + with: + repository: KeyvalueSoftwareSystems/siren-infra + ref: main + token: ${{secrets.SIREN_PAT}} + sparse-checkout: | + k8s/siren-services/${{ env.SERVICE_NAME }}/${{ needs.prepare-env.outputs.ENV }}-values.yaml + sparse-checkout-cone-mode: false + + - name: Rename values.yaml for Helm + shell: bash + run: | + cp k8s/siren-services/${{ env.SERVICE_NAME }}/${{ needs.prepare-env.outputs.ENV }}-values.yaml ./values.yaml + cat ./values.yaml + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ vars[env.AWS_ACCOUNT_ID] }}:role/github-actions + aws-region: ${{ env.AWS_REGION }} + + - name: Deploy to Kubernetes + shell: bash + run: | + aws eks update-kubeconfig --name ${{ needs.prepare-env.outputs.K8S_CLUSTER }} + aws ecr get-login-password --region ${{ env.AWS_REGION }} | helm registry login --username AWS --password-stdin ${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com + + # Construct base Helm command + HELM_CMD="helm upgrade --install ${{ env.SERVICE_NAME }} oci://${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.HELM_CHART_NAME }} \ + --namespace ${{ needs.prepare-env.outputs.ENV }} \ + --values values.yaml \ + --set default.image.repository='${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPOSITORY }}' \ + --set default.image.tag='${{ github.event.inputs.tag }}'" + + # Run the Helm command + echo "Running: $HELM_CMD" + eval $HELM_CMD diff --git a/.github/workflows/cd-ui.yaml b/.github/workflows/cd-ui.yaml new file mode 100644 index 0000000000..c5d84fc2eb --- /dev/null +++ b/.github/workflows/cd-ui.yaml @@ -0,0 +1,162 @@ +name: Deploy Conductor UI + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - stg + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) +permissions: + id-token: write + contents: write + packages: read + actions: read +env: + SERVICE_NAME: conductor-ui + AWS_REGION: "ap-south-1" + HELM_CHART_NAME: "application-helm-chart" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + K8S_CLUSTER: ${{ steps.vars.outputs.K8S_CLUSTER }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + AWS_ACCOUNT_ID: ${{ steps.vars.outputs.AWS_ACCOUNT_ID}} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + IMAGE_TAG=${{ github.event.inputs.tag }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-prd-mb-prime" >> $GITHUB_OUTPUT + elif [ $ENV == 'stg' ] + then + echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-dev-mb-prime" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-dev-mb-prime" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + echo ":label: Image Tag - $IMAGE_TAG " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-conductor-ui" >> $GITHUB_OUTPUT + + # Deploy Conductor UI Image to ECS + deploy-to-k8s: + name: Deploy to k8s + runs-on: ubuntu-latest + container: + image: public.ecr.aws/kvsiren-dev/pipeline/helm-deploy:latest + timeout-minutes: 15 + permissions: + id-token: write + pull-requests: write + contents: read + needs: + - prepare-env + env: + AWS_ACCOUNT_ID: ${{ needs.prepare-env.outputs.AWS_ACCOUNT_ID }} + ECR_REPOSITORY: ${{ needs.prepare-env.outputs.ECR_REPOSITORY }} + steps: + - name: Checkout code from action + uses: actions/checkout@v2 + + - name: Checkout values.yaml from siren-infra + uses: actions/checkout@v4 + with: + repository: KeyvalueSoftwareSystems/siren-infra + ref: main + token: ${{secrets.SIREN_PAT}} + sparse-checkout: | + k8s/siren-services/${{ env.SERVICE_NAME }}/${{ needs.prepare-env.outputs.ENV }}-values.yaml + sparse-checkout-cone-mode: false + + - name: Rename values.yaml for Helm + shell: bash + run: | + cp k8s/siren-services/${{ env.SERVICE_NAME }}/${{ needs.prepare-env.outputs.ENV }}-values.yaml ./values.yaml + cat ./values.yaml + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ vars[env.AWS_ACCOUNT_ID] }}:role/github-actions + aws-region: ${{ env.AWS_REGION }} + + - name: Deploy to Kubernetes + shell: bash + run: | + aws eks update-kubeconfig --name ${{ needs.prepare-env.outputs.K8S_CLUSTER }} + aws ecr get-login-password --region ${{ env.AWS_REGION }} | helm registry login --username AWS --password-stdin ${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com + + # Construct base Helm command + HELM_CMD="helm upgrade --install ${{ env.SERVICE_NAME }} oci://${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.HELM_CHART_NAME }} \ + --namespace ${{ needs.prepare-env.outputs.ENV }} \ + --values values.yaml \ + --set default.image.repository='${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPOSITORY }}' \ + --set default.image.tag='${{ github.event.inputs.tag }}'" + + # Run the Helm command + echo "Running: $HELM_CMD" + eval $HELM_CMD diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml new file mode 100644 index 0000000000..4cce9dae38 --- /dev/null +++ b/.github/workflows/ci-server-.yaml @@ -0,0 +1,186 @@ +name: Build & Publish Conductor Server + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - stg + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) + +env: + SERVICE_NAME: conductor-server + AWS_REGION: "ap-south-1" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + PUBLIC_CONDUCTOR_ECR: ${{ steps.vars.outputs.PUBLIC_CONDUCTOR_ECR}} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + IMAGE_TAG=${{ github.event.inputs.tag }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'stg' ] + then + echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "PUBLIC_CONDUCTOR_ECR=sandbox-conductor-server" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + echo ":label: Image Tag - $IMAGE_TAG " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Building and Pushing Conductor Server Image to ECR + build-push-image: + name: Build and Push Server Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + IMAGE_TAG: ${{ github.event.inputs.tag }} + PUBLIC_CONDUCTOR_ECR: ${{needs.prepare-env.outputs.PUBLIC_CONDUCTOR_ECR}} + outputs: + ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} + APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Build and push to Amazon ECR + id: build + uses: docker/build-push-action@v5.1.0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: ./Dockerfile + push: true + provenance: false + platforms: linux/amd64 + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + - name: Configure AWS credentials for Public ECR + if: ${{ env.ENV == 'dev'}} + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: us-east-1 + + - name: Login to Amazon ECR Public + if: ${{ env.ENV == 'dev'}} + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Build and push to Public ECR + if: ${{ env.ENV == 'dev'}} + id: build-public + uses: docker/build-push-action@v5.1.0 + env: + PUBLIC_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} + PUBLIC_REGISTRY_ALIAS: kvsiren-${{ env.ENV }} + with: + context: . + file: ./Dockerfile + push: true + provenance: false + platforms: linux/amd64 + tags: ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} + + - name: Image name + id: image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT + diff --git a/.github/workflows/ci-ui.yaml b/.github/workflows/ci-ui.yaml new file mode 100644 index 0000000000..5e7ffebf18 --- /dev/null +++ b/.github/workflows/ci-ui.yaml @@ -0,0 +1,175 @@ +name: Build & Publish Conductor UI + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - stg + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) + +env: + SERVICE_NAME: conductor-ui + AWS_REGION: "ap-south-1" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + DEFAULT_CONF: ${{ steps.vars.outputs.DEFAULT_CONF }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + IMAGE_TAG=${{ github.event.inputs.tag }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "DEFAULT_CONF=default-prd.conf" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'stg' ] + then + echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "DEFAULT_CONF=default-stg.conf" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "DEFAULT_CONF=default-dev.conf" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + echo ":label: Image Tag - $IMAGE_TAG " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Building and Pushing Conductor UI Image to ECR + build-push-ui-image: + name: Build and Push UI Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + DEFAULT_CONF: ${{needs.prepare-env.outputs.DEFAULT_CONF}} + IMAGE_TAG: ${{ github.event.inputs.tag }} + outputs: + ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} + APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download S3 file + run: | + aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-ui/.env ./ui/.env + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Set Up Node + uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Yarn Build + run: | + cd ui/ + mv ./${{ env.DEFAULT_CONF }} ./default.conf + yarn install && yarn build + + - name: Build and push to Amazon ECR + id: build + uses: docker/build-push-action@v5.1.0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: ./ui/ + file: ./ui/Dockerfile + push: true + provenance: false + platforms: linux/amd64 + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + - name: Image name + id: image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index a25c97dd2e..0000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: CI - -on: [ push, pull_request ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - name: Gradle wrapper validation - uses: gradle/wrapper-validation-action@v1 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Cache SonarCloud packages - uses: actions/cache@v3 - with: - path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - name: Cache Gradle packages - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - name: Build with Gradle - if: github.ref != 'refs/heads/main' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - ./gradlew build --scan - - name: Build and Publish snapshot - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - run: | - echo "Running build for commit ${{ github.sha }}" - ./gradlew build - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() - with: - report_paths: '**/build/test-results/test/TEST-*.xml' - - name: Upload build artifacts - uses: actions/upload-artifact@v3 - with: - name: build-artifacts - path: '**/build/reports' - - name: Store Buildscan URL - uses: actions/upload-artifact@v3 - with: - name: build-scan - path: 'buildscan.log' - build-ui: - runs-on: ubuntu-latest - container: cypress/browsers:node14.17.6-chrome100-ff98 - defaults: - run: - working-directory: ui - steps: - - uses: actions/checkout@v3 - - - name: Install Dependencies - run: yarn install - - - name: Build UI - run: yarn run build - - - name: Run E2E Tests - uses: cypress-io/github-action@v4 - with: - working-directory: ui - install: false - start: yarn run serve-build - wait-on: 'http://localhost:5000' - - - name: Run Component Tests - uses: cypress-io/github-action@v4 - with: - working-directory: ui - install: false - component: true - - - name: Archive test screenshots - uses: actions/upload-artifact@v2 - if: failure() - with: - name: cypress-screenshots - path: ui/cypress/screenshots - - - name: Archive test videos - uses: actions/upload-artifact@v2 - if: always() - with: - name: cypress-videos - path: ui/cypress/videos - diff --git a/.github/workflows/generate_gh_pages.yml b/.github/workflows/generate_gh_pages.yml deleted file mode 100644 index 8c429e1b8e..0000000000 --- a/.github/workflows/generate_gh_pages.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Publish docs via GitHub Pages -on: - workflow_dispatch - -jobs: - build: - name: Deploy docs - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v2 - - - name: Deploy docs - uses: mhausenblas/mkdocs-deploy-gh-pages@master - env: - GITHUB_TOKEN: ${{ secrets.DOCSITE_TOKEN }} - CONFIG_FILE: mkdocs.yml - REQUIREMENTS: requirements.txt diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 51b514ffa3..0000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Publish Conductor OSS toMaven Central -on: - release: - types: - - released - - prereleased - -permissions: - contents: read - -jobs: - publish: - runs-on: ubuntu-latest - name: Gradle Build and Publish - steps: - - uses: actions/checkout@v3 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Cache Gradle packages - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Publish release - run: | - export VERSION="${{github.ref_name}}" - export PUBLISH_VERSION=`echo ${VERSION:1}` - echo Publishing version $PUBLISH_VERSION - ./gradlew publish -Pversion=$PUBLISH_VERSION -Pusername=${{ secrets.SONATYPE_USERNAME }} -Ppassword=${{ secrets.SONATYPE_PASSWORD }} - env: - ORG_GRADLE_PROJECT_signingKeyId: ${{ secrets.SIGNING_KEY_ID }} - ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SIGNING_KEY }} - ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SIGNING_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/release_draft.yml b/.github/workflows/release_draft.yml deleted file mode 100644 index 2f185417d7..0000000000 --- a/.github/workflows/release_draft.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Release Drafter - -on: - push: - branches: - - main - -permissions: - contents: read - -jobs: - update_release_draft: - permissions: - contents: write # for release-drafter/release-drafter to create a github release - pull-requests: write # for release-drafter/release-drafter to add label to PR - runs-on: ubuntu-latest - steps: - - uses: release-drafter/release-drafter@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..935be7dd78 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +# +# conductor:server - Netflix conductor server +# + +# =========================================================================================================== +# 0. Builder stage +# =========================================================================================================== +FROM eclipse-temurin:17-jdk-focal AS builder + +LABEL maintainer="Netflix OSS " + +# Copy the project directly onto the image +COPY . /conductor +WORKDIR /conductor + +# Build the server on run +RUN ./gradlew build -x test --stacktrace + +# =========================================================================================================== +# 1. Bin stage +# =========================================================================================================== +FROM eclipse-temurin:17-jre-focal + +LABEL maintainer="Netflix OSS " + +# Make app folders +RUN mkdir -p /app/config /app/logs /app/libs + +# Copy the compiled output to new image +COPY --from=builder /conductor/docker/server/bin /app +COPY --from=builder /conductor/docker/server/config /app/config +COPY --from=builder /conductor/server/build/libs/*boot*.jar /app/libs/conductor-server.jar + +# Copy the files for the server into the app folders +RUN chmod +x /app/startup.sh + +HEALTHCHECK --interval=60s --timeout=30s --retries=10 CMD curl -I -XGET http://localhost:8080/health || exit 1 + +CMD [ "/app/startup.sh" ] +ENTRYPOINT [ "/bin/sh"] diff --git a/awss3-storage/build.gradle b/awss3-storage/build.gradle index 57e9d4fc3a..350ae8bd97 100644 --- a/awss3-storage/build.gradle +++ b/awss3-storage/build.gradle @@ -18,4 +18,7 @@ dependencies { implementation "com.amazonaws:aws-java-sdk-s3:${revAwsSdk}" implementation "org.apache.commons:commons-lang3" + implementation 'software.amazon.awssdk:s3:2.20.146' + implementation 'software.amazon.awssdk:sts:2.20.146' + implementation 'com.amazonaws:aws-java-sdk-sts:1.12.782' } diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java index b14d79395a..44dc8a43ed 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java @@ -21,6 +21,7 @@ import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.s3.storage.S3PayloadStorage; +import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -41,6 +42,17 @@ public ExternalPayloadStorage s3ExternalPayloadStorage( matchIfMissing = true) @Bean public AmazonS3 amazonS3(S3Properties properties) { + if (properties.getWebIdentity()) { + return AmazonS3ClientBuilder.standard() + .withCredentials(WebIdentityTokenCredentialsProvider.builder().build()) + .build(); + } return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build(); + // TODO: Add localstack support to test locally + // return AmazonS3ClientBuilder.standard() + // .withEndpointConfiguration( + // new AwsClientBuilder.EndpointConfiguration( + // properties.getEndpoint(), properties.getRegion())) + // .build(); } } diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java index 9c41b4a107..df28f94c30 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java @@ -24,6 +24,8 @@ public class S3Properties { /** The s3 bucket name where the payloads will be stored */ private String bucketName = "conductor_payloads"; + private Boolean webIdentity = false; + /** The time (in seconds) for which the signed url will be valid */ @DurationUnit(ChronoUnit.SECONDS) private Duration signedUrlExpirationDuration = Duration.ofSeconds(5); @@ -54,4 +56,18 @@ public String getRegion() { public void setRegion(String region) { this.region = region; } + + public Boolean getWebIdentity() { + return webIdentity; + } + // TODO: Add localstack support to test locally + // private String endpoint = "http://s3.localhost.localstack.cloud:4566"; + // + // public String getEndpoint() { + // return endpoint; + // } + // + // public void setEndpoint(String endpoint) { + // this.endpoint = endpoint; + // } } diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java index 19ac68d27d..4a70ac033e 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java @@ -132,7 +132,9 @@ public void upload(String path, InputStream payload, long payloadSize) { } catch (SdkClientException e) { String msg = String.format( - "Error uploading to S3 - path:%s, payloadSize: %d", path, payloadSize); + "Error uploading to S3 - path:%s, payloadSize: %d, bucketName: %s", + path, payloadSize, bucketName); + e.printStackTrace(); LOGGER.error(msg, e); throw new TransientException(msg, e); } diff --git a/core/build.gradle b/core/build.gradle index beef8f7a25..065a9fe26a 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -43,6 +43,8 @@ dependencies { implementation "org.openjdk.nashorn:nashorn-core:15.4" + implementation "com.netflix.spectator:spectator-reg-metrics3:${version_spectator}" + // JAXB is not bundled with Java 11, dependencies added explicitly // These are needed by Apache BVAL implementation "jakarta.xml.bind:jakarta.xml.bind-api:${revJAXB}" diff --git a/dependencies.gradle b/dependencies.gradle index ed1c08fa10..32407c30da 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -67,5 +67,5 @@ ext { revNatsStreaming = '2.6.5' revNats = '2.15.6' revStan = '2.2.3' - + version_spectator='0.60.0' } diff --git a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java index d85fa24840..efa5bb2f63 100644 --- a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java +++ b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +25,7 @@ import org.springframework.http.*; import org.springframework.stereotype.Component; import org.springframework.util.MultiValueMap; +import org.springframework.web.client.HttpStatusCodeException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -117,7 +119,13 @@ public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor execu if (response.body != null) { task.setReasonForIncompletion(response.body.toString()); } else { - task.setReasonForIncompletion("No response from the remote service"); + task.setReasonForIncompletion( + String.format( + "%d: %s", + response.statusCode, + Objects.requireNonNullElse( + response.reasonPhrase, + "No response from the remote service"))); } task.setStatus(TaskModel.Status.FAILED); } @@ -175,12 +183,16 @@ protected HttpResponse httpCall(Input input) throws Exception { if (responseEntity.getStatusCode().is2xxSuccessful() && responseEntity.hasBody()) { response.body = extractBody(responseEntity.getBody()); } - response.statusCode = responseEntity.getStatusCodeValue(); response.reasonPhrase = HttpStatus.valueOf(responseEntity.getStatusCode().value()).getReasonPhrase(); response.headers = responseEntity.getHeaders(); return response; + } catch (HttpStatusCodeException ex) { + response.headers = ex.getResponseHeaders(); + response.statusCode = ex.getStatusCode().value(); + response.reasonPhrase = ex.getStatusText(); + return response; } catch (RestClientException ex) { LOGGER.error( String.format( diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V10_update_archival_procedure.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V10_update_archival_procedure.sql new file mode 100644 index 0000000000..fdb67f93ad --- /dev/null +++ b/postgres-persistence/src/main/resources/db/migration_postgres/V10_update_archival_procedure.sql @@ -0,0 +1,109 @@ + +CREATE TABLE IF NOT EXISTS archival_logs ( + id SERIAL PRIMARY KEY, + log_time TIMESTAMP NOT NULL DEFAULT now(), + log_message TEXT NOT NULL, + archival_date DATE NOT NULL +); + +CREATE OR REPLACE PROCEDURE public.conductor_archive(IN archival_date date) + LANGUAGE plpgsql +AS $procedure$ +DECLARE + deleted_workflows INT := 0; + deleted_wf_def_links INT := 0; + deleted_wf_to_task INT := 0; + deleted_tasks INT := 0; + deleted_task_scheduled INT := 0; + deleted_workflow_index INT := 0; + deleted_task_index INT := 0; + total_deleted INT := 0; + log_message TEXT; +BEGIN + -- Step 1: Collect workflow IDs eligible for deletion + CREATE TEMP TABLE temp_workflows_to_delete ON COMMIT DROP AS + SELECT workflow_id + FROM workflow + WHERE created_on < archival_date + AND (json_data::jsonb ->> 'status') IN ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED'); + + ALTER TABLE temp_workflows_to_delete ADD PRIMARY KEY (workflow_id); + ANALYZE temp_workflows_to_delete; + + -- Step 2: Cascade deletes + + -- workflow_def_to_workflow + DELETE FROM workflow_def_to_workflow wdw + USING temp_workflows_to_delete tw + WHERE wdw.workflow_id = tw.workflow_id; + GET DIAGNOSTICS deleted_wf_def_links = ROW_COUNT; + + -- workflow_index + DELETE FROM workflow_index wi + USING temp_workflows_to_delete tw + WHERE wi.workflow_id = tw.workflow_id; + GET DIAGNOSTICS deleted_workflow_index = ROW_COUNT; + + -- workflow_to_task + CREATE TEMP TABLE temp_tasks_to_delete ON COMMIT DROP AS + SELECT wt.task_id + FROM workflow_to_task wt + JOIN temp_workflows_to_delete tw ON wt.workflow_id = tw.workflow_id; + + ALTER TABLE temp_tasks_to_delete ADD PRIMARY KEY (task_id); + ANALYZE temp_tasks_to_delete; + + DELETE FROM workflow_to_task wt + USING temp_workflows_to_delete tw + WHERE wt.workflow_id = tw.workflow_id; + GET DIAGNOSTICS deleted_wf_to_task = ROW_COUNT; + + -- task_scheduled + DELETE FROM task_scheduled ts + USING temp_tasks_to_delete tt + WHERE ts.task_id = tt.task_id; + GET DIAGNOSTICS deleted_task_scheduled = ROW_COUNT; + + -- task_index + DELETE FROM task_index ti + USING temp_tasks_to_delete tt + WHERE ti.task_id = tt.task_id; + GET DIAGNOSTICS deleted_task_index = ROW_COUNT; + + -- task + DELETE FROM task t + USING temp_tasks_to_delete tt + WHERE t.task_id = tt.task_id; + GET DIAGNOSTICS deleted_tasks = ROW_COUNT; + + -- workflow + DELETE FROM workflow w + USING temp_workflows_to_delete tw + WHERE w.workflow_id = tw.workflow_id; + GET DIAGNOSTICS deleted_workflows = ROW_COUNT; + + -- Step 3: Logging + total_deleted := deleted_workflows + deleted_wf_def_links + deleted_workflow_index + + deleted_wf_to_task + deleted_tasks + deleted_task_scheduled + deleted_task_index; + + log_message := 'Cleanup completed successfully for COMPLETED, FAILED, TIMED_OUT, and TERMINATED workflows before ' || archival_date || '. ' || + 'Total deleted: ' || total_deleted || ' | Breakdown: ' || + 'workflow: ' || deleted_workflows || ', ' || + 'workflow_def_to_workflow: ' || deleted_wf_def_links || ', ' || + 'workflow_index: ' || deleted_workflow_index || ', ' || + 'workflow_to_task: ' || deleted_wf_to_task || ', ' || + 'task: ' || deleted_tasks || ', ' || + 'task_scheduled: ' || deleted_task_scheduled || ', ' || + 'task_index: ' || deleted_task_index; + + INSERT INTO archival_logs(log_message, archival_date) + VALUES (log_message, archival_date); + +EXCEPTION + WHEN OTHERS THEN + INSERT INTO archival_logs(log_message, archival_date) + VALUES ('Error in cleanup_completed_workflows: ' || SQLERRM, archival_date); + RAISE; +END; +$procedure$ +; diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V9__archival_procedure.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V9__archival_procedure.sql new file mode 100644 index 0000000000..83ae483c42 --- /dev/null +++ b/postgres-persistence/src/main/resources/db/migration_postgres/V9__archival_procedure.sql @@ -0,0 +1,55 @@ +CREATE OR REPLACE PROCEDURE public.conductor_archive(IN archival_date date) + LANGUAGE plpgsql +AS $procedure$ +BEGIN + + --CREATING TEMP TABLE FOR TASK IDs + CREATE TEMP TABLE temp_task_ids ON COMMIT DROP AS + SELECT task_id FROM task WHERE created_on < archival_date; + + ALTER TABLE temp_task_ids ADD PRIMARY KEY (task_id); + ANALYZE temp_task_ids; + + --CREATING TEMP TABLE FOR WORKFLOW IDs + CREATE TEMP TABLE temp_workflow_ids ON COMMIT DROP AS + SELECT workflow_id FROM workflow WHERE created_on < archival_date; + + ALTER TABLE temp_workflow_ids ADD PRIMARY KEY (workflow_id); + ANALYZE temp_workflow_ids; + + --CREATING TEMP TABLE FOR temp_workflow_def_to_workflow IDs + CREATE TEMP TABLE temp_workflow_def_to_workflow_ids ON COMMIT DROP AS + SELECT wdt.workflow_id + FROM workflow_def_to_workflow wdt + JOIN temp_workflow_ids tw ON tw.workflow_id = wdt.workflow_id; + + ALTER TABLE temp_workflow_def_to_workflow_ids ADD PRIMARY KEY (workflow_id); + ANALYZE temp_workflow_def_to_workflow_ids; + + --CREATING TEMP TABLES FOR workflow_to_task IDs + CREATE TEMP TABLE temp_workflow_to_task_ids ON COMMIT DROP AS + SELECT w.task_id + FROM workflow_to_task w + JOIN temp_task_ids t ON t.task_id = w.task_id; + + ALTER TABLE temp_workflow_to_task_ids ADD PRIMARY KEY(task_id); + ANALYZE temp_workflow_to_task_ids; + + --CREATING TEMP TABLES FOR task_scheduled IDs + CREATE TEMP TABLE temp_task_scheduled_ids ON COMMIT DROP AS + SELECT ts.task_id + FROM task_scheduled ts + JOIN temp_task_ids t ON t.task_id = ts.task_id; + + ALTER TABLE temp_task_scheduled_ids ADD PRIMARY KEY(task_id); + ANALYZE temp_task_scheduled_ids; + + DELETE FROM task t USING temp_task_ids tti WHERE t.task_id = tti.task_id; + DELETE FROM workflow w USING temp_workflow_ids t WHERE w.workflow_id = t.workflow_id; + DELETE FROM workflow_def_to_workflow w USING temp_workflow_def_to_workflow_ids t WHERE w.workflow_id = t.workflow_id; + DELETE FROM workflow_to_task w USING temp_workflow_to_task_ids t WHERE w.task_id = t.task_id; + DELETE FROM task_scheduled t USING temp_task_scheduled_ids tts WHERE t.task_id = tts.task_id; + DELETE FROM event_execution WHERE created_on < archival_date; +END; +$procedure$ +; diff --git a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java new file mode 100644 index 0000000000..7a3fc396b6 --- /dev/null +++ b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java @@ -0,0 +1,140 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.rest.startup; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.context.event.EventListener; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.stereotype.Component; +import org.springframework.util.FileCopyUtils; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import static org.springframework.http.HttpHeaders.CONTENT_TYPE; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + +@Component +public class SirenInitializer { + private static final String ALREADY_EXISTS_KEYWORD = "already exists"; + private static final Logger LOGGER = LoggerFactory.getLogger(SirenInitializer.class); + + private final RestTemplate restTemplate; + + @Value("${loadSirenResources:false}") + private boolean loadSirenResources; + + @Value("${server.url:http://localhost:8080}") + private String url; + + @Value("classpath:./siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json") + private Resource finalizeWorkflowExecutionEventHandler; + + @Value("classpath:./siren/tasks/finalizeWorkflowExecutionTask.json") + private Resource finalizeWorkflowExecutionTask; + + @Value("classpath:./siren/tasks/sendNotificationTask.json") + private Resource sendNotificationTask; + + @Value("classpath:./siren/tasks/sendWebhookTask.json") + private Resource sendWebhookTask; + + @Value("classpath:./siren/workflows/sirenFinalizeExecutionWorkflow.json") + private Resource sirenFinalizeExecutionWorkflow; + + @Value("classpath:./siren/workflows/sirenWebhookWorkflow.json") + private Resource sirenWebhookWorkflow; + + @Value("classpath:./siren/workflows/sirenCampaignWorkflow.json") + private Resource sirenCampaignWorkflow; + + public SirenInitializer(RestTemplateBuilder restTemplateBuilder) { + this.restTemplate = restTemplateBuilder.build(); + } + + @EventListener(ApplicationReadyEvent.class) + public void setupSirenResources() { + if (loadSirenResources) { + LOGGER.info("Loading siren resources"); + createSirenResources(); + } + } + + private void createSirenResources() { + MultiValueMap headers = new LinkedMultiValueMap<>(); + headers.add(CONTENT_TYPE, APPLICATION_JSON_VALUE); + createWorkflow(sirenFinalizeExecutionWorkflow, headers); + createWorkflow(sirenWebhookWorkflow, headers); + createWorkflow(sirenCampaignWorkflow, headers); + LOGGER.info("Siren workflows are created"); + + updateTask(sendNotificationTask, headers); + updateTask(sendWebhookTask, headers); + updateTask(finalizeWorkflowExecutionTask, headers); + LOGGER.info("Siren tasks are created"); + + createEventHandler(finalizeWorkflowExecutionEventHandler, headers); + LOGGER.info("Siren event handlers are created"); + } + + private void createWorkflow(Resource resource, MultiValueMap headers) { + try { + HttpEntity request = new HttpEntity<>(readToString(resource), headers); + restTemplate.postForEntity(url + "/api/metadata/workflow", request, Map.class); + } catch (RestClientException e) { + handleException(e); + } + } + + private void updateTask(Resource resource, MultiValueMap headers) { + HttpEntity request = new HttpEntity<>(readToString(resource), headers); + restTemplate.postForEntity(url + "/api/metadata/taskdefs", request, Map.class); + } + + private void createEventHandler(Resource resource, MultiValueMap headers) { + try { + HttpEntity request = new HttpEntity<>(readToString(resource), headers); + restTemplate.postForEntity(url + "/api/event", request, Map.class); + } catch (RestClientException e) { + handleException(e); + } + } + + private void handleException(RestClientException e) { + if (e.getMessage().contains(ALREADY_EXISTS_KEYWORD)) { + LOGGER.info("Skipping creation: {}", e.getMessage()); + } else { + LOGGER.error("Error while creation ", e); + throw e; + } + } + + private String readToString(Resource resource) { + try { + return FileCopyUtils.copyToString(new InputStreamReader(resource.getInputStream())); + } catch (IOException e) { + LOGGER.error("Error while loading siren resources", e); + throw new RuntimeException("Error reading resources", e); + } + } +} diff --git a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json new file mode 100644 index 0000000000..d14435c114 --- /dev/null +++ b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json @@ -0,0 +1,25 @@ +{ + "name": "finalize_workflow_execution_event_handler", + "event": "conductor:finalize_workflow_execution_event", + "actions": [ + { + "action": "start_workflow", + "start_workflow": { + "name": "siren_finalize_execution_workflow", + "input": { + "workflowType": "${workflowType}", + "version": "${version}", + "workflowId": "${workflowId}", + "correlationId": "${correlationId}", + "status": "${status}", + "output": "${output}", + "reasonForIncompletion": "${reasonForIncompletion}", + "executionTime": "${executionTime}", + "event": "${event}" + } + }, + "expandInlineJSON": true + } + ], + "active": true +} diff --git a/rest/src/main/resources/siren/tasks/finalizeWorkflowExecutionTask.json b/rest/src/main/resources/siren/tasks/finalizeWorkflowExecutionTask.json new file mode 100644 index 0000000000..eb19e7a45e --- /dev/null +++ b/rest/src/main/resources/siren/tasks/finalizeWorkflowExecutionTask.json @@ -0,0 +1,21 @@ +[ + { + "createdBy": "", + "accessPolicy": {}, + "name": "finalize_workflow_execution_task", + "description": "Finalize Workflow Execution Task", + "retryCount": 5, + "timeoutSeconds": 3600, + "inputKeys": [], + "outputKeys": [], + "timeoutPolicy": "TIME_OUT_WF", + "retryLogic": "EXPONENTIAL_BACKOFF", + "retryDelaySeconds": 10, + "responseTimeoutSeconds": 600, + "inputTemplate": {}, + "rateLimitPerFrequency": 0, + "rateLimitFrequencyInSeconds": 1, + "ownerEmail": "admin@sirenapp.io", + "backoffScaleFactor": 1 + } +] diff --git a/rest/src/main/resources/siren/tasks/sendNotificationTask.json b/rest/src/main/resources/siren/tasks/sendNotificationTask.json new file mode 100644 index 0000000000..4d331f70d5 --- /dev/null +++ b/rest/src/main/resources/siren/tasks/sendNotificationTask.json @@ -0,0 +1,21 @@ +[ + { + "createdBy": "", + "accessPolicy": {}, + "name": "send_notification_task", + "description": "Send Notification Task", + "retryCount": 5, + "timeoutSeconds": 3600, + "inputKeys": [], + "outputKeys": [], + "timeoutPolicy": "TIME_OUT_WF", + "retryLogic": "EXPONENTIAL_BACKOFF", + "retryDelaySeconds": 10, + "responseTimeoutSeconds": 600, + "inputTemplate": {}, + "rateLimitPerFrequency": 0, + "rateLimitFrequencyInSeconds": 1, + "ownerEmail": "admin@sirenapp.io", + "backoffScaleFactor": 1 + } +] diff --git a/rest/src/main/resources/siren/tasks/sendWebhookTask.json b/rest/src/main/resources/siren/tasks/sendWebhookTask.json new file mode 100644 index 0000000000..1878c2aaa0 --- /dev/null +++ b/rest/src/main/resources/siren/tasks/sendWebhookTask.json @@ -0,0 +1,21 @@ +[ + { + "createdBy": "", + "accessPolicy": {}, + "name": "send_webhook_task", + "description": "Send Webhook Task", + "retryCount": 5, + "timeoutSeconds": 3600, + "inputKeys": [], + "outputKeys": [], + "timeoutPolicy": "TIME_OUT_WF", + "retryLogic": "EXPONENTIAL_BACKOFF", + "retryDelaySeconds": 10, + "responseTimeoutSeconds": 600, + "inputTemplate": {}, + "rateLimitPerFrequency": 0, + "rateLimitFrequencyInSeconds": 1, + "ownerEmail": "admin@sirenapp.io", + "backoffScaleFactor": 1 + } +] diff --git a/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json b/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json new file mode 100644 index 0000000000..aa5b54afef --- /dev/null +++ b/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json @@ -0,0 +1,57 @@ +{ + "accessPolicy": {}, + "name": "siren_campaign_workflow", + "description": "Workflow for executing campaigns", + "version": 1, + "tasks": [ + { + "name": "template_fork", + "taskReferenceName": "temp_pSzeL9pM3AMtpm7hD-WcA_template_fork", + "inputParameters": {}, + "type": "FORK_JOIN", + "forkTasks": [ + [ + { + "name": "send_notification_task", + "taskReferenceName": "prov_j7ooKIGve0iG2n2tH-1Ds_send_notification_task", + "inputParameters": { + "workflowExecutionId": "${workflow.input.workflowExecutionId}", + "providerIntegrationId": "${workflow.input.providerIntegrationId}", + "templateId": "${workflow.input.templateId}", + "notifyVariables": "${workflow.input.notifyVariables}", + "templateVariables": "${workflow.input.templateVariables}" + }, + "type": "SIMPLE", + "startDelay": 0, + "optional": true, + "asyncComplete": false, + "permissive": false + } + ] + ], + "startDelay": 0, + "optional": false, + "asyncComplete": false, + "permissive": false + }, + { + "name": "template_join", + "taskReferenceName": "temp_pSzeL9pM3AMtpm7hD-WcA_template_join", + "inputParameters": {}, + "type": "JOIN", + "startDelay": 0, + "optional": false, + "asyncComplete": false, + "permissive": false + } + ], + "inputParameters": [], + "outputParameters": {}, + "schemaVersion": 2, + "restartable": true, + "workflowStatusListenerEnabled": true, + "ownerEmail": "admin@sirenapp.io", + "timeoutSeconds": 0, + "variables": {}, + "inputTemplate": {} +} \ No newline at end of file diff --git a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json new file mode 100644 index 0000000000..3c0373cb7c --- /dev/null +++ b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json @@ -0,0 +1,31 @@ +{ + "accessPolicy": {}, + "name": "siren_finalize_execution_workflow", + "description": "Workflow for finalizing workflow execution", + "version": 1, + "tasks": [ + { + "name": "finalize_workflow_execution_task", + "taskReferenceName": "finalize_workflow_execution_task_ref", + "inputParameters": { + "status": "${workflow.input.status}", + "externalExecutionId": "${workflow.input.workflowId}" + }, + "type": "SIMPLE", + "startDelay": 0, + "optional": false, + "asyncComplete": false, + "permissive": false + } + ], + "inputParameters": [], + "outputParameters": {}, + "schemaVersion": 2, + "restartable": true, + "workflowStatusListenerEnabled": false, + "ownerEmail": "admin@sirenapp.io", + "timeoutPolicy": "ALERT_ONLY", + "timeoutSeconds": 0, + "variables": {}, + "inputTemplate": {} +} diff --git a/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json new file mode 100644 index 0000000000..450505f423 --- /dev/null +++ b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json @@ -0,0 +1,31 @@ +{ + "accessPolicy": {}, + "name": "siren_webhook_workflow", + "description": "Workflow for sending webhook", + "version": 1, + "tasks": [ + { + "name": "send_webhook_task", + "taskReferenceName": "send_webhook_task_ref", + "inputParameters": { + "notificationAuditId": "${workflow.input.notificationAuditId}", + "workflowExecutionId": "${workflow.input.workflowExecutionId}" + }, + "type": "SIMPLE", + "startDelay": 0, + "optional": true, + "asyncComplete": false, + "permissive": false + } + ], + "inputParameters": [], + "outputParameters": {}, + "schemaVersion": 2, + "restartable": true, + "workflowStatusListenerEnabled": false, + "ownerEmail": "admin@sirenapp.io", + "timeoutPolicy": "ALERT_ONLY", + "timeoutSeconds": 0, + "variables": {}, + "inputTemplate": {} +} diff --git a/server/build.gradle b/server/build.gradle index dc9dd8303e..a387b10d9b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -70,7 +70,10 @@ dependencies { implementation "io.orkes.queues:orkes-conductor-queues:${revOrkesQueues}" implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:${revSpringDoc}" + runtimeOnly group: 'com.netflix.conductor', name: 'conductor-postgres-persistence', version: '3.9.1' + implementation "com.netflix.spectator:spectator-reg-metrics3:${version_spectator}" + implementation 'javax.xml.bind:jaxb-api:2.3.1' runtimeOnly "org.glassfish.jaxb:jaxb-runtime:${revJAXB}" diff --git a/ui/Dockerfile b/ui/Dockerfile new file mode 100644 index 0000000000..240bdb712e --- /dev/null +++ b/ui/Dockerfile @@ -0,0 +1,3 @@ +FROM nginx +COPY default.conf /etc/nginx/conf.d/default.conf +COPY build/ /usr/share/nginx/html \ No newline at end of file diff --git a/ui/default-dev.conf b/ui/default-dev.conf new file mode 100644 index 0000000000..180f83dee3 --- /dev/null +++ b/ui/default-dev.conf @@ -0,0 +1,57 @@ + +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.dev.svc.cluster.local:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.dev.svc.cluster.local:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.dev.svc.cluster.local:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{"status":"UP"}'; + } +} \ No newline at end of file diff --git a/ui/default-local.conf b/ui/default-local.conf new file mode 100644 index 0000000000..fa8f0871d0 --- /dev/null +++ b/ui/default-local.conf @@ -0,0 +1,50 @@ +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } +} \ No newline at end of file diff --git a/ui/default-prd.conf b/ui/default-prd.conf new file mode 100644 index 0000000000..256c0cce2c --- /dev/null +++ b/ui/default-prd.conf @@ -0,0 +1,57 @@ + +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.prd.svc.cluster.local:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.prd.svc.cluster.local:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.prd.svc.cluster.local:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{"status":"UP"}'; + } +} \ No newline at end of file diff --git a/ui/default-stg.conf b/ui/default-stg.conf new file mode 100644 index 0000000000..f88374776e --- /dev/null +++ b/ui/default-stg.conf @@ -0,0 +1,57 @@ + +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.stg.svc.cluster.local:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.stg.svc.cluster.local:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://conductor.stg.svc.cluster.local:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{"status":"UP"}'; + } +} \ No newline at end of file