Skip to content
This repository was archived by the owner on Apr 1, 2025. It is now read-only.

Commit c98db51

Browse files
Merge pull request #93 from microsoft/feature/create-e2e-deployment
Feature/create e2e deployment
2 parents aab814e + 17f0a15 commit c98db51

File tree

108 files changed

+4607
-755
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+4607
-755
lines changed

.github/workflows/build-docker.yaml

Lines changed: 69 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -47,34 +47,78 @@ jobs:
4747
- name: Build NAR file
4848
run: ./gradlew nar
4949

50-
# until improved container support is enabled for the repo, we'll stick with V1 of the action
51-
- name: Push to GitHub Packages
52-
uses: docker/build-push-action@v1
50+
- name: Set up QEMU
51+
uses: docker/setup-qemu-action@v1
52+
53+
- name: Set up Docker Buildx
54+
uses: docker/setup-buildx-action@v1
55+
56+
- name: Login to DockerHub
57+
uses: docker/login-action@v1
5358
with:
54-
path: external/nifi
55-
username: ${{ github.actor }}
56-
password: ${{ secrets.GITHUB_TOKEN }}
57-
registry: docker.pkg.github.com
58-
repository: microsoft/data-appliance-gx/nifi
59-
tag_with_ref: false
59+
username: ${{ secrets.DOCKERHUB_USERNAME }}
60+
password: ${{ secrets.DOCKERHUB_TOKEN }}
61+
62+
- name: Build and push
63+
uses: docker/build-push-action@v2
64+
with:
65+
context: external/nifi
66+
push: true
67+
platforms: linux/amd64,linux/arm64
68+
cache-to: type=inline
69+
tags: beardyinc/nifi:latest
6070

6171

6272
# we can enable the v2 of the docker build once the improved container support has been enabled for the repo:
6373
# https://bit.ly/3hB5sJ6
6474

65-
# - name: Login to Github Container Registry
66-
# uses: docker/login-action@v1
67-
# with:
68-
# registry: ghcr.io
69-
# username: ${{ github.repository_owner }}
70-
# password: ${{ secrets.GITHUB_TOKEN }}
71-
#
72-
# - name: Build and push
73-
# uses: docker/build-push-action@v2
74-
# with:
75-
# context: external/nifi
76-
# pull: true
77-
# push: true
78-
# # cache-from: type=registry,ref=microsoft/data-appliance-gx/nifi:latest
79-
# cache-to: type=inline
80-
# tags: ghcr.io/microsoft/data-appliance-gx/nifi:latest
75+
# - name: Login to Github Container Registry
76+
# uses: docker/login-action@v1
77+
# with:
78+
# registry: ghcr.io
79+
# username: ${{ github.repository_owner }}
80+
# password: ${{ secrets.GITHUB_TOKEN }}
81+
#
82+
# - name: Build and push
83+
# uses: docker/build-push-action@v2
84+
# with:
85+
# context: external/nifi
86+
# pull: true
87+
# push: true
88+
# # cache-from: type=registry,ref=microsoft/data-appliance-gx/nifi:latest
89+
# cache-to: type=inline
90+
# tags: ghcr.io/microsoft/data-appliance-gx/nifi:latest
91+
92+
build-connector-image:
93+
runs-on: ubuntu-latest
94+
steps:
95+
- uses: actions/checkout@v2
96+
- name: Set up JDK 11
97+
uses: actions/setup-java@v1
98+
with:
99+
java-version: '11'
100+
101+
- name: Build Azure Image
102+
run: ./gradlew clean shadowJar
103+
104+
- name: Set up QEMU
105+
uses: docker/setup-qemu-action@v1
106+
107+
- name: Set up Docker Buildx
108+
uses: docker/setup-buildx-action@v1
109+
110+
- name: Login to DockerHub
111+
uses: docker/login-action@v1
112+
with:
113+
username: ${{ secrets.DOCKERHUB_USERNAME }}
114+
password: ${{ secrets.DOCKERHUB_TOKEN }}
115+
116+
- name: Build and push
117+
uses: docker/build-push-action@v2
118+
with:
119+
context: distributions/demo-e2e
120+
pull: true
121+
push: true
122+
platforms: linux/amd64,linux/arm64
123+
cache-to: type=inline
124+
tags: beardyinc/dagx-demo:latest

.github/workflows/terraform.yaml

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# This workflow installs the latest version of Terraform CLI and configures the Terraform CLI configuration file
2+
# with an API token for Terraform Cloud (app.terraform.io). On pull request events, this workflow will run
3+
# `terraform init`, `terraform fmt`, and `terraform plan` (speculative plan via Terraform Cloud). On push events
4+
# to the main branch, `terraform apply` will be executed.
5+
#
6+
# Documentation for `hashicorp/setup-terraform` is located here: https://github.com/hashicorp/setup-terraform
7+
#
8+
# To use this workflow, you will need to complete the following setup steps.
9+
#
10+
# 1. Create a `main.tf` file in the root of this repository with the `remote` backend and one or more resources defined.
11+
# Example `main.tf`:
12+
# # The configuration for the `remote` backend.
13+
# terraform {
14+
# backend "remote" {
15+
# # The name of your Terraform Cloud organization.
16+
# organization = "example-organization"
17+
#
18+
# # The name of the Terraform Cloud workspace to store Terraform state files in.
19+
# workspaces {
20+
# name = "example-workspace"
21+
# }
22+
# }
23+
# }
24+
#
25+
# # An example resource that does nothing.
26+
# resource "null_resource" "example" {
27+
# triggers = {
28+
# value = "A example resource that does nothing!"
29+
# }
30+
# }
31+
#
32+
#
33+
# 2. Generate a Terraform Cloud user API token and store it as a GitHub secret (e.g. TF_API_TOKEN) on this repository.
34+
# Documentation:
35+
# - https://www.terraform.io/docs/cloud/users-teams-organizations/api-tokens.html
36+
# - https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets
37+
#
38+
# 3. Reference the GitHub secret in step using the `hashicorp/setup-terraform` GitHub Action.
39+
# Example:
40+
# - name: Setup Terraform
41+
# uses: hashicorp/setup-terraform@v1
42+
# with:
43+
# cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
44+
45+
name: 'Terraform'
46+
47+
on:
48+
push:
49+
branches:
50+
- main
51+
pull_request:
52+
53+
jobs:
54+
terraform:
55+
name: 'Terraform'
56+
runs-on: ubuntu-latest
57+
environment: production
58+
env:
59+
ARM_CLIENT_ID: ${{ secrets.AZURE_AD_CIENT_ID }}
60+
ARM_CLIENT_SECRET: ${{ secrets.AZURE_AD_CLIENT_SECRET }}
61+
ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_AD_SUBSCRIPTION_ID }}
62+
ARM_TENANT_ID: ${{ secrets.AZURE_AD_TENANT_ID }}
63+
TF_VAR_CERTIFICATE: ${{ secrets.PRIMARY_APP_ID_CERT }}
64+
AWS_DEFAULT_REGION: "us-east-1"
65+
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
66+
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
67+
68+
# Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
69+
defaults:
70+
run:
71+
shell: bash
72+
73+
steps:
74+
- name: Checkout
75+
uses: actions/checkout@v2
76+
77+
- name: Setup Terraform
78+
uses: hashicorp/setup-terraform@v1
79+
with:
80+
# terraform_version: 0.13.0:
81+
cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
82+
83+
- name: Terraform Format
84+
id: fmt
85+
run: terraform -chdir=scripts fmt -check
86+
87+
- name: Terraform Init
88+
id: init
89+
run: terraform -chdir=scripts init
90+
91+
- name: Terraform Validate
92+
id: validate
93+
run: terraform -chdir=scripts validate -no-color
94+
95+
- name: Terraform Plan
96+
id: plan
97+
if: github.event_name == 'pull_request'
98+
run: terraform -chdir=scripts plan -no-color -var "resourcesuffix=dev" -var "backend_account_key=${{ secrets.TF_BACKEND_KEY }}"
99+
100+
101+
- name: Taint the connector instance
102+
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
103+
run: terraform -chdir=scripts taint azurerm_container_group.connector-instance
104+
continue-on-error: true
105+
106+
- name: Terraform Apply
107+
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
108+
run: terraform -chdir=scripts apply -var "resourcesuffix=dev" -var "backend_account_key=${{ secrets.TF_BACKEND_KEY }}" -auto-approve

.github/workflows/verify.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@ jobs:
99
steps:
1010
- uses: actions/checkout@v2
1111
- name: Set up JDK 11
12+
id: setup-jdk
1213
uses: actions/setup-java@v1
1314
with:
1415
java-version: '11'
1516

1617
- name: Gradle Test Core
18+
id: test
1719
env:
1820
AZ_STORAGE_SAS: ${{ secrets.AZ_STORAGE_SAS }}
1921
AZ_STORAGE_KEY: ${{ secrets.AZ_STORAGE_KEY }}
@@ -23,6 +25,7 @@ jobs:
2325
run: ./gradlew clean check -x :extensions:catalog:catalog-atlas:check -x :extensions:security:security-azure:check -x :extensions:transfer:transfer-nifi:check -x :external:nifi:processors:check
2426

2527
- name: Publish Unit Test Results
28+
id: publish-results
2629
uses: EnricoMi/publish-unit-test-result-action@v1
2730
if: always()
2831
with:
@@ -48,6 +51,7 @@ jobs:
4851
steps:
4952
- uses: actions/checkout@v2
5053
- name: Set up JDK 11
54+
id: setup-jdk
5155
uses: actions/setup-java@v1
5256
with:
5357
java-version: '11'
@@ -62,6 +66,7 @@ jobs:
6266
run: ./gradlew extensions:transfer:transfer-nifi:check
6367

6468
- name: Test Atlas Integration
69+
id: atlas-integration
6570
run: ./gradlew extensions:catalog:catalog-atlas:check
6671

6772

@@ -76,13 +81,15 @@ jobs:
7681
java-version: '11'
7782

7883
- name: Test Custom Nifi Processors
84+
id: custom-nifi-processors
7985
env:
8086
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
8187
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
8288
AZ_STORAGE_SAS: ${{ secrets.AZ_STORAGE_SAS }}
8389
run: ./gradlew external:nifi:processors:check
8490

8591
- name: Test Azure Vault Integration
92+
id: azure-vault-tests
8693
env:
8794
AZ_STORAGE_SAS: ${{ secrets.AZ_STORAGE_SAS }}
8895
run: ./gradlew extensions:security:security-azure:check

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,5 @@ distributions/azure/azure.properties
4646

4747
.DS_Store
4848
**/secrets
49+
kubeconfig
50+
**/dagx-config.properties

build.gradle.kts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ plugins {
88
}
99

1010
repositories {
11-
jcenter()
11+
mavenCentral()
1212
}
1313

1414
val jetBrainsAnnotationsVersion: String by project
@@ -21,7 +21,7 @@ val configFs by extra { System.getProperty("configuration.fs", "disabled") }
2121
subprojects {
2222

2323
repositories {
24-
jcenter()
24+
mavenCentral()
2525
maven {
2626
url = uri("https://maven.iais.fraunhofer.de/artifactory/eis-ids-public/")
2727
}

common/src/main/java/com/microsoft/dagx/common/string/StringUtils.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,8 @@ public static boolean isNullOrBlank(String str) {
2424
public static boolean equalsIgnoreCase(String str1, String str2) {
2525
return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2);
2626
}
27+
28+
public static String toString(Object nullable) {
29+
return nullable != null ? nullable.toString() : null;
30+
}
2731
}

core/src/main/java/com/microsoft/dagx/system/DefaultServiceExtensionContext.java

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,7 @@
1313
import com.microsoft.dagx.spi.types.TypeManager;
1414
import com.microsoft.dagx.util.TopologicalSort;
1515

16-
import java.util.ArrayList;
17-
import java.util.Collections;
18-
import java.util.HashMap;
19-
import java.util.List;
20-
import java.util.Map;
21-
import java.util.Set;
16+
import java.util.*;
2217
import java.util.stream.Collectors;
2318

2419
import static com.microsoft.dagx.spi.system.ServiceExtension.LoadPhase.DEFAULT;
@@ -48,8 +43,8 @@ public DefaultServiceExtensionContext(TypeManager typeManager, Monitor monitor,
4843
this.monitor = monitor;
4944
this.serviceLocator = serviceLocator;
5045
// register as services
51-
this.services.put(TypeManager.class, typeManager);
52-
this.services.put(Monitor.class, monitor);
46+
services.put(TypeManager.class, typeManager);
47+
services.put(Monitor.class, monitor);
5348
}
5449

5550
public void initialize() {

core/src/main/java/com/microsoft/dagx/system/ExtensionLoader.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public static void loadVault(DefaultServiceExtensionContext context) {
4545
context.getMonitor().info("Secrets vault not configured. Defaulting to null vault.");
4646
}
4747
vaultExtension.initialize(context.getMonitor());
48+
vaultExtension.intializeVault(context);
4849
context.registerService(Vault.class, vaultExtension.getVault());
4950
context.registerService(PrivateKeyResolver.class, vaultExtension.getPrivateKeyResolver());
5051
context.registerService(CertificateResolver.class, vaultExtension.getCertificateResolver());

distributions/azure/build.gradle.kts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ dependencies {
3737
implementation(project(":extensions:configuration:configuration-fs"))
3838

3939
implementation(project(":extensions:catalog:catalog-atlas"))
40-
implementation(project(":extensions:catalog:catalog-atlas-dataseed"))
40+
implementation(project(":extensions:dataseed:dataseed-atlas"))
4141
implementation(project(":extensions:transfer:transfer-store-memory"))
4242

4343
implementation(project(":extensions:policy:policy-registry-memory"))

distributions/demo-e2e/Dockerfile

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
FROM openjdk:11-jre-slim
2+
3+
WORKDIR /app
4+
COPY ./build/libs/dagx-demo-e2e.jar /app
5+
#COPY ./dagx-config-docker.properties /app/dagx-config.properties
6+
#COPY ./cert.pfx /cert
7+
8+
EXPOSE 8181
9+
10+
ENTRYPOINT java \
11+
-Ddagx.vault.clientid=${CLIENTID} \
12+
-Ddagx.vault.tenantid=${TENANTID} \
13+
-Ddagx.vault.certificate=/cert/cert.pfx \
14+
-Ddagx.vault.name=${VAULTNAME} \
15+
-Ddagx.atlas.url=${ATLAS_URL} \
16+
-Ddagx.nifi.url=${NIFI_URL} \
17+
-Ddagx.nifi.flow.url=${NIFI_FLOW_URL} \
18+
-Djava.security.edg=file:/dev/.urandom -jar dagx-demo-e2e.jar

0 commit comments

Comments
 (0)