Skip to content

Commit 5a00b98

Browse files
pantherman594pantherman594
authored
AoU Environment Variables (#250)
* WIP * WIP config jsons * Cleanup * Update dockerfiles * Remove test DC * Lint * Fix dockerfile * Fix script * Address feedback
1 parent e980c46 commit 5a00b98

File tree

21 files changed

+1514
-0
lines changed

21 files changed

+1514
-0
lines changed

‎src/aou-common/aou-common-compose.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,6 @@ services:
88
aou-common-extension-builder:
99
build:
1010
context: ../aou-common/extension-builder
11+
aou-common-load-envs:
12+
build:
13+
context: ../aou-common/load-envs

‎src/aou-common/load-envs/Dockerfile‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
FROM golang:1.23-bookworm
2+
WORKDIR /source
3+
RUN --mount=type=bind,source=.,target=/source,rw \
4+
mkdir -p /dist && \
5+
go build -v -o /dist/load-env .
6+
7+
COPY load-env.sh /dist/load-env.sh
8+
RUN chmod +x /dist/load-env.sh
9+
10+
ENTRYPOINT ["echo", "This image is only a build layer and should not be run directly."]

‎src/aou-common/load-envs/configs/cdr_config_prod.json‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"accessTiers": [
3+
{
4+
"shortName": "registered",
5+
"datasetsBucket": "",
6+
"artifactRegistryRepo": "us-central1-docker.pkg.dev/all-of-us-rw-prod/aou-rw-gar-remote-repo-docker-prod"
7+
},
8+
{
9+
"shortName": "controlled",
10+
"datasetsBucket": "gs://vwb-aou-datasets-controlled/",
11+
"artifactRegistryRepo": "us-central1-docker.pkg.dev/all-of-us-rw-prod/aou-rw-gar-remote-repo-docker-prod"
12+
}
13+
],
14+
"cdrVersions": [
15+
{
16+
"name": "All of Us Registered Tier Dataset v8",
17+
"bigqueryProject": "wb-affable-acorn-7941",
18+
"bigqueryDataset": "R2024Q3R5",
19+
"accessTier": "registered",
20+
"dcVersionName": "cdrv8"
21+
},
22+
{
23+
"name": "All of Us Controlled Tier Dataset v8",
24+
"bigqueryProject": "wb-silky-artichoke-2408",
25+
"bigqueryDataset": "C2024Q3R5",
26+
"accessTier": "controlled",
27+
"storageBasePath": "v8",
28+
"wgsCramManifestPath": "wgs/cram/manifest.csv",
29+
"microarrayHailStoragePath": "microarray/hail.mt",
30+
"microarrayVcfManifestPath": "microarray/vcf/manifest.csv",
31+
"microarrayIdatManifestPath": "microarray/idat/manifest.csv",
32+
"wgsVdsPath": "wgs/short_read/snpindel/vds/hail.vds",
33+
"wgsExomeMultiHailPath": "wgs/short_read/snpindel/exome/multiMT/hail.mt",
34+
"wgsExomeSplitHailPath": "wgs/short_read/snpindel/exome/splitMT/hail.mt",
35+
"wgsExomeVcfPath": "wgs/short_read/snpindel/exome/vcf",
36+
"wgsAcafThresholdMultiHailPath": "wgs/short_read/snpindel/acaf_threshold/multiMT/hail.mt",
37+
"wgsAcafThresholdSplitHailPath": "wgs/short_read/snpindel/acaf_threshold/splitMT/hail.mt",
38+
"wgsAcafThresholdVcfPath": "wgs/short_read/snpindel/acaf_threshold/vcf",
39+
"wgsClinvarMultiHailPath": "wgs/short_read/snpindel/clinvar/multiMT/hail.mt",
40+
"wgsClinvarSplitHailPath": "wgs/short_read/snpindel/clinvar/splitMT/hail.mt",
41+
"wgsClinvarVcfPath": "wgs/short_read/snpindel/clinvar/vcf",
42+
"wgsCMRGVcfPath": "wgs/short_read/snpindel/cmrg",
43+
"dcVersionName": "cdrv8"
44+
}
45+
]
46+
}

‎src/aou-common/load-envs/configs/cdr_config_stable.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"accessTiers": [
3+
{
4+
"shortName": "registered",
5+
"datasetsBucket": "",
6+
"artifactRegistryRepo": "us-central1-docker.pkg.dev/all-of-us-rw-stable/aou-rw-gar-remote-repo-docker-stable"
7+
},
8+
{
9+
"shortName": "controlled",
10+
"datasetsBucket": "",
11+
"artifactRegistryRepo": "us-central1-docker.pkg.dev/all-of-us-rw-stable/aou-rw-gar-remote-repo-docker-stable"
12+
}
13+
],
14+
"cdrVersions": [
15+
{
16+
"name": "Synthetic Dataset v8",
17+
"bigqueryProject": "wb-ardent-leek-9025",
18+
"bigqueryDataset": "SR2023Q3R2",
19+
"accessTier": "registered",
20+
"dcVersionName": "cdrv8"
21+
},
22+
{
23+
"name": "Synthetic Dataset in the Controlled Tier v8",
24+
"bigqueryProject": "wb-agile-corn-797",
25+
"bigqueryDataset": "aou_test_data_SC2023Q3R2",
26+
"accessTier": "controlled",
27+
"dcVersionName": "cdrv8"
28+
}
29+
]
30+
}

‎src/aou-common/load-envs/configs/cdr_config_test.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"accessTiers": [
3+
{
4+
"shortName": "registered",
5+
"datasetsBucket": "",
6+
"artifactRegistryRepo": "us-central1-docker.pkg.dev/all-of-us-workbench-test/aou-rw-gar-remote-repo-docker-test"
7+
},
8+
{
9+
"shortName": "controlled",
10+
"datasetsBucket": "gs://cdrv1genomic-vwb-dev-buoyant-apple-6238/",
11+
"artifactRegistryRepo": "us-central1-docker.pkg.dev/all-of-us-workbench-test/aou-rw-gar-remote-repo-docker-test"
12+
}
13+
],
14+
"cdrVersions": [
15+
{
16+
"name": "Synthetic Dataset v4",
17+
"bigqueryProject": "vwb-dev-rapid-kale-4145",
18+
"bigqueryDataset": "aou_synthetic_SR2023Q3R2",
19+
"accessTier": "registered",
20+
"dcVersionName": "cdrv8"
21+
},
22+
{
23+
"name": "Synthetic Dataset in the Controlled Tier v8",
24+
"bigqueryProject": "vwb-dev-buoyant-apple-6238",
25+
"bigqueryDataset": "aou_test_data_SC2023Q3R2",
26+
"storageBasePath": "v8",
27+
"wgsVcfMergedStoragePath": "wgs/vcf/merged",
28+
"wgsCramManifestPath": "wgs/cram/manifest.csv",
29+
"microarrayVcfSingleSampleStoragePath": "microarray/vcf/single_sample",
30+
"accessTier": "controlled",
31+
"dcVersionName": "cdrv8"
32+
}
33+
]
34+
}

‎src/aou-common/load-envs/configs/configs.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package configs
2+
3+
import _ "embed"
4+
5+
//go:embed datacollection_mapping.json
6+
var DatacollectionMapping []byte
7+
8+
//go:embed cdr_config_test.json
9+
var CdrConfigTest []byte
10+
11+
//go:embed cdr_config_stable.json
12+
var CdrConfigStable []byte
13+
14+
//go:embed cdr_config_prod.json
15+
var CdrConfigProd []byte

‎src/aou-common/load-envs/configs/datacollection_mapping.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
[
2+
{
3+
"dataCollectionUuid": "505eb5d3-5b17-42d2-82ed-d3c223ed9e02",
4+
"cdrEnv": "test",
5+
"accessTier": "registered"
6+
},
7+
{
8+
"dataCollectionUuid": "7d7b32f1-3481-4e60-b496-4b6175833f3f",
9+
"cdrEnv": "test",
10+
"accessTier": "controlled"
11+
},
12+
{
13+
"dataCollectionUuid": "7bb6ae49-0891-4ae8-8f39-3fdacc45f2e8",
14+
"cdrEnv": "stable",
15+
"accessTier": "registered"
16+
},
17+
{
18+
"dataCollectionUuid": "91613bc6-3e48-4f22-a6d1-4a80338e1fb5",
19+
"cdrEnv": "stable",
20+
"accessTier": "controlled"
21+
},
22+
{
23+
"dataCollectionUuid": "698c6700-afbe-454a-b73a-c675e629336c",
24+
"cdrEnv": "prod",
25+
"accessTier": "registered"
26+
},
27+
{
28+
"dataCollectionUuid": "3d83ef80-77d7-43e8-a479-52946619b769",
29+
"cdrEnv": "prod",
30+
"accessTier": "controlled"
31+
}
32+
]

‎src/aou-common/load-envs/go.mod‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
module github.com/verily-src/workbench-app-devcontainers/src/aou-common/load-envs
2+
3+
go 1.23
4+
5+
require (
6+
github.com/deepmap/oapi-codegen v1.16.3
7+
github.com/google/uuid v1.6.0
8+
github.com/oapi-codegen/runtime v1.1.2
9+
)
10+
11+
require github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect

‎src/aou-common/load-envs/go.sum‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
2+
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
3+
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
4+
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
5+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
7+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
8+
github.com/deepmap/oapi-codegen v1.16.3 h1:GT9G86SbQtT1r8ZB+4Cybi9VGdu1P5ieNvNdEoCSbrA=
9+
github.com/deepmap/oapi-codegen v1.16.3/go.mod h1:JD6ErqeX0nYnhdciLc61Konj3NBASREMlkHOgHn8WAM=
10+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
11+
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
12+
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
13+
github.com/oapi-codegen/runtime v1.1.2 h1:P2+CubHq8fO4Q6fV1tqDBZHCwpVpvPg7oKiYzQgXIyI=
14+
github.com/oapi-codegen/runtime v1.1.2/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
15+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
16+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
17+
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
18+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
19+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
20+
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
21+
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
22+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
23+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎src/aou-common/load-envs/internal/envvars/cdrversion.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package envvars
2+
3+
// AccessTier represents an access tier configuration
4+
type AccessTier struct {
5+
ShortName string `json:"shortName"`
6+
DatasetsBucket string `json:"datasetsBucket"`
7+
ArtifactRegistryRepo string `json:"artifactRegistryRepo"`
8+
}
9+
10+
// CdrConfig represents the configuration for CDR versions and access tiers
11+
type CdrConfig struct {
12+
AccessTiers []AccessTier `json:"accessTiers"`
13+
CdrVersions []CdrVersion `json:"cdrVersions"`
14+
}
15+
16+
// CdrVersion represents a CDR (Curated Data Repository) version with all associated metadata
17+
type CdrVersion struct {
18+
Name string `json:"name"`
19+
AccessTier string `json:"accessTier"`
20+
BigqueryProject string `json:"bigqueryProject"`
21+
BigqueryDataset string `json:"bigqueryDataset"`
22+
DCVersionName string `json:"dcVersionName"`
23+
StorageBasePath string `json:"storageBasePath"`
24+
WgsVcfMergedStoragePath string `json:"wgsVcfMergedStoragePath" env:"WGS_VCF_MERGED_STORAGE_PATH"`
25+
WgsHailStoragePath string `json:"wgsHailStoragePath" env:"WGS_HAIL_STORAGE_PATH"`
26+
WgsCramManifestPath string `json:"wgsCramManifestPath" env:"WGS_CRAM_MANIFEST_PATH"`
27+
MicroarrayHailStoragePath string `json:"microarrayHailStoragePath" env:"MICROARRAY_HAIL_STORAGE_PATH"`
28+
MicroarrayVcfSingleSampleStoragePath string `json:"microarrayVcfSingleSampleStoragePath" env:"MICROARRAY_VCF_SINGLE_SAMPLE_STORAGE_PATH"`
29+
MicroarrayVcfManifestPath string `json:"microarrayVcfManifestPath" env:"MICROARRAY_VCF_MANIFEST_PATH"`
30+
MicroarrayIdatManifestPath string `json:"microarrayIdatManifestPath" env:"MICROARRAY_IDAT_MANIFEST_PATH"`
31+
WgsVdsPath string `json:"wgsVdsPath" env:"WGS_VDS_PATH"`
32+
WgsExomeMultiHailPath string `json:"wgsExomeMultiHailPath" env:"WGS_EXOME_MULTI_HAIL_PATH"`
33+
WgsExomeSplitHailPath string `json:"wgsExomeSplitHailPath" env:"WGS_EXOME_SPLIT_HAIL_PATH"`
34+
WgsExomeVcfPath string `json:"wgsExomeVcfPath" env:"WGS_EXOME_VCF_PATH"`
35+
WgsAcafThresholdMultiHailPath string `json:"wgsAcafThresholdMultiHailPath" env:"WGS_ACAF_THRESHOLD_MULTI_HAIL_PATH"`
36+
WgsAcafThresholdSplitHailPath string `json:"wgsAcafThresholdSplitHailPath" env:"WGS_ACAF_THRESHOLD_SPLIT_HAIL_PATH"`
37+
WgsAcafThresholdVcfPath string `json:"wgsAcafThresholdVcfPath" env:"WGS_ACAF_THRESHOLD_VCF_PATH"`
38+
WgsClinvarMultiHailPath string `json:"wgsClinvarMultiHailPath" env:"WGS_CLINVAR_MULTI_HAIL_PATH"`
39+
WgsClinvarSplitHailPath string `json:"wgsClinvarSplitHailPath" env:"WGS_CLINVAR_SPLIT_HAIL_PATH"`
40+
WgsClinvarVcfPath string `json:"wgsClinvarVcfPath" env:"WGS_CLINVAR_VCF_PATH"`
41+
WgsLongReadsManifestPath string `json:"wgsLongReadsManifestPath" env:"LONG_READS_MANIFEST_PATH"`
42+
WgsLongReadsHailGRCh38 string `json:"wgsLongReadsHailGRCh38" env:"WGS_LONGREADS_HAIL_GRCH38_PATH"`
43+
WgsLongReadsHailT2T string `json:"wgsLongReadsHailT2T" env:"WGS_LONGREADS_HAIL_T2T_PATH"`
44+
WgsLongReadsJointVcfGRCh38 string `json:"wgsLongReadsJointVcfGRCh38" env:"WGS_LONGREADS_JOINT_SNP_INDEL_VCF_GRCH38_PATH"`
45+
WgsLongReadsJointVcfT2T string `json:"wgsLongReadsJointVcfT2T" env:"WGS_LONGREADS_JOINT_SNP_INDEL_VCF_T2T_PATH"`
46+
WgsCMRGVcfPath string `json:"wgsCMRGVcfPath" env:"WGS_CMRG_VCF_PATH"`
47+
}

0 commit comments

Comments
 (0)