Skip to content

Commit 492e281

Browse files
pantherman594pantherman594
authored
Install workbench-tools using conda (#234)
* Install workbench-tools using conda * Source bashrc in test * Lint * Fix test * Fix test
1 parent 85e07e9 commit 492e281

File tree

20 files changed

+338
-509
lines changed

20 files changed

+338
-509
lines changed

‎.github/actions/smoke-test/action.yaml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,15 @@ inputs:
33
template:
44
description: 'Template to test'
55
required: true
6+
user:
7+
description: 'Username of container user'
8+
required: true
9+
workbench_tools:
10+
description: 'Whether to test workbench tools'
11+
required: true
12+
options:
13+
description: 'JSON string of additional options for building the app'
14+
required: true
615

716
runs:
817
using: composite
@@ -23,9 +32,11 @@ runs:
2332
- name: Build template
2433
id: build_template
2534
shell: bash
26-
run: .github/actions/smoke-test/build.sh ${{ inputs.template }}
35+
env:
36+
OPTIONS: ${{ inputs.options }}
37+
run: .github/actions/smoke-test/build.sh "${{ inputs.template }}" "$OPTIONS"
2738

2839
- name: Test template
2940
id: test_template
3041
shell: bash
31-
run: .github/actions/smoke-test/test.sh ${{ inputs.template }}
42+
run: .github/actions/smoke-test/test.sh "${{ inputs.template }}" "${{ inputs.user }}" "${{ inputs.workbench_tools }}"

‎.github/actions/smoke-test/build.sh‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ set -o errexit
1111
set -o nounset
1212

1313
readonly TEMPLATE_ID="$1"
14+
readonly INPUT_OPTIONS="$2"
1415

1516
# include hidden files because devcontainer configs
1617
# are in .devcontainer/ or .devcontainer.json file.
@@ -36,11 +37,15 @@ if [[ "${OPTION_PROPERTY}" != "" ]] && [[ "${OPTION_PROPERTY}" != "null" ]]; the
3637
echo "(!) Configuring template options for '${TEMPLATE_ID}'"
3738
for OPTION in "${OPTIONS[@]}"; do
3839
OPTION_KEY="\${templateOption:$OPTION}"
39-
OPTION_VALUE=$(jq -r ".options | .${OPTION} | .default" devcontainer-template.json)
40+
OPTION_VALUE="$(jq -r ".${OPTION}" <<< "$INPUT_OPTIONS")"
4041

4142
if [[ "${OPTION_VALUE}" == "" ]] || [[ "${OPTION_VALUE}" == "null" ]]; then
42-
echo "Template '${TEMPLATE_ID}' is missing a default value for option '${OPTION}'"
43-
exit 1
43+
OPTION_VALUE=$(jq -r ".options | .${OPTION} | .default" devcontainer-template.json)
44+
45+
if [[ "${OPTION_VALUE}" == "" ]] || [[ "${OPTION_VALUE}" == "null" ]]; then
46+
echo "Template '${TEMPLATE_ID}' is missing a default value for option '${OPTION}'"
47+
exit 1
48+
fi
4449
fi
4550

4651
echo "(!) Replacing '${OPTION_KEY}' with '${OPTION_VALUE}'"

‎.github/actions/smoke-test/test.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ set -o errexit
99
set -o nounset
1010

1111
readonly TEMPLATE_ID="$1"
12+
readonly TEST_USER="$2"
13+
readonly WORKBENCH_TOOLS="$3"
1214
readonly SRC_DIR="/tmp/${TEMPLATE_ID}"
1315

1416
echo "Running Smoke Test"
@@ -27,10 +29,10 @@ SCRIPT="$(printf '\
2729
else \
2830
sudo chmod +x test.sh; \
2931
fi && \
30-
./test.sh %q; \
32+
./test.sh %q %q %q; \
3133
else \
3234
ls -a; \
33-
fi' "${TEMPLATE_ID}")"
35+
fi' "${TEMPLATE_ID}" "${TEST_USER}" "${WORKBENCH_TOOLS}")"
3436
readonly SCRIPT
3537

3638
devcontainer exec \

‎.github/workflows/test-pr.yaml‎

Lines changed: 87 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -14,64 +14,109 @@ jobs:
1414
SHELLCHECK_OPTS: -e SC1090 -e SC1091
1515

1616
detect-changes:
17+
name: "Detect changes"
1718
runs-on: ubuntu-latest
18-
outputs:
19-
apps: ${{ steps.output.outputs.apps }}
20-
steps:
21-
- uses: dorny/paths-filter@v2
22-
id: filter
23-
with:
24-
list-files: json
25-
# BENCH-4080: Re-enable cirrocumulus test
26-
filters: |
27-
common: &common
28-
- '.github/workflows/test-pr.yaml'
29-
- '.github/actions/smoke-test/**'
30-
- 'startupscript/**'
31-
- 'test/**'
32-
- 'features/src/workbench-tools/**'
33-
jupyter-template:
34-
- *common
35-
- './src/jupyter-template/**'
19+
strategy:
20+
matrix:
21+
common_filters:
22+
-
23+
- '.github/workflows/test-pr.yaml'
24+
- '.github/actions/smoke-test/**'
25+
- 'startupscript/**'
26+
- 'test/**'
27+
config:
28+
# BENCH-4080: Re-enable cirrocumulus test
29+
- jupyter-template:
30+
user: jupyter
3631
jupyter:
37-
- *common
38-
- './src/jupyter/**'
32+
user: jovyan
3933
shiny:
40-
- *common
41-
- './src/shiny/**'
34+
user: shiny
4235
r-analysis:
43-
- *common
44-
- './src/r-analysis/**'
36+
user: rstudio
37+
workbench_tools: true
4538
vscode:
46-
- *common
47-
- './src/vscode/**'
39+
user: abc
40+
workbench_tools: true
41+
workbench-jupyter:
42+
template: custom-workbench-jupyter-template
43+
user: jupyter
44+
maximize_build_space: true
45+
workbench_tools: true
46+
options:
47+
containerImage: 'us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-workbench-jupyter:ov-dev-stable'
4848
jupyter-aou:
49-
- *common
50-
- './src/jupyter-aou/**'
49+
user: jupyter
50+
maximize_build_space: true
51+
workbench_tools: true
5152
nemo_jupyter:
52-
- *common
53-
- './src/nemo_jupyter/**'
53+
user: jupyter
54+
maximize_build_space: true
55+
workbench_tools: true
5456
nemo_jupyter_aou:
55-
- *common
56-
- './src/nemo_jupyter_aou/**'
57+
user: jupyter
58+
maximize_build_space: true
59+
workbench_tools: true
60+
filters:
61+
- 'src/nemo_jupyter/**'
5762
workbench-jupyter-parabricks:
58-
- *common
59-
- './src/workbench-jupyter-parabricks/**'
63+
user: jupyter
64+
workbench_tools: true
6065
workbench-jupyter-parabricks-aou:
61-
- *common
62-
- './src/workbench-jupyter-parabricks-aou/**'
66+
user: jupyter
67+
workbench_tools: true
68+
filters:
69+
- 'src/workbench-jupyter-parabricks/**'
70+
outputs:
71+
apps: ${{ steps.output.outputs.apps }}
72+
steps:
73+
- name: Build filters
74+
id: build-filters
75+
run: |
76+
CONFIG='${{ toJson(matrix.config) }}'
77+
COMMON='${{ toJson(matrix.common_filters) }}'
78+
79+
echo "filters<<EOF" >> $GITHUB_OUTPUT
80+
jq -n \
81+
--argjson config "$CONFIG" \
82+
--argjson common "$COMMON" \
83+
'[$config
84+
| to_entries
85+
| .[]
86+
| .value as $v
87+
| .value = $common
88+
| .value += ["src/" + ($v.template // .key) + "/**"]
89+
| .value += if $v.workbench_tools then ["features/src/workbench-tools/**"] else [] end
90+
| .value += ($v.filters // [])
91+
] | from_entries' | yq -P >> $GITHUB_OUTPUT
92+
echo "EOF" >> $GITHUB_OUTPUT
93+
- uses: dorny/paths-filter@v2
94+
id: filter
95+
with:
96+
list-files: json
97+
filters: ${{ steps.build-filters.outputs.filters }}
6398
- name: Output changed templates as JSON object
6499
id: output
65100
run: |
66-
MAXIMIZE_BUILD_SPACE='["jupyter-aou", "nemo_jupyter", "nemo_jupyter_aou"]'
67101
CHANGED_TEMPLATES='${{ steps.filter.outputs.changes }}'
102+
CONFIG='${{ toJson(matrix.config) }}'
68103
69104
# output to github output
70105
echo "apps<<EOF" >> $GITHUB_OUTPUT
71106
jq -n \
72107
--argjson changed "$CHANGED_TEMPLATES" \
73-
--argjson maximize "$MAXIMIZE_BUILD_SPACE" \
74-
'[$changed[] as $t | select($t != "common") | { template: $t, maximize_build_space: $maximize | (index($t) != null) }]' >> $GITHUB_OUTPUT
108+
--argjson config "$CONFIG" \
109+
'[$changed[] as $t
110+
| $config[$t]
111+
| select(. != null)
112+
| {
113+
template: (.template // $t),
114+
user,
115+
maximize_build_space: (.maximize_build_space // false),
116+
workbench_tools: (.workbench_tools // false),
117+
options: (.options // {}),
118+
}
119+
]' >> $GITHUB_OUTPUT
75120
echo "EOF" >> $GITHUB_OUTPUT
76121
77122
@@ -111,3 +156,6 @@ jobs:
111156
uses: ./.github/actions/smoke-test
112157
with:
113158
template: "${{ matrix.app.template }}"
159+
user: "${{ matrix.app.user }}"
160+
workbench_tools: "${{ matrix.app.workbench_tools }}"
161+
options: "${{ toJSON(matrix.app.options) }}"

‎features/src/workbench-tools/README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,9 @@ Installs common tools for Workbench Apps. Currently it only supports Debian-base
1515

1616
| Options Id | Description | Type | Default Value |
1717
|-----|-----|-----|-----|
18-
| cloud | Cloud provider to install CLI tools for. If set to \"gcp\", installFromSource will default to true. | string | "" |
19-
| installFromSource | Install tools that require building from source. This may take a long time. | boolean | false |
18+
| cloud | Cloud provider to install CLI tools for. If set to "gcp", installFromSource will default to true. | string | - |
19+
| username | Username of the container user. | string | root |
20+
| userHomeDir | Home directory of the container user. | string | /root |
2021

2122
## Versions
2223

‎features/src/workbench-tools/devcontainer-feature.json‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,22 @@
99
"default": "",
1010
"description": "Cloud provider to install CLI tools for. If set to \"gcp\", installFromSource will default to true."
1111
},
12-
"installFromSource": {
13-
"type": "boolean",
14-
"default": false,
15-
"description": "Install tools that require building from source. This may take a long time."
12+
"username": {
13+
"type": "string",
14+
"default": "root",
15+
"description": "Username of the container user."
16+
},
17+
"userHomeDir": {
18+
"type": "string",
19+
"default": "/root",
20+
"description": "Home directory of the container user."
1621
}
1722
},
1823
"installsAfter": [
1924
"ghcr.io/devcontainers/features/common-utils",
2025
"ghcr.io/devcontainers/features/python",
21-
"ghcr.io/devcontainers/features/conda"
26+
"ghcr.io/devcontainers/features/anaconda",
27+
"ghcr.io/devcontainers/features/conda",
28+
"ghcr.io/rocker-org/devcontainer-features/miniforge"
2229
]
2330
}

‎features/src/workbench-tools/install-conda.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
3+
# install-conda.sh installs conda and mamba into the devcontainer.
4+
# Installation steps taken from https://github.com/rocker-org/devcontainer-features/blob/main/src/miniforge/install.sh
5+
#
6+
# This script is expected to be sourced from install.sh, with USERNAME and
7+
# USER_HOME_DIR variables set. When it completes, conda will be installed for
8+
# the specified user and also added to the current shell.
9+
10+
set -o errexit
11+
set -o nounset
12+
set -o pipefail
13+
set -o xtrace
14+
15+
readonly CONDA_DIR="/opt/conda"
16+
17+
install_miniforge() {
18+
local conda_dir=$1
19+
local download_url
20+
download_url="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh"
21+
22+
check_packages curl ca-certificates
23+
mkdir -p /tmp/miniforge
24+
(
25+
cd /tmp/miniforge
26+
curl -sLo miniforge.sh "${download_url}"
27+
chmod +x miniforge.sh
28+
/bin/bash miniforge.sh -b -p "${conda_dir}"
29+
)
30+
rm -rf /tmp/miniforge
31+
"${conda_dir}/bin/conda" clean -yaf
32+
}
33+
34+
check_packages curl ca-certificates
35+
36+
# Ensure that login shells get the correct path if the user updated the PATH using ENV.
37+
rm -f /etc/profile.d/00-restore-env.sh
38+
echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" >/etc/profile.d/00-restore-env.sh
39+
chmod +x /etc/profile.d/00-restore-env.sh
40+
41+
# Install the miniforge
42+
echo "Downloading Miniforge3..."
43+
install_miniforge "${CONDA_DIR}"
44+
45+
CONDA_SCRIPT="${CONDA_DIR}/etc/profile.d/conda.sh"
46+
# shellcheck source=/dev/null
47+
source "${CONDA_SCRIPT}"
48+
conda config --set env_prompt '({name})'
49+
50+
echo "source ${CONDA_SCRIPT}" >> "${USER_HOME_DIR}/.bashrc"
51+
chown -R "${USERNAME}:" "${USER_HOME_DIR}/.bashrc"
52+
53+
chown -R "${USERNAME}:" "${CONDA_DIR}"
54+
chmod -R g+r+w "${CONDA_DIR}"
55+
find "${CONDA_DIR}" -type d -print0 | xargs -n 1 -0 chmod g+s

0 commit comments

Comments
 (0)