Skip to content

Commit cd5316e

Browse files
josephmachadojosephmachado
authored
Merge pull request #5 from luutuankiet/devcontainer-setup
setup workspace with devcontainer
2 parents e33816b + c0769b0 commit cd5316e

File tree

8 files changed

+237
-0
lines changed

8 files changed

+237
-0
lines changed

‎.devcontainer/devcontainer.json‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
2+
// README at: https://github.com/devcontainers/templates/tree/main/src/python
3+
{
4+
"name": "Python 3",
5+
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
6+
"image": "mcr.microsoft.com/devcontainers/python:1-3.9-bookworm",
7+
"runArgs": [
8+
"--name",
9+
"spark-Learn"
10+
],
11+
"remoteUser": "root",
12+
"postCreateCommand": "bash -i .devcontainer/postCreateCommand.sh",
13+
"features": {
14+
"ghcr.io/devcontainers/features/docker-in-docker:2": {
15+
"moby": "false",
16+
"azureDnsAutoDetection": "false",
17+
"installDockerBuildx": "false",
18+
"installDockerComposeSwitch": "false",
19+
"version": "latest",
20+
"dockerDashComposeVersion": "latest"
21+
}
22+
},
23+
"customizations": {
24+
"vscode": {
25+
"extensions": [
26+
"bastienboutonnet.vscode-dbt",
27+
"chrisdias.vscode-opennewinstance",
28+
"cweijan.vscode-office",
29+
"dvirtz.parquet-viewer",
30+
"henriblancke.vscode-dbt-formatter",
31+
"innoverio.vscode-dbt-power-user",
32+
"mechatroner.rainbow-csv",
33+
"mhutchie.git-graph",
34+
"ms-python.debugpy",
35+
"ms-python.python",
36+
"ms-python.vscode-pylance",
37+
"ms-toolsai.jupyter",
38+
"ms-toolsai.jupyter-keymap",
39+
"ms-toolsai.jupyter-renderers",
40+
"ms-toolsai.vscode-jupyter-cell-tags",
41+
"ms-toolsai.vscode-jupyter-slideshow",
42+
"mtxr.sqltools",
43+
"mtxr.sqltools-driver-pg",
44+
"mtxr.sqltools-driver-sqlite",
45+
"mutantdino.resourcemonitor",
46+
"randomfractalsinc.duckdb-sql-tools",
47+
"samuelcolvin.jinjahtml",
48+
"sourcegraph.cody-ai",
49+
"taoklerks.poor-mans-t-sql-formatter-vscode",
50+
"uloco.theme-bluloco-dark",
51+
"visualstudioexptteam.intellicode-api-usage-examples",
52+
"visualstudioexptteam.vscodeintellicode",
53+
"ms-azuretools.vscode-docker",
54+
"redhat.vscode-yaml",
55+
"esbenp.prettier-vscode",
56+
"vscode-icons-team.vscode-icons"
57+
],
58+
"settings": {
59+
"terminal.integrated.env.osx": {
60+
"PYTHONPATH": "${env:PYTHONPATH}"
61+
},
62+
"terminal.integrated.env.linux": {
63+
"PYTHONPATH": "${env:PYTHONPATH}"
64+
},
65+
"terminal.integrated.env.windows": {
66+
"PYTHONPATH": "${env:PYTHONPATH}"
67+
},
68+
"python.defaultInterpreterPath": ".venv/bin/python",
69+
"python.venvFolders": [
70+
"${workspaceFolder}/.venv"
71+
],
72+
"python.envFile": "${workspaceFolder}/.env",
73+
"python.terminal.activateEnvironment": true
74+
}
75+
}
76+
}
77+
}

‎.devcontainer/postCreateCommand.sh‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/bin/bash
2+
# Function to print a section header
3+
print_section() {
4+
local section_title=$1
5+
echo
6+
echo "================================="
7+
echo "================================="
8+
echo "================================="
9+
echo " $section_title"
10+
echo "================================="
11+
echo "================================="
12+
echo "================================="
13+
echo
14+
}
15+
16+
17+
18+
19+
##### install npm
20+
print_section "INSTALL NPM"
21+
22+
# installs nvm (Node Version Manager)
23+
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
24+
25+
# download and install Node.js
26+
nvm install 20
27+
28+
# verifies the right Node.js version is in the environment
29+
node -v # should print `v20.13.1`
30+
31+
# verifies the right NPM version is in the environment
32+
npm -v # should print `10.5.2`
33+
34+
35+
# deps for sqltools - duckdb driver
36+
npm install [email protected]
37+
38+
39+
40+
### sets up zsh terminal
41+
print_section "SETUP ZSH TERMINAL"
42+
curl -o- https://gist.githubusercontent.com/luutuankiet/fbb70fca0f7f948c4e102442d76c363e/raw/boilerplate-dev-env | bash
43+
44+
45+
#### sets up python
46+
print_section "SETUP PYTHON"
47+
apt-get update && apt-get install -y python3-venv
48+
49+
# init then source env vars
50+
print_section "INIT & SOURCE ENV VARS"
51+
chmod +x ./env_init.sh
52+
chmod +x ./source_env.sh
53+
54+
. ./env_init.sh
55+
source source_env.sh
56+
57+
58+
59+
##### TODO: uncomment this for a true rebuild from scratch. currenlty broken due to packages deps in requirements file.
60+
# create env
61+
print_section "CREATE VENV & INSTALL REQUIREMENTS"
62+
python3 -m venv --clear $VIRTUAL_ENV
63+
64+
65+
66+
67+
# # add virt env to PATH which allows the next part of script to install packages directly to venv
68+
# export PATH="$VIRTUAL_ENV/bin:$PATH"
69+
70+
# # install reqs. each lines is a separate process hence neeeds a source .venv in front
71+
source .venv/bin/activate && \
72+
pip install -r data-processing-spark/1-lab-setup/containers/spark/requirements.txt
73+
74+
75+
# fix for deactivate script : https://github.com/microsoft/vscode-python/wiki/Fixing-%22deactivate%22-command-for-Virtual-Environments
76+
ENV_WORK_DIR=$(pwd)
77+
curl -o $ENV_WORK_DIR/deactivate https://gist.githubusercontent.com/karrtikr/963469ba74c9b7632d2c43224ffa2f25/raw/deactivate
78+
echo "source $ENV_WORK_DIR/deactivate" >> ~/.zshrc
79+
80+
# remember to add the following to gitignore
81+
# .venv
82+
# node_modules
83+
# package-lock.json
84+
# package.json
85+
86+
87+
# optional : download code cli to then access dev container from a browser
88+
curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz
89+
tar -xf vscode_cli.tar.gz
90+
91+
# usage: ./code tunnel

‎.env‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
VIRTUAL_ENV="/workspaces/efficient_data_processing_spark/.venv"
2+
PYTHONPATH="/workspaces/efficient_data_processing_spark/data-processing-spark"

‎.github/dependabot.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# To get started with Dependabot version updates, you'll need to specify which
2+
# package ecosystems to update and where the package manifests are located.
3+
# Please see the documentation for more information:
4+
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
5+
# https://containers.dev/guide/dependabot
6+
7+
version: 2
8+
updates:
9+
- package-ecosystem: "devcontainers"
10+
directory: "/"
11+
schedule:
12+
interval: weekly

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,9 @@ derby.log
66
*.pyc
77
__pycache__/
88

9+
.venv
10+
node_modules
11+
package-lock.json
12+
package.json
13+
vscode_cli.tar.gz
14+
code

‎deactivate‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# >>> Virtual env deactivate hook >>>
2+
3+
# Same as deactivate in "<venv>/bin/activate"
4+
deactivate () {
5+
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
6+
PATH="${_OLD_VIRTUAL_PATH:-}"
7+
export PATH
8+
unset _OLD_VIRTUAL_PATH
9+
fi
10+
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
11+
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
12+
export PYTHONHOME
13+
unset _OLD_VIRTUAL_PYTHONHOME
14+
fi
15+
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
16+
hash -r 2> /dev/null
17+
fi
18+
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
19+
PS1="${_OLD_VIRTUAL_PS1:-}"
20+
export PS1
21+
unset _OLD_VIRTUAL_PS1
22+
fi
23+
unset VIRTUAL_ENV
24+
unset VIRTUAL_ENV_PROMPT
25+
if [ ! "${1:-}" = "nondestructive" ] ; then
26+
unset -f deactivate
27+
fi
28+
}
29+
30+
# Initialize the variables required by deactivate function
31+
_OLD_VIRTUAL_PS1="${PS1:-}"
32+
_OLD_VIRTUAL_PATH="$PATH"
33+
if [ -n "${PYTHONHOME:-}" ] ; then
34+
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
35+
fi
36+
37+
# <<< Virtual env deactivate hook <<<

‎env_init.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
# parse dir
3+
ENV_WORK_DIR=$(pwd)
4+
5+
cat <<EOF > .env
6+
VIRTUAL_ENV="$ENV_WORK_DIR/.venv"
7+
PYTHONPATH="$ENV_WORK_DIR/data-processing-spark"
8+
EOF

‎source_env.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
set -a # Automatically export all variables
3+
source .env
4+
set +a # Stop automatically exporting variables

0 commit comments

Comments
 (0)