diff --git a/.actrc b/.actrc new file mode 100644 index 000000000..9a0f250d0 --- /dev/null +++ b/.actrc @@ -0,0 +1,2 @@ +-P ubuntu-latest=node:22-bookworm +--container-options=--memory=32g --network=host diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..83342269d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Exclude large directories that aren't needed for build +vscode/node_modules + +# Exclude build artifacts +*.tar.gz +*.log + +# Exclude development files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Exclude OS files +.DS_Store +Thumbs.db diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7ecf8d2b..fc42ca9b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,7 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + # The main job for building the application build: name: Build sagemaker-code-editor runs-on: ubuntu-latest @@ -37,7 +38,13 @@ jobs: - name: Apply patches (if any) run: | if [ -d patches ] && [ "$(ls -A patches)" ]; then - quilt push -a || true + { + quilt push -a --leave-rejects --color=auto + } || { + printf "\nPatching error, review logs!\n" + find ./vscode -name "*.rej" + exit 1 + } fi - name: Set Development Version @@ -53,18 +60,17 @@ jobs: cd vscode export DISABLE_V8_COMPILE_CACHE=1 export UV_THREADPOOL_SIZE=4 + + npm install -g node-gyp + # Install dependencies using npm, skip optional and native modules npm install - VSCODE_RIPGREP_VERSION=$(jq -r '.dependencies."@vscode/ripgrep"' package.json) - mv package.json package.json.orig - jq 'del(.dependencies."@vscode/ripgrep")' package.json.orig > package.json - - npm install - npm install --ignore-scripts "@vscode/ripgrep@${VSCODE_RIPGREP_VERSION}" - + # Run the gulp build task with memory optimizations ARCH_ALIAS=linux-x64 - npx gulp vscode-reh-web-${ARCH_ALIAS}-min + node --max-old-space-size=32768 --optimize-for-size \ + ./node_modules/gulp/bin/gulp.js \ + "vscode-reh-web-${ARCH_ALIAS}-min" - name: Find build output id: find_output @@ -96,6 +102,7 @@ jobs: tar czf $TARBALL -C "$PARENT_DIR" "$BUILD_DIR_NAME" - name: Upload build artifact + if: env.ACT == '' uses: actions/upload-artifact@v4 with: name: npm-package diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f0eeb7124..f90e622fa 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,23 +16,20 @@ jobs: runs-on: ubuntu-latest if: github.event.workflow_run.conclusion == 'success' || github.event_name == 'pull_request' steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - name: Checkout repo with submodules + uses: actions/checkout@v4 with: - node-version: 20 - cache: npm - - run: npm ci - - name: Download build artifact - if: github.event_name != 'pull_request' - uses: actions/download-artifact@v4 + submodules: recursive + - name: Set up Node.js + uses: actions/setup-node@v4 with: - name: build-package - github-token: ${{ secrets.GITHUB_TOKEN }} - run-id: ${{ github.event.workflow_run.id }} + node-version: 20 + cache: 'npm' + cache-dependency-path: '**/package-lock.json' - name: Run E2E tests run: | # Add your actual E2E test commands here - echo "Running E2E tests..." + echo "[PLACEHOLDER] Running E2E tests..." # Example: npm run test:e2e - uses: actions/upload-artifact@v4 if: failure() diff --git a/.gitignore b/.gitignore index eab08afe3..b5e3fb045 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .DS_Store -.pc \ No newline at end of file +.pc +.artifacts +bin \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..42121fea4 --- /dev/null +++ b/Makefile @@ -0,0 +1,82 @@ +.PHONY: build-cache build install-act run-github run-local clean-vscode clean + +build-cache: + @echo "Building SageMaker Code Editor (multi-stage npm cache)..." + docker buildx build \ + --platform linux/amd64 \ + --progress=plain \ + --memory=32g \ + -t npm-cache:latest \ + -f scripts/Dockerfile.build.cache . + +build: + @echo "Building SageMaker Code Editor (original)..." + docker buildx build \ + --platform linux/amd64 \ + --progress=plain \ + --memory=32g \ + --output type=local,dest=./.artifacts \ + -t localbuild:latest \ + -f scripts/Dockerfile.build . + +install-act: + @echo "Installing act (GitHub Actions runner)..." + @if ! command -v act >/dev/null 2>&1 && [ ! -f ./bin/act ]; then \ + curl -s https://raw.githubusercontent.com/nektos/act/master/install.sh | bash; \ + echo "act installed successfully"; \ + else \ + echo "act is already available"; \ + fi + +run-github: install-act + @echo "Running complete GitHub Actions workflow locally..." + @echo "Available workflows:" + @ls -la .github/workflows/ + @echo "" + @echo "Running full build.yml workflow..." + @if command -v act >/dev/null 2>&1; then \ + act push -W .github/workflows/build.yml --platform ubuntu-22.04=catthehacker/ubuntu:act-22.04 --container-options "--memory=32g --memory-swap=32g"; \ + else \ + ./bin/act push -W .github/workflows/build.yml --platform ubuntu-22.04=catthehacker/ubuntu:act-22.04 --container-options "--memory=32g --memory-swap=32g"; \ + fi + +run-local: + @if [ -z "$(TARBALL)" ]; then \ + echo "Building and running SageMaker Code Editor locally on port 8888..."; \ + docker build -f scripts/Dockerfile.dev -t local-code-editor-dev . || exit 1; \ + echo "Stopping container..."; \ + docker stop local-code-editor-dev; \ + sleep 2; \ + echo "Starting container on http://localhost:8888"; \ + docker run --rm -d -p 8888:8000 -v .:/workspace -u $(id -u):$(id -g) --entrypoint /workspace/scripts/run-code-editor-dev.sh --name local-code-editor-dev local-code-editor-dev || exit 1; \ + docker logs -f local-code-editor-dev; \ + else \ + echo "Building and running SageMaker Code Editor locally on port 8888..."; \ + docker build -f scripts/Dockerfile.run --build-arg TARBALL=$(TARBALL) -t local-code-editor . || exit 1; \ + echo "Stopping container..."; \ + docker stop local-code-editor; \ + sleep 2; \ + echo "Starting container on http://localhost:8888"; \ + docker run --rm -d -p 8888:8000 --name local-code-editor local-code-editor || exit 1; \ + docker logs -f local-code-editor; \ + fi + +clean-vscode: + @echo "Cleaning VSCode node_modules..." + @find . -type d -name "node_modules" -exec rm -rf {} + 2>/dev/null || true + @rm -rf vscode/out/* 2>/dev/null || true + @echo "VSCode cleanup completed" + +clean: clean-vscode + @echo "Cleaning act temporary files and Docker images..." + @echo "Removing act cache..." + @rm -rf ~/.cache/act 2>/dev/null || true + @echo "Act cleanup completed" + +reset-vscode: + @echo "Resetting vscode submodule..." + git submodule update --init --recursive + git submodule foreach --recursive "git reset --hard HEAD && sudo git clean -fd" + @echo "Resetting patches..." + sudo rm -rf .pc/* + diff --git a/README.md b/README.md index cc5b8da21..c3742b3ff 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,25 @@ This script will: - Run `yarn watch` from within the `vscode` folder - Open a new terminal and run `./vscode/scripts/code-server.sh --launch` +## Make Commands + +Available make targets for building and testing: + +### 1. When making local changes to iterate faster where tarball generation is not required [each run takes 10-20 mins] +- `make run-local` - Build and run SageMaker Code Editor locally from source and does not require a TARBALL; this process runs a watcher so changes are automatically picked from local workspace +- `make clean-vscode` - Cleans node_modules and out files + +### 2. Once local changes are tested; follow this process to generate minified tarball [each run takes ~40 mins to build] +- `make build-cache` - Build SageMaker Code Editor with multi-stage npm cache; Run once and layer gets cached with node_modules +- `make build` - Build SageMaker Code Editor and output artifacts (tarball) to ./artifacts +- `make run-local TARBALL=` - Build and run SageMaker Code Editor locally on port 8888 using specified tarball from previos step. Example: `make run-local TARBALL=sagemaker-code-editor-1.101.2.tar.gz` + +### 3. This process is used to test and simulate github workflows locally [each run takes ~60 mins] +- `make run-github` - Run complete GitHub Actions workflow locally using act + +### 4. Cleanup +- `make clean` - Cleans node_modules, out files, and act temporary files + ## Troubleshooting and Feedback For any issues that customers would like to report, please route to the `amazon-sagemaker-feedback` repository: https://github.com/aws/amazon-sagemaker-feedback diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build new file mode 100644 index 000000000..b8fff5098 --- /dev/null +++ b/scripts/Dockerfile.build @@ -0,0 +1,27 @@ +FROM npm-cache:latest AS builder + +WORKDIR /workspace + +# Declare a build argument to control the minify flag +ARG ARGS + +# Copy source files except vscode so new patches are picked up without cache busting +COPY patches ./patches +COPY resources ./resources +COPY .git ./.git +COPY scripts/ ./scripts/ +COPY .pc ./.pc +COPY LICENSE . +COPY LICENSE-THIRD-PARTY . + +# Apply patches and build +RUN chmod +x scripts/docker-install.sh && \ + ./scripts/docker-install.sh -t "$(cat vscode/package.json | grep '"version"' | cut -d'"' -f4)" + +RUN echo "Build arguments: $ARGS" +RUN chmod +x scripts/create-code-editor-tarball.sh && \ + ./scripts/create-code-editor-tarball.sh -v "$(cat vscode/package.json | grep '"version"' | cut -d'"' -f4)" + +# Final stage: Minimal image with only the build artifacts to copy from +FROM scratch +COPY --from=builder /workspace/artifacts . \ No newline at end of file diff --git a/scripts/Dockerfile.build.cache b/scripts/Dockerfile.build.cache new file mode 100644 index 000000000..1085ed1e1 --- /dev/null +++ b/scripts/Dockerfile.build.cache @@ -0,0 +1,43 @@ +FROM ubuntu:24.04 AS base-image + +# Install required tools and Node.js +RUN apt-get update && apt-get install -y \ + git \ + python3 \ + python3-pip \ + build-essential \ + curl \ + quilt \ + pkg-config \ + libx11-dev \ + libxkbfile-dev \ + libsecret-1-dev \ + libkrb5-dev \ + libgssapi-krb5-2 \ + time \ + && rm -rf /var/lib/apt/lists/* + +# Install Node.js 22 (latest LTS) +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \ + apt-get install -y nodejs + +# Stage 2: NPM Dependencies Cache +# This stage uses the pre-configured base-image to build the cache. +FROM base-image AS npm-cache + +WORKDIR /workspace + +COPY vscode ./vscode +COPY patches ./patches +COPY resources ./resources +COPY .git ./.git +COPY scripts/postinstall.sh ./scripts/postinstall.sh +COPY scripts/docker-install.sh ./scripts/docker-install.sh +COPY scripts/copy-resources.sh ./scripts/copy-resources.sh +COPY .pc ./.pc +COPY LICENSE . +COPY LICENSE-THIRD-PARTY . + +# Apply patches and build +RUN chmod +x scripts/docker-install.sh && \ + ./scripts/docker-install.sh -t "$(cat vscode/package.json | grep '"version"' | cut -d'"' -f4)" \ No newline at end of file diff --git a/scripts/Dockerfile.dev b/scripts/Dockerfile.dev new file mode 100644 index 000000000..bbc34c3ec --- /dev/null +++ b/scripts/Dockerfile.dev @@ -0,0 +1,36 @@ +FROM ubuntu:24.04 AS base-image + +# Install required tools and Node.js +RUN apt-get update && apt-get install -y \ + git \ + python3 \ + python3-pip \ + build-essential \ + curl \ + quilt \ + pkg-config \ + libx11-dev \ + libxkbfile-dev \ + libsecret-1-dev \ + libkrb5-dev \ + libgssapi-krb5-2 \ + time \ + && rm -rf /var/lib/apt/lists/* + +# Install Node.js 22 (latest LTS) +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \ + apt-get install -y nodejs + +# Stage 2: NPM Dependencies Cache +# This stage uses the pre-configured base-image to build the cache. +FROM base-image AS npm-cache + +# Install supervisord +RUN apt-get update && \ + apt-get install -y supervisor + +WORKDIR /workspace + +EXPOSE 8080 + +# Following is the entrypoint script for local runs: /workspace/scripts/run-code-editor-dev.sh \ No newline at end of file diff --git a/scripts/Dockerfile.run b/scripts/Dockerfile.run new file mode 100644 index 000000000..6ab0e2a17 --- /dev/null +++ b/scripts/Dockerfile.run @@ -0,0 +1,24 @@ +FROM public.ecr.aws/sagemaker/sagemaker-distribution:latest-cpu + +# Accept tarball name as build argument +ARG TARBALL=code-editor1.8.0b5.tar.gz + +# Switch to root to install +USER root + +# Copy and extract the code editor to /tmp +COPY .artifacts/${TARBALL} /tmp/ +RUN cd /tmp && tar -xzf ${TARBALL} + +# Move to final location and set permissions +RUN mv /tmp/sagemaker-code-editor /opt/ && \ + chmod +x /opt/sagemaker-code-editor/bin/code-server-oss + +# Add to PATH +ENV PATH="/opt/sagemaker-code-editor/bin:$PATH" + +# Expose port +EXPOSE 8000 + +# Run code-server-oss with host binding to all interfaces +CMD ["code-server-oss", "--host", "0.0.0.0", "--without-connection-token"] diff --git a/scripts/create-code-editor-tarball.sh b/scripts/create-code-editor-tarball.sh new file mode 100755 index 000000000..8427c28cd --- /dev/null +++ b/scripts/create-code-editor-tarball.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +while getopts "v:m" opt; do + case $opt in + v) version="$OPTARG" + ;; + ?) usage; exit 1 ;; + esac +done + +if [[ -z $version ]]; then + echo "Please provide version using '-v'"; + exit 1 +fi + +VERSION=$version + +# Set current project root +PROJ_ROOT=$(pwd) + +pushd ${PROJ_ROOT}/vscode + +printf "\n======== Running gulp build task ========\n" +export DISABLE_V8_COMPILE_CACHE=1 +export UV_THREADPOOL_SIZE=4 +export ARCH_ALIAS=linux-x64 +export NODE_ENV=production +export MINIFY=true +node --max-old-space-size=16384 --optimize-for-size \ + ./node_modules/gulp/bin/gulp.js \ + "vscode-reh-web-${ARCH_ALIAS}${MINIFY:+-min}" + +popd + +TARBALL="sagemaker-code-editor-${VERSION}.tar.gz" +BUILD_DIR_PATH=.artifacts + +mv vscode-reh-web-linux-x64 sagemaker-code-editor +mkdir ${BUILD_DIR_PATH} +tar -czf ${BUILD_DIR_PATH}/${TARBALL} sagemaker-code-editor +sha256sum ${BUILD_DIR_PATH}/${TARBALL} diff --git a/scripts/create_code_editor_tarball.sh b/scripts/create_code_editor_tarball.sh deleted file mode 100644 index 6ae444092..000000000 --- a/scripts/create_code_editor_tarball.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -while getopts "v:" opt; do - case $opt in - v) version="$OPTARG" - ;; - \?) echo "Invalid option -$OPTARG" >&2 - exit 1 - ;; - esac - - case $OPTARG in - -*) echo "Option $opt needs a valid argument" - exit 1 - ;; - esac -done - -if [[ -z $version ]]; then - echo "Please provide version using '-v'"; - exit 1 -fi - -VERSION=$version -# Set current project root -PROJ_ROOT=$(pwd) - -mkdir -p ${PROJ_ROOT}/sagemaker-code-editor/code-editor${VERSION} -rm -rf ${PROJ_ROOT}/sagemaker-code-editor/code-editor${VERSION}/src -mkdir -p ${PROJ_ROOT}/sagemaker-code-editor/code-editor${VERSION}/src -cp -a ${PROJ_ROOT}/vscode/. ${PROJ_ROOT}/sagemaker-code-editor/code-editor${VERSION}/src/ -rm -rf ${PROJ_ROOT}/sagemaker-code-editor/code-editor${VERSION}.tar.gz -cd ${PROJ_ROOT}/sagemaker-code-editor -tar -czf code-editor${VERSION}.tar.gz code-editor${VERSION} -cd ${PROJ_ROOT} -cp ${PROJ_ROOT}/sagemaker-code-editor/code-editor${VERSION}.tar.gz ${PROJ_ROOT}/ -rm -rf ${PROJ_ROOT}/sagemaker-code-editor -sha256sum ${PROJ_ROOT}/code-editor${VERSION}.tar.gz diff --git a/scripts/docker-install.sh b/scripts/docker-install.sh new file mode 100755 index 000000000..38ccb5bf6 --- /dev/null +++ b/scripts/docker-install.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# ONLY FOR LOCAL BUILD USECASE +# This script is intended to be run inside the docker container +# It will setup the environment and build the project within the container + +# set +e to prevent quilt from exiting when no patches popped +set +e + +# Set current project root +PROJ_ROOT=$(pwd) + +# Clean out patches +printf "\n======== Cleaning out patches ========\n" +quilt pop -a +rm -rf .pc + +# re-enable -e to allow exiting on error +set -e + +# make sure module is current +printf "\n======== Updating submodule ========\n" +git submodule update --init + +# Apply patches +printf "\n======== Applying patches ========\n" + +if [ -d patches ] && [ "$(ls -A patches)" ]; then + { + quilt push -a --leave-rejects --color=auto + } || { + printf "\nPatching error, review logs!\n" + find ./vscode -name "*.rej" + exit 1 + } +fi + + +# Generate Licenses +printf "\n======== Generate Licenses ========\n" +cd ${PROJ_ROOT}/vscode +cp LICENSE.txt LICENSE.vscode.txt +cp ThirdPartyNotices.txt LICENSE-THIRD-PARTY.vscode.txt +cp ../LICENSE-THIRD-PARTY . + +cd ${PROJ_ROOT} +# Comment out breaking lines in postinstall.js +printf "\n======== Comment out breaking git config lines in postinstall.js ========\n" +sh ${PROJ_ROOT}/scripts/postinstall.sh + +# Copy resources +printf "\n======== Copy resources ========\n" +${PROJ_ROOT}/scripts/copy-resources.sh + +# Build the project +printf "\n======== Building project in ${PROJ_ROOT}/vscode ========\n" +cd ${PROJ_ROOT}/vscode +npm config set cache ~/.npm --global +npm install -g node-gyp +npm install --use-cache --no-audit --no-fund --prefer-offline --verbose --no-optional +npm run download-builtin-extensions --prefer-offline --verbose diff --git a/scripts/install.sh b/scripts/install.sh old mode 100644 new mode 100755 diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh old mode 100644 new mode 100755 diff --git a/scripts/run-code-editor-dev.sh b/scripts/run-code-editor-dev.sh new file mode 100755 index 000000000..5833c8dd1 --- /dev/null +++ b/scripts/run-code-editor-dev.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# ONLY FOR LOCAL DEV USECASE +# This script is intended to be run inside the docker container +# It will setup the environment and start the code editor in watch mode within the container + +set +e + +# Set current project root +PROJ_ROOT=$(pwd) + +# Clean out patches +printf "\n======== Cleaning out patches ========\n" +quilt pop -a +rm -rf .pc + +# re-enable -e to allow exiting on error +set -e + +# make sure module is current +printf "\n======== Updating submodule ========\n" +git config --global --add safe.directory /workspace +git submodule update --init + +# Apply patches +printf "\n======== Applying patches ========\n" +{ + quilt push -a --leave-rejects --color=auto +} || { + printf "\nPatching error, review logs!\n" + find ./vscode -name "*.rej" + exit 1 +} + +cd ${PROJ_ROOT} +# Comment out breaking lines in postinstall.js +printf "\n======== Comment out breaking git config lines in postinstall.js ========\n" +sh ${PROJ_ROOT}/scripts/postinstall.sh + +# Copy resources +printf "\n======== Copy resources ========\n" +${PROJ_ROOT}/scripts/copy-resources.sh + +# Build the project +printf "\n======== Installing dependencies ========\n" +cd ${PROJ_ROOT}/vscode + +# Install dependencies +npm install +npm run download-builtin-extensions + +printf "\n======== Starting Code Editor and Watch process using supervisord ========\n" + +npm run watch & + +# Sleep for npm watch to start and stabilize +sleep 30 + +# This sciript can be directly used to run code server without supervisord +# ./scripts/code-server.sh --host 0.0.0.0 --port 8000 --without-connection-token + +# Create supervisord config to run code server +cat > /etc/supervisor/supervisord.conf << EOF +[unix_http_server] +file=/var/run/supervisor.sock + +[supervisord] +logfile=/var/log/supervisord.log +pidfile=/var/run/supervisord.pid +nodaemon=false + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + +[include] +files = /etc/supervisor/conf.d/*.conf +EOF + +# Start supervisor daemon +/usr/bin/supervisord -c /etc/supervisor/supervisord.conf & +sleep 2 + +# Create supervisor config file for code server +cat > /etc/supervisor/conf.d/vscode-server.conf << EOF +[program:vscode-server] +directory=/workspace/vscode +command=bash -c "while true; do ./scripts/code-server.sh --host 0.0.0.0 --port 8000 --without-connection-token; sleep 20; done" +autostart=true +autorestart=true +startretries=999999 +stderr_logfile=/var/log/vscode-server.err.log +stdout_logfile=/var/log/vscode-server.out.log +EOF + +# Reload supervisor config +supervisorctl reread +supervisorctl update + +supervisorctl start vscode-server + +# Keep container running and show logs +tail -f /var/log/vscode-*.log \ No newline at end of file