From 3c146bed9f65530a8eec8adf86390bdd812e00d0 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Mon, 14 Jul 2025 16:00:32 +0200 Subject: [PATCH 01/10] Add e2e test example --- Makefile | 36 ++++++ e2e/fixtures/table_format/with_models.txt | 2 + e2e/shells/bash_test.sh | 128 ++++++++++++++++++++++ e2e/shells/cmd_test.bat | 0 e2e/shells/mintty_test.sh | 0 e2e/shells/zsh_test.sh | 0 6 files changed, 166 insertions(+) create mode 100644 e2e/fixtures/table_format/with_models.txt create mode 100755 e2e/shells/bash_test.sh create mode 100644 e2e/shells/cmd_test.bat create mode 100644 e2e/shells/mintty_test.sh create mode 100755 e2e/shells/zsh_test.sh diff --git a/Makefile b/Makefile index c1d46622..663ece3a 100644 --- a/Makefile +++ b/Makefile @@ -63,9 +63,45 @@ unit-tests: @go test -v ./... @echo "Unit tests completed!" +e2e-tests: build e2e-shell-bash e2e-shell-zsh e2e-shell-cmd e2e-shell-mintty + @echo "All shell-specific tests completed!" + +e2e-shell-bash: + @echo "Running bash-specific tests..." + @if command -v bash >/dev/null 2>&1; then \ + ./e2e/shells/bash_test.sh; \ + else \ + echo "Bash not available, skipping bash tests"; \ + fi + +e2e-shell-zsh: + @echo "Running zsh-specific tests..." + @if command -v zsh >/dev/null 2>&1; then \ + ./e2e/shells/zsh_test.sh; \ + else \ + echo "Zsh not available, skipping zsh tests"; \ + fi + +e2e-shell-cmd: + @echo "Running cmd.exe-specific tests..." + @if [ "$(shell uname -s)" = "MINGW64_NT" ] || [ "$(shell uname -s)" = "MSYS_NT" ]; then \ + cmd.exe /c e2e\\shells\\cmd_test.bat; \ + else \ + echo "Windows cmd.exe not available, skipping cmd tests"; \ + fi + +e2e-shell-mintty: + @echo "Running mintty bash-specific tests..." + @if [ -n "$$MSYSTEM" ] || [ "$$TERM" = "xterm" ]; then \ + ./e2e/shells/mintty_test.sh; \ + else \ + echo "Mintty/Git Bash not detected, skipping mintty tests"; \ + fi + clean: @echo "Cleaning up..." @rm -f $(BINARY_NAME) + @rm -f model-cli-test @echo "Cleaned!" docs: diff --git a/e2e/fixtures/table_format/with_models.txt b/e2e/fixtures/table_format/with_models.txt new file mode 100644 index 00000000..ba35d408 --- /dev/null +++ b/e2e/fixtures/table_format/with_models.txt @@ -0,0 +1,2 @@ +MODEL NAME PARAMETERS QUANTIZATION ARCHITECTURE MODEL ID CREATED SIZE +ai/smollm2 361.82 M IQ2_XXS/Q4_K_M llama 354bf30d0aa3 3 months ago 256.35 MiB diff --git a/e2e/shells/bash_test.sh b/e2e/shells/bash_test.sh new file mode 100755 index 00000000..1c787075 --- /dev/null +++ b/e2e/shells/bash_test.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# Bash-specific e2e test for docker model list command with fixture validation +# This script validates the output against the expected fixture + +set -e + +TEST_MODEL="ai/smollm2" +FAILED_TESTS=0 +TOTAL_TESTS=0 + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[PASS]${NC} $1" +} + +log_error() { + echo -e "${RED}[FAIL]${NC} $1" + FAILED_TESTS=$((FAILED_TESTS + 1)) +} + +# Load fixture file and normalize lines +load_fixture() { + local fixture_path="$1" + if [ ! -f "$fixture_path" ]; then + log_error "Fixture file not found: $fixture_path" + return 1 + fi + # Remove trailing whitespace + sed 's/[[:space:]]*$//' "$fixture_path" +} + +run_test_with_fixture_validation() { + local test_name="$1" + local command="$2" + local fixture_path="$3" + local expected_exit_code="${4:-0}" + + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + log_info "Running test: $test_name" + + # Load expected output from fixture + expected_output=$(load_fixture "$fixture_path") + + if [ $? -ne 0 ]; then + log_error "$test_name - Failed to load fixture" + return 1 + fi + + # Run command and capture output and exit code + set +e + output=$(eval "$command" 2>&1) + exit_code=$? + set -e + + # Normalize output: remove leading/trailing blank lines and trailing spaces + normalized_output=$(echo "$output" | sed 's/[[:space:]]*$//') + + if [ $exit_code -eq $expected_exit_code ] && [ "$normalized_output" = "$expected_output" ]; then + log_success "$test_name" + return 0 + else + log_error "$test_name" + echo "Command: $command" + echo "Expected exit code: $expected_exit_code, got: $exit_code" + echo "Expected output:" + echo "$expected_output" + echo "Actual normalized output:" + echo "$normalized_output" + return 1 + fi +} + +# Check prerequisites +log_info "Checking prerequisites..." + +# Ensure we're running from the correct directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$PROJECT_ROOT" + +# Check if docker model is available +if ! command -v docker &> /dev/null; then + log_error "docker command not found" + exit 1 +fi + +# Check if model runner is running +if ! docker model status | grep -q "Docker Model Runner is running"; then + log_error "Docker Model Runner is not running" + exit 1 +fi + +# Check if test model is available +if ! docker model list | grep -q "$TEST_MODEL"; then + log_info "Test model $TEST_MODEL not found, pulling..." + docker model pull "$TEST_MODEL" +fi + +log_info "Starting fixture validation test..." + +# Test: Fixture format validation +run_test_with_fixture_validation "Fixture format validation" \ + "docker model list" \ + "e2e/fixtures/table_format/with_models.txt" + +# Summary +echo +log_info "Test Summary:" +echo "Total tests: $TOTAL_TESTS" +echo "Failed tests: $FAILED_TESTS" +echo "Passed tests: $((TOTAL_TESTS - FAILED_TESTS))" + +if [ $FAILED_TESTS -eq 0 ]; then + log_success "Fixture validation test passed!" + exit 0 +else + log_error "$FAILED_TESTS tests failed" + exit 1 +fi diff --git a/e2e/shells/cmd_test.bat b/e2e/shells/cmd_test.bat new file mode 100644 index 00000000..e69de29b diff --git a/e2e/shells/mintty_test.sh b/e2e/shells/mintty_test.sh new file mode 100644 index 00000000..e69de29b diff --git a/e2e/shells/zsh_test.sh b/e2e/shells/zsh_test.sh new file mode 100755 index 00000000..e69de29b From d4b76ca4f6df13caae7a211513658ca191467ab6 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Tue, 15 Jul 2025 15:54:33 +0200 Subject: [PATCH 02/10] Adds Github workflow --- .github/workflows/e2e-tests.yml | 151 ++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 .github/workflows/e2e-tests.yml diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 00000000..634955ba --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,151 @@ +name: E2E Shell Tests + +on: + push: + pull_request: + branches: [ "main" ] + workflow_dispatch: + inputs: + test_model: + description: "Model to test with" + required: false + default: "ai/smollm2" + +jobs: + e2e-shell-tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false # Continue other tests even if one fails + matrix: + include: + # Linux shells + - os: ubuntu-latest + shell-name: bash + shell-type: bash + test-script: ./e2e/shells/bash_test.sh + + - os: ubuntu-latest + shell-name: zsh + shell-type: bash + test-script: ./e2e/shells/zsh_test.sh + + # Windows shells + - os: windows-latest + shell-name: cmd + shell-type: cmd + test-script: e2e\shells\cmd_test.bat + + - os: windows-latest + shell-name: git-bash + shell-type: bash + test-script: ./e2e/shells/mintty_test.sh + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # Cross-platform Docker setup + - name: Set up Docker + uses: docker/setup-docker-action@v4 + + - name: Install model-cli as Docker plugin + shell: bash + run: | + echo "Installing model-cli as Docker plugin..." + make install + echo "Installation completed successfully" + + - name: Test docker model version + shell: bash + run: | + echo "Testing docker model version command..." + docker model version + + # Verify the command returns successfully + if [ $? -eq 0 ]; then + echo "✅ docker model version command works correctly" + else + echo "❌ docker model version command failed" + exit 1 + fi + + - name: Test model pull and run + shell: bash + run: | + MODEL="${{ github.event.inputs.test_model || 'ai/smollm2' }}" + echo "Testing with model: $MODEL" + + # Test model pull + echo "Pulling model..." + docker model pull "$MODEL" + + if [ $? -eq 0 ]; then + echo "✅ Model pull successful" + else + echo "❌ Model pull failed" + exit 1 + fi + + # Test basic model run (with timeout to avoid hanging) + echo "Testing docker model run..." + if [ "$RUNNER_OS" = "Windows" ]; then + # Windows doesn't have timeout command, use PowerShell + powershell -Command "& { Start-Process -FilePath 'docker' -ArgumentList 'model', 'run', '$MODEL', 'Give me a fact about whales.' -Wait -TimeoutSec 60 }" || { + exit_code=$? + if [ $exit_code -eq 1 ]; then + echo "✅ Model run test completed (timed out as expected for non-interactive test)" + else + echo "❌ Model run failed with exit code: $exit_code" + exit 1 + fi + } + else + timeout 60s docker model run "$MODEL" "Give me a fact about whales." || { + exit_code=$? + if [ $exit_code -eq 124 ]; then + echo "✅ Model run test completed (timed out as expected for non-interactive test)" + else + echo "❌ Model run failed with exit code: $exit_code" + exit 1 + fi + } + fi + + # Shell-specific setup + - name: Install Zsh (Linux) + if: matrix.shell-name == 'zsh' && runner.os == 'Linux' + shell: bash + run: sudo apt-get update && sudo apt-get install -y zsh + + # Shell-specific test execution + - name: Run Bash E2E Tests + if: matrix.shell-name == 'bash' + shell: bash + run: | + chmod +x ${{ matrix.test-script }} + ${{ matrix.test-script }} + + - name: Run Zsh E2E Tests + if: matrix.shell-name == 'zsh' + shell: bash + run: | + chmod +x ${{ matrix.test-script }} + zsh ${{ matrix.test-script }} + + - name: Run CMD E2E Tests + if: matrix.shell-name == 'cmd' + shell: cmd + run: ${{ matrix.test-script }} + + - name: Run Git Bash E2E Tests + if: matrix.shell-name == 'git-bash' + shell: bash + run: | + chmod +x ${{ matrix.test-script }} + ${{ matrix.test-script }} From acb717026876497f469adfdd9d4b51a5c652d632 Mon Sep 17 00:00:00 2001 From: Ignasi Date: Tue, 15 Jul 2025 15:57:21 +0200 Subject: [PATCH 03/10] Potential fix for code scanning alert no. 5: Workflow does not contain permissions Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/e2e-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 634955ba..891ee177 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -11,6 +11,9 @@ on: required: false default: "ai/smollm2" +permissions: + contents: read + jobs: e2e-shell-tests: runs-on: ${{ matrix.os }} From 69e96945108085054aaf72b80536aa935cb96dd7 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Tue, 15 Jul 2025 16:09:05 +0200 Subject: [PATCH 04/10] Install docker model plugin in Windows --- .github/workflows/e2e-tests.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 891ee177..81680ecd 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -57,13 +57,23 @@ jobs: - name: Set up Docker uses: docker/setup-docker-action@v4 - - name: Install model-cli as Docker plugin + - name: Install model-cli as Docker plugin (Linux/macOS) + if: runner.os != 'Windows' shell: bash run: | echo "Installing model-cli as Docker plugin..." make install echo "Installation completed successfully" + - name: Install model-cli as Docker plugin (Windows) + if: runner.os == 'Windows' + run: | + set PLUGIN_DIR=%ProgramData%\Docker\cli-plugins + mkdir "%PLUGIN_DIR%" || echo already exists + make install + dir "%PLUGIN_DIR%" + shell: cmd + - name: Test docker model version shell: bash run: | From 2a77ea59aa16b80fb6c28bc6c0c97c50127f56c8 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Tue, 15 Jul 2025 16:12:20 +0200 Subject: [PATCH 05/10] Make the env var PLUGIN_DIR visible to make --- .github/workflows/e2e-tests.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 81680ecd..ea5aae1e 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -2,8 +2,6 @@ name: E2E Shell Tests on: push: - pull_request: - branches: [ "main" ] workflow_dispatch: inputs: test_model: @@ -68,10 +66,9 @@ jobs: - name: Install model-cli as Docker plugin (Windows) if: runner.os == 'Windows' run: | - set PLUGIN_DIR=%ProgramData%\Docker\cli-plugins - mkdir "%PLUGIN_DIR%" || echo already exists - make install - dir "%PLUGIN_DIR%" + mkdir "%ProgramData%\Docker\cli-plugins" || echo already exists + make install PLUGIN_DIR="%ProgramData%\Docker\cli-plugins" + dir "%ProgramData%\Docker\cli-plugins" shell: cmd - name: Test docker model version From 9a54dc3ba689bef8b3556c6818f402976e6adf45 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Tue, 15 Jul 2025 16:15:33 +0200 Subject: [PATCH 06/10] Override binary name --- .github/workflows/e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index ea5aae1e..75336517 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -67,7 +67,7 @@ jobs: if: runner.os == 'Windows' run: | mkdir "%ProgramData%\Docker\cli-plugins" || echo already exists - make install PLUGIN_DIR="%ProgramData%\Docker\cli-plugins" + make BINARY_NAME=docker-model.exe install PLUGIN_DIR="%ProgramData%\Docker\cli-plugins" dir "%ProgramData%\Docker\cli-plugins" shell: cmd From 1192aeb74302b1547db5d39141eb00c6af995493 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Tue, 15 Jul 2025 16:20:50 +0200 Subject: [PATCH 07/10] Override plugin name --- .github/workflows/e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 75336517..80673248 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -67,7 +67,7 @@ jobs: if: runner.os == 'Windows' run: | mkdir "%ProgramData%\Docker\cli-plugins" || echo already exists - make BINARY_NAME=docker-model.exe install PLUGIN_DIR="%ProgramData%\Docker\cli-plugins" + make PLUGIN_NAME=docker-model.exe install PLUGIN_DIR="%ProgramData%\Docker\cli-plugins" dir "%ProgramData%\Docker\cli-plugins" shell: cmd From 030a444e66107c235fc3ca11f7ec72b23a9c2468 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Tue, 15 Jul 2025 16:58:14 +0200 Subject: [PATCH 08/10] Skip setup docker step on windows to check if docker is already setup in Windows runners to run linux containers --- .github/workflows/e2e-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 80673248..09fc1bb6 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -54,6 +54,7 @@ jobs: # Cross-platform Docker setup - name: Set up Docker uses: docker/setup-docker-action@v4 + if: runner.os != 'Windows' - name: Install model-cli as Docker plugin (Linux/macOS) if: runner.os != 'Windows' From 7ebd270e3a69752ef323cd1dff97b2bc9c70a6d2 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Wed, 16 Jul 2025 12:10:09 +0200 Subject: [PATCH 09/10] Disable Windows tests as we need to build a Windows-based DMR image first --- .github/workflows/e2e-tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 09fc1bb6..d157be18 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -30,16 +30,16 @@ jobs: shell-type: bash test-script: ./e2e/shells/zsh_test.sh - # Windows shells - - os: windows-latest - shell-name: cmd - shell-type: cmd - test-script: e2e\shells\cmd_test.bat - - - os: windows-latest - shell-name: git-bash - shell-type: bash - test-script: ./e2e/shells/mintty_test.sh + # Windows shells - DISABLED: Need to build a Windows-based DMR image first + # - os: windows-latest + # shell-name: cmd + # shell-type: cmd + # test-script: e2e\shells\cmd_test.bat + + # - os: windows-latest + # shell-name: git-bash + # shell-type: bash + # test-script: ./e2e/shells/mintty_test.sh steps: - name: Checkout repository From 42ef67b09f7f1ef585fdb88f8b7e6589f36419a0 Mon Sep 17 00:00:00 2001 From: ilopezluna Date: Wed, 16 Jul 2025 12:13:39 +0200 Subject: [PATCH 10/10] Adds macOS runner --- .github/workflows/e2e-tests.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index d157be18..b4f100d4 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -30,6 +30,17 @@ jobs: shell-type: bash test-script: ./e2e/shells/zsh_test.sh + # macOS shells + - os: macos-latest + shell-name: bash + shell-type: bash + test-script: ./e2e/shells/bash_test.sh + + - os: macos-latest + shell-name: zsh + shell-type: bash + test-script: ./e2e/shells/zsh_test.sh + # Windows shells - DISABLED: Need to build a Windows-based DMR image first # - os: windows-latest # shell-name: cmd