[WIP] Add e2e test example

.github/workflows/e2e-tests.yml

@@ -0,0 +1,173 @@
+name: E2E Shell Tests
+
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      test_model:
+        description: "Model to test with"
+        required: false
+        default: "ai/smollm2"
+
+permissions:
+  contents: read
+
+jobs:
+  e2e-shell-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false  # Continue other tests even if one fails
+      matrix:
+        include:
+          # Linux shells
+          - os: ubuntu-latest
+            shell-name: bash
+            shell-type: bash
+            test-script: ./e2e/shells/bash_test.sh
+
+          - os: ubuntu-latest
+            shell-name: zsh
+            shell-type: bash
+            test-script: ./e2e/shells/zsh_test.sh
+
+          # macOS shells
+          - os: macos-latest
+            shell-name: bash
+            shell-type: bash
+            test-script: ./e2e/shells/bash_test.sh
+
+          - os: macos-latest
+            shell-name: zsh
+            shell-type: bash
+            test-script: ./e2e/shells/zsh_test.sh
+
+          # Windows shells - DISABLED: Need to build a Windows-based DMR image first
+          # - os: windows-latest
+          #   shell-name: cmd
+          #   shell-type: cmd
+          #   test-script: e2e\shells\cmd_test.bat
+
+          # - os: windows-latest
+          #   shell-name: git-bash
+          #   shell-type: bash
+          #   test-script: ./e2e/shells/mintty_test.sh
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      # Cross-platform Docker setup
+      - name: Set up Docker
+        uses: docker/setup-docker-action@v4
+        if: runner.os != 'Windows'
+
+      - name: Install model-cli as Docker plugin (Linux/macOS)
+        if: runner.os != 'Windows'
+        shell: bash
+        run: |
+          echo "Installing model-cli as Docker plugin..."
+          make install
+          echo "Installation completed successfully"
+
+      - name: Install model-cli as Docker plugin (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          mkdir "%ProgramData%\Docker\cli-plugins" || echo already exists
+          make PLUGIN_NAME=docker-model.exe install PLUGIN_DIR="%ProgramData%\Docker\cli-plugins"
+          dir "%ProgramData%\Docker\cli-plugins"
+        shell: cmd
+
+      - name: Test docker model version
+        shell: bash
+        run: |
+          echo "Testing docker model version command..."
+          docker model version
+
+          # Verify the command returns successfully
+          if [ $? -eq 0 ]; then
+            echo "✅ docker model version command works correctly"
+          else
+            echo "❌ docker model version command failed"
+            exit 1
+          fi
+
+      - name: Test model pull and run
+        shell: bash
+        run: |
+          MODEL="${{ github.event.inputs.test_model || 'ai/smollm2' }}"
+          echo "Testing with model: $MODEL"
+
+          # Test model pull
+          echo "Pulling model..."
+          docker model pull "$MODEL"
+
+          if [ $? -eq 0 ]; then
+            echo "✅ Model pull successful"
+          else
+            echo "❌ Model pull failed"
+            exit 1
+          fi
+
+          # Test basic model run (with timeout to avoid hanging)
+          echo "Testing docker model run..."
+          if [ "$RUNNER_OS" = "Windows" ]; then
+            # Windows doesn't have timeout command, use PowerShell
+            powershell -Command "& { Start-Process -FilePath 'docker' -ArgumentList 'model', 'run', '$MODEL', 'Give me a fact about whales.' -Wait -TimeoutSec 60 }" || {
+              exit_code=$?
+              if [ $exit_code -eq 1 ]; then
+                echo "✅ Model run test completed (timed out as expected for non-interactive test)"
+              else
+                echo "❌ Model run failed with exit code: $exit_code"
+                exit 1
+              fi
+            }
+          else
+            timeout 60s docker model run "$MODEL" "Give me a fact about whales." || {
+              exit_code=$?
+              if [ $exit_code -eq 124 ]; then
+                echo "✅ Model run test completed (timed out as expected for non-interactive test)"
+              else
+                echo "❌ Model run failed with exit code: $exit_code"
+                exit 1
+              fi
+            }
+          fi
+
+      # Shell-specific setup
+      - name: Install Zsh (Linux)
+        if: matrix.shell-name == 'zsh' && runner.os == 'Linux'
+        shell: bash
+        run: sudo apt-get update && sudo apt-get install -y zsh
+
+      # Shell-specific test execution
+      - name: Run Bash E2E Tests
+        if: matrix.shell-name == 'bash'
+        shell: bash
+        run: |
+          chmod +x ${{ matrix.test-script }}
+          ${{ matrix.test-script }}
+
+      - name: Run Zsh E2E Tests
+        if: matrix.shell-name == 'zsh'
+        shell: bash
+        run: |
+          chmod +x ${{ matrix.test-script }}
+          zsh ${{ matrix.test-script }}
+
+      - name: Run CMD E2E Tests
+        if: matrix.shell-name == 'cmd'
+        shell: cmd
+        run: ${{ matrix.test-script }}
+
+      - name: Run Git Bash E2E Tests
+        if: matrix.shell-name == 'git-bash'
+        shell: bash
+        run: |
+          chmod +x ${{ matrix.test-script }}
+          ${{ matrix.test-script }}

Makefile

@@ -63,9 +63,45 @@ unit-tests:
 	@go test -v ./...
 	@echo "Unit tests completed!"
 
+e2e-tests: build e2e-shell-bash e2e-shell-zsh e2e-shell-cmd e2e-shell-mintty
+	@echo "All shell-specific tests completed!"
+
+e2e-shell-bash:
+	@echo "Running bash-specific tests..."
+	@if command -v bash >/dev/null 2>&1; then \
+		./e2e/shells/bash_test.sh; \
+	else \
+		echo "Bash not available, skipping bash tests"; \
+	fi
+
+e2e-shell-zsh:
+	@echo "Running zsh-specific tests..."
+	@if command -v zsh >/dev/null 2>&1; then \
+		./e2e/shells/zsh_test.sh; \
+	else \
+		echo "Zsh not available, skipping zsh tests"; \
+	fi
+
+e2e-shell-cmd:
+	@echo "Running cmd.exe-specific tests..."
+	@if [ "$(shell uname -s)" = "MINGW64_NT" ] || [ "$(shell uname -s)" = "MSYS_NT" ]; then \
+		cmd.exe /c e2e\\shells\\cmd_test.bat; \
+	else \
+		echo "Windows cmd.exe not available, skipping cmd tests"; \
+	fi
+
+e2e-shell-mintty:
+	@echo "Running mintty bash-specific tests..."
+	@if [ -n "$$MSYSTEM" ] || [ "$$TERM" = "xterm" ]; then \
+		./e2e/shells/mintty_test.sh; \
+	else \
+		echo "Mintty/Git Bash not detected, skipping mintty tests"; \
+	fi
+
 clean:
 	@echo "Cleaning up..."
 	@rm -f $(BINARY_NAME)
+	@rm -f model-cli-test
 	@echo "Cleaned!"
 
 docs:

e2e/fixtures/table_format/with_models.txt

@@ -0,0 +1,2 @@
+MODEL NAME  PARAMETERS  QUANTIZATION    ARCHITECTURE  MODEL ID      CREATED       SIZE
+ai/smollm2  361.82 M    IQ2_XXS/Q4_K_M  llama         354bf30d0aa3  3 months ago  256.35 MiB

e2e/shells/bash_test.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Bash-specific e2e test for docker model list command with fixture validation
+# This script validates the output against the expected fixture
+
+set -e
+
+TEST_MODEL="ai/smollm2"
+FAILED_TESTS=0
+TOTAL_TESTS=0
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[PASS]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[FAIL]${NC} $1"
+    FAILED_TESTS=$((FAILED_TESTS + 1))
+}
+
+# Load fixture file and normalize lines
+load_fixture() {
+    local fixture_path="$1"
+    if [ ! -f "$fixture_path" ]; then
+        log_error "Fixture file not found: $fixture_path"
+        return 1
+    fi
+    # Remove trailing whitespace
+    sed 's/[[:space:]]*$//' "$fixture_path"
+}
+
+run_test_with_fixture_validation() {
+    local test_name="$1"
+    local command="$2"
+    local fixture_path="$3"
+    local expected_exit_code="${4:-0}"
+
+    TOTAL_TESTS=$((TOTAL_TESTS + 1))
+    log_info "Running test: $test_name"
+
+    # Load expected output from fixture
+    expected_output=$(load_fixture "$fixture_path")
+
+    if [ $? -ne 0 ]; then
+        log_error "$test_name - Failed to load fixture"
+        return 1
+    fi
+
+    # Run command and capture output and exit code
+    set +e
+    output=$(eval "$command" 2>&1)
+    exit_code=$?
+    set -e
+
+    # Normalize output: remove leading/trailing blank lines and trailing spaces
+    normalized_output=$(echo "$output" | sed 's/[[:space:]]*$//')
+
+    if [ $exit_code -eq $expected_exit_code ] && [ "$normalized_output" = "$expected_output" ]; then
+        log_success "$test_name"
+        return 0
+    else
+        log_error "$test_name"
+        echo "Command: $command"
+        echo "Expected exit code: $expected_exit_code, got: $exit_code"
+        echo "Expected output:"
+        echo "$expected_output"
+        echo "Actual normalized output:"
+        echo "$normalized_output"
+        return 1
+    fi
+}
+
+# Check prerequisites
+log_info "Checking prerequisites..."
+
+# Ensure we're running from the correct directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+cd "$PROJECT_ROOT"
+
+# Check if docker model is available
+if ! command -v docker &> /dev/null; then
+    log_error "docker command not found"
+    exit 1
+fi
+
+# Check if model runner is running
+if ! docker model status | grep -q "Docker Model Runner is running"; then
+    log_error "Docker Model Runner is not running"
+    exit 1
+fi
+
+# Check if test model is available
+if ! docker model list | grep -q "$TEST_MODEL"; then
+    log_info "Test model $TEST_MODEL not found, pulling..."
+    docker model pull "$TEST_MODEL"
+fi
+
+log_info "Starting fixture validation test..."
+
+# Test: Fixture format validation
+run_test_with_fixture_validation "Fixture format validation" \
+    "docker model list" \
+    "e2e/fixtures/table_format/with_models.txt"
+
+# Summary
+echo
+log_info "Test Summary:"
+echo "Total tests: $TOTAL_TESTS"
+echo "Failed tests: $FAILED_TESTS"
+echo "Passed tests: $((TOTAL_TESTS - FAILED_TESTS))"
+
+if [ $FAILED_TESTS -eq 0 ]; then
+    log_success "Fixture validation test passed!"
+    exit 0
+else
+    log_error "$FAILED_TESTS tests failed"
+    exit 1
+fi