diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 00000000..b4f100d4 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,173 @@ +name: E2E Shell Tests + +on: + push: + workflow_dispatch: + inputs: + test_model: + description: "Model to test with" + required: false + default: "ai/smollm2" + +permissions: + contents: read + +jobs: + e2e-shell-tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false # Continue other tests even if one fails + matrix: + include: + # Linux shells + - os: ubuntu-latest + shell-name: bash + shell-type: bash + test-script: ./e2e/shells/bash_test.sh + + - os: ubuntu-latest + shell-name: zsh + shell-type: bash + test-script: ./e2e/shells/zsh_test.sh + + # macOS shells + - os: macos-latest + shell-name: bash + shell-type: bash + test-script: ./e2e/shells/bash_test.sh + + - os: macos-latest + shell-name: zsh + shell-type: bash + test-script: ./e2e/shells/zsh_test.sh + + # Windows shells - DISABLED: Need to build a Windows-based DMR image first + # - os: windows-latest + # shell-name: cmd + # shell-type: cmd + # test-script: e2e\shells\cmd_test.bat + + # - os: windows-latest + # shell-name: git-bash + # shell-type: bash + # test-script: ./e2e/shells/mintty_test.sh + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # Cross-platform Docker setup + - name: Set up Docker + uses: docker/setup-docker-action@v4 + if: runner.os != 'Windows' + + - name: Install model-cli as Docker plugin (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: | + echo "Installing model-cli as Docker plugin..." + make install + echo "Installation completed successfully" + + - name: Install model-cli as Docker plugin (Windows) + if: runner.os == 'Windows' + run: | + mkdir "%ProgramData%\Docker\cli-plugins" || echo already exists + make PLUGIN_NAME=docker-model.exe install PLUGIN_DIR="%ProgramData%\Docker\cli-plugins" + dir "%ProgramData%\Docker\cli-plugins" + shell: cmd + + - name: Test docker model version + shell: bash + run: | + echo "Testing docker model version command..." + docker model version + + # Verify the command returns successfully + if [ $? -eq 0 ]; then + echo "✅ docker model version command works correctly" + else + echo "❌ docker model version command failed" + exit 1 + fi + + - name: Test model pull and run + shell: bash + run: | + MODEL="${{ github.event.inputs.test_model || 'ai/smollm2' }}" + echo "Testing with model: $MODEL" + + # Test model pull + echo "Pulling model..." + docker model pull "$MODEL" + + if [ $? -eq 0 ]; then + echo "✅ Model pull successful" + else + echo "❌ Model pull failed" + exit 1 + fi + + # Test basic model run (with timeout to avoid hanging) + echo "Testing docker model run..." + if [ "$RUNNER_OS" = "Windows" ]; then + # Windows doesn't have timeout command, use PowerShell + powershell -Command "& { Start-Process -FilePath 'docker' -ArgumentList 'model', 'run', '$MODEL', 'Give me a fact about whales.' -Wait -TimeoutSec 60 }" || { + exit_code=$? + if [ $exit_code -eq 1 ]; then + echo "✅ Model run test completed (timed out as expected for non-interactive test)" + else + echo "❌ Model run failed with exit code: $exit_code" + exit 1 + fi + } + else + timeout 60s docker model run "$MODEL" "Give me a fact about whales." || { + exit_code=$? + if [ $exit_code -eq 124 ]; then + echo "✅ Model run test completed (timed out as expected for non-interactive test)" + else + echo "❌ Model run failed with exit code: $exit_code" + exit 1 + fi + } + fi + + # Shell-specific setup + - name: Install Zsh (Linux) + if: matrix.shell-name == 'zsh' && runner.os == 'Linux' + shell: bash + run: sudo apt-get update && sudo apt-get install -y zsh + + # Shell-specific test execution + - name: Run Bash E2E Tests + if: matrix.shell-name == 'bash' + shell: bash + run: | + chmod +x ${{ matrix.test-script }} + ${{ matrix.test-script }} + + - name: Run Zsh E2E Tests + if: matrix.shell-name == 'zsh' + shell: bash + run: | + chmod +x ${{ matrix.test-script }} + zsh ${{ matrix.test-script }} + + - name: Run CMD E2E Tests + if: matrix.shell-name == 'cmd' + shell: cmd + run: ${{ matrix.test-script }} + + - name: Run Git Bash E2E Tests + if: matrix.shell-name == 'git-bash' + shell: bash + run: | + chmod +x ${{ matrix.test-script }} + ${{ matrix.test-script }} diff --git a/Makefile b/Makefile index c1d46622..663ece3a 100644 --- a/Makefile +++ b/Makefile @@ -63,9 +63,45 @@ unit-tests: @go test -v ./... @echo "Unit tests completed!" +e2e-tests: build e2e-shell-bash e2e-shell-zsh e2e-shell-cmd e2e-shell-mintty + @echo "All shell-specific tests completed!" + +e2e-shell-bash: + @echo "Running bash-specific tests..." + @if command -v bash >/dev/null 2>&1; then \ + ./e2e/shells/bash_test.sh; \ + else \ + echo "Bash not available, skipping bash tests"; \ + fi + +e2e-shell-zsh: + @echo "Running zsh-specific tests..." + @if command -v zsh >/dev/null 2>&1; then \ + ./e2e/shells/zsh_test.sh; \ + else \ + echo "Zsh not available, skipping zsh tests"; \ + fi + +e2e-shell-cmd: + @echo "Running cmd.exe-specific tests..." + @if [ "$(shell uname -s)" = "MINGW64_NT" ] || [ "$(shell uname -s)" = "MSYS_NT" ]; then \ + cmd.exe /c e2e\\shells\\cmd_test.bat; \ + else \ + echo "Windows cmd.exe not available, skipping cmd tests"; \ + fi + +e2e-shell-mintty: + @echo "Running mintty bash-specific tests..." + @if [ -n "$$MSYSTEM" ] || [ "$$TERM" = "xterm" ]; then \ + ./e2e/shells/mintty_test.sh; \ + else \ + echo "Mintty/Git Bash not detected, skipping mintty tests"; \ + fi + clean: @echo "Cleaning up..." @rm -f $(BINARY_NAME) + @rm -f model-cli-test @echo "Cleaned!" docs: diff --git a/e2e/fixtures/table_format/with_models.txt b/e2e/fixtures/table_format/with_models.txt new file mode 100644 index 00000000..ba35d408 --- /dev/null +++ b/e2e/fixtures/table_format/with_models.txt @@ -0,0 +1,2 @@ +MODEL NAME PARAMETERS QUANTIZATION ARCHITECTURE MODEL ID CREATED SIZE +ai/smollm2 361.82 M IQ2_XXS/Q4_K_M llama 354bf30d0aa3 3 months ago 256.35 MiB diff --git a/e2e/shells/bash_test.sh b/e2e/shells/bash_test.sh new file mode 100755 index 00000000..1c787075 --- /dev/null +++ b/e2e/shells/bash_test.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# Bash-specific e2e test for docker model list command with fixture validation +# This script validates the output against the expected fixture + +set -e + +TEST_MODEL="ai/smollm2" +FAILED_TESTS=0 +TOTAL_TESTS=0 + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[PASS]${NC} $1" +} + +log_error() { + echo -e "${RED}[FAIL]${NC} $1" + FAILED_TESTS=$((FAILED_TESTS + 1)) +} + +# Load fixture file and normalize lines +load_fixture() { + local fixture_path="$1" + if [ ! -f "$fixture_path" ]; then + log_error "Fixture file not found: $fixture_path" + return 1 + fi + # Remove trailing whitespace + sed 's/[[:space:]]*$//' "$fixture_path" +} + +run_test_with_fixture_validation() { + local test_name="$1" + local command="$2" + local fixture_path="$3" + local expected_exit_code="${4:-0}" + + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + log_info "Running test: $test_name" + + # Load expected output from fixture + expected_output=$(load_fixture "$fixture_path") + + if [ $? -ne 0 ]; then + log_error "$test_name - Failed to load fixture" + return 1 + fi + + # Run command and capture output and exit code + set +e + output=$(eval "$command" 2>&1) + exit_code=$? + set -e + + # Normalize output: remove leading/trailing blank lines and trailing spaces + normalized_output=$(echo "$output" | sed 's/[[:space:]]*$//') + + if [ $exit_code -eq $expected_exit_code ] && [ "$normalized_output" = "$expected_output" ]; then + log_success "$test_name" + return 0 + else + log_error "$test_name" + echo "Command: $command" + echo "Expected exit code: $expected_exit_code, got: $exit_code" + echo "Expected output:" + echo "$expected_output" + echo "Actual normalized output:" + echo "$normalized_output" + return 1 + fi +} + +# Check prerequisites +log_info "Checking prerequisites..." + +# Ensure we're running from the correct directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$PROJECT_ROOT" + +# Check if docker model is available +if ! command -v docker &> /dev/null; then + log_error "docker command not found" + exit 1 +fi + +# Check if model runner is running +if ! docker model status | grep -q "Docker Model Runner is running"; then + log_error "Docker Model Runner is not running" + exit 1 +fi + +# Check if test model is available +if ! docker model list | grep -q "$TEST_MODEL"; then + log_info "Test model $TEST_MODEL not found, pulling..." + docker model pull "$TEST_MODEL" +fi + +log_info "Starting fixture validation test..." + +# Test: Fixture format validation +run_test_with_fixture_validation "Fixture format validation" \ + "docker model list" \ + "e2e/fixtures/table_format/with_models.txt" + +# Summary +echo +log_info "Test Summary:" +echo "Total tests: $TOTAL_TESTS" +echo "Failed tests: $FAILED_TESTS" +echo "Passed tests: $((TOTAL_TESTS - FAILED_TESTS))" + +if [ $FAILED_TESTS -eq 0 ]; then + log_success "Fixture validation test passed!" + exit 0 +else + log_error "$FAILED_TESTS tests failed" + exit 1 +fi diff --git a/e2e/shells/cmd_test.bat b/e2e/shells/cmd_test.bat new file mode 100644 index 00000000..e69de29b diff --git a/e2e/shells/mintty_test.sh b/e2e/shells/mintty_test.sh new file mode 100644 index 00000000..e69de29b diff --git a/e2e/shells/zsh_test.sh b/e2e/shells/zsh_test.sh new file mode 100755 index 00000000..e69de29b