diff --git a/.gitignore b/.gitignore index 30a43a92..8a0a35b5 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,7 @@ dist/ build/ # Entrypoint for the application -!/cmd/d8 \ No newline at end of file +!/cmd/d8 + +# E2E test logs +testing/e2e/.logs/ \ No newline at end of file diff --git a/.golangci.yaml b/.golangci.yaml index dcff2e5a..d4fe847e 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -103,6 +103,7 @@ linters: - examples$ - _test\.go$ - deepcopy\.go$ + - testing/ formatters: enable: - gci diff --git a/Taskfile.yml b/Taskfile.yml index 82be4790..31d5f06e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -266,6 +266,41 @@ tasks: cmds: - task: _test:go + test:e2e:mirror: + desc: Run full cycle E2E mirror test (platform + modules + security) + summary: | + E2E test for d8 mirror pull/push commands. + Required: E2E_LICENSE_TOKEN env variable + + Example: E2E_LICENSE_TOKEN=xxx task test:e2e:mirror + cmds: + - task build + - go test -v -count=1 -timeout 180m -tags=e2e -run 'TestFullCycleE2E' ./testing/e2e/mirror {{ .CLI_ARGS }} + + test:e2e:mirror:platform: + desc: Run platform E2E test + cmds: + - task build + - go test -v -count=1 -timeout 120m -tags=e2e -run 'TestPlatform' ./testing/e2e/mirror {{ .CLI_ARGS }} + + test:e2e:mirror:modules: + desc: Run all modules E2E test + cmds: + - task build + - go test -v -count=1 -timeout 120m -tags=e2e -run 'TestModulesE2E$' ./testing/e2e/mirror {{ .CLI_ARGS }} + + + test:e2e:mirror:security: + desc: Run security E2E test + cmds: + - task build + - go test -v -count=1 -timeout 30m -tags=e2e -run 'TestSecurityE2E' ./testing/e2e/mirror {{ .CLI_ARGS }} + + test:e2e:mirror:logs:clean: + desc: Clean E2E test logs + cmds: + - rm -rf ./testing/e2e/.logs/* + lint: desc: Run golangci-lint with auto-fix cmds: diff --git a/go.mod b/go.mod index 8d07317d..821dc8e2 100644 --- a/go.mod +++ b/go.mod @@ -134,6 +134,11 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/chanced/caps v1.0.2 // indirect + github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.8.0 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cilium/ebpf v0.12.3 // indirect github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible // indirect diff --git a/go.sum b/go.sum index 243393f9..9d86d774 100644 --- a/go.sum +++ b/go.sum @@ -300,6 +300,16 @@ github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiw github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= github.com/chanced/caps v1.0.2 h1:RELvNN4lZajqSXJGzPaU7z8B4LK2+o2Oc/upeWdgMOA= github.com/chanced/caps v1.0.2/go.mod h1:SJhRzeYLKJ3OmzyQXhdZ7Etj7lqqWoPtQ1zcSJRtQjs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE= +github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= diff --git a/pkg/libmirror/images/digests_test.go b/pkg/libmirror/images/digests_test.go index 772bd414..7359667a 100644 --- a/pkg/libmirror/images/digests_test.go +++ b/pkg/libmirror/images/digests_test.go @@ -47,8 +47,6 @@ func TestExtractImageDigestsFromDeckhouseInstaller(t *testing.T) { installersLayout := createOCILayoutWithInstallerImage(t, "nonexistent.registry.com/deckhouse", installerTag, expectedImages) client := mock.NewRegistryClientMock(t) - client.GetRegistryMock.Return("nonexistent.registry.com") - client.WithSegmentMock.Return(client) client.CheckImageExistsMock.Return(nil) images, err := ExtractImageDigestsFromDeckhouseInstaller( ¶ms.PullParams{BaseParams: params.BaseParams{ diff --git a/pkg/libmirror/layouts/push_test.go b/pkg/libmirror/layouts/push_test.go index 3e2f3311..813a339f 100644 --- a/pkg/libmirror/layouts/push_test.go +++ b/pkg/libmirror/layouts/push_test.go @@ -38,7 +38,9 @@ func TestPushLayoutToRepoWithParallelism(t *testing.T) { const totalImages, layersPerImage = 10, 3 imagesLayout := createEmptyOCILayout(t) - host, repoPath, _ := mirrorTestUtils.SetupEmptyRegistryRepo(false) + reg := mirrorTestUtils.SetupTestRegistry(false) + defer reg.Close() + repoPath := reg.Host + "/deckhouse/ee" client := mock.NewRegistryClientMock(t) client.PushImageMock.Return(nil) @@ -50,7 +52,7 @@ func TestPushLayoutToRepoWithParallelism(t *testing.T) { digest, err := img.Digest() s.NoError(err) err = imagesLayout.AppendImage(img, platformOpt, layout.WithAnnotations(map[string]string{ - "org.opencontainers.image.ref.name": host + repoPath + "@" + digest.String(), + "org.opencontainers.image.ref.name": repoPath + "@" + digest.String(), "io.deckhouse.image.short_tag": digest.Hex, })) s.NoError(err) @@ -59,7 +61,7 @@ func TestPushLayoutToRepoWithParallelism(t *testing.T) { err := PushLayoutToRepo( client, imagesLayout, - host+repoPath, // Images repo + repoPath, authn.Anonymous, log.NewSLogger(slog.LevelDebug), params.ParallelismConfig{ @@ -81,7 +83,9 @@ func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { const totalImages, layersPerImage = 10, 3 imagesLayout := createEmptyOCILayout(t) - host, repoPath, _ := mirrorTestUtils.SetupEmptyRegistryRepo(false) + reg := mirrorTestUtils.SetupTestRegistry(false) + defer reg.Close() + repoPath := reg.Host + "/deckhouse/ee" client := mock.NewRegistryClientMock(t) client.PushImageMock.Return(nil) @@ -93,7 +97,7 @@ func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { digest, err := img.Digest() s.NoError(err) err = imagesLayout.AppendImage(img, platformOpt, layout.WithAnnotations(map[string]string{ - "org.opencontainers.image.ref.name": host + repoPath + "@" + digest.String(), + "org.opencontainers.image.ref.name": repoPath + "@" + digest.String(), "io.deckhouse.image.short_tag": digest.Hex, })) s.NoError(err) @@ -102,7 +106,7 @@ func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { err := PushLayoutToRepo( client, imagesLayout, - host+repoPath, // Images repo + repoPath, authn.Anonymous, log.NewSLogger(slog.LevelDebug), params.ParallelismConfig{ @@ -121,7 +125,9 @@ func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { func TestPushEmptyLayoutToRepo(t *testing.T) { s := require.New(t) - host, repoPath, blobHandler := mirrorTestUtils.SetupEmptyRegistryRepo(false) + reg := mirrorTestUtils.SetupTestRegistry(false) + defer reg.Close() + repoPath := reg.Host + "/deckhouse/ee" client := mock.NewRegistryClientMock(t) @@ -129,7 +135,7 @@ func TestPushEmptyLayoutToRepo(t *testing.T) { err := PushLayoutToRepo( client, emptyLayout, - host+repoPath, + repoPath, authn.Anonymous, log.NewSLogger(slog.LevelDebug), params.DefaultParallelism, @@ -137,5 +143,5 @@ func TestPushEmptyLayoutToRepo(t *testing.T) { false, // TLS verification irrelevant to HTTP requests ) s.NoError(err, "Push should not fail") - s.Len(blobHandler.ListBlobs(), 0, "No blobs should be pushed to registry") + s.Len(reg.ListBlobs(), 0, "No blobs should be pushed to registry") } diff --git a/testing/e2e/mirror/README.md b/testing/e2e/mirror/README.md new file mode 100644 index 00000000..48fc88ad --- /dev/null +++ b/testing/e2e/mirror/README.md @@ -0,0 +1,182 @@ +# E2E Tests for d8 mirror + +End-to-end tests for the `d8 mirror pull` and `d8 mirror push` commands. + +## Overview + +These tests perform **complete mirror cycles with verification** to ensure: +1. All expected images are downloaded from source +2. All images are correctly pushed to target registry +3. All images match between source and target (deep comparison) + +## Test Types + +| Test | Description | Timeout | Command | +|------|-------------|---------|---------| +| **Full Cycle** | Platform + Modules + Security | 3h | `task test:e2e:mirror` | +| **Platform** | Deckhouse core only | 2h | `task test:e2e:mirror:platform` | +| **Platform Stable** | Only stable channel | 2h | `task test:e2e:mirror:platform` + `E2E_DECKHOUSE_TAG=stable` | +| **Modules** | All modules | 2h | `task test:e2e:mirror:modules` | +| **Single Module** | One module (fast) | 2h | `task test:e2e:mirror:modules` + `E2E_INCLUDE_MODULES=module-name` | +| **Security** | Security DBs only | 30m | `task test:e2e:mirror:security` | + +## Verification Approach + +### Step 1: Read Expected Images from Source +Before pulling, we independently read what SHOULD be downloaded: +- Release channel versions from source registry +- `images_digests.json` from each installer image +- Module list and versions + +### Step 2: Pull & Push +Execute `d8 mirror pull` and `d8 mirror push` + +### Step 3: Verify +Compare expected images with what's actually in target: +- All expected digests must exist in target +- All images in target must match source (manifest, config, layers) + +This catches: +- **Pull bugs** - if pull forgets to download an image +- **Push bugs** - if push fails to upload an image +- **Data corruption** - if any digest doesn't match + +## Running Tests + +### Quick Start + +```bash +# Full cycle with license token +E2E_LICENSE_TOKEN=xxx task test:e2e:mirror + +# Platform only (faster) +E2E_LICENSE_TOKEN=xxx E2E_DECKHOUSE_TAG=stable task test:e2e:mirror:platform + +# Single module (fastest) +E2E_LICENSE_TOKEN=xxx E2E_INCLUDE_MODULES=module-name task test:e2e:mirror:modules +``` + +### Using Environment Variables + +```bash +# Official registry with license +E2E_LICENSE_TOKEN=your_license_token \ +task test:e2e:mirror + +# Local registry +E2E_SOURCE_REGISTRY=localhost:443/deckhouse \ +E2E_SOURCE_USER=admin \ +E2E_SOURCE_PASSWORD=secret \ +E2E_TLS_SKIP_VERIFY=true \ +task test:e2e:mirror:platform + +# Specific release channel +E2E_LICENSE_TOKEN=xxx \ +E2E_DECKHOUSE_TAG=stable \ +task test:e2e:mirror:platform + +# Specific modules only +E2E_LICENSE_TOKEN=xxx \ +E2E_INCLUDE_MODULES="pod-reloader,neuvector" \ +task test:e2e:mirror:modules +``` + +### Configuration Options + +| Flag | Environment Variable | Default | Description | +|------|---------------------|---------|-------------| +| `-source-registry` | `E2E_SOURCE_REGISTRY` | `registry.deckhouse.ru/deckhouse/fe` | Source registry | +| `-source-user` | `E2E_SOURCE_USER` | | Source registry username | +| `-source-password` | `E2E_SOURCE_PASSWORD` | | Source registry password | +| `-license-token` | `E2E_LICENSE_TOKEN` | | License token | +| `-target-registry` | `E2E_TARGET_REGISTRY` | `""` (in-memory) | Target registry | +| `-target-user` | `E2E_TARGET_USER` | | Target registry username | +| `-target-password` | `E2E_TARGET_PASSWORD` | | Target registry password | +| `-tls-skip-verify` | `E2E_TLS_SKIP_VERIFY` | `false` | Skip TLS verification | +| `-deckhouse-tag` | `E2E_DECKHOUSE_TAG` | | Specific tag/channel (e.g., `stable`, `v1.65.8`) | +| `-no-modules` | `E2E_NO_MODULES` | `false` | Skip modules | +| `-no-platform` | `E2E_NO_PLATFORM` | `false` | Skip platform | +| `-no-security` | `E2E_NO_SECURITY` | `false` | Skip security DBs | +| `-include-modules` | `E2E_INCLUDE_MODULES` | | Comma-separated module list | +| `-keep-bundle` | `E2E_KEEP_BUNDLE` | `false` | Keep bundle after test | +| `-existing-bundle` | `E2E_EXISTING_BUNDLE` | | Path to existing bundle (skip pull) | +| `-d8-binary` | `E2E_D8_BINARY` | `bin/d8` | Path to d8 binary | +| `-new-pull` | `E2E_NEW_PULL` | `false` | Use experimental pull | + +## Test Artifacts + +Tests produce artifacts in `testing/e2e/.logs/-/`: + +``` +testing/e2e/.logs/TestFullCycleE2E-20251226-114128/ +├── test.log # Full command output (pull/push) +├── report.txt # Test summary report +└── comparison.txt # Detailed registry comparison +``` + +### Cleaning Up Logs + +```bash +task test:e2e:mirror:logs:clean +``` + +## Requirements + +- Built `d8` binary (run `task build`) +- Valid credentials for source registry +- Network access +- Disk space for bundle (20-50 GB depending on scope) + +## What Gets Verified + +### Platform Test +1. Release channels exist (alpha, beta, stable, rock-solid) +2. Install images for each version +3. All digests from `images_digests.json` exist in target + +### Modules Test +1. Module list matches expected +2. Each module has release tags +3. Module images match source + +### Security Test +1. All security databases exist (trivy-db, trivy-bdu, etc.) +2. Tags match source + +## Troubleshooting + +### "Source authentication not provided" +```bash +E2E_LICENSE_TOKEN=your_token task test:e2e:mirror +``` + +### "Pull failed" +1. Check `d8` binary: `task build` +2. Check network access +3. For self-signed certs: `E2E_TLS_SKIP_VERIFY=true` + +### "Verification failed" +Check `comparison.txt`: +- **Missing in target**: Pull or push didn't transfer the image +- **Digest mismatch**: Data corruption or version skew + +### Running Against Local Registry +```bash +E2E_SOURCE_REGISTRY=localhost:5000/deckhouse \ +E2E_SOURCE_USER=admin \ +E2E_SOURCE_PASSWORD=admin \ +E2E_TLS_SKIP_VERIFY=true \ +task test:e2e:mirror:platform +``` + +### Keep Bundle for Debugging +```bash +E2E_KEEP_BUNDLE=true E2E_LICENSE_TOKEN=xxx task test:e2e:mirror:platform +# Bundle location shown in test output +``` + +### Use Existing Bundle (Skip Pull) +```bash +E2E_EXISTING_BUNDLE=/path/to/bundle E2E_LICENSE_TOKEN=xxx task test:e2e:mirror:platform +# Test will skip pull step and use existing bundle +``` diff --git a/testing/e2e/mirror/fullcycle_test.go b/testing/e2e/mirror/fullcycle_test.go new file mode 100644 index 00000000..c2661d5a --- /dev/null +++ b/testing/e2e/mirror/fullcycle_test.go @@ -0,0 +1,210 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func TestFullCycleE2E(t *testing.T) { + cfg := internal.GetConfig() + + if !cfg.HasSourceAuth() { + t.Skip("Skipping: no source authentication configured (set E2E_LICENSE_TOKEN)") + } + + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + runFullCycleTest(t, cfg, env) + printFinalReport(t, env) +} + +func runFullCycleTest(t *testing.T, cfg *internal.Config, env *testEnv) { + ctx, cancel := context.WithTimeout(context.Background(), internal.FullCycleTestTimeout) + defer cancel() + + internal.PrintHeader("FULL CYCLE E2E TEST") + + internal.PrintStep(1, "Reading expected images from source registry") + expected := readAllExpectedImages(t, ctx, cfg) + env.Report.ExpectedModules = getModuleNames(expected) + + if cfg.HasExistingBundle() { + t.Logf("Using existing bundle: %s (skipping pull)", env.BundleDir) + env.Report.AddStep("Pull (existing bundle)", "SKIP", 0, nil) + } else { + internal.PrintStep(2, "Pulling images to bundle") + runPullStep(t, cfg, env) + } + + internal.PrintStep(3, "Pushing bundle to target registry") + runPushStep(t, cfg, env) + + internal.PrintStep(4, "Verifying expected images in target") + runVerificationStep(t, ctx, cfg, env, expected) + + internal.PrintSuccessBox(env.Report.MatchedImages, env.Report.FoundAttTags) +} + +func readAllExpectedImages(t *testing.T, ctx context.Context, cfg *internal.Config) *internal.ExpectedImages { + t.Helper() + + reader := createSourceReader(t, cfg) + result := &internal.ExpectedImages{} + + if !cfg.NoPlatform { + t.Log("Reading platform images...") + channels, err := reader.ReadReleaseChannels(ctx) + if err != nil { + t.Logf("Warning: failed to read release channels: %v", err) + } else { + platform, err := reader.ReadPlatformDigests(ctx, channels) + if err != nil { + t.Logf("Warning: failed to read platform digests: %v", err) + } else { + result.Platform = platform + t.Logf("Platform: %d versions, %d digests", len(platform.Versions), len(platform.ImageDigests)) + } + } + } + + if !cfg.NoModules { + t.Log("Reading modules...") + modules, err := reader.ReadModulesList(ctx) + if err != nil { + t.Logf("Warning: failed to read modules: %v", err) + } else { + modules = filterModules(modules, cfg.IncludeModules) + + for _, moduleName := range modules { + info, err := reader.ReadModuleDigests(ctx, moduleName) + if err != nil { + t.Logf("Warning: failed to read module %s: %v", moduleName, err) + continue + } + result.Modules = append(result.Modules, info) + } + t.Logf("Modules: %d", len(result.Modules)) + } + } + + if !cfg.NoSecurity { + t.Log("Reading security databases...") + security, err := reader.ReadSecurityDigests(ctx) + if err != nil { + t.Logf("Warning: failed to read security: %v", err) + } else { + result.Security = security + t.Logf("Security: %d databases", len(security.Databases)) + } + } + + return result +} + +func getModuleNames(expected *internal.ExpectedImages) []string { + if expected == nil || len(expected.Modules) == 0 { + return nil + } + names := make([]string, len(expected.Modules)) + for i, m := range expected.Modules { + names[i] = m.Name + } + return names +} + +func runVerificationStep(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expected *internal.ExpectedImages) { + t.Helper() + stepStart := time.Now() + + verifier := createVerifier(t, cfg, env) + result, err := verifier.VerifyFull(ctx, cfg.DeckhouseTag, cfg.IncludeModules) + if err != nil { + env.Report.AddStep("Verification", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Verification failed") + } + + saveVerificationReport(t, env.ComparisonFile, result) + + env.Report.TotalImages = len(result.ExpectedDigests) + env.Report.MatchedImages = len(result.FoundDigests) + env.Report.MissingImages = len(result.MissingDigests) + env.Report.ExpectedAttTags = len(result.ExpectedAttTags) + env.Report.FoundAttTags = len(result.FoundAttTags) + env.Report.MissingAttTags = len(result.MissingAttTags) + env.Report.ModulesExpected = result.ModulesExpected + env.Report.ModulesFound = result.ModulesFound + env.Report.ModulesMissing = len(result.ModulesMissing) + env.Report.SecurityExpected = result.SecurityExpected + env.Report.SecurityFound = result.SecurityFound + env.Report.SecurityMissing = len(result.SecurityMissing) + env.Report.SourceImageCount = len(result.ExpectedDigests) + len(result.ExpectedAttTags) + env.Report.TargetImageCount = len(result.FoundDigests) + len(result.FoundAttTags) + + t.Log("") + t.Log(result.Summary()) + + var failures []string + if len(result.MissingDigests) > 0 { + failures = append(failures, fmt.Sprintf("missing %d digests in target", len(result.MissingDigests))) + } + if len(result.MissingAttTags) > 0 { + failures = append(failures, fmt.Sprintf("missing %d .att tags in target", len(result.MissingAttTags))) + } + + if len(failures) > 0 { + env.Report.AddStep( + fmt.Sprintf("Verification (%d/%d digests, %d/%d .att)", + len(result.FoundDigests), len(result.ExpectedDigests), + len(result.FoundAttTags), len(result.ExpectedAttTags)), + "FAIL", time.Since(stepStart), + fmt.Errorf("%v", failures), + ) + + require.Empty(t, failures, + "Mirror verification FAILED!\n\n%s\n\nSee %s for details", + result.Summary(), env.ComparisonFile) + } + + env.Report.AddStep( + fmt.Sprintf("Verification (%d digests, %d .att tags)", + len(result.FoundDigests), len(result.FoundAttTags)), + "PASS", time.Since(stepStart), nil, + ) +} + +func printFinalReport(t *testing.T, env *testEnv) { + env.Report.EndTime = time.Now() + report := env.Report.String() + + t.Log("") + t.Log(report) + + saveReport(t, env.ReportFile, env.Report) + t.Logf("Report written to: %s", env.ReportFile) +} + diff --git a/testing/e2e/mirror/helpers_test.go b/testing/e2e/mirror/helpers_test.go new file mode 100644 index 00000000..15a2afb1 --- /dev/null +++ b/testing/e2e/mirror/helpers_test.go @@ -0,0 +1,215 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func createVerifier(t *testing.T, cfg *internal.Config, env *testEnv) *internal.DigestVerifier { + t.Helper() + + sourceReader := internal.NewSourceReader(cfg.SourceRegistry, cfg.GetSourceAuth(), cfg.TLSSkipVerify) + verifier := internal.NewDigestVerifier( + sourceReader, + env.TargetRegistry, + cfg.GetTargetAuth(), + cfg.TLSSkipVerify, + ) + verifier.SetProgressCallback(func(msg string) { + t.Logf(" %s", msg) + }) + return verifier +} + +func createSourceReader(t *testing.T, cfg *internal.Config) *internal.SourceReader { + t.Helper() + + reader := internal.NewSourceReader(cfg.SourceRegistry, cfg.GetSourceAuth(), cfg.TLSSkipVerify) + reader.SetProgressCallback(func(msg string) { + t.Logf(" %s", msg) + }) + return reader +} + +func filterModules(modules []string, include []string) []string { + if len(include) == 0 { + return modules + } + includeSet := make(map[string]bool, len(include)) + for _, m := range include { + includeSet[m] = true + } + var filtered []string + for _, m := range modules { + if includeSet[m] { + filtered = append(filtered, m) + } + } + return filtered +} + +func verifyPlatformImages(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, deckhouseTag string) { + t.Helper() + stepStart := time.Now() + + verifier := createVerifier(t, cfg, env) + result, err := verifier.VerifyPlatform(ctx, deckhouseTag) + require.NoError(t, err, "Verification failed") + + saveVerificationReport(t, env.ComparisonFile, result) + + env.Report.TotalImages = len(result.ExpectedDigests) + env.Report.MatchedImages = len(result.FoundDigests) + env.Report.MissingImages = len(result.MissingDigests) + env.Report.ExpectedAttTags = len(result.ExpectedAttTags) + env.Report.FoundAttTags = len(result.FoundAttTags) + env.Report.MissingAttTags = len(result.MissingAttTags) + env.Report.SourceImageCount = len(result.ExpectedDigests) + len(result.ExpectedAttTags) + env.Report.TargetImageCount = len(result.FoundDigests) + len(result.FoundAttTags) + + t.Log("") + t.Log(result.Summary()) + + var failures []string + if len(result.MissingDigests) > 0 { + failures = append(failures, fmt.Sprintf("missing %d digests in target", len(result.MissingDigests))) + } + if len(result.MissingAttTags) > 0 { + failures = append(failures, fmt.Sprintf("missing %d .att tags in target", len(result.MissingAttTags))) + } + + if len(failures) > 0 { + env.Report.AddStep( + fmt.Sprintf("Verification (%d/%d digests, %d/%d .att)", + len(result.FoundDigests), len(result.ExpectedDigests), + len(result.FoundAttTags), len(result.ExpectedAttTags)), + "FAIL", time.Since(stepStart), + fmt.Errorf("%v", failures), + ) + + require.Empty(t, failures, + "Platform verification FAILED!\n\n%s\n\nSee %s for details", + result.Summary(), env.ComparisonFile) + return + } + + env.Report.AddStep( + fmt.Sprintf("Verification (%d digests, %d .att tags)", + len(result.FoundDigests), len(result.FoundAttTags)), + "PASS", time.Since(stepStart), nil, + ) +} + +func verifyModulesImages(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expectedModules []string) { + t.Helper() + stepStart := time.Now() + + verifier := createVerifier(t, cfg, env) + result, err := verifier.VerifyModules(ctx, expectedModules) + require.NoError(t, err, "Modules verification failed") + + t.Logf("Found %d/%d modules in target", result.ModulesFound, result.ModulesExpected) + + for _, missing := range result.ModulesMissing { + t.Logf(" ✗ %s", missing) + } + + if len(result.ModulesMissing) > 0 { + env.Report.AddStep( + fmt.Sprintf("Modules Verification (%d/%d found)", + result.ModulesFound, result.ModulesExpected), + "FAIL", time.Since(stepStart), + fmt.Errorf("missing %d modules: %v", len(result.ModulesMissing), result.ModulesMissing), + ) + require.Empty(t, result.ModulesMissing, "Some modules are missing in target") + return + } + + env.Report.ModulesExpected = result.ModulesExpected + env.Report.ModulesFound = result.ModulesFound + env.Report.ModulesMissing = len(result.ModulesMissing) + + env.Report.AddStep( + fmt.Sprintf("Modules Verification (%d modules)", result.ModulesFound), + "PASS", time.Since(stepStart), nil, + ) + t.Log("Modules verification passed") +} + +func verifySecurityImages(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv) { + t.Helper() + stepStart := time.Now() + + verifier := createVerifier(t, cfg, env) + result, err := verifier.VerifySecurity(ctx) + require.NoError(t, err, "Security verification failed") + + t.Logf("Found %d/%d security databases in target", result.SecurityFound, result.SecurityExpected) + + for _, missing := range result.SecurityMissing { + t.Logf(" ✗ %s", missing) + } + + if len(result.SecurityMissing) > 0 { + env.Report.AddStep( + fmt.Sprintf("Security Verification (%d/%d found)", + result.SecurityFound, result.SecurityExpected), + "FAIL", time.Since(stepStart), + fmt.Errorf("missing %d security databases: %v", len(result.SecurityMissing), result.SecurityMissing), + ) + require.Empty(t, result.SecurityMissing, "Some security databases are missing in target") + return + } + + env.Report.SecurityExpected = result.SecurityExpected + env.Report.SecurityFound = result.SecurityFound + env.Report.SecurityMissing = len(result.SecurityMissing) + + env.Report.AddStep( + fmt.Sprintf("Security Verification (%d databases)", result.SecurityFound), + "PASS", time.Since(stepStart), nil, + ) + t.Log("Security verification passed") +} + +func saveVerificationReport(t *testing.T, path string, result *internal.VerificationResult) { + t.Helper() + + report := result.DetailedReport() + err := internal.WriteFile(path, []byte(report)) + if err != nil { + t.Logf("Warning: failed to write verification report: %v", err) + } +} + +func saveReport(t *testing.T, path string, report *internal.TestReport) { + t.Helper() + if err := report.WriteToFile(path); err != nil { + t.Logf("Warning: failed to write report: %v", err) + } +} + diff --git a/testing/e2e/mirror/internal/commands.go b/testing/e2e/mirror/internal/commands.go new file mode 100644 index 00000000..f9c3f27e --- /dev/null +++ b/testing/e2e/mirror/internal/commands.go @@ -0,0 +1,135 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "fmt" + "io" + "os" + "os/exec" + "testing" + "time" +) + +func BuildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { + args := []string{ + "mirror", "pull", + "--source", cfg.SourceRegistry, + "--force", + } + + if cfg.SourceUser != "" { + args = append(args, "--source-login", cfg.SourceUser) + args = append(args, "--source-password", cfg.SourcePassword) + } else if cfg.LicenseToken != "" { + args = append(args, "--license", cfg.LicenseToken) + } + + if cfg.TLSSkipVerify { + args = append(args, "--tls-skip-verify") + } + + if cfg.DeckhouseTag != "" { + args = append(args, "--deckhouse-tag", cfg.DeckhouseTag) + } + if cfg.NoModules { + args = append(args, "--no-modules") + } + if cfg.NoPlatform { + args = append(args, "--no-platform") + } + if cfg.NoSecurity { + args = append(args, "--no-security-db") + } + for _, module := range cfg.IncludeModules { + args = append(args, "--include-module", module) + } + + args = append(args, bundleDir) + + cmd := exec.Command(cfg.D8Binary, args...) + cmd.Env = os.Environ() + + if cfg.NewPull { + cmd.Env = append(cmd.Env, "NEW_PULL=true") + } + + return cmd +} + +func BuildPushCommand(cfg *Config, bundleDir, targetRegistry string) *exec.Cmd { + args := []string{ + "mirror", "push", + bundleDir, + targetRegistry, + } + + if cfg.TLSSkipVerify { + args = append(args, "--tls-skip-verify") + } + + if cfg.TargetUser != "" { + args = append(args, "--registry-login", cfg.TargetUser) + args = append(args, "--registry-password", cfg.TargetPassword) + } + + cmd := exec.Command(cfg.D8Binary, args...) + cmd.Env = os.Environ() + + // Ensure HOME is set - some tools (like ssh) require it + if os.Getenv("HOME") == "" { + if homeDir, err := os.UserHomeDir(); err == nil { + cmd.Env = append(cmd.Env, "HOME="+homeDir) + } + } + + if cfg.NewPull { + cmd.Env = append(cmd.Env, "NEW_PULL=true") + } + + return cmd +} + +func RunCommandWithLog(t *testing.T, cmd *exec.Cmd, logFile string) error { + t.Helper() + + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + t.Logf("Warning: could not open log file %s: %v", logFile, err) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + defer f.Close() + + fmt.Fprintf(f, "\n\n========== COMMAND: %s ==========\n", cmd.String()) + fmt.Fprintf(f, "Started: %s\n\n", time.Now().Format(time.RFC3339)) + + cmd.Stdout = io.MultiWriter(os.Stdout, f) + cmd.Stderr = io.MultiWriter(os.Stderr, f) + + cmdErr := cmd.Run() + + if cmdErr != nil { + fmt.Fprintf(f, "\n\n========== COMMAND FAILED: %v ==========\n", cmdErr) + } else { + fmt.Fprintf(f, "\n\n========== COMMAND SUCCEEDED ==========\n") + } + + return cmdErr +} + diff --git a/testing/e2e/mirror/internal/config.go b/testing/e2e/mirror/internal/config.go new file mode 100644 index 00000000..9edd9e72 --- /dev/null +++ b/testing/e2e/mirror/internal/config.go @@ -0,0 +1,227 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "flag" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" +) + +const ( + SecurityTestTimeout = 30 * time.Minute + PlatformTestTimeout = 2 * time.Hour + ModulesTestTimeout = 2 * time.Hour + FullCycleTestTimeout = 3 * time.Hour +) + +var ( + sourceRegistry = flag.String("source-registry", + getEnvOrDefault("E2E_SOURCE_REGISTRY", "registry.deckhouse.ru/deckhouse/fe"), + "Reference registry to pull from") + sourceUser = flag.String("source-user", + getEnvOrDefault("E2E_SOURCE_USER", ""), + "Source registry username (alternative to license-token)") + sourcePassword = flag.String("source-password", + getEnvOrDefault("E2E_SOURCE_PASSWORD", ""), + "Source registry password (alternative to license-token)") + licenseToken = flag.String("license-token", + getEnvOrDefault("E2E_LICENSE_TOKEN", ""), + "License token for source registry authentication (shortcut for source-user=license-token)") + + targetRegistry = flag.String("target-registry", + getEnvOrDefault("E2E_TARGET_REGISTRY", ""), + "Target registry to push to (empty = use in-memory registry)") + targetUser = flag.String("target-user", + getEnvOrDefault("E2E_TARGET_USER", ""), + "Target registry username") + targetPassword = flag.String("target-password", + getEnvOrDefault("E2E_TARGET_PASSWORD", ""), + "Target registry password") + + tlsSkipVerify = flag.Bool("tls-skip-verify", + getEnvOrDefault("E2E_TLS_SKIP_VERIFY", "") == "true", + "Skip TLS certificate verification (for self-signed certs)") + keepBundle = flag.Bool("keep-bundle", + getEnvOrDefault("E2E_KEEP_BUNDLE", "") == "true", + "Keep bundle directory after test") + existingBundle = flag.String("existing-bundle", + getEnvOrDefault("E2E_EXISTING_BUNDLE", ""), + "Path to existing bundle directory (skip pull step)") + d8Binary = flag.String("d8-binary", + getEnvOrDefault("E2E_D8_BINARY", "bin/d8"), + "Path to d8 binary") + + deckhouseTag = flag.String("deckhouse-tag", + getEnvOrDefault("E2E_DECKHOUSE_TAG", ""), + "Specific Deckhouse tag or release channel (e.g., 'stable', 'v1.65.8')") + noModules = flag.Bool("no-modules", + getEnvOrDefault("E2E_NO_MODULES", "") == "true", + "Skip modules during pull") + noPlatform = flag.Bool("no-platform", + getEnvOrDefault("E2E_NO_PLATFORM", "") == "true", + "Skip platform during pull") + noSecurity = flag.Bool("no-security", + getEnvOrDefault("E2E_NO_SECURITY", "") == "true", + "Skip security databases during pull") + includeModules = flag.String("include-modules", + getEnvOrDefault("E2E_INCLUDE_MODULES", ""), + "Comma-separated list of modules to include (empty = all)") + + newPull = flag.Bool("new-pull", + getEnvOrDefault("E2E_NEW_PULL", "") == "true", + "Use new pull implementation") +) + +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +type Config struct { + SourceRegistry string + SourceUser string + SourcePassword string + LicenseToken string + + TargetRegistry string + TargetUser string + TargetPassword string + + TLSSkipVerify bool + KeepBundle bool + ExistingBundle string + D8Binary string + + DeckhouseTag string + NoModules bool + NoPlatform bool + NoSecurity bool + IncludeModules []string + + NewPull bool +} + +func GetConfig() *Config { + cfg := &Config{ + SourceRegistry: *sourceRegistry, + SourceUser: *sourceUser, + SourcePassword: *sourcePassword, + LicenseToken: *licenseToken, + TargetRegistry: *targetRegistry, + TargetUser: *targetUser, + TargetPassword: *targetPassword, + TLSSkipVerify: *tlsSkipVerify, + KeepBundle: *keepBundle, + ExistingBundle: *existingBundle, + D8Binary: resolveD8Binary(*d8Binary), + DeckhouseTag: *deckhouseTag, + NoModules: *noModules, + NoPlatform: *noPlatform, + NoSecurity: *noSecurity, + NewPull: *newPull, + } + + if *includeModules != "" { + cfg.IncludeModules = parseCommaSeparated(*includeModules) + } + + return cfg +} + +func resolveD8Binary(path string) string { + if filepath.IsAbs(path) { + return path + } + projectRoot := FindProjectRoot() + return filepath.Join(projectRoot, path) +} + +func FindProjectRoot() string { + dir, _ := os.Getwd() + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + dir, _ = os.Getwd() + return dir + } + dir = parent + } +} + + + +func parseCommaSeparated(s string) []string { + if s == "" { + return nil + } + var parts []string + for _, part := range strings.Split(s, ",") { + if trimmed := strings.TrimSpace(part); trimmed != "" { + parts = append(parts, trimmed) + } + } + return parts +} + +func (c *Config) GetSourceAuth() authn.Authenticator { + if c.SourceUser != "" { + return authn.FromConfig(authn.AuthConfig{ + Username: c.SourceUser, + Password: c.SourcePassword, + }) + } + if c.LicenseToken != "" { + return authn.FromConfig(authn.AuthConfig{ + Username: "license-token", + Password: c.LicenseToken, + }) + } + return authn.Anonymous +} + +func (c *Config) HasSourceAuth() bool { + return c.SourceUser != "" || c.LicenseToken != "" +} + +func (c *Config) GetTargetAuth() authn.Authenticator { + if c.TargetUser != "" { + return authn.FromConfig(authn.AuthConfig{ + Username: c.TargetUser, + Password: c.TargetPassword, + }) + } + return authn.Anonymous +} + +func (c *Config) UseInMemoryRegistry() bool { + return c.TargetRegistry == "" +} + +func (c *Config) HasExistingBundle() bool { + return c.ExistingBundle != "" +} + diff --git a/testing/e2e/mirror/internal/output.go b/testing/e2e/mirror/internal/output.go new file mode 100644 index 00000000..6549bcab --- /dev/null +++ b/testing/e2e/mirror/internal/output.go @@ -0,0 +1,144 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "fmt" + "os" + "strings" + "sync" + + "github.com/charmbracelet/lipgloss" + "github.com/muesli/termenv" + "golang.org/x/term" +) + +var ( + ColorCyan = lipgloss.Color("6") + ColorGreen = lipgloss.Color("2") + ColorRed = lipgloss.Color("1") + ColorYellow = lipgloss.Color("3") + ColorBlue = lipgloss.Color("4") + ColorWhite = lipgloss.Color("15") + ColorGray = lipgloss.Color("8") +) + +type styles struct { + Title lipgloss.Style + Header lipgloss.Style + Label lipgloss.Style + Value lipgloss.Style + Dim lipgloss.Style + Success lipgloss.Style + Error lipgloss.Style + StepNum lipgloss.Style + StepText lipgloss.Style + Separator lipgloss.Style +} + +func initStyles() styles { + return styles{ + Title: lipgloss.NewStyle().Bold(true).Foreground(ColorWhite), + Header: lipgloss.NewStyle().Bold(true).Foreground(ColorCyan), + Label: lipgloss.NewStyle().Foreground(ColorGray), + Value: lipgloss.NewStyle().Foreground(ColorWhite), + Dim: lipgloss.NewStyle().Foreground(ColorGray), + Success: lipgloss.NewStyle().Foreground(ColorGreen), + Error: lipgloss.NewStyle().Foreground(ColorRed), + StepNum: lipgloss.NewStyle().Bold(true).Foreground(ColorBlue), + StepText: lipgloss.NewStyle().Bold(true).Foreground(ColorWhite), + Separator: lipgloss.NewStyle().Foreground(ColorCyan), + } +} + +var defaultStyles = initStyles() + +var ( + StyleTitle = defaultStyles.Title + StyleHeader = defaultStyles.Header + StyleLabel = defaultStyles.Label + StyleValue = defaultStyles.Value + StyleDim = defaultStyles.Dim + StyleSuccess = defaultStyles.Success + StyleError = defaultStyles.Error +) + +var ( + BadgeOK = defaultStyles.Success.Copy().Bold(true).Render("[OK]") + BadgeFail = defaultStyles.Error.Copy().Bold(true).Render("[FAIL]") + BadgeSkip = lipgloss.NewStyle().Foreground(ColorYellow).Render("[SKIP]") +) + +var output = os.Stderr +var colorInitOnce sync.Once + +func EnsureColorInit() { + colorInitOnce.Do(func() { + if term.IsTerminal(int(os.Stderr.Fd())) || os.Getenv("FORCE_COLOR") != "" || os.Getenv("TERM") != "" { + lipgloss.DefaultRenderer().SetColorProfile(termenv.TrueColor) + } + }) +} + +func WriteLinef(format string, args ...interface{}) { + EnsureColorInit() + fmt.Fprintf(output, format+"\n", args...) +} + +func WriteRawf(format string, args ...interface{}) { + EnsureColorInit() + fmt.Fprintf(output, format, args...) +} + +func PrintStep(num int, description string) { + badge := defaultStyles.StepNum.Render(fmt.Sprintf("[STEP %d]", num)) + text := defaultStyles.StepText.Render(description) + WriteLinef("\n%s %s", badge, text) +} + +const separatorWidth = 80 + +func Separator(char string) string { + return defaultStyles.Separator.Render(strings.Repeat(char, separatorWidth)) +} + +func PrintHeader(title string) { + WriteLinef("") + WriteLinef(Separator("═")) + WriteLinef(" %s", StyleTitle.Render(title)) + WriteLinef(Separator("═")) +} + +func PrintSuccessBox(matchedDigests, matchedAttTags int) { + box := lipgloss.NewStyle(). + Border(lipgloss.DoubleBorder()). + BorderForeground(ColorGreen). + Padding(0, 2). + Foreground(ColorGreen) + + WriteLinef("") + WriteLinef(box.Render(fmt.Sprintf( + "SUCCESS: ALL EXPECTED IMAGES VERIFIED\n\nVerified: %d digests, %d .att tags\nAll expected images are present in target registry!", + matchedDigests, + matchedAttTags, + ))) +} + +func WriteFile(path string, data []byte) error { + return os.WriteFile(path, data, 0644) +} + diff --git a/testing/e2e/mirror/internal/report.go b/testing/e2e/mirror/internal/report.go new file mode 100644 index 00000000..f69ffda5 --- /dev/null +++ b/testing/e2e/mirror/internal/report.go @@ -0,0 +1,277 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/charmbracelet/lipgloss" +) + +type TestReport struct { + TestName string + StartTime time.Time + EndTime time.Time + SourceRegistry string + TargetRegistry string + LogDir string + + SourceImageCount int + TargetImageCount int + + TotalImages int + MatchedImages int + MissingImages int + + ExpectedAttTags int + FoundAttTags int + MissingAttTags int + + ModulesExpected int + ModulesFound int + ModulesMissing int + + SecurityExpected int + SecurityFound int + SecurityMissing int + + BundleSize int64 + ExpectedModules []string + + Steps []StepResult +} + +type StepResult struct { + Name string + Status string + Duration time.Duration + Error string +} + +func (r *TestReport) AddStep(name, status string, duration time.Duration, err error) { + errStr := "" + if err != nil { + errStr = err.Error() + } + r.Steps = append(r.Steps, StepResult{ + Name: name, + Status: status, + Duration: duration, + Error: errStr, + }) +} + +func (r *TestReport) WriteToFile(path string) error { + content := r.String() + return os.WriteFile(path, []byte(content), 0644) +} + +func (r *TestReport) formatSection(useColors bool, label string, found, expected, missing int) string { + if expected == 0 && missing == 0 { + return "" + } + + var badge, value string + if useColors { + if missing > 0 { + badge = BadgeFail + value = StyleError.Render(fmt.Sprintf("%d / %d", found, expected)) + } else { + badge = BadgeOK + value = StyleSuccess.Render(fmt.Sprintf("%d / %d", found, expected)) + } + label = StyleLabel.Render(label) + return fmt.Sprintf(" %s %s %s\n", badge, label, value) + } + + status := "[OK]" + if missing > 0 { + status = "[FAIL]" + } + labelText := strings.TrimSuffix(label, ":") + result := fmt.Sprintf(" %s %s: %d / %d\n", status, labelText, found, expected) + if missing > 0 { + result += fmt.Sprintf(" Missing %s: %d\n", strings.ToLower(labelText), missing) + } + return result +} + +func (r *TestReport) formatStep(useColors bool, step StepResult) string { + dur := step.Duration.Round(time.Millisecond).String() + if useColors { + dur = StyleDim.Render(fmt.Sprintf("(%s)", dur)) + switch step.Status { + case "PASS": + return fmt.Sprintf(" %s %s %s\n", BadgeOK, step.Name, dur) + case "FAIL": + result := fmt.Sprintf(" %s %s %s\n", BadgeFail, step.Name, dur) + if step.Error != "" { + result += " " + StyleError.Render("ERROR: "+step.Error) + "\n" + } + return result + default: + return fmt.Sprintf(" %s %s\n", BadgeSkip, step.Name) + } + } + + switch step.Status { + case "PASS": + return fmt.Sprintf(" [PASS] %s (%s)\n", step.Name, dur) + case "FAIL": + result := fmt.Sprintf(" [FAIL] %s (%s)\n", step.Name, dur) + if step.Error != "" { + result += fmt.Sprintf(" ERROR: %s\n", step.Error) + } + return result + default: + return fmt.Sprintf(" [SKIP] %s\n", step.Name) + } +} + +func (r *TestReport) formatSummary(useColors bool, passCount, failCount int) string { + var parts []string + if r.MatchedImages > 0 { + parts = append(parts, fmt.Sprintf("%d platform digests", r.MatchedImages)) + } + if r.FoundAttTags > 0 { + parts = append(parts, fmt.Sprintf("%d attestations", r.FoundAttTags)) + } + if r.ModulesFound > 0 { + parts = append(parts, fmt.Sprintf("%d modules", r.ModulesFound)) + } + if r.SecurityFound > 0 { + parts = append(parts, fmt.Sprintf("%d security databases", r.SecurityFound)) + } + + if useColors { + if failCount > 0 { + resultStyle := lipgloss.NewStyle().Bold(true).Foreground(ColorRed) + return " " + resultStyle.Render("RESULT: FAILED") + fmt.Sprintf(" (%d passed, %d failed)\n", passCount, failCount) + } + resultStyle := lipgloss.NewStyle().Bold(true).Foreground(ColorGreen) + result := " " + resultStyle.Render("RESULT: PASSED") + "\n" + if len(parts) > 0 { + result += " " + StyleSuccess.Render(strings.Join(parts, ", ")+" verified") + "\n" + } + return result + } + + if failCount > 0 { + return fmt.Sprintf("RESULT: FAILED (%d passed, %d failed)\n", passCount, failCount) + } + result := "RESULT: PASSED\n" + if len(parts) > 0 { + result += fmt.Sprintf(" %s verified\n", strings.Join(parts, ", ")) + } + return result +} + +func (r *TestReport) format(useColors bool) string { + duration := r.EndTime.Sub(r.StartTime) + if r.EndTime.IsZero() { + duration = time.Since(r.StartTime) + } + + var b strings.Builder + + if useColors { + b.WriteString("\n") + b.WriteString(Separator("═") + "\n") + b.WriteString(" " + StyleTitle.Render("E2E TEST REPORT") + "\n") + b.WriteString(Separator("═") + "\n\n") + b.WriteString(" " + StyleLabel.Render("Duration: ") + StyleDim.Render(duration.Round(time.Second).String()) + "\n\n") + b.WriteString(" " + StyleHeader.Render("REGISTRIES") + "\n") + b.WriteString(" " + StyleLabel.Render("Source: ") + StyleValue.Render(r.SourceRegistry) + "\n") + b.WriteString(" " + StyleLabel.Render("Target: ") + StyleValue.Render(r.TargetRegistry) + "\n\n") + b.WriteString(" " + StyleHeader.Render("VERIFICATION") + "\n") + } else { + b.WriteString("================================================================================\n") + b.WriteString(fmt.Sprintf("E2E TEST REPORT: %s\n", r.TestName)) + b.WriteString("================================================================================\n\n") + b.WriteString("EXECUTION:\n") + b.WriteString(fmt.Sprintf(" Started: %s\n", r.StartTime.Format(time.RFC3339))) + b.WriteString(fmt.Sprintf(" Finished: %s\n", r.EndTime.Format(time.RFC3339))) + b.WriteString(fmt.Sprintf(" Duration: %s\n", duration.Round(time.Second))) + b.WriteString(fmt.Sprintf(" Log dir: %s\n\n", r.LogDir)) + b.WriteString("REGISTRIES:\n") + b.WriteString(fmt.Sprintf(" Source: %s\n", r.SourceRegistry)) + b.WriteString(fmt.Sprintf(" Target: %s\n\n", r.TargetRegistry)) + b.WriteString("VERIFICATION:\n") + } + + if section := r.formatSection(useColors, "Platform digests: ", r.MatchedImages, r.TotalImages, r.MissingImages); section != "" { + b.WriteString(section) + } + if section := r.formatSection(useColors, "Attestation tags: ", r.FoundAttTags, r.ExpectedAttTags, r.MissingAttTags); section != "" { + b.WriteString(section) + } + if r.ModulesExpected > 0 { + b.WriteString(r.formatSection(useColors, "Modules verified: ", r.ModulesFound, r.ModulesExpected, r.ModulesMissing)) + } + if r.SecurityExpected > 0 { + b.WriteString(r.formatSection(useColors, "Security verified:", r.SecurityFound, r.SecurityExpected, r.SecurityMissing)) + } + b.WriteString("\n") + + if useColors { + b.WriteString(" " + StyleHeader.Render("STEPS") + "\n") + } else { + b.WriteString("STEPS:\n") + } + passCount, failCount := r.countSteps() + for _, step := range r.Steps { + b.WriteString(r.formatStep(useColors, step)) + } + b.WriteString("\n") + + if useColors { + b.WriteString(Separator("─") + "\n") + b.WriteString(r.formatSummary(useColors, passCount, failCount)) + b.WriteString(Separator("═") + "\n") + } else { + b.WriteString("================================================================================\n") + b.WriteString(r.formatSummary(useColors, passCount, failCount)) + b.WriteString("================================================================================\n") + } + + return b.String() +} + +func (r *TestReport) Print() { + WriteRawf("%s", r.format(true)) +} + +func (r *TestReport) countSteps() (int, int) { + var passed, failed int + for _, step := range r.Steps { + switch step.Status { + case "PASS": + passed++ + case "FAIL": + failed++ + } + } + return passed, failed +} + +func (r *TestReport) String() string { + return r.format(false) +} + diff --git a/testing/e2e/mirror/internal/source.go b/testing/e2e/mirror/internal/source.go new file mode 100644 index 00000000..0b9b3a20 --- /dev/null +++ b/testing/e2e/mirror/internal/source.go @@ -0,0 +1,444 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "archive/tar" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "path" + "sort" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + + d8internal "github.com/deckhouse/deckhouse-cli/internal" +) + +func InsecureTransport() http.RoundTripper { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + return transport +} + +type SourceReader struct { + registry string + auth authn.Authenticator + opts []remote.Option + progressFn func(string) +} + +func NewSourceReader(registry string, auth authn.Authenticator, tlsSkipVerify bool) *SourceReader { + opts := []remote.Option{remote.WithAuth(auth)} + if tlsSkipVerify { + opts = append(opts, remote.WithTransport(InsecureTransport())) + } + + return &SourceReader{ + registry: registry, + auth: auth, + opts: opts, + } +} + +func (r *SourceReader) SetProgressCallback(fn func(string)) { + r.progressFn = fn +} + +func (r *SourceReader) Registry() string { + return r.registry +} + +func (r *SourceReader) RemoteOpts() []remote.Option { + return r.opts +} + +func (r *SourceReader) progress(format string, args ...interface{}) { + if r.progressFn != nil { + r.progressFn(fmt.Sprintf(format, args...)) + } +} + +type ReleaseChannelInfo struct { + Channel string + Version string +} + +func (r *SourceReader) ReadReleaseChannels(ctx context.Context) ([]ReleaseChannelInfo, error) { + channels := d8internal.GetAllDefaultReleaseChannels() + result := make([]ReleaseChannelInfo, 0, len(channels)) + + for _, channel := range channels { + r.progress("Reading release channel: %s", channel) + + ref := path.Join(r.registry, d8internal.ReleaseChannelSegment) + ":" + channel + version, err := r.readReleaseChannelVersion(ctx, ref) + if err != nil { + r.progress(" Warning: failed to read %s: %v", channel, err) + continue + } + + result = append(result, ReleaseChannelInfo{ + Channel: channel, + Version: version, + }) + r.progress(" %s -> %s", channel, version) + } + + return result, nil +} + +func (r *SourceReader) readReleaseChannelVersion(ctx context.Context, ref string) (string, error) { + imgRef, err := name.ParseReference(ref) + if err != nil { + return "", fmt.Errorf("parse reference: %w", err) + } + + img, err := remote.Image(imgRef, r.opts...) + if err != nil { + return "", fmt.Errorf("get image: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return "", fmt.Errorf("get layers: %w", err) + } + + for _, layer := range layers { + rc, err := layer.Uncompressed() + if err != nil { + continue + } + + version, err := extractVersionFromTar(rc) + rc.Close() + if err == nil && version != "" { + return version, nil + } + } + + return "", fmt.Errorf("version.json not found in image") +} + +func extractVersionFromTar(rc io.Reader) (string, error) { + tr := tar.NewReader(rc) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + + if strings.HasSuffix(hdr.Name, "version.json") { + var meta struct { + Version string `json:"version"` + } + if err := json.NewDecoder(tr).Decode(&meta); err != nil { + return "", err + } + return meta.Version, nil + } + } + return "", nil +} + +type PlatformDigests struct { + Versions []string + InstallDigests map[string]string + ImageDigests []string +} + +func (r *SourceReader) ReadPlatformDigests(ctx context.Context, channels []ReleaseChannelInfo) (*PlatformDigests, error) { + result := &PlatformDigests{ + InstallDigests: make(map[string]string), + ImageDigests: make([]string, 0), + } + + versionSet := make(map[string]bool) + for _, ch := range channels { + versionSet[ch.Version] = true + } + + for version := range versionSet { + result.Versions = append(result.Versions, version) + } + sort.Strings(result.Versions) + + digestSet := make(map[string]bool) + + for _, version := range result.Versions { + r.progress("Reading install:%s digests...", version) + + tag := version + if !strings.HasPrefix(tag, "v") { + if _, err := semver.NewVersion(version); err == nil { + tag = "v" + tag + } + } + + installRef := path.Join(r.registry, d8internal.InstallSegment) + ":" + tag + digests, err := r.readInstallDigests(ctx, installRef) + if err != nil { + r.progress(" Warning: failed to read install:%s: %v", tag, err) + continue + } + + r.progress(" Found %d digests", len(digests)) + for _, d := range digests { + digestSet[d] = true + } + } + + for d := range digestSet { + result.ImageDigests = append(result.ImageDigests, d) + } + sort.Strings(result.ImageDigests) + + return result, nil +} + +func (r *SourceReader) readInstallDigests(ctx context.Context, ref string) ([]string, error) { + imgRef, err := name.ParseReference(ref) + if err != nil { + return nil, fmt.Errorf("parse reference: %w", err) + } + + desc, err := remote.Get(imgRef, r.opts...) + if err != nil { + return nil, fmt.Errorf("get descriptor: %w", err) + } + + var img v1.Image + + if desc.MediaType.IsIndex() { + idx, err := desc.ImageIndex() + if err != nil { + return nil, fmt.Errorf("get index: %w", err) + } + + manifest, err := idx.IndexManifest() + if err != nil { + return nil, fmt.Errorf("get index manifest: %w", err) + } + + if len(manifest.Manifests) == 0 { + return nil, fmt.Errorf("index has no manifests") + } + + img, err = idx.Image(manifest.Manifests[0].Digest) + if err != nil { + return nil, fmt.Errorf("get image from index: %w", err) + } + } else { + img, err = desc.Image() + if err != nil { + return nil, fmt.Errorf("get image: %w", err) + } + } + + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("get layers: %w", err) + } + + for _, layer := range layers { + rc, err := layer.Uncompressed() + if err != nil { + continue + } + + digests, err := extractDigestsFromTar(rc) + rc.Close() + if err == nil && len(digests) > 0 { + return digests, nil + } + } + + return nil, fmt.Errorf("images_digests.json not found") +} + +func extractDigestsFromTar(rc io.Reader) ([]string, error) { + tr := tar.NewReader(rc) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if strings.HasSuffix(hdr.Name, "deckhouse/candi/images_digests.json") || + hdr.Name == "deckhouse/candi/images_digests.json" { + var digestsByModule map[string]map[string]string + if err := json.NewDecoder(tr).Decode(&digestsByModule); err != nil { + return nil, err + } + + var result []string + for _, images := range digestsByModule { + for _, digest := range images { + result = append(result, digest) + } + } + return result, nil + } + } + return nil, nil +} + +type ModuleInfo struct { + Name string + Versions []string + Digests []string +} + +func (r *SourceReader) listTags(ctx context.Context, repoPath string) ([]string, error) { + ref, err := name.ParseReference(repoPath + ":latest") + if err != nil { + return nil, fmt.Errorf("parse reference: %w", err) + } + + repo := ref.Context() + opts := append(r.opts, remote.WithContext(ctx)) + tags, err := remote.List(repo, opts...) + if err != nil { + return nil, fmt.Errorf("list tags: %w", err) + } + + return tags, nil +} + +func (r *SourceReader) ReadModulesList(ctx context.Context) ([]string, error) { + r.progress("Discovering modules...") + + modulesRef := path.Join(r.registry, d8internal.ModulesSegment) + tags, err := r.listTags(ctx, modulesRef) + if err != nil { + return nil, fmt.Errorf("list modules: %w", err) + } + + r.progress("Found %d modules", len(tags)) + return tags, nil +} + +func (r *SourceReader) ReadModuleDigests(ctx context.Context, moduleName string) (*ModuleInfo, error) { + r.progress("Reading module %s...", moduleName) + + info := &ModuleInfo{ + Name: moduleName, + Versions: make([]string, 0), + Digests: make([]string, 0), + } + + moduleReleaseRef := path.Join(r.registry, d8internal.ModulesSegment, moduleName, "release") + tags, err := r.listTags(ctx, moduleReleaseRef) + if err != nil { + r.progress(" No release tags found") + return info, nil + } + + info.Versions = tags + r.progress(" Found %d release tags", len(tags)) + + return info, nil +} + +type SecurityDigests struct { + Databases map[string][]string +} + +func (r *SourceReader) ReadSecurityDigests(ctx context.Context) (*SecurityDigests, error) { + r.progress("Reading security databases...") + + result := &SecurityDigests{ + Databases: make(map[string][]string), + } + + databases := []string{ + d8internal.SecurityTrivyDBSegment, + d8internal.SecurityTrivyBDUSegment, + d8internal.SecurityTrivyJavaDBSegment, + d8internal.SecurityTrivyChecksSegment, + } + + for _, db := range databases { + dbRef := path.Join(r.registry, d8internal.SecuritySegment, db) + tags, err := r.listTags(ctx, dbRef) + if err != nil { + r.progress(" %s: not found", db) + continue + } + + result.Databases[db] = tags + r.progress(" %s: %d tags", db, len(tags)) + } + + return result, nil +} + +type ExpectedImages struct { + Platform *PlatformDigests + Modules []*ModuleInfo + Security *SecurityDigests +} + +func (r *SourceReader) ReadAllExpected(ctx context.Context) (*ExpectedImages, error) { + result := &ExpectedImages{} + + channels, err := r.ReadReleaseChannels(ctx) + if err != nil { + return nil, fmt.Errorf("read release channels: %w", err) + } + + result.Platform, err = r.ReadPlatformDigests(ctx, channels) + if err != nil { + return nil, fmt.Errorf("read platform digests: %w", err) + } + + modules, err := r.ReadModulesList(ctx) + if err != nil { + r.progress("Warning: failed to read modules: %v", err) + } else { + for _, moduleName := range modules { + info, err := r.ReadModuleDigests(ctx, moduleName) + if err != nil { + r.progress("Warning: failed to read module %s: %v", moduleName, err) + continue + } + result.Modules = append(result.Modules, info) + } + } + + result.Security, err = r.ReadSecurityDigests(ctx) + if err != nil { + r.progress("Warning: failed to read security: %v", err) + } + + return result, nil +} + diff --git a/testing/e2e/mirror/internal/verify.go b/testing/e2e/mirror/internal/verify.go new file mode 100644 index 00000000..5378c73a --- /dev/null +++ b/testing/e2e/mirror/internal/verify.go @@ -0,0 +1,677 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "fmt" + "path" + "strings" + "sync" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + + d8internal "github.com/deckhouse/deckhouse-cli/internal" +) + +type DigestVerifier struct { + sourceReader *SourceReader + targetReg string + targetAuth authn.Authenticator + targetOpts []remote.Option + progressFn func(string) +} + +func NewDigestVerifier( + sourceReader *SourceReader, + targetReg string, + targetAuth authn.Authenticator, + tlsSkipVerify bool, +) *DigestVerifier { + opts := []remote.Option{remote.WithAuth(targetAuth)} + if tlsSkipVerify { + opts = append(opts, remote.WithTransport(InsecureTransport())) + } + + return &DigestVerifier{ + sourceReader: sourceReader, + targetReg: targetReg, + targetAuth: targetAuth, + targetOpts: opts, + } +} + +func (v *DigestVerifier) SetProgressCallback(fn func(string)) { + v.progressFn = fn + v.sourceReader.SetProgressCallback(fn) +} + +func (v *DigestVerifier) logProgressf(format string, args ...interface{}) { + if v.progressFn != nil { + v.progressFn(fmt.Sprintf(format, args...)) + } +} + +type VerificationResult struct { + StartTime time.Time + EndTime time.Time + + ExpectedDigests []string + ExpectedAttTags []string + ReleaseChannels []ReleaseChannelInfo + Versions []string + + FoundDigests []string + MissingDigests []string + FoundAttTags []string + MissingAttTags []string + + ModulesExpected int + ModulesFound int + ModulesMissing []string + + SecurityExpected int + SecurityFound int + SecurityMissing []string + + Errors []string + + TotalExpected int + TotalFound int + TotalMissing int +} + +func (r *VerificationResult) IsSuccess() bool { + return len(r.MissingDigests) == 0 && + len(r.MissingAttTags) == 0 && + len(r.ModulesMissing) == 0 && + len(r.SecurityMissing) == 0 +} + +func (r *VerificationResult) Summary() string { + var sb strings.Builder + + sb.WriteString("VERIFICATION SUMMARY\n") + sb.WriteString("====================\n\n") + + sb.WriteString(fmt.Sprintf("Duration: %v\n", r.EndTime.Sub(r.StartTime).Round(time.Second))) + sb.WriteString(fmt.Sprintf("Release channels: %d\n", len(r.ReleaseChannels))) + sb.WriteString(fmt.Sprintf("Versions: %d\n", len(r.Versions))) + sb.WriteString("\n") + + sb.WriteString("EXPECTED FROM SOURCE:\n") + sb.WriteString(fmt.Sprintf(" Platform digests: %d\n", len(r.ExpectedDigests))) + sb.WriteString(fmt.Sprintf(" Attestation tags (.att): %d\n", len(r.ExpectedAttTags))) + if r.ModulesExpected > 0 { + sb.WriteString(fmt.Sprintf(" Modules: %d\n", r.ModulesExpected)) + } + if r.SecurityExpected > 0 { + sb.WriteString(fmt.Sprintf(" Security databases: %d\n", r.SecurityExpected)) + } + sb.WriteString("\n") + + sb.WriteString("VERIFICATION RESULTS:\n") + sb.WriteString(fmt.Sprintf(" ✓ Platform digests: %d / %d\n", len(r.FoundDigests), len(r.ExpectedDigests))) + sb.WriteString(fmt.Sprintf(" ✗ Missing digests: %d\n", len(r.MissingDigests))) + sb.WriteString(fmt.Sprintf(" ✓ Attestation tags: %d / %d\n", len(r.FoundAttTags), len(r.ExpectedAttTags))) + sb.WriteString(fmt.Sprintf(" ✗ Missing .att tags: %d\n", len(r.MissingAttTags))) + if r.ModulesExpected > 0 { + sb.WriteString(fmt.Sprintf(" ✓ Modules: %d / %d\n", r.ModulesFound, r.ModulesExpected)) + if len(r.ModulesMissing) > 0 { + sb.WriteString(fmt.Sprintf(" ✗ Missing modules: %d\n", len(r.ModulesMissing))) + } + } + if r.SecurityExpected > 0 { + sb.WriteString(fmt.Sprintf(" ✓ Security databases: %d / %d\n", r.SecurityFound, r.SecurityExpected)) + if len(r.SecurityMissing) > 0 { + sb.WriteString(fmt.Sprintf(" ✗ Missing security: %d\n", len(r.SecurityMissing))) + } + } + sb.WriteString("\n") + + if r.IsSuccess() { + sb.WriteString("STATUS: ✓ PASSED - All expected images found in target\n") + } else { + sb.WriteString("STATUS: ✗ FAILED - Some images missing in target\n") + } + + return sb.String() +} + +func (r *VerificationResult) DetailedReport() string { + var sb strings.Builder + + sb.WriteString(r.Summary()) + sb.WriteString("\n") + + sb.WriteString("RELEASE CHANNELS:\n") + for _, ch := range r.ReleaseChannels { + sb.WriteString(fmt.Sprintf(" %s -> %s\n", ch.Channel, ch.Version)) + } + sb.WriteString("\n") + + if len(r.MissingDigests) > 0 { + sb.WriteString("MISSING DIGESTS:\n") + for _, d := range r.MissingDigests { + sb.WriteString(fmt.Sprintf(" - %s\n", d)) + } + sb.WriteString("\n") + } + + if len(r.MissingAttTags) > 0 { + sb.WriteString("MISSING ATTESTATION TAGS:\n") + for _, t := range r.MissingAttTags { + sb.WriteString(fmt.Sprintf(" - %s\n", t)) + } + sb.WriteString("\n") + } + + if len(r.ModulesMissing) > 0 { + sb.WriteString("MISSING MODULES:\n") + for _, m := range r.ModulesMissing { + sb.WriteString(fmt.Sprintf(" - %s\n", m)) + } + sb.WriteString("\n") + } + + if len(r.SecurityMissing) > 0 { + sb.WriteString("MISSING SECURITY DATABASES:\n") + for _, s := range r.SecurityMissing { + sb.WriteString(fmt.Sprintf(" - %s\n", s)) + } + sb.WriteString("\n") + } + + if len(r.Errors) > 0 { + sb.WriteString("ERRORS:\n") + for _, e := range r.Errors { + sb.WriteString(fmt.Sprintf(" - %s\n", e)) + } + sb.WriteString("\n") + } + + return sb.String() +} + +func (v *DigestVerifier) VerifyPlatform(ctx context.Context, deckhouseTag string) (*VerificationResult, error) { + result := &VerificationResult{ + StartTime: time.Now(), + } + + v.logProgressf("Reading release channels from source...") + var channels []ReleaseChannelInfo + var err error + + if deckhouseTag != "" { + v.logProgressf(" Using specified tag: %s", deckhouseTag) + channels = []ReleaseChannelInfo{{Channel: deckhouseTag, Version: deckhouseTag}} + } else { + channels, err = v.sourceReader.ReadReleaseChannels(ctx) + if err != nil { + return nil, fmt.Errorf("read release channels: %w", err) + } + } + result.ReleaseChannels = channels + v.logProgressf(" Found %d release channels", len(channels)) + + v.logProgressf("Reading platform digests from install images...") + platformDigests, err := v.sourceReader.ReadPlatformDigests(ctx, channels) + if err != nil { + return nil, fmt.Errorf("read platform digests: %w", err) + } + + result.Versions = platformDigests.Versions + result.ExpectedDigests = platformDigests.ImageDigests + v.logProgressf(" Found %d unique digests across %d versions", len(result.ExpectedDigests), len(result.Versions)) + + if len(result.ExpectedDigests) == 0 { + tag := "unknown" + if len(channels) > 0 { + tag = channels[0].Version + } + return nil, fmt.Errorf("found 0 platform digests - verification cannot proceed (check install image for tag %s)", tag) + } + + v.logProgressf("Finding .att tags for expected digests...") + allAttTags := v.getAttTagsFromSource(ctx) + result.ExpectedAttTags = v.filterAttTagsForDigests(allAttTags, result.ExpectedDigests) + v.logProgressf(" Found %d .att tags in source (%d total, %d match our digests)", + len(result.ExpectedAttTags), len(allAttTags), len(result.ExpectedAttTags)) + + v.logProgressf("Verifying digests in target registry...") + v.verifyDigests(ctx, result) + v.logProgressf(" Found: %d, Missing: %d", len(result.FoundDigests), len(result.MissingDigests)) + + v.logProgressf("Verifying .att tags in target registry...") + v.verifyAttTags(ctx, result) + v.logProgressf(" Found: %d, Missing: %d", len(result.FoundAttTags), len(result.MissingAttTags)) + + result.EndTime = time.Now() + result.TotalExpected = len(result.ExpectedDigests) + len(result.ExpectedAttTags) + result.TotalFound = len(result.FoundDigests) + len(result.FoundAttTags) + result.TotalMissing = len(result.MissingDigests) + len(result.MissingAttTags) + + return result, nil +} + + +func (v *DigestVerifier) VerifyModules(ctx context.Context, moduleNames []string) (*VerificationResult, error) { + result := &VerificationResult{ + StartTime: time.Now(), + } + + v.logProgressf("Getting module list...") + var modules []string + var err error + + if len(moduleNames) > 0 && moduleNames[0] != "" { + modules = moduleNames + } else { + modules, err = v.sourceReader.ReadModulesList(ctx) + if err != nil { + return nil, fmt.Errorf("read modules list: %w", err) + } + } + v.logProgressf(" Found %d modules to verify", len(modules)) + + result.ModulesExpected = len(modules) + + releaseChannels := d8internal.GetAllDefaultReleaseChannels() + sourceOpts := v.sourceReader.RemoteOpts() + + v.logProgressf("Verifying modules...") + + for _, moduleName := range modules { + v.logProgressf(" Checking module: %s", moduleName) + + sourceReleaseRepo := path.Join(v.sourceReader.Registry(), d8internal.ModulesSegment, moduleName, "release") + targetReleaseRepo := path.Join(v.targetReg, d8internal.ModulesSegment, moduleName, "release") + + targetRef, err := name.ParseReference(targetReleaseRepo + ":latest") + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("module %s: invalid target reference: %v", moduleName, err)) + continue + } + + targetOpts := append(v.targetOpts, remote.WithContext(ctx)) + targetTags, err := remote.List(targetRef.Context(), targetOpts...) + if err != nil { + v.logProgressf(" Target: no tags found (module not mirrored)") + result.ModulesMissing = append(result.ModulesMissing, moduleName) + continue + } + + // Filter to only release channels + targetChannels := []string{} + for _, tag := range targetTags { + for _, channel := range releaseChannels { + if tag == channel { + targetChannels = append(targetChannels, tag) + break + } + } + } + + if len(targetChannels) == 0 { + v.logProgressf(" Target: no release channels found (module not properly mirrored)") + result.ModulesMissing = append(result.ModulesMissing, moduleName) + continue + } + + v.logProgressf(" Target has %d channels: %v", len(targetChannels), targetChannels) + + matchedChannels := []string{} + mismatchedChannels := []string{} + sourceNotFoundChannels := []string{} + + for _, channel := range targetChannels { + targetTagRef := targetReleaseRepo + ":" + channel + targetImgRef, err := name.ParseReference(targetTagRef) + if err != nil { + continue + } + + targetDesc, err := remote.Head(targetImgRef, targetOpts...) + if err != nil { + continue // Already listed, should exist + } + targetDigest := targetDesc.Digest.String() + + sourceTagRef := sourceReleaseRepo + ":" + channel + sourceImgRef, err := name.ParseReference(sourceTagRef) + if err != nil { + sourceNotFoundChannels = append(sourceNotFoundChannels, channel) + continue + } + + sourceOptsWithCtx := append(sourceOpts, remote.WithContext(ctx)) + sourceDesc, err := remote.Head(sourceImgRef, sourceOptsWithCtx...) + if err != nil { + // Channel exists in target but not in source - might be removed upstream + v.logProgressf(" ⚠ %s: exists in target but not in source (may be removed upstream)", channel) + sourceNotFoundChannels = append(sourceNotFoundChannels, channel) + continue + } + sourceDigest := sourceDesc.Digest.String() + + if sourceDigest != targetDigest { + v.logProgressf(" ✗ %s: DIGEST MISMATCH!", channel) + v.logProgressf(" Source: %s", sourceDigest) + v.logProgressf(" Target: %s", targetDigest) + mismatchedChannels = append(mismatchedChannels, channel) + result.Errors = append(result.Errors, + fmt.Sprintf("module %s/%s: digest mismatch (source=%s, target=%s)", + moduleName, channel, sourceDigest[:19], targetDigest[:19])) + } else { + v.logProgressf(" ✓ %s: digest match %s", channel, sourceDigest[:19]) + matchedChannels = append(matchedChannels, channel) + } + } + + if len(mismatchedChannels) > 0 { + v.logProgressf(" Result: %d matched, %d MISMATCHED, %d source-not-found", + len(matchedChannels), len(mismatchedChannels), len(sourceNotFoundChannels)) + result.ModulesFound++ + } else if len(matchedChannels) > 0 { + v.logProgressf(" ✓ All %d channels verified with matching digests", len(matchedChannels)) + result.ModulesFound++ + } else if len(sourceNotFoundChannels) > 0 { + v.logProgressf(" ⚠ All %d channels exist only in target (removed from source?)", len(sourceNotFoundChannels)) + result.ModulesFound++ + } + } + + result.EndTime = time.Now() + return result, nil +} + +func (v *DigestVerifier) VerifySecurity(ctx context.Context) (*VerificationResult, error) { + result := &VerificationResult{ + StartTime: time.Now(), + } + + expectedTags := map[string]string{ + d8internal.SecurityTrivyDBSegment: "2", + d8internal.SecurityTrivyBDUSegment: "1", + d8internal.SecurityTrivyJavaDBSegment: "1", + d8internal.SecurityTrivyChecksSegment: "0", + } + + result.SecurityExpected = len(expectedTags) + sourceOpts := v.sourceReader.RemoteOpts() + sourceReg := v.sourceReader.Registry() + + v.logProgressf("Verifying security databases...") + + for db, expectedTag := range expectedTags { + v.logProgressf(" Checking %s:%s...", db, expectedTag) + + sourceRef := path.Join(sourceReg, d8internal.SecuritySegment, db) + ":" + expectedTag + sourceImgRef, err := name.ParseReference(sourceRef) + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("db %s: invalid source reference: %v", db, err)) + result.SecurityMissing = append(result.SecurityMissing, db) + continue + } + + sourceOptsWithCtx := append(sourceOpts, remote.WithContext(ctx)) + sourceDesc, err := remote.Head(sourceImgRef, sourceOptsWithCtx...) + if err != nil { + v.logProgressf(" ⚠ Cannot read source %s:%s: %v (skipping)", db, expectedTag, err) + result.SecurityExpected-- + continue + } + sourceDigest := sourceDesc.Digest.String() + v.logProgressf(" Source digest: %s", sourceDigest[:19]) + + targetRef := path.Join(v.targetReg, d8internal.SecuritySegment, db) + ":" + expectedTag + targetImgRef, err := name.ParseReference(targetRef) + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("db %s: invalid target reference: %v", db, err)) + result.SecurityMissing = append(result.SecurityMissing, db) + continue + } + + targetOpts := append(v.targetOpts, remote.WithContext(ctx)) + targetDesc, err := remote.Head(targetImgRef, targetOpts...) + if err != nil { + v.logProgressf(" ✗ %s:%s NOT FOUND in target", db, expectedTag) + result.SecurityMissing = append(result.SecurityMissing, db) + continue + } + targetDigest := targetDesc.Digest.String() + + if sourceDigest != targetDigest { + v.logProgressf(" ✗ %s:%s DIGEST MISMATCH!", db, expectedTag) + v.logProgressf(" Source: %s", sourceDigest) + v.logProgressf(" Target: %s", targetDigest) + result.Errors = append(result.Errors, + fmt.Sprintf("db %s: digest mismatch (source=%s, target=%s)", + db, sourceDigest[:19], targetDigest[:19])) + result.SecurityMissing = append(result.SecurityMissing, db) + continue + } + + v.logProgressf(" ✓ %s:%s digest match %s", db, expectedTag, sourceDigest[:19]) + result.SecurityFound++ + } + + result.EndTime = time.Now() + return result, nil +} + +type verifyItem struct { + ref string + onErr func(string, error) + onFound func(string) + onMissing func(string) +} + +func (v *DigestVerifier) verifyInParallel(ctx context.Context, items []verifyItem) { + var wg sync.WaitGroup + var mu sync.Mutex + sem := make(chan struct{}, 10) + + for _, item := range items { + wg.Add(1) + go func(it verifyItem) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + imgRef, err := name.ParseReference(it.ref) + if err != nil { + mu.Lock() + it.onErr(it.ref, err) + mu.Unlock() + return + } + + opts := append(v.targetOpts, remote.WithContext(ctx)) + _, err = remote.Head(imgRef, opts...) + mu.Lock() + if err != nil { + it.onMissing(it.ref) + } else { + it.onFound(it.ref) + } + mu.Unlock() + }(item) + } + + wg.Wait() +} + +func (v *DigestVerifier) verifyDigests(ctx context.Context, result *VerificationResult) { + items := make([]verifyItem, 0, len(result.ExpectedDigests)) + for _, digest := range result.ExpectedDigests { + digest := digest // capture loop variable + ref := v.targetReg + "@" + digest + items = append(items, verifyItem{ + ref: ref, + onErr: func(ref string, err error) { + result.Errors = append(result.Errors, fmt.Sprintf("invalid digest ref %s: %v", digest, err)) + }, + onFound: func(ref string) { + result.FoundDigests = append(result.FoundDigests, digest) + }, + onMissing: func(ref string) { + result.MissingDigests = append(result.MissingDigests, digest) + }, + }) + } + v.verifyInParallel(ctx, items) +} + +func (v *DigestVerifier) getAttTagsFromSource(ctx context.Context) []string { + sourceReg := v.sourceReader.Registry() + sourceOpts := append(v.sourceReader.RemoteOpts(), remote.WithContext(ctx)) + + ref, err := name.ParseReference(sourceReg + ":latest") + if err != nil { + v.logProgressf(" Warning: failed to parse source registry: %v", err) + return nil + } + + repo := ref.Context() + tags, err := remote.List(repo, sourceOpts...) + if err != nil { + v.logProgressf(" Warning: failed to list source tags: %v", err) + return nil + } + + var attTags []string + for _, tag := range tags { + if strings.HasSuffix(tag, ".att") { + attTags = append(attTags, tag) + } + } + + return attTags +} + +func (v *DigestVerifier) filterAttTagsForDigests(attTags []string, digests []string) []string { + expectedPrefixes := make(map[string]bool) + for _, digest := range digests { + if strings.HasPrefix(digest, "sha256:") { + hash := strings.TrimPrefix(digest, "sha256:") + prefix := "sha256-" + hash + expectedPrefixes[prefix] = true + } else { + digestPreview := digest + if len(digest) > 20 { + digestPreview = digest[:20] + } + v.logProgressf(" Warning: non-sha256 digest found: %s (skipping .att tag matching)", digestPreview) + } + } + + var filtered []string + for _, tag := range attTags { + if strings.HasSuffix(tag, ".att") { + prefix := strings.TrimSuffix(tag, ".att") + if expectedPrefixes[prefix] { + filtered = append(filtered, tag) + } + } + } + + return filtered +} + +func (v *DigestVerifier) verifyAttTags(ctx context.Context, result *VerificationResult) { + items := make([]verifyItem, 0, len(result.ExpectedAttTags)) + for _, attTag := range result.ExpectedAttTags { + attTag := attTag // capture loop variable + ref := v.targetReg + ":" + attTag + items = append(items, verifyItem{ + ref: ref, + onErr: func(ref string, err error) { + result.Errors = append(result.Errors, fmt.Sprintf("invalid att ref %s: %v", attTag, err)) + }, + onFound: func(ref string) { + result.FoundAttTags = append(result.FoundAttTags, attTag) + }, + onMissing: func(ref string) { + result.MissingAttTags = append(result.MissingAttTags, attTag) + }, + }) + } + v.verifyInParallel(ctx, items) +} + +func (v *DigestVerifier) VerifyFull(ctx context.Context, deckhouseTag string, moduleNames []string) (*VerificationResult, error) { + result := &VerificationResult{ + StartTime: time.Now(), + } + + v.logProgressf("=== PLATFORM VERIFICATION ===") + platformResult, err := v.VerifyPlatform(ctx, deckhouseTag) + if err != nil { + return nil, fmt.Errorf("platform verification: %w", err) + } + mergeResults(result, platformResult) + + v.logProgressf("\n=== MODULES VERIFICATION ===") + modulesResult, err := v.VerifyModules(ctx, moduleNames) + if err != nil { + return nil, fmt.Errorf("modules verification: %w", err) + } + mergeResults(result, modulesResult) + + v.logProgressf("\n=== SECURITY VERIFICATION ===") + securityResult, err := v.VerifySecurity(ctx) + if err != nil { + return nil, fmt.Errorf("security verification: %w", err) + } + mergeResults(result, securityResult) + + result.EndTime = time.Now() + result.TotalExpected = len(result.ExpectedDigests) + len(result.ExpectedAttTags) + + result.ModulesExpected + result.SecurityExpected + result.TotalFound = len(result.FoundDigests) + len(result.FoundAttTags) + + result.ModulesFound + result.SecurityFound + result.TotalMissing = len(result.MissingDigests) + len(result.MissingAttTags) + + len(result.ModulesMissing) + len(result.SecurityMissing) + + return result, nil +} + +func mergeResults(dst, src *VerificationResult) { + dst.ExpectedDigests = append(dst.ExpectedDigests, src.ExpectedDigests...) + dst.ExpectedAttTags = append(dst.ExpectedAttTags, src.ExpectedAttTags...) + dst.ReleaseChannels = append(dst.ReleaseChannels, src.ReleaseChannels...) + dst.Versions = append(dst.Versions, src.Versions...) + dst.FoundDigests = append(dst.FoundDigests, src.FoundDigests...) + dst.MissingDigests = append(dst.MissingDigests, src.MissingDigests...) + dst.FoundAttTags = append(dst.FoundAttTags, src.FoundAttTags...) + dst.MissingAttTags = append(dst.MissingAttTags, src.MissingAttTags...) + + dst.ModulesExpected += src.ModulesExpected + dst.ModulesFound += src.ModulesFound + dst.ModulesMissing = append(dst.ModulesMissing, src.ModulesMissing...) + + dst.SecurityExpected += src.SecurityExpected + dst.SecurityFound += src.SecurityFound + dst.SecurityMissing = append(dst.SecurityMissing, src.SecurityMissing...) + + dst.Errors = append(dst.Errors, src.Errors...) +} + diff --git a/testing/e2e/mirror/mirror_e2e_test.go b/testing/e2e/mirror/mirror_e2e_test.go deleted file mode 100644 index 44f46e21..00000000 --- a/testing/e2e/mirror/mirror_e2e_test.go +++ /dev/null @@ -1,251 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mirror - -import ( - "encoding/json" - "fmt" - "math/rand" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/Masterminds/semver/v3" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/crane" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/random" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/stretchr/testify/require" - "sigs.k8s.io/yaml" - - "github.com/deckhouse/deckhouse-cli/internal" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" -) - -func TestMirrorE2E(t *testing.T) { - t.SkipNow() -} - -func createDeckhouseReleaseChannelsInRegistry(t *testing.T, repo string) { - t.Helper() - - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.AlphaChannel, "v1.56.5") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.BetaChannel, "v1.56.5") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.EarlyAccessChannel, "v1.55.7") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.StableChannel, "v1.55.7") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.RockSolidChannel, "v1.55.7") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", "v1.55.7", "v1.55.7") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", "v1.56.5", "v1.56.5") -} - -func createTrivyVulnerabilityDatabasesInRegistry(t *testing.T, repo string, insecure, useTLS bool) { - t.Helper() - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authn.Anonymous, insecure, useTLS) - - images := []string{ - repo + "/security/trivy-db:2", - repo + "/security/trivy-bdu:1", - repo + "/security/trivy-java-db:1", - repo + "/security/trivy-checks:0", - } - - for _, image := range images { - ref, err := name.ParseReference(image, nameOpts...) - require.NoError(t, err) - wantImage, err := random.Image(256, 1) - require.NoError(t, err) - require.NoError(t, remote.Write(ref, wantImage, remoteOpts...)) - } -} - -func createDeckhouseControllersAndInstallersInRegistry(t *testing.T, repo string) { - t.Helper() - - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(nil, true, false) - - createRandomImageInRegistry(t, repo+":"+internal.AlphaChannel) - createRandomImageInRegistry(t, repo+":"+internal.BetaChannel) - createRandomImageInRegistry(t, repo+":"+internal.EarlyAccessChannel) - createRandomImageInRegistry(t, repo+":"+internal.StableChannel) - createRandomImageInRegistry(t, repo+":"+internal.RockSolidChannel) - createRandomImageInRegistry(t, repo+":v1.56.5") - createRandomImageInRegistry(t, repo+":v1.55.7") - - installers := map[string]v1.Image{ - "v1.56.5": createSyntheticInstallerImage(t, "v1.56.5", repo), - "v1.55.7": createSyntheticInstallerImage(t, "v1.55.7", repo), - } - installers[internal.AlphaChannel] = installers["v1.56.5"] - installers[internal.BetaChannel] = installers["v1.56.5"] - installers[internal.EarlyAccessChannel] = installers["v1.55.7"] - installers[internal.StableChannel] = installers["v1.55.7"] - installers[internal.RockSolidChannel] = installers["v1.55.7"] - - for shortTag, installer := range installers { - ref, err := name.ParseReference(repo+"/install:"+shortTag, nameOpts...) - require.NoError(t, err) - - err = remote.Write(ref, installer, remoteOpts...) - require.NoError(t, err) - - ref, err = name.ParseReference(repo+"/install-standalone:"+shortTag, nameOpts...) - require.NoError(t, err) - - err = remote.Write(ref, installer, remoteOpts...) - require.NoError(t, err) - } -} - -func createSyntheticInstallerImage(t *testing.T, version, repo string) v1.Image { - t.Helper() - - // FROM scratch - base := empty.Image - layers := make([]v1.Layer, 0) - - // COPY ./version /deckhouse/version - // COPY ./images_digests.json /deckhouse/candi/images_digests.json - imagesDigests, err := json.Marshal( - map[string]map[string]string{ - "common": { - "alpine": createRandomImageInRegistry(t, repo+":alpine"+version), - }, - "nodeManager": { - "bashibleApiserver": createRandomImageInRegistry(t, repo+":bashibleApiserver"+version), - }, - }) - require.NoError(t, err) - l, err := crane.Layer(map[string][]byte{ - "deckhouse/version": []byte(version), - "deckhouse/candi/images_digests.json": imagesDigests, - }) - require.NoError(t, err) - layers = append(layers, l) - - img, err := mutate.AppendLayers(base, layers...) - require.NoError(t, err) - - // ENTRYPOINT ["/bin/bash"] - img, err = mutate.Config(img, v1.Config{ - Entrypoint: []string{"/bin/bash"}, - }) - require.NoError(t, err) - - return img -} - -func createRandomImageInRegistry(t *testing.T, tag string) (digest string) { - t.Helper() - - img, err := random.Image(int64(rand.Intn(1024)+1), int64(rand.Intn(5)+1)) - require.NoError(t, err) - - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(nil, true, false) - ref, err := name.ParseReference(tag, nameOpts...) - require.NoError(t, err) - - err = remote.Write(ref, img, remoteOpts...) - require.NoError(t, err) - - digestHash, err := img.Digest() - require.NoError(t, err) - - return digestHash.String() -} - -func createDeckhouseReleaseChannelImageInRegistry(t *testing.T, repo, tag, version string) (digest string) { - t.Helper() - - // FROM scratch - base := empty.Image - layers := make([]v1.Layer, 0) - - // COPY ./version.json /version.json - changelog, err := yaml.JSONToYAML([]byte(`{"candi":{"fixes":[{"summary":"Fix deckhouse containerd start after installing new containerd-deckhouse package.","pull_request":"https://github.com/deckhouse/deckhouse/pull/6329"}]}}`)) - require.NoError(t, err) - versionInfo := fmt.Sprintf( - `{"disruptions":{"1.56":["ingressNginx"]},"requirements":{"containerdOnAllNodes":"true","ingressNginx":"1.1","k8s":"1.23.0","nodesMinimalOSVersionUbuntu":"18.04"},"version":%q}`, - "v"+version, - ) - l, err := crane.Layer(map[string][]byte{ - "version.json": []byte(versionInfo), - "changelog.yaml": changelog, - }) - layers = append(layers, l) - - img, err := mutate.AppendLayers(base, layers...) - require.NoError(t, err) - - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(nil, true, false) - ref, err := name.ParseReference(repo+":"+tag, nameOpts...) - require.NoError(t, err) - - err = remote.Write(ref, img, remoteOpts...) - require.NoError(t, err) - - digestHash, err := img.Digest() - require.NoError(t, err) - - return digestHash.String() -} - -func validateDeckhouseReleasesManifests(t *testing.T, pullCtx *params.PullParams, versions []semver.Version) { - t.Helper() - deckhouseReleasesManifestsFilepath := filepath.Join(pullCtx.BundleDir, "deckhousereleases.yaml") - actualManifests, err := os.ReadFile(deckhouseReleasesManifestsFilepath) - require.NoError(t, err) - - expectedManifests := strings.Builder{} - for _, version := range versions { - expectedManifests.WriteString(fmt.Sprintf(`--- -apiVersion: deckhouse.io/v1alpha1 -approved: false -kind: DeckhouseRelease -metadata: - creationTimestamp: null - name: v%[1]s -spec: - changelog: - candi: - fixes: - - summary: Fix deckhouse containerd start after installing new containerd-deckhouse package. - pull_request: https://github.com/deckhouse/deckhouse/pull/6329 - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v%[1]s - disruptions: - - ingressNginx - requirements: - containerdOnAllNodes: 'true' - ingressNginx: '1.1' - k8s: 1.23.0 - nodesMinimalOSVersionUbuntu: '18.04' - version: v%[1]s -status: - approved: false - message: "" - transitionTime: "0001-01-01T00:00:00Z" -`, version.String())) - } - - require.FileExists(t, deckhouseReleasesManifestsFilepath, "deckhousereleases.yaml should be generated next tar bundle") - require.YAMLEq(t, expectedManifests.String(), string(actualManifests)) -} diff --git a/testing/e2e/mirror/modules_test.go b/testing/e2e/mirror/modules_test.go new file mode 100644 index 00000000..90645a55 --- /dev/null +++ b/testing/e2e/mirror/modules_test.go @@ -0,0 +1,84 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func TestModulesE2E(t *testing.T) { + cfg := internal.GetConfig() + + if !cfg.HasSourceAuth() { + t.Skip("Skipping: no source authentication configured (set E2E_LICENSE_TOKEN)") + } + + cfg.NoPlatform = true + cfg.NoSecurity = true + + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), internal.ModulesTestTimeout) + defer cancel() + + runModulesTest(t, ctx, cfg, env) +} + +func runModulesTest(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv) { + internal.PrintHeader("MODULES E2E TEST") + + internal.PrintStep(1, "Reading expected modules from source") + expectedModules := readExpectedModules(t, ctx, cfg) + + internal.PrintStep(2, "Pulling modules") + runPullStep(t, cfg, env) + + internal.PrintStep(3, "Pushing to target registry") + runPushStep(t, cfg, env) + + internal.PrintStep(4, "Verifying modules in target") + verifyModulesInTarget(t, ctx, cfg, env, expectedModules) + + fmt.Printf("\n✅ Modules test passed: %d modules\n", len(expectedModules)) +} + +func readExpectedModules(t *testing.T, ctx context.Context, cfg *internal.Config) []string { + t.Helper() + + reader := createSourceReader(t, cfg) + modules, err := reader.ReadModulesList(ctx) + require.NoError(t, err, "Failed to read modules list") + + modules = filterModules(modules, cfg.IncludeModules) + + t.Logf("Expected %d modules: %v", len(modules), modules) + return modules +} + +func verifyModulesInTarget(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expectedModules []string) { + t.Helper() + verifyModulesImages(t, ctx, cfg, env, expectedModules) +} diff --git a/testing/e2e/mirror/platform_test.go b/testing/e2e/mirror/platform_test.go new file mode 100644 index 00000000..47187e39 --- /dev/null +++ b/testing/e2e/mirror/platform_test.go @@ -0,0 +1,118 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func TestPlatformE2E(t *testing.T) { + cfg := internal.GetConfig() + + if !cfg.HasSourceAuth() { + t.Skip("Skipping: no source authentication configured (set E2E_LICENSE_TOKEN)") + } + + cfg.NoModules = true + cfg.NoSecurity = true + + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), internal.PlatformTestTimeout) + defer cancel() + + runPlatformTest(t, ctx, cfg, env) +} + +func runPlatformTest(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv) { + internal.PrintHeader("PLATFORM E2E TEST") + + internal.PrintStep(1, "Reading expected platform images from source") + expected := readExpectedPlatformImages(t, ctx, cfg) + + if cfg.HasExistingBundle() { + t.Logf("Using existing bundle: %s (skipping pull)", env.BundleDir) + env.Report.AddStep("Pull (existing bundle)", "SKIP", 0, nil) + } else { + internal.PrintStep(2, "Pulling platform images") + runPullStep(t, cfg, env) + } + + internal.PrintStep(3, "Pushing to target registry") + runPushStep(t, cfg, env) + + internal.PrintStep(4, "Verifying expected images in target") + verifyExpectedInTarget(t, ctx, cfg, env, expected) + + internal.PrintSuccessBox(env.Report.MatchedImages, env.Report.FoundAttTags) +} + +type ExpectedPlatformImages struct { + Channels []internal.ReleaseChannelInfo + Versions []string + Digests []string +} + +func readExpectedPlatformImages(t *testing.T, ctx context.Context, cfg *internal.Config) *ExpectedPlatformImages { + t.Helper() + + reader := createSourceReader(t, cfg) + result := &ExpectedPlatformImages{} + + channels, err := reader.ReadReleaseChannels(ctx) + require.NoError(t, err, "Failed to read release channels") + result.Channels = channels + + t.Logf("Found %d release channels:", len(channels)) + for _, ch := range channels { + t.Logf(" %s -> %s", ch.Channel, ch.Version) + } + + if cfg.DeckhouseTag != "" { + for _, ch := range channels { + if ch.Channel == cfg.DeckhouseTag { + channels = []internal.ReleaseChannelInfo{ch} + break + } + } + } + + platform, err := reader.ReadPlatformDigests(ctx, channels) + require.NoError(t, err, "Failed to read platform digests") + + result.Versions = platform.Versions + result.Digests = platform.ImageDigests + + t.Logf("Expected: %d versions, %d digests", len(result.Versions), len(result.Digests)) + + return result +} + +func verifyExpectedInTarget(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expected *ExpectedPlatformImages) { + t.Helper() + verifyPlatformImages(t, ctx, cfg, env, cfg.DeckhouseTag) + t.Logf("Platform verification passed: %d digests, %d .att tags", env.Report.MatchedImages, env.Report.FoundAttTags) +} + diff --git a/testing/e2e/mirror/security_test.go b/testing/e2e/mirror/security_test.go new file mode 100644 index 00000000..09b73240 --- /dev/null +++ b/testing/e2e/mirror/security_test.go @@ -0,0 +1,91 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func TestSecurityE2E(t *testing.T) { + cfg := internal.GetConfig() + + if !cfg.HasSourceAuth() { + t.Skip("Skipping: no source authentication configured (set E2E_LICENSE_TOKEN)") + } + + cfg.NoModules = true + cfg.NoPlatform = true + + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), internal.SecurityTestTimeout) + defer cancel() + + runSecurityTest(t, ctx, cfg, env) +} + +func runSecurityTest(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv) { + internal.PrintHeader("SECURITY E2E TEST") + + internal.PrintStep(1, "Reading expected security databases from source") + expected := readExpectedSecurityImages(t, ctx, cfg) + + internal.PrintStep(2, "Pulling security databases") + runPullStep(t, cfg, env) + + internal.PrintStep(3, "Pushing to target registry") + runPushStep(t, cfg, env) + + internal.PrintStep(4, "Verifying security databases in target") + verifySecurityInTarget(t, ctx, cfg, env, expected) + + totalTags := 0 + for _, tags := range expected.Databases { + totalTags += len(tags) + } + fmt.Printf("\n✅ Security test passed: %d databases, %d tags\n", len(expected.Databases), totalTags) +} + +func readExpectedSecurityImages(t *testing.T, ctx context.Context, cfg *internal.Config) *internal.SecurityDigests { + t.Helper() + + reader := createSourceReader(t, cfg) + security, err := reader.ReadSecurityDigests(ctx) + require.NoError(t, err, "Failed to read security databases") + + t.Logf("Found %d security databases:", len(security.Databases)) + for db, tags := range security.Databases { + t.Logf(" %s: %d tags", db, len(tags)) + } + + return security +} + +func verifySecurityInTarget(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expected *internal.SecurityDigests) { + t.Helper() + verifySecurityImages(t, ctx, cfg, env) +} + diff --git a/testing/e2e/mirror/testenv.go b/testing/e2e/mirror/testenv.go new file mode 100644 index 00000000..1ef53f52 --- /dev/null +++ b/testing/e2e/mirror/testenv.go @@ -0,0 +1,241 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" + mirrorutil "github.com/deckhouse/deckhouse-cli/testing/util/mirror" +) + +type testEnv struct { + LogDir string + LogFile string + ReportFile string + ComparisonFile string + BundleDir string + TargetRegistry string + Report *internal.TestReport + Cleanup func() +} + +func setupTestEnvironment(t *testing.T, cfg *internal.Config) *testEnv { + t.Helper() + + logDir := getLogDir(t.Name()) + require.NoError(t, os.MkdirAll(logDir, 0755)) + + targetHost, targetPath, registryCleanup := setupTargetRegistry(t, cfg) + targetRegistry := targetHost + targetPath + t.Logf("Target registry: %s", targetRegistry) + + bundleDir := setupBundleDir(t, cfg) + + report := &internal.TestReport{ + TestName: t.Name(), + StartTime: time.Now(), + SourceRegistry: cfg.SourceRegistry, + TargetRegistry: targetRegistry, + LogDir: logDir, + } + + env := &testEnv{ + LogDir: logDir, + LogFile: filepath.Join(logDir, "test.log"), + ReportFile: filepath.Join(logDir, "report.txt"), + ComparisonFile: filepath.Join(logDir, "comparison.txt"), + BundleDir: bundleDir, + TargetRegistry: targetRegistry, + Report: report, + } + + env.Cleanup = func() { + registryCleanup() + finalizeReport(t, env) + } + + return env +} + +func setupBundleDir(t *testing.T, cfg *internal.Config) string { + t.Helper() + + if cfg.ExistingBundle != "" { + if _, err := os.Stat(cfg.ExistingBundle); os.IsNotExist(err) { + t.Fatalf("Existing bundle directory not found: %s", cfg.ExistingBundle) + } + t.Logf("Using existing bundle: %s", cfg.ExistingBundle) + return cfg.ExistingBundle + } + + if cfg.KeepBundle { + bundleDir := filepath.Join(os.TempDir(), fmt.Sprintf("d8-mirror-e2e-%d", time.Now().Unix())) + require.NoError(t, os.MkdirAll(bundleDir, 0755)) + t.Logf("Bundle directory (will be kept): %s", bundleDir) + return bundleDir + } + + bundleDir := t.TempDir() + t.Logf("Bundle directory: %s", bundleDir) + return bundleDir +} + +func setupTargetRegistry(t *testing.T, cfg *internal.Config) (host, path string, cleanup func()) { + t.Helper() + + if cfg.UseInMemoryRegistry() { + reg := mirrorutil.SetupTestRegistry(false) + repoPath := "/deckhouse/ee" + t.Logf("Started test registry at %s%s", reg.Host, repoPath) + return reg.Host, repoPath, reg.Close + } + + return cfg.TargetRegistry, "", func() {} +} + +func finalizeReport(t *testing.T, env *testEnv) { + t.Helper() + + env.Report.EndTime = time.Now() + env.Report.Print() + + if err := env.Report.WriteToFile(env.ReportFile); err != nil { + t.Logf("Warning: failed to write report: %v", err) + } else { + t.Logf("Report written to: %s", env.ReportFile) + } +} + +func runPullStep(t *testing.T, cfg *internal.Config, env *testEnv) { + t.Helper() + stepStart := time.Now() + + cmd := internal.BuildPullCommand(cfg, env.BundleDir) + t.Logf("Running: %s", cmd.String()) + + err := internal.RunCommandWithLog(t, cmd, env.LogFile) + if err != nil { + env.Report.AddStep("Pull images", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Pull failed") + } + + bundleInfo := validateBundle(t, env.BundleDir, cfg) + env.Report.BundleSize = bundleInfo.TotalSize + if len(bundleInfo.Modules) > 0 { + env.Report.ExpectedModules = bundleInfo.Modules + } + + env.Report.AddStep( + fmt.Sprintf("Pull images (%.2f GB bundle)", float64(bundleInfo.TotalSize)/(1024*1024*1024)), + "PASS", time.Since(stepStart), nil, + ) + t.Logf("Pull completed: %.2f GB total", float64(bundleInfo.TotalSize)/(1024*1024*1024)) +} + +func runPushStep(t *testing.T, cfg *internal.Config, env *testEnv) { + t.Helper() + stepStart := time.Now() + + cmd := internal.BuildPushCommand(cfg, env.BundleDir, env.TargetRegistry) + t.Logf("Running: %s", cmd.String()) + + err := internal.RunCommandWithLog(t, cmd, env.LogFile) + if err != nil { + env.Report.AddStep("Push to registry", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Push failed") + } + + env.Report.AddStep("Push to registry", "PASS", time.Since(stepStart), nil) + t.Log("Push completed successfully") +} + +type BundleInfo struct { + TotalSize int64 + Modules []string + HasPlatform bool + HasSecurity bool +} + +func validateBundle(t *testing.T, bundleDir string, cfg *internal.Config) *BundleInfo { + t.Helper() + + files, err := os.ReadDir(bundleDir) + require.NoError(t, err, "Failed to read bundle directory") + + info := &BundleInfo{} + + for _, f := range files { + if f.IsDir() { + continue + } + + finfo, err := f.Info() + require.NoError(t, err) + info.TotalSize += finfo.Size() + + name := f.Name() + t.Logf(" %s (%.2f MB)", name, float64(finfo.Size())/(1024*1024)) + + switch { + case name == "platform.tar" || strings.HasPrefix(name, "platform."): + info.HasPlatform = true + case name == "security.tar" || strings.HasPrefix(name, "security."): + info.HasSecurity = true + case strings.HasPrefix(name, "module-") && strings.Contains(name, ".tar"): + moduleName := strings.TrimPrefix(name, "module-") + moduleName = strings.Split(moduleName, ".")[0] + if moduleName != "" && !slices.Contains(info.Modules, moduleName) { + info.Modules = append(info.Modules, moduleName) + } + } + } + + if !cfg.NoPlatform { + require.True(t, info.HasPlatform, "Bundle missing platform.tar - pull may have failed!") + } + if !cfg.NoSecurity { + require.True(t, info.HasSecurity, "Bundle missing security.tar - pull may have failed!") + } + if !cfg.NoModules && len(cfg.IncludeModules) == 0 { + require.NotEmpty(t, info.Modules, "Bundle has no modules - pull may have failed!") + } + + if len(info.Modules) > 0 { + t.Logf("Bundle contains %d modules: %v", len(info.Modules), info.Modules) + } + + return info +} + +func getLogDir(testName string) string { + projectRoot := internal.FindProjectRoot() + timestamp := time.Now().Format("20060102-150405") + safeName := strings.ReplaceAll(testName, "/", "-") + return filepath.Join(projectRoot, "testing", "e2e", ".logs", fmt.Sprintf("%s-%s", safeName, timestamp)) +} + diff --git a/testing/util/mirror/registry.go b/testing/util/mirror/registry.go index e7dbf56a..5e0efb43 100644 --- a/testing/util/mirror/registry.go +++ b/testing/util/mirror/registry.go @@ -17,59 +17,118 @@ limitations under the License. package mirror import ( - "context" "io" + "io/fs" golog "log" "net/http/httptest" + "os" + "path/filepath" "strings" - "sync" "github.com/google/go-containerregistry/pkg/registry" - v1 "github.com/google/go-containerregistry/pkg/v1" ) -type ListableBlobHandler struct { - registry.BlobHandler - registry.BlobPutHandler +// TestRegistry is a disk-based container registry for e2e testing. +// Blobs are stored on disk to avoid memory exhaustion when mirroring large images. +type TestRegistry struct { + server *httptest.Server + storageDir string - mu sync.Mutex - ingestedBlobs []string + Host string // e.g. "127.0.0.1:12345" } -func (h *ListableBlobHandler) Get(ctx context.Context, repo string, hash v1.Hash) (io.ReadCloser, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.ingestedBlobs = append(h.ingestedBlobs, hash.String()) - - return h.BlobHandler.Get(ctx, repo, hash) -} - -func (h *ListableBlobHandler) ListBlobs() []string { - return h.ingestedBlobs -} +// NewTestRegistry creates a new disk-based test registry. +// Storage is created in a temporary directory that will be cleaned up on Close(). +func NewTestRegistry(useTLS bool) (*TestRegistry, error) { + storageDir, err := os.MkdirTemp("", "test-registry-*") + if err != nil { + return nil, err + } -func SetupEmptyRegistryRepo(useTLS bool) ( /*host*/ string /*repoPath*/, string, *ListableBlobHandler) { - var host, repoPath string + blobHandler := registry.NewDiskBlobHandler(storageDir) - memBlobHandler := registry.NewInMemoryBlobHandler() - bh := &ListableBlobHandler{ - BlobHandler: memBlobHandler, - BlobPutHandler: memBlobHandler.(registry.BlobPutHandler), - } - registryHandler := registry.New(registry.WithBlobHandler(bh), registry.Logger(golog.New(io.Discard, "", 0))) + handler := registry.New( + registry.WithBlobHandler(blobHandler), + registry.Logger(golog.New(io.Discard, "", 0)), + ) - server := httptest.NewUnstartedServer(registryHandler) + server := httptest.NewUnstartedServer(handler) if useTLS { server.StartTLS() } else { server.Start() } - host = strings.TrimPrefix(server.URL, "http://") - repoPath = "/deckhouse/ee" + host := strings.TrimPrefix(server.URL, "http://") if useTLS { host = strings.TrimPrefix(server.URL, "https://") } - return host, repoPath, bh + return &TestRegistry{ + server: server, + storageDir: storageDir, + Host: host, + }, nil +} + +// Close stops the registry server and removes all stored data. +func (r *TestRegistry) Close() { + if r.server != nil { + r.server.Close() + } + if r.storageDir != "" { + os.RemoveAll(r.storageDir) + } +} + +// StoragePath returns the path to the on-disk blob storage. +// Useful for debugging or inspecting stored data. +func (r *TestRegistry) StoragePath() string { + return r.storageDir +} + +// BlobCount returns the number of blobs currently stored in the registry. +func (r *TestRegistry) BlobCount() int { + count := 0 + _ = filepath.WalkDir(r.storageDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if !d.IsDir() { + count++ + } + return nil + }) + return count +} + +// ListBlobs returns a list of blob digests stored in the registry. +// This is useful for verifying what was pushed. +func (r *TestRegistry) ListBlobs() []string { + var blobs []string + _ = filepath.WalkDir(r.storageDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if !d.IsDir() { + // Extract digest from path (format: storageDir/sha256/abc123...) + rel, _ := filepath.Rel(r.storageDir, path) + parts := strings.Split(rel, string(filepath.Separator)) + if len(parts) == 2 { + blobs = append(blobs, parts[0]+":"+parts[1]) + } + } + return nil + }) + return blobs +} + +// SetupTestRegistry creates a disk-based registry for testing. +// Returns *TestRegistry - use reg.Host to get the address, then append your own repo path. +func SetupTestRegistry(useTLS bool) *TestRegistry { + reg, err := NewTestRegistry(useTLS) + if err != nil { + panic("failed to create test registry: " + err.Error()) + } + return reg }