fix(ci): e2e app airgap restore image pull backoff (#1842)

emosbaugh · web-flow · commit 9d1ec8df4f2c · 2025-02-07T09:24:50.000-08:00
diff --git a/cmd/installer/cli/testing/assets/release-restore-newdr/restore.yaml b/cmd/installer/cli/testing/assets/release-restore-newdr/restore.yaml
@@ -17,7 +17,7 @@ spec:
       - init:
           initContainers:
           - name: restore-hook-init1
-            image: proxy.replicated.com/anonymous/nginx:1.24-alpine
+            image: 'repl{{ LocalImageName "proxy.replicated.com/anonymous/nginx:1.24-alpine" }}'
             command:
             - /bin/ash
             - -c
diff --git a/e2e/kots-release-install-failing-preflights/restore.yaml b/e2e/kots-release-install-failing-preflights/restore.yaml
@@ -20,7 +20,7 @@ spec:
       - init:
           initContainers:
           - name: restore-hook-init1
-            image: proxy.replicated.com/anonymous/nginx:1.24-alpine
+            image: 'repl{{ LocalImageName "proxy.replicated.com/anonymous/nginx:1.24-alpine" }}'
             command:
             - /bin/ash
             - -c
diff --git a/e2e/kots-release-install-warning-preflights/restore.yaml b/e2e/kots-release-install-warning-preflights/restore.yaml
@@ -20,7 +20,7 @@ spec:
       - init:
           initContainers:
           - name: restore-hook-init1
-            image: proxy.replicated.com/anonymous/nginx:1.24-alpine
+            image: 'repl{{ LocalImageName "proxy.replicated.com/anonymous/nginx:1.24-alpine" }}'
             command:
             - /bin/ash
             - -c
diff --git a/e2e/kots-release-install/restore.yaml b/e2e/kots-release-install/restore.yaml
@@ -20,7 +20,7 @@ spec:
       - init:
           initContainers:
           - name: restore-hook-init1
-            image: proxy.replicated.com/anonymous/nginx:1.24-alpine
+            image: 'repl{{ LocalImageName "proxy.replicated.com/anonymous/nginx:1.24-alpine" }}'
             command:
             - /bin/ash
             - -c
diff --git a/e2e/kots-release-unsupported-overrides/restore.yaml b/e2e/kots-release-unsupported-overrides/restore.yaml
@@ -20,7 +20,7 @@ spec:
       - init:
           initContainers:
           - name: restore-hook-init1
-            image: proxy.replicated.com/anonymous/nginx:1.24-alpine
+            image: 'repl{{ LocalImageName "proxy.replicated.com/anonymous/nginx:1.24-alpine" }}'
             command:
             - /bin/ash
             - -c
diff --git a/e2e/kots-release-upgrade/restore.yaml b/e2e/kots-release-upgrade/restore.yaml
@@ -20,7 +20,7 @@ spec:
       - init:
           initContainers:
           - name: restore-hook-init1
-            image: proxy.replicated.com/anonymous/nginx:1.24-alpine
+            image: 'repl{{ LocalImageName "proxy.replicated.com/anonymous/nginx:1.24-alpine" }}'
             command:
             - /bin/ash
             - -c
diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -1,5 +1,5 @@
 import { test, expect, Page, FrameLocator } from '@playwright/test';
-import { login } from '../shared';
+import { login, vaidateAppAndClusterReady } from '../shared';
 
 test('deploy upgrade', async ({ page }) => {
   test.setTimeout(15 * 60 * 1000); // 15 minutes
@@ -59,7 +59,5 @@ async function verifyUpgradeSuccess(page: Page) {
   await expect(page.locator('.VersionHistoryRow', { hasText: process.env.APP_UPGRADE_VERSION })).toContainText('Currently deployed version', { timeout: 90 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
-  await expect(page.locator('#app')).toContainText('Currently deployed version');
-  await expect(page.locator('#app')).toContainText('Ready', { timeout: 30 * 1000 });
-  await expect(page.locator('#app')).toContainText('Up to date');
+  await vaidateAppAndClusterReady(page, expect, 10 * 1000);
 }
diff --git a/e2e/playwright/tests/shared/app-state.ts b/e2e/playwright/tests/shared/app-state.ts
@@ -0,0 +1,8 @@
+import { Page, Expect } from '@playwright/test';
+
+export const vaidateAppAndClusterReady = async (page: Page, expect: Expect, initialTimeout: number) => {
+  await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
+  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: initialTimeout });
+  await expect(page.locator('#app')).toContainText('Ready', { timeout: 45000 });
+  await expect(page.locator('#app')).toContainText('Up to date');
+};
diff --git a/e2e/playwright/tests/shared/deploy-app.ts b/e2e/playwright/tests/shared/deploy-app.ts
@@ -1,4 +1,5 @@
 import { Page, Expect } from '@playwright/test';
+import { vaidateAppAndClusterReady } from '.';
 
 export const deployApp = async (page: Page, expect: Expect) => {
   await expect(page.getByText('Optionally add nodes to the cluster'),).toBeVisible();
@@ -16,7 +17,5 @@ export const deployApp = async (page: Page, expect: Expect) => {
   await expect(page.getByRole('button', { name: 'Rerun' })).toBeVisible({ timeout: 10 * 1000 });
   await expect(page.locator('#app')).toContainText('The Volume Snapshots CRD exists');
   await page.getByRole('button', { name: 'Deploy' }).click();
-  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 90000 });
-  await expect(page.locator('#app')).toContainText('Ready', { timeout: 45000 });
-  await expect(page.locator('#app')).toContainText('Up to date');
+  await vaidateAppAndClusterReady(page, expect, 90000);
 };
diff --git a/e2e/playwright/tests/shared/deploy-ec18-app-version.ts b/e2e/playwright/tests/shared/deploy-ec18-app-version.ts
@@ -1,20 +1,19 @@
 import { Page, Expect } from '@playwright/test';
+import { vaidateAppAndClusterReady } from '.';
 
 export const deployEC18AppVersion = async (page: Page, expect: Expect) => {
-    await expect(page.getByRole('button', { name: 'Add node', exact: true })).toBeVisible();
-    await page.getByRole('button', { name: 'Continue' }).click();
-    await expect(page.locator('h3')).toContainText('The First Config Group');
-    await page.locator('input[type="text"]').click();
-    await page.locator('input[type="text"]').fill('initial-hostname.com');
-    await page.locator('input[type="password"]').click();
-    await page.locator('input[type="password"]').fill('password');
-    await page.getByRole('button', { name: 'Continue' }).click();
-    await expect(page.getByText('Preflight checks', { exact: true })).toBeVisible({ timeout: 10 * 1000 });
-    await expect(page.getByRole('button', { name: 'Re-run' })).toBeVisible({ timeout: 10 * 1000 });
-    await expect(page.locator('#app')).toContainText('Embedded Cluster Installation CRD exists');
-    await expect(page.locator('#app')).toContainText('Embedded Cluster Config CRD exists');
-    await page.getByRole('button', { name: 'Deploy' }).click();
-    await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 90000 });
-    await expect(page.locator('#app')).toContainText('Ready', { timeout: 45000 });
-    await expect(page.locator('#app')).toContainText('Up to date');
-  };
+  await expect(page.getByRole('button', { name: 'Add node', exact: true })).toBeVisible();
+  await page.getByRole('button', { name: 'Continue' }).click();
+  await expect(page.locator('h3')).toContainText('The First Config Group');
+  await page.locator('input[type="text"]').click();
+  await page.locator('input[type="text"]').fill('initial-hostname.com');
+  await page.locator('input[type="password"]').click();
+  await page.locator('input[type="password"]').fill('password');
+  await page.getByRole('button', { name: 'Continue' }).click();
+  await expect(page.getByText('Preflight checks', { exact: true })).toBeVisible({ timeout: 10 * 1000 });
+  await expect(page.getByRole('button', { name: 'Re-run' })).toBeVisible({ timeout: 10 * 1000 });
+  await expect(page.locator('#app')).toContainText('Embedded Cluster Installation CRD exists');
+  await expect(page.locator('#app')).toContainText('Embedded Cluster Config CRD exists');
+  await page.getByRole('button', { name: 'Deploy' }).click();
+  await vaidateAppAndClusterReady(page, expect, 90000);
+};
diff --git a/e2e/playwright/tests/shared/index.ts b/e2e/playwright/tests/shared/index.ts
@@ -1,3 +1,4 @@
 export * from './login';
 export * from './deploy-app';
-export * from './deploy-ec18-app-version';
+export * from './deploy-ec18-app-version';
+export * from './app-state';
diff --git a/e2e/playwright/tests/validate-restore-app/test.spec.ts b/e2e/playwright/tests/validate-restore-app/test.spec.ts
@@ -0,0 +1,8 @@
+import { test, expect } from '@playwright/test';
+import { login, vaidateAppAndClusterReady } from '../shared';
+
+test('validate restore app', async ({ page }) => {
+  test.setTimeout(5 * 60 * 1000); // 5 minutes
+  await login(page);
+  await vaidateAppAndClusterReady(page, expect, 90 * 1000);
+});
diff --git a/e2e/restore_test.go b/e2e/restore_test.go
@@ -91,6 +91,14 @@ func TestSingleNodeDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
+	if err := tc.SetupPlaywright(); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil {
+		t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
+	}
+
 	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
 	testArgs = []string{appUpgradeVersion}
 
@@ -190,6 +198,14 @@ func TestSingleNodeLegacyDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
+	if err := tc.SetupPlaywright(); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil {
+		t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
+	}
+
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
@@ -279,6 +295,14 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) {
 		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
+	if err := tc.SetupPlaywright(); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil {
+		t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
+	}
+
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
@@ -350,6 +374,20 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
+	line = []string{"check-post-restore.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
+	if err := tc.SetupPlaywright(); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil {
+		t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
+	}
+
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
@@ -461,6 +499,14 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
+	if err := tc.SetupPlaywright(); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil {
+		t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
+	}
+
 	t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
 	line = []string{"airgap-update.sh"}
 	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
@@ -671,6 +717,14 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
+	if err := tc.SetupPlaywright(); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil {
+		t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
+	}
+
 	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
 	testArgs = []string{appUpgradeVersion}
 
@@ -947,6 +1001,14 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: validating restored app", time.Now().Format(time.RFC3339))
+	if err := tc.SetupPlaywright(withEnv); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := tc.RunPlaywrightTest("validate-restore-app"); err != nil {
+		t.Fatalf("fail to run playwright test validate-restore-app: %v", err)
+	}
+
 	t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
 	line = []string{"airgap-update.sh"}
 	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
