Proposal for improved e2e test coverage via natural language instructions

.github/workflows/storycheck-example.yml

@@ -0,0 +1,30 @@
+# File: .github/workflows/storycheck-example.yml
+name: StoryCheck Example - ENS Registration Commit
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 * * *'
+  push:
+    branches: [dev]
+  pull_request:
+    branches: [dev]
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up pnpm # Setup pnpm to use the ENS dApp specificed packageManager version in package.json
+        uses: pnpm/action-setup@v4
+      - name: Run StoryCheck
+        id: storycheck
+        uses: GuardianUI/storycheck@v0.1.13
+        env: 
+          ALCHEMY_API_KEY: ${{ secrets.ALCHEMY_API_KEY }}        
+        with:
+          storypath: 'user-stories/ens_reg_commit'
+          output-dir: 'results'
+      - name: Check if passed
+        if: ${{ steps.storycheck.outputs.passed != 'true' }}
+        run: echo "StoryCheck failed!" && exit 1

user-stories/ens_reg_commit/story.md

@@ -0,0 +1,35 @@
+# Register a new ENS domain
+
+This use story walks through an initial domain name search "storychecktest" and successful registration on a mobile device.
+
+## Prerequisites
+
+1. Chain
+   - Id 1
+   - RPC https://lb.drpc.org/ogrpc?network=ethereum&dkey=AnmpasF2C0JBqeAEzxVO8aTteiMlrW4R75hpDonbV6cR
+   - Block 23086523   
+2. Browser
+   - Pixel 7
+
+## User Steps
+
+1. Browse to https://app.ens.domains/
+1. Click Accept
+1. Click on search box
+1. Type storychecktest
+1. Press Enter
+1. Click Connect
+1. Click Browser Wallet
+1. Scroll down
+1. Select Ethereum payment method
+1. Click Next
+1. Click Skip Profile
+1. Scroll down
+1. Click Begin
+1. Click "Open Wallet"
+1. Wait 10 seconds
+
+## Expected Results
+
+- Verify commitment transaction succeeded [verifier](verifiers/tx_success.py)
+- Verify commitment timestamp set [verifier](verifiers/commitment_timestamp.py)

user-stories/ens_reg_commit/verifiers/commitment_timestamp.py

@@ -0,0 +1,46 @@
+# examples/ens_reg_commit/verifiers/commitment_timestamp.py
+from loguru import logger
+import json
+from pathlib import Path
+
+def verify(results_dir):
+    logger.info(f"results_dir: {results_dir}")
+    manifest_path = Path(results_dir) / "manifest.json"
+    with open(manifest_path, 'r') as f:
+        manifest = json.load(f)
+    tx_snapshot_path = Path(results_dir) / manifest["tx_snapshot"]
+    with open(tx_snapshot_path, 'r') as f:
+        tx_log = json.load(f)
+    logger.info("[Verifier: commitment_timestamp] Starting verification of commitment timestamp.")
+
+    if not tx_log:
+        logger.error("[Verifier: commitment_timestamp] No transactions in log")
+        return {'passed': False, 'error': 'No transactions in log'}
+
+    expected_selector = '0xf14fcbc8'  # ENS commit function selector
+    passed = True
+    error = None
+
+    for tx in tx_log:
+        if tx.get('writeTxException') is not None:
+            logger.error(f"[Verifier: commitment_timestamp] Tx failed, timestamp not set: {tx['writeTxException']}")
+            passed = False
+            error = 'Tx failed, timestamp not set'
+            break
+
+        params = tx.get('writeTx', {}).get('params', [{}])[0]
+        data = params.get('data', '')
+        if not data.startswith(expected_selector):
+            logger.error(f"[Verifier: commitment_timestamp] Unexpected tx data: {data}")
+            passed = False
+            error = 'Tx data does not match ENS commit call'
+            break
+
+        # Infer timestamp set since tx succeeded (deterministic)
+
+    if passed:
+        logger.info("[Verifier: commitment_timestamp] Verification passed. Commitment timestamp verified as set via successful commit tx. Transaction log: {tx_log}")
+    else:
+        logger.error(f"[Verifier: commitment_timestamp] Verification failed. Error: {error}. Transaction log: {tx_log}")
+
+    return {'passed': passed, 'error': error}

user-stories/ens_reg_commit/verifiers/tx_success.py

@@ -0,0 +1,30 @@
+# examples/ens_reg_commit/verifiers/tx_success.py
+from loguru import logger
+import json
+from pathlib import Path
+
+def verify(results_dir):
+    logger.info(f"results_dir: {results_dir}")
+    manifest_path = Path(results_dir) / "manifest.json"
+    with open(manifest_path, 'r') as f:
+        manifest = json.load(f)
+    tx_snapshot_path = Path(results_dir) / manifest["tx_snapshot"]
+    with open(tx_snapshot_path, 'r') as f:
+        tx_log = json.load(f)
+    logger.info("[Verifier: tx_success] Starting verification of transaction success.")
+    if not tx_log:
+        logger.error("[Verifier: tx_success] Verification failed. No transactions in log.")
+        return False
+    passed = True
+    for tx in tx_log:
+        if tx.get('writeTxException') is not None:
+            logger.error(f"[Verifier: tx_success] Transaction failed: {tx['writeTxException']}")
+            passed = False
+        if not tx.get('writeTxResult'):
+            logger.error(f"[Verifier: tx_success] No transaction result found in: {tx}")
+            passed = False
+    if passed:
+        logger.info(f"[Verifier: tx_success] Verification passed. Transaction log: {tx_log}")
+    else:
+        logger.error(f"[Verifier: tx_success] Verification failed. Transaction log: {tx_log}")
+    return passed