From dc5c14589eb0fab1eb3a83723960e6b2e4d740ea Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Mon, 4 Aug 2025 20:36:15 -0700 Subject: [PATCH 01/15] Migrate from heroku to github actions --- .github/workflows/hubcap-scheduler.yml | 124 +++++++++++++++++++ CONTRIBUTING.md | 13 ++ GITHUB_ACTIONS_SETUP.md | 162 +++++++++++++++++++++++++ RELEASE.md | 33 ++++- 4 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/hubcap-scheduler.yml create mode 100644 GITHUB_ACTIONS_SETUP.md diff --git a/.github/workflows/hubcap-scheduler.yml b/.github/workflows/hubcap-scheduler.yml new file mode 100644 index 0000000..7a99bed --- /dev/null +++ b/.github/workflows/hubcap-scheduler.yml @@ -0,0 +1,124 @@ +name: Hubcap Scheduler + +on: + schedule: + # Run every hour at :00 for production + - cron: '0 * * * *' + workflow_dispatch: + inputs: + environment: + description: 'Environment to run against' + required: true + default: 'test' + type: choice + options: + - 'test' + - 'production' + dry_run: + description: 'Run in dry-run mode (no PRs created)' + required: false + default: 'true' + type: choice + options: + - 'true' + - 'false' + +jobs: + hubcap: + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: ${{ github.event.inputs.environment || 'production' }} + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + + - name: Configure environment + env: + HUBCAP_CONFIG: ${{ secrets.HUBCAP_CONFIG }} + INPUT_DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} + ENVIRONMENT: ${{ github.event.inputs.environment || 'production' }} + run: | + echo "Using $ENVIRONMENT environment configuration" + echo "$HUBCAP_CONFIG" > base_config.json + + # Modify config for dry run if requested + if [ "$INPUT_DRY_RUN" = "true" ]; then + echo "Enabling dry-run mode (push_branches = false)" + jq '.push_branches = false' base_config.json > config.json + else + echo "Running in live mode (push_branches = true)" + cp base_config.json config.json + fi + + # Set the CONFIG environment variable for hubcap.py + echo "CONFIG<> $GITHUB_ENV + cat config.json >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + # Log the configuration (without sensitive data) + echo "Configuration summary:" + jq 'del(.user.token)' config.json + + - name: Run hubcap + run: | + echo "Starting hubcap execution..." + python3 hubcap.py 2>&1 | tee hubcap.log + + - name: Check execution results + if: always() + run: | + echo "=== Execution Summary ===" + + # Check for errors + if grep -q 'ERROR' hubcap.log; then + echo "❌ Found ERROR events in execution:" + grep 'ERROR' hubcap.log + echo "EXECUTION_STATUS=failed" >> $GITHUB_ENV + else + echo "✅ No ERROR-level log entries found" + fi + + # Check for successful completion + if grep -q 'hubcap execution completed successfully' hubcap.log; then + echo "✅ Hubcap completed successfully" + echo "EXECUTION_STATUS=success" >> $GITHUB_ENV + elif grep -q 'No packages have new versions to update' hubcap.log; then + echo "✅ Hubcap completed successfully (no updates needed)" + echo "EXECUTION_STATUS=success" >> $GITHUB_ENV + else + echo "❌ Hubcap did not complete successfully" + echo "EXECUTION_STATUS=failed" >> $GITHUB_ENV + fi + + # Summary of what was processed + if grep -q 'Pushing branches:' hubcap.log; then + echo "📝 Branches processed:" + grep 'Pushing branches:' hubcap.log + fi + + - name: Upload execution artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: hubcap-execution-${{ github.event.inputs.environment || 'production' }}-${{ github.run_number }} + path: | + hubcap.log + target/ + retention-days: 30 + + - name: Fail job if execution failed + if: always() && env.EXECUTION_STATUS == 'failed' + run: | + echo "❌ Hubcap execution failed - check logs above" + exit 1 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f56eaec..f5275a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,6 +39,7 @@ export CONFIG=$( Environments** and create two environments: + +#### Production Environment +- **Name**: `production` +- **Protection rules**: + - ✅ Required reviewers (recommended for production safety) + - ✅ Wait timer: 0 minutes +- **Environment secrets**: See step 2 below + +#### Test Environment +- **Name**: `test` +- **Protection rules**: None (allows automatic execution) +- **Environment secrets**: See step 2 below + +### 2. Configure Environment Secrets + +Each environment needs a `HUBCAP_CONFIG` secret with the appropriate configuration. + +#### Production Environment Secret +**Secret name**: `HUBCAP_CONFIG` +**Secret value**: +```json +{ + "user": { + "name": "dbt-hubcap", + "email": "buildbot@fishtownanalytics.com", + "token": "ghp_your_production_token_here" + }, + "org": "dbt-labs", + "repo": "hub.getdbt.com", + "push_branches": true, + "one_branch_per_repo": true +} +``` + +#### Test Environment Secret +**Secret name**: `HUBCAP_CONFIG` +**Secret value**: +```json +{ + "user": { + "name": "dbt-hubcap-test", + "email": "buildbot+test@fishtownanalytics.com", + "token": "ghp_your_test_token_here" + }, + "org": "dbt-labs", + "repo": "hub.getdbt.com-test", + "push_branches": true, + "one_branch_per_repo": true +} +``` + +### 3. GitHub Personal Access Tokens + +Create two GitHub Personal Access Tokens: + +#### Production Token +- **Scopes**: `repo`, `workflow` +- **Expiration**: Set appropriate expiration +- **Access**: Must have write access to `dbt-labs/hub.getdbt.com` + +#### Test Token +- **Scopes**: `repo`, `workflow` +- **Expiration**: Set appropriate expiration +- **Access**: Must have write access to `dbt-labs/hub.getdbt.com-test` + +## Usage + +### Automatic Execution (Production) +The workflow runs automatically every hour at `:00` in production mode. + +### Manual Execution +You can manually trigger the workflow with different options: + +#### Test Environment (Dry Run) +```bash +gh workflow run "Hubcap Scheduler" \ + --field environment=test \ + --field dry_run=true +``` + +#### Test Environment (Live) +```bash +gh workflow run "Hubcap Scheduler" \ + --field environment=test \ + --field dry_run=false +``` + +#### Production (Manual) +```bash +gh workflow run "Hubcap Scheduler" \ + --field environment=production \ + --field dry_run=false +``` + +### Via GitHub Web Interface +1. Go to **Actions > Hubcap Scheduler** +2. Click **Run workflow** +3. Select: + - **Environment**: `test` or `production` + - **Dry run**: `true` (no PRs) or `false` (create PRs) +4. Click **Run workflow** + +## Monitoring + +### Workflow Status +- View execution history in **Actions** tab +- Each run shows environment, duration, and status +- Failed runs will show error details in logs + +### Artifacts +Each execution saves: +- `hubcap.log`: Complete execution log +- `target/`: Cloned repositories and generated files +- Retention: 30 days + +### Notifications +Configure notifications in repository settings: +- **Settings > Notifications** +- Enable workflow failure notifications +- Set up Slack/email integration if needed + +## Troubleshooting + +### Common Issues + +**Token Permission Errors** +- Verify token has `repo` and `workflow` scopes +- Check token has write access to target repository +- Ensure token hasn't expired + +**Configuration Errors** +- Validate JSON syntax in `HUBCAP_CONFIG` secrets +- Check repository names match intended targets +- Verify user email and name are correct + +**Execution Failures** +- Check workflow logs for detailed error messages +- Review `hubcap.log` artifact for application-specific errors +- Verify target repository structure and accessibility + +### Getting Help +- Check workflow execution logs +- Review artifacts from failed runs +- Test with dry-run mode first +- Use test environment for debugging \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md index ef0fb79..7d296f0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,6 +1,8 @@ # Release instructions -This application is hosted on [Heroku](https://www.heroku.com), but it can also be executed in production mode locally. +This application runs on [GitHub Actions](https://github.com/features/actions) and can also be executed in production mode locally. + +**Note**: This application was previously hosted on Heroku. See [GitHub Actions Setup Guide](GITHUB_ACTIONS_SETUP.md) for current deployment instructions. ## Design overview It is designed to do the following: @@ -84,3 +86,32 @@ heroku https://git.heroku.com/dbt-hubcap.git (push) origin git@github.com:dbt-labs/hubcap.git (fetch) origin git@github.com:dbt-labs/hubcap.git (push) ``` + +## GitHub Actions production setup + +**Current deployment method**. See [GITHUB_ACTIONS_SETUP.md](GITHUB_ACTIONS_SETUP.md) for detailed setup instructions. + +### Overview +- **Schedule**: Runs automatically every hour at `:00` +- **Environments**: Separate `production` and `test` environments +- **Configuration**: Environment-specific secrets containing JSON configuration +- **Manual execution**: Can be triggered manually via GitHub UI or CLI + +### Quick setup +1. Create `production` and `test` environments in repository settings +2. Add `HUBCAP_CONFIG` secret to each environment with appropriate JSON configuration +3. The workflow runs automatically on schedule or can be triggered manually + +### Manual execution examples +```bash +# Test environment with dry run (no PRs created) +gh workflow run "Hubcap Scheduler" --field environment=test --field dry_run=true + +# Production environment (live) +gh workflow run "Hubcap Scheduler" --field environment=production --field dry_run=false +``` + +### Monitoring +- View execution logs in repository **Actions** tab +- Each run creates artifacts with logs and generated files +- Failed executions send notifications (if configured) From 0ce6d623f5095b093b27ff98ced3ade9666cf342 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Mon, 4 Aug 2025 20:41:46 -0700 Subject: [PATCH 02/15] pre-commit fixes --- .github/workflows/hubcap-scheduler.yml | 16 ++++++++-------- GITHUB_ACTIONS_SETUP.md | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/hubcap-scheduler.yml b/.github/workflows/hubcap-scheduler.yml index 7a99bed..34fb94e 100644 --- a/.github/workflows/hubcap-scheduler.yml +++ b/.github/workflows/hubcap-scheduler.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 environment: ${{ github.event.inputs.environment || 'production' }} - + steps: - name: Check out the repository uses: actions/checkout@v4 @@ -51,7 +51,7 @@ jobs: run: | echo "Using $ENVIRONMENT environment configuration" echo "$HUBCAP_CONFIG" > base_config.json - + # Modify config for dry run if requested if [ "$INPUT_DRY_RUN" = "true" ]; then echo "Enabling dry-run mode (push_branches = false)" @@ -60,12 +60,12 @@ jobs: echo "Running in live mode (push_branches = true)" cp base_config.json config.json fi - + # Set the CONFIG environment variable for hubcap.py echo "CONFIG<> $GITHUB_ENV cat config.json >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - + # Log the configuration (without sensitive data) echo "Configuration summary:" jq 'del(.user.token)' config.json @@ -79,7 +79,7 @@ jobs: if: always() run: | echo "=== Execution Summary ===" - + # Check for errors if grep -q 'ERROR' hubcap.log; then echo "❌ Found ERROR events in execution:" @@ -88,7 +88,7 @@ jobs: else echo "✅ No ERROR-level log entries found" fi - + # Check for successful completion if grep -q 'hubcap execution completed successfully' hubcap.log; then echo "✅ Hubcap completed successfully" @@ -100,7 +100,7 @@ jobs: echo "❌ Hubcap did not complete successfully" echo "EXECUTION_STATUS=failed" >> $GITHUB_ENV fi - + # Summary of what was processed if grep -q 'Pushing branches:' hubcap.log; then echo "📝 Branches processed:" @@ -121,4 +121,4 @@ jobs: if: always() && env.EXECUTION_STATUS == 'failed' run: | echo "❌ Hubcap execution failed - check logs above" - exit 1 \ No newline at end of file + exit 1 diff --git a/GITHUB_ACTIONS_SETUP.md b/GITHUB_ACTIONS_SETUP.md index 5736f4b..2a4b502 100644 --- a/GITHUB_ACTIONS_SETUP.md +++ b/GITHUB_ACTIONS_SETUP.md @@ -159,4 +159,4 @@ Configure notifications in repository settings: - Check workflow execution logs - Review artifacts from failed runs - Test with dry-run mode first -- Use test environment for debugging \ No newline at end of file +- Use test environment for debugging From 0a5db9e10ce5759aa37bca864cad9ffb14beae71 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Tue, 5 Aug 2025 00:00:47 -0700 Subject: [PATCH 03/15] Add 'fusion-schema-compat' to index.json and version.json --- .github/workflows/hubcap-scheduler.yml | 4 + .github/workflows/main.yml | 50 ++-- .gitignore | 1 + hubcap/records.py | 164 +++++++++++-- test_fusion_compatibility.py | 308 +++++++++++++++++++++++++ test_fusion_integration.py | 290 +++++++++++++++++++++++ 6 files changed, 786 insertions(+), 31 deletions(-) create mode 100644 test_fusion_compatibility.py create mode 100644 test_fusion_integration.py diff --git a/.github/workflows/hubcap-scheduler.yml b/.github/workflows/hubcap-scheduler.yml index 34fb94e..4152f2d 100644 --- a/.github/workflows/hubcap-scheduler.yml +++ b/.github/workflows/hubcap-scheduler.yml @@ -43,6 +43,10 @@ jobs: python -m pip install --upgrade pip python -m pip install -r requirements.txt + - name: Install dbt fusion + run: | + curl -fsSL https://public.cdn.getdbt.com/fs/install/install.sh | sh -s -- --update + - name: Configure environment env: HUBCAP_CONFIG: ${{ secrets.HUBCAP_CONFIG }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 422dd67..b24cbd7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,11 +23,15 @@ jobs: with: python-version: '3.9' - - name: Install dependencies + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Install dependencies with uv run: | - python -m pip install --user --upgrade pip - python -m pip install -r requirements.txt - python -m pip install -r requirements-dev.txt + uv pip install -r requirements.txt + uv pip install -r requirements-dev.txt - name: Create test config run: | @@ -84,14 +88,17 @@ jobs: with: python-version: '3.9' - - name: Install python dependencies + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Install python dependencies with uv run: | - python -m pip install --user --upgrade pip - python -m pip --version - python -m pip install pre-commit + uv pip install pre-commit pre-commit --version - python -m pip install -r requirements.txt - python -m pip install -r requirements-dev.txt + uv pip install -r requirements.txt + uv pip install -r requirements-dev.txt - name: Run pre-commit hooks run: pre-commit run --all-files --show-diff-on-failure @@ -116,11 +123,20 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install python tools + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Install dependencies with uv + run: | + uv pip install -r requirements.txt + uv pip install -r requirements-dev.txt + + - name: Install dbt fusion (for integration tests) + run: | + curl -fsSL https://public.cdn.getdbt.com/fs/install/install.sh | sh -s -- --update + + - name: Run tests with pytest run: | - python -m pip install --user --upgrade pip - python -m pip --version - python -m pip install tox - tox --version - - name: Run tests - run: tox + uv run pytest tests/ test_fusion_compatibility.py test_fusion_integration.py -v --tb=short diff --git a/.gitignore b/.gitignore index 6aab5a0..925f338 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ target/ __pycache__/ config.json hub.current.json +logs/ diff --git a/hubcap/records.py b/hubcap/records.py index 9937ec4..48b2c22 100644 --- a/hubcap/records.py +++ b/hubcap/records.py @@ -16,6 +16,124 @@ from hubcap import version +def check_fusion_schema_compatibility(repo_path: Path) -> bool: + """ + Check if a dbt package is fusion schema compatible by running 'dbtf parse'. + + Args: + repo_path: Path to the dbt package repository + + Returns: + True if fusion compatible (dbtf parse exits with code 0), False otherwise + """ + # Add a test profiles.yml to the current directory + profiles_path = repo_path / Path("profiles.yml") + try: + with open(profiles_path, "a") as f: + f.write( + "\n" + "test_schema_compat:\n" + " target: dev\n" + " outputs:\n" + " dev:\n" + " type: postgres\n" + " host: localhost\n" + " port: 5432\n" + " user: postgres\n" + " password: postgres\n" + " dbname: postgres\n" + " schema: public\n" + ) + + # Ensure the `_DBT_FUSION_STRICT_MODE` is set (this will ensure fusion errors on schema violations) + os.environ["_DBT_FUSION_STRICT_MODE"] = "1" + + # Run dbtf parse command (try dbtf first, fall back to dbt) + try: + # Try dbtf first (without shell=True to get proper FileNotFoundError) + result = subprocess.run( + [ + "dbtf", + "parse", + "--profile", + "test_schema_compat", + "--project-dir", + str(repo_path), + ], + capture_output=True, + timeout=60, + ) + # If dbtf command exists but returns error mentioning it's not found, fall back to dbt + if ( + result.returncode != 0 + and result.stderr + and b"not found" in result.stderr + ): + raise FileNotFoundError("dbtf command not found") + except FileNotFoundError: + # Fall back to dbt command, but validate that this is dbt-fusion + version_result = subprocess.run( + ["dbt", "--version"], capture_output=True, timeout=60 + ) + if b"dbt-fusion" not in version_result.stdout: + raise FileNotFoundError( + "dbt-fusion command not found - regular dbt-core detected instead" + ) + + # Run dbt parse since we have dbt-fusion + result = subprocess.run( + [ + "dbt", + "parse", + "--profile", + "test_schema_compat", + "--project-dir", + str(repo_path), + ], + capture_output=True, + timeout=60, + ) + + # Return True if exit code is 0 (success) + is_compatible = result.returncode == 0 + + if is_compatible: + logging.info(f"Package at {repo_path} is fusion schema compatible") + else: + logging.info(f"Package at {repo_path} is not fusion schema compatible") + + # Remove the test profile + os.remove(profiles_path) + + return is_compatible + + except subprocess.TimeoutExpired: + logging.warning(f"dbtf parse timed out for package at {repo_path}") + try: + os.remove(profiles_path) + except Exception: + pass + return False + except FileNotFoundError: + logging.warning( + f"dbtf command not found - skipping fusion compatibility check for {repo_path}" + ) + try: + os.remove(profiles_path) + except Exception: + pass + return False + except Exception as e: + logging.warning( + f"Error checking fusion compatibility for {repo_path}: {str(e)}" + ) + try: + os.remove(profiles_path) + except Exception: + pass + return False + + class PullRequestStrategy(ABC): @abstractmethod def pull_request_title(self, org: str, repo: str) -> str: @@ -89,6 +207,8 @@ def __init__( self.package_name = package_name self.existing_tags = existing_tags self.new_tags = new_tags + # Track fusion compatibility for each tag + self.fusion_compatibility = {} def run(self, main_dir, pr_strategy): os.chdir(main_dir) @@ -101,16 +221,6 @@ def run(self, main_dir, pr_strategy): index_filepath = ( Path(os.path.dirname(self.hub_version_index_path)) / "index.json" ) - new_index_entry = self.make_index( - self.github_username, - self.github_repo_name, - self.package_name, - self.fetch_index_file_contents(index_filepath), - set(self.new_tags) | set(self.existing_tags), - ) - with open(index_filepath, "w") as f: - logging.info(f"writing index.json to {index_filepath}") - f.write(str(json.dumps(new_index_entry, indent=4))) # create a version spec for each tag for tag in self.new_tags: @@ -119,9 +229,11 @@ def run(self, main_dir, pr_strategy): git_helper.run_cmd(f"git checkout tags/{tag}") packages = package.parse_pkgs(Path(os.getcwd())) require_dbt_version = package.parse_require_dbt_version(Path(os.getcwd())) - - # return to hub and build spec os.chdir(main_dir) + # check fusion compatibility + is_fusion_compatible = check_fusion_schema_compatibility(Path(os.getcwd())) + self.fusion_compatibility[tag] = is_fusion_compatible + # return to hub and build spec package_spec = self.make_spec( self.github_username, self.github_repo_name, @@ -129,6 +241,7 @@ def run(self, main_dir, pr_strategy): packages, require_dbt_version, tag, + is_fusion_compatible, ) version_path = self.hub_version_index_path / Path(f"{tag}.json") @@ -141,6 +254,18 @@ def run(self, main_dir, pr_strategy): git_helper.run_cmd("git add -A") subprocess.run(args=["git", "commit", "-am", f"{msg}"], capture_output=True) + new_index_entry = self.make_index( + self.github_username, + self.github_repo_name, + self.package_name, + self.fetch_index_file_contents(index_filepath), + set(self.new_tags) | set(self.existing_tags), + self.fusion_compatibility, + ) + with open(index_filepath, "w") as f: + logging.info(f"writing index.json to {index_filepath}") + f.write(str(json.dumps(new_index_entry, indent=4))) + # if succesful return branchname return branch_name, self.github_username, self.github_repo_name @@ -159,7 +284,9 @@ def cut_version_branch(self, pr_strategy): return branch_name - def make_index(self, org_name, repo, package_name, existing, tags): + def make_index( + self, org_name, repo, package_name, existing, tags, fusion_compatibility + ): description = "dbt models for {}".format(repo) assets = {"logo": "logos/placeholder.svg"} @@ -176,6 +303,7 @@ def make_index(self, org_name, repo, package_name, existing, tags): "namespace": org_name, "description": description, "latest": latest_version.replace("=", ""), # LOL + "fusion-schema-compat": fusion_compatibility[latest_version], "assets": assets, } @@ -210,7 +338,14 @@ def get_sha1(self, url): return digest def make_spec( - self, org, repo, package_name, packages, require_dbt_version, version + self, + org, + repo, + package_name, + packages, + require_dbt_version, + version, + fusion_schema_compat=False, ): """The hub needs these specs for packages to be discoverable by deps and on the web""" tarball_url = "https://codeload.github.com/{}/{}/tar.gz/{}".format( @@ -235,4 +370,5 @@ def make_spec( ), }, "downloads": {"tarball": tarball_url, "format": "tgz", "sha1": sha1}, + "fusion-schema-compat": fusion_schema_compat, } diff --git a/test_fusion_compatibility.py b/test_fusion_compatibility.py new file mode 100644 index 0000000..dec2e0c --- /dev/null +++ b/test_fusion_compatibility.py @@ -0,0 +1,308 @@ +"""Unit tests for the check_fusion_schema_compatibility function""" + +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch, MagicMock +import subprocess + +# Import the actual function from the hubcap.records module +from hubcap.records import check_fusion_schema_compatibility + + +class TestCheckFusionSchemaCompatibility(unittest.TestCase): + """Test cases for check_fusion_schema_compatibility function""" + + def setUp(self): + """Set up test fixtures""" + # Create a temporary directory for each test + self.temp_dir = tempfile.mkdtemp() + self.repo_path = Path(self.temp_dir) + + # Create a minimal dbt_project.yml file + dbt_project_content = """ +name: 'test_package' +version: '1.0.0' +profile: 'test_schema_compat' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" +""" + with open(self.repo_path / "dbt_project.yml", "w") as f: + f.write(dbt_project_content) + + # Store original environment state + self.original_env = os.environ.get("_DBT_FUSION_STRICT_MODE") + + def tearDown(self): + """Clean up test fixtures""" + # Restore original environment + if self.original_env is not None: + os.environ["_DBT_FUSION_STRICT_MODE"] = self.original_env + elif "_DBT_FUSION_STRICT_MODE" in os.environ: + del os.environ["_DBT_FUSION_STRICT_MODE"] + + # Clean up temp directory + import shutil + + shutil.rmtree(self.temp_dir, ignore_errors=True) + + @patch("hubcap.records.subprocess.run") + def test_fusion_compatible_success(self, mock_subprocess): + """Test successful fusion compatibility check""" + # Mock successful dbtf parse command + mock_result = MagicMock() + mock_result.returncode = 0 + mock_subprocess.return_value = mock_result + + # Run the function + result = check_fusion_schema_compatibility(self.repo_path) + + # Assertions + self.assertTrue(result) + + # Verify subprocess was called with correct arguments + mock_subprocess.assert_called_once_with( + [ + "dbtf", + "parse", + "--profile", + "test_schema_compat", + "--project-dir", + str(self.repo_path), + ], + capture_output=True, + timeout=60, + ) + + # Verify environment variable was set + self.assertEqual(os.environ.get("_DBT_FUSION_STRICT_MODE"), "1") + + # Verify profiles.yml was cleaned up + profiles_path = self.repo_path / "profiles.yml" + self.assertFalse(profiles_path.exists()) + + @patch("hubcap.records.subprocess.run") + def test_fusion_incompatible_failure(self, mock_subprocess): + """Test failed fusion compatibility check""" + # Mock failed dbtf parse command + mock_result = MagicMock() + mock_result.returncode = 1 + mock_subprocess.return_value = mock_result + + # Run the function + result = check_fusion_schema_compatibility(self.repo_path) + + # Assertions + self.assertFalse(result) + + # Verify subprocess was called + mock_subprocess.assert_called_once() + + # Verify profiles.yml was cleaned up + profiles_path = self.repo_path / "profiles.yml" + self.assertFalse(profiles_path.exists()) + + @patch("hubcap.records.subprocess.run") + def test_profiles_yml_creation_and_content(self, mock_subprocess): + """Test that profiles.yml is created with correct content""" + # Mock successful command + mock_result = MagicMock() + mock_result.returncode = 0 + mock_subprocess.return_value = mock_result + + # Capture the profiles.yml content during execution + profiles_content = None + + def capture_profiles_content(*args, **kwargs): + nonlocal profiles_content + profiles_path = self.repo_path / "profiles.yml" + if profiles_path.exists(): + with open(profiles_path, "r") as f: + profiles_content = f.read() + return mock_result + + mock_subprocess.side_effect = capture_profiles_content + + # Run the function + check_fusion_schema_compatibility(self.repo_path) + + # Verify profiles.yml content + self.assertIsNotNone(profiles_content) + self.assertIn("test_schema_compat:", profiles_content) + self.assertIn("type: postgres", profiles_content) + self.assertIn("host: localhost", profiles_content) + self.assertIn("port: 5432", profiles_content) + self.assertIn("user: postgres", profiles_content) + self.assertIn("password: postgres", profiles_content) + self.assertIn("dbname: postgres", profiles_content) + self.assertIn("schema: public", profiles_content) + + @patch("hubcap.records.subprocess.run") + def test_timeout_handling(self, mock_subprocess): + """Test timeout scenario""" + # Mock timeout exception + mock_subprocess.side_effect = subprocess.TimeoutExpired( + cmd=["dbtf", "parse"], timeout=60 + ) + + # Run the function + result = check_fusion_schema_compatibility(self.repo_path) + + # Assertions + self.assertFalse(result) + + # Verify profiles.yml was cleaned up even after timeout + profiles_path = self.repo_path / "profiles.yml" + self.assertFalse(profiles_path.exists()) + + @patch("hubcap.records.subprocess.run") + def test_file_not_found_handling(self, mock_subprocess): + """Test FileNotFoundError scenario (dbtf command not available)""" + # Mock FileNotFoundError + mock_subprocess.side_effect = FileNotFoundError("dbtf command not found") + + # Run the function + result = check_fusion_schema_compatibility(self.repo_path) + + # Assertions + self.assertFalse(result) + + # Verify profiles.yml was cleaned up + profiles_path = self.repo_path / "profiles.yml" + self.assertFalse(profiles_path.exists()) + + @patch("hubcap.records.subprocess.run") + def test_general_exception_handling(self, mock_subprocess): + """Test general exception handling""" + # Mock a general exception + mock_subprocess.side_effect = Exception("Unexpected error") + + # Run the function + result = check_fusion_schema_compatibility(self.repo_path) + + # Assertions + self.assertFalse(result) + + # Verify profiles.yml was cleaned up + profiles_path = self.repo_path / "profiles.yml" + self.assertFalse(profiles_path.exists()) + + @patch("hubcap.records.subprocess.run") + def test_existing_profiles_yml_handling(self, mock_subprocess): + """Test behavior when profiles.yml already exists""" + # Create an existing profiles.yml file + existing_content = "existing_profile:\n target: dev\n" + profiles_path = self.repo_path / "profiles.yml" + with open(profiles_path, "w") as f: + f.write(existing_content) + + # Mock successful command + mock_result = MagicMock() + mock_result.returncode = 0 + mock_subprocess.return_value = mock_result + + # Run the function + result = check_fusion_schema_compatibility(self.repo_path) + + # Assertions + self.assertTrue(result) + + # Verify the original file is gone (it gets removed) + self.assertFalse(profiles_path.exists()) + + @patch("hubcap.records.logging.info") + @patch("hubcap.records.subprocess.run") + def test_logging_success(self, mock_subprocess, mock_logging): + """Test that success is logged correctly""" + # Mock successful command + mock_result = MagicMock() + mock_result.returncode = 0 + mock_subprocess.return_value = mock_result + + # Run the function + check_fusion_schema_compatibility(self.repo_path) + + # Verify logging was called with success message + mock_logging.assert_called_with( + f"Package at {self.repo_path} is fusion schema compatible" + ) + + @patch("hubcap.records.logging.info") + @patch("hubcap.records.subprocess.run") + def test_logging_failure(self, mock_subprocess, mock_logging): + """Test that failure is logged correctly""" + # Mock failed command + mock_result = MagicMock() + mock_result.returncode = 1 + mock_subprocess.return_value = mock_result + + # Run the function + check_fusion_schema_compatibility(self.repo_path) + + # Verify logging was called with failure message + mock_logging.assert_called_with( + f"Package at {self.repo_path} is not fusion schema compatible" + ) + + @patch("hubcap.records.logging.warning") + @patch("hubcap.records.subprocess.run") + def test_logging_timeout(self, mock_subprocess, mock_logging): + """Test that timeout is logged correctly""" + # Mock timeout + mock_subprocess.side_effect = subprocess.TimeoutExpired( + cmd=["dbtf", "parse"], timeout=60 + ) + + # Run the function + check_fusion_schema_compatibility(self.repo_path) + + # Verify logging was called with timeout message + mock_logging.assert_called_with( + f"dbtf parse timed out for package at {self.repo_path}" + ) + + def test_environment_variable_set(self): + """Test that _DBT_FUSION_STRICT_MODE environment variable is properly set""" + with patch("hubcap.records.subprocess.run") as mock_subprocess: + mock_result = MagicMock() + mock_result.returncode = 0 + mock_subprocess.return_value = mock_result + + # Ensure env var is not set initially + if "_DBT_FUSION_STRICT_MODE" in os.environ: + del os.environ["_DBT_FUSION_STRICT_MODE"] + + # Run the function + check_fusion_schema_compatibility(self.repo_path) + + # Verify environment variable was set to "1" + self.assertEqual(os.environ.get("_DBT_FUSION_STRICT_MODE"), "1") + + def test_profiles_yml_cleanup_on_file_creation_failure(self): + """Test that cleanup works even if file creation fails""" + # Make the directory read-only to cause file creation to fail + self.repo_path.chmod(0o444) + + try: + result = check_fusion_schema_compatibility(self.repo_path) + # Should return False due to the exception + self.assertFalse(result) + finally: + # Restore permissions for cleanup + self.repo_path.chmod(0o755) + + +if __name__ == "__main__": + # Run with verbose output + unittest.main(verbosity=2) diff --git a/test_fusion_integration.py b/test_fusion_integration.py new file mode 100644 index 0000000..1ed0e59 --- /dev/null +++ b/test_fusion_integration.py @@ -0,0 +1,290 @@ +"""Integration tests for the check_fusion_schema_compatibility function""" + +import os +import tempfile +import unittest +import shutil +from pathlib import Path + +# Import the actual function from the hubcap.records module +from hubcap.records import check_fusion_schema_compatibility + + +class TestFusionSchemaCompatibilityIntegration(unittest.TestCase): + """Integration test cases that actually run dbtf parse""" + + def setUp(self): + """Set up test fixtures""" + # Create a temporary directory for each test + self.temp_dir = tempfile.mkdtemp() + self.repo_path = Path(self.temp_dir) + + # Store original environment state + self.original_env = os.environ.get("_DBT_FUSION_STRICT_MODE") + + def tearDown(self): + """Clean up test fixtures""" + # Restore original environment + if self.original_env is not None: + os.environ["_DBT_FUSION_STRICT_MODE"] = self.original_env + elif "_DBT_FUSION_STRICT_MODE" in os.environ: + del os.environ["_DBT_FUSION_STRICT_MODE"] + + # Clean up temp directory + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _create_basic_dbt_project(self, project_name="test_package"): + """Create a basic dbt project structure""" + # Create dbt_project.yml + dbt_project_content = f""" +name: '{project_name}' +version: '1.0.0' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +models: + {project_name}: + +schema: public + +materialized: table + +vars: + # Variables for testing + test_var: "test_value" +""" + with open(self.repo_path / "dbt_project.yml", "w") as f: + f.write(dbt_project_content) + + # Create directories + for dir_name in ["models", "tests", "macros", "seeds", "snapshots", "analyses"]: + (self.repo_path / dir_name).mkdir(exist_ok=True) + + def _create_simple_model(self, model_name="test_model"): + """Create a simple dbt model""" + models_dir = self.repo_path / "models" + models_dir.mkdir(exist_ok=True) + + model_content = """ +{{ config(materialized='table') }} + +select + 1 as id, + 'test' as name, + current_timestamp as created_at +""" + with open(models_dir / f"{model_name}.sql", "w") as f: + f.write(model_content) + + def _create_fusion_compatible_model(self): + """Create a model that should be fusion schema compatible""" + models_dir = self.repo_path / "models" + models_dir.mkdir(exist_ok=True) + + # Simple select statement that should be fusion compatible + model_content = """ +{{ config(materialized='table') }} + +select + cast(1 as integer) as id, + cast('test' as varchar(50)) as name, + cast(current_timestamp as timestamp) as created_at +""" + with open(models_dir / "fusion_compatible_model.sql", "w") as f: + f.write(model_content) + + def _create_potentially_incompatible_model(self): + """Create a model that might have fusion compatibility issues""" + models_dir = self.repo_path / "models" + models_dir.mkdir(exist_ok=True) + + # Model with complex transformations that might cause fusion issues + model_content = """ +{{ config(materialized='table') }} + +select + id, + name, + case + when length(name) > 10 then 'long' + else 'short' + end as name_category, + row_number() over (partition by name order by id) as row_num +from ( + select 1 as id, 'test' as name + union all + select 2 as id, 'another_test_name' as name +) base_data +""" + with open(models_dir / "complex_model.sql", "w") as f: + f.write(model_content) + + def test_dbtf_command_available(self): + """Test if dbtf command or dbt-fusion is available in the environment""" + import subprocess + + try: + # Try dbtf first + result = subprocess.run( + ["dbtf", "--version"], capture_output=True, timeout=10 + ) + if result.returncode == 0: + print(f"dbtf version: {result.stdout.decode().strip()}") + return True + except FileNotFoundError: + pass + + try: + # Fall back to dbt command, but it must be dbt-fusion + result = subprocess.run( + ["dbt", "--version"], capture_output=True, timeout=10 + ) + if result.returncode == 0: + output = result.stdout.decode().strip() + print(f"dbt version: {output}") + if "dbt-fusion" in output: + return True + else: + self.fail( + "dbt-fusion is required for fusion compatibility integration tests, but found regular dbt-core instead" + ) + else: + self.fail("dbt command returned non-zero exit code") + except FileNotFoundError: + self.fail( + "Neither dbtf nor dbt command found in PATH - dbt-fusion is required for integration tests" + ) + except subprocess.TimeoutExpired: + self.fail("dbt command timed out") + + def test_fusion_compatibility_simple_project(self): + """Test fusion compatibility with a simple dbt project""" + # Check if dbtf is available + self.test_dbtf_command_available() + + # Create a basic project + self._create_basic_dbt_project("simple_test") + self._create_fusion_compatible_model() + + # Test fusion compatibility + result = check_fusion_schema_compatibility(self.repo_path) + + # Since we have dbt-fusion available and this is a simple valid project, it should be compatible + self.assertTrue(result, "Simple dbt project should be fusion schema compatible") + print(f"Simple project fusion compatibility: {result}") + + def test_fusion_compatibility_complex_project(self): + """Test fusion compatibility with a more complex dbt project""" + # Check if dbtf is available + self.test_dbtf_command_available() + + # Create a project with potentially complex models + self._create_basic_dbt_project("complex_test") + self._create_fusion_compatible_model() + self._create_potentially_incompatible_model() + + # Test fusion compatibility + result = check_fusion_schema_compatibility(self.repo_path) + + # Since we have dbt-fusion available and this is a valid project, it should be compatible + self.assertTrue( + result, "Complex dbt project should be fusion schema compatible" + ) + print(f"Complex project fusion compatibility: {result}") + + def test_fusion_compatibility_empty_project(self): + """Test fusion compatibility with an empty dbt project""" + # Check if dbtf is available + self.test_dbtf_command_available() + + # Create a basic project with no models + self._create_basic_dbt_project("empty_test") + + # Test fusion compatibility - empty project should be compatible + result = check_fusion_schema_compatibility(self.repo_path) + + self.assertTrue(result, "Empty dbt project should be fusion schema compatible") + print(f"Empty project fusion compatibility: {result}") + + def test_profiles_yml_creation_and_cleanup(self): + """Test that profiles.yml is created and cleaned up properly""" + # Check if dbtf is available + self.test_dbtf_command_available() + + self._create_basic_dbt_project("cleanup_test") + self._create_simple_model() + + profiles_path = self.repo_path / "profiles.yml" + + # Ensure profiles.yml doesn't exist before + self.assertFalse(profiles_path.exists()) + + # Run the function + result = check_fusion_schema_compatibility(self.repo_path) + + # Ensure profiles.yml is cleaned up after + self.assertFalse(profiles_path.exists()) + # Since this is a valid project with dbt-fusion available, it should be compatible + self.assertTrue(result, "Valid dbt project should be fusion schema compatible") + + def test_environment_variable_setting(self): + """Test that _DBT_FUSION_STRICT_MODE is set during execution""" + # Check if dbtf is available + self.test_dbtf_command_available() + + self._create_basic_dbt_project("env_test") + self._create_simple_model() + + # Clear environment variable if set + if "_DBT_FUSION_STRICT_MODE" in os.environ: + del os.environ["_DBT_FUSION_STRICT_MODE"] + + # Run the function + result = check_fusion_schema_compatibility(self.repo_path) + + # Since this is a valid project with dbt-fusion available, it should be compatible + self.assertTrue(result, "Valid dbt project should be fusion schema compatible") + # The function should have set the environment variable during execution + self.assertEqual(os.environ.get("_DBT_FUSION_STRICT_MODE"), "1") + + def test_invalid_dbt_project(self): + """Test behavior with an invalid dbt project""" + # Check if dbtf is available + self.test_dbtf_command_available() + + # Create an invalid dbt_project.yml + invalid_content = "invalid: yaml: content:" + with open(self.repo_path / "dbt_project.yml", "w") as f: + f.write(invalid_content) + + # This should return False due to the invalid project + result = check_fusion_schema_compatibility(self.repo_path) + + # Should return False for invalid projects + self.assertFalse(result) + + def test_missing_dbt_project_yml(self): + """Test behavior when dbt_project.yml is missing""" + # Check if dbtf is available + self.test_dbtf_command_available() + + # Don't create dbt_project.yml - just use empty directory + + # This should return False due to missing dbt_project.yml + result = check_fusion_schema_compatibility(self.repo_path) + + # Should return False for missing project file + self.assertFalse(result) + + +if __name__ == "__main__": + # Run with verbose output + unittest.main(verbosity=2) From ebfd3c0801c55cc80b48b424383219581801bcfd Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Tue, 5 Aug 2025 00:11:52 -0700 Subject: [PATCH 04/15] Add testing for UpdateTask and move tests into appropriate location --- .github/workflows/main.yml | 2 +- hubcap/records.py | 4 +- test_update_task_fusion.py | 500 ++++++++++++++++++ .../test_fusion_compatibility.py | 0 .../test_fusion_integration.py | 0 tests/test_update_task_fusion.py | 500 ++++++++++++++++++ 6 files changed, 1004 insertions(+), 2 deletions(-) create mode 100644 test_update_task_fusion.py rename test_fusion_compatibility.py => tests/test_fusion_compatibility.py (100%) rename test_fusion_integration.py => tests/test_fusion_integration.py (100%) create mode 100644 tests/test_update_task_fusion.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b24cbd7..7d2d3f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -139,4 +139,4 @@ jobs: - name: Run tests with pytest run: | - uv run pytest tests/ test_fusion_compatibility.py test_fusion_integration.py -v --tb=short + uv run pytest tests/ -v --tb=short diff --git a/hubcap/records.py b/hubcap/records.py index 48b2c22..7e0b6eb 100644 --- a/hubcap/records.py +++ b/hubcap/records.py @@ -231,7 +231,9 @@ def run(self, main_dir, pr_strategy): require_dbt_version = package.parse_require_dbt_version(Path(os.getcwd())) os.chdir(main_dir) # check fusion compatibility - is_fusion_compatible = check_fusion_schema_compatibility(Path(os.getcwd())) + is_fusion_compatible = check_fusion_schema_compatibility( + self.local_path_to_repo + ) self.fusion_compatibility[tag] = is_fusion_compatible # return to hub and build spec package_spec = self.make_spec( diff --git a/test_update_task_fusion.py b/test_update_task_fusion.py new file mode 100644 index 0000000..62cc7bf --- /dev/null +++ b/test_update_task_fusion.py @@ -0,0 +1,500 @@ +"""Tests for UpdateTask fusion compatibility JSON generation""" + +import json +import os +import tempfile +import unittest +import shutil +from pathlib import Path +from unittest.mock import patch, MagicMock + +# Import the classes we need to test +from hubcap.records import UpdateTask + + +class TestUpdateTaskFusionCompatibility(unittest.TestCase): + """Test UpdateTask's generation of index.json and version.json with fusion compatibility""" + + def setUp(self): + """Set up test fixtures""" + # Create temporary directories + self.temp_dir = tempfile.mkdtemp() + self.hub_dir = Path(self.temp_dir) / "hub.getdbt.com" + self.package_dir = Path(self.temp_dir) / "test_org_test_package" + + # Create hub directory structure + self.hub_dir.mkdir(parents=True) + self.package_dir.mkdir(parents=True) + + # Create a basic dbt project in the package directory + self._create_test_package() + + # Create UpdateTask with test data + self.update_task = UpdateTask( + github_username="test_org", + github_repo_name="test_package", + local_path_to_repo=self.package_dir, + package_name="test_package", + existing_tags=[], + new_tags=["1.0.0", "1.1.0"], + hub_repo="hub.getdbt.com", + ) + + def tearDown(self): + """Clean up test fixtures""" + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _create_test_package(self): + """Create a test dbt package structure""" + # Create dbt_project.yml + dbt_project_content = """ +name: 'test_package' +version: '1.0.0' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +models: + test_package: + +schema: public + +materialized: table +""" + with open(self.package_dir / "dbt_project.yml", "w") as f: + f.write(dbt_project_content) + + # Create models directory and a simple model + models_dir = self.package_dir / "models" + models_dir.mkdir(exist_ok=True) + + model_content = """ +{{ config(materialized='table') }} + +select + 1 as id, + 'test' as name, + current_timestamp as created_at +""" + with open(models_dir / "test_model.sql", "w") as f: + f.write(model_content) + + # Create a packages.yml file (sometimes needed) + packages_content = """ +packages: + - package: dbt-labs/dbt_utils + version: ">=1.0.0" +""" + with open(self.package_dir / "packages.yml", "w") as f: + f.write(packages_content) + + @patch("hubcap.records.git_helper.run_cmd") + @patch("hubcap.records.subprocess.run") + @patch("hubcap.records.package.parse_pkgs") + @patch("hubcap.records.package.parse_require_dbt_version") + @patch("hubcap.records.check_fusion_schema_compatibility") + @patch("hubcap.records.UpdateTask.get_sha1") + def test_fusion_compatible_package_json_generation( + self, + mock_sha1, + mock_fusion_check, + mock_parse_dbt_version, + mock_parse_pkgs, + mock_subprocess, + mock_git_cmd, + ): + """Test that fusion-compatible packages generate correct JSON files""" + # Mock returns + mock_sha1.return_value = "abc123def456" + mock_fusion_check.return_value = True # Fusion compatible + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_parse_pkgs.return_value = [ + {"name": "dbt-labs/dbt_utils", "version": ">=1.0.0"} + ] + mock_subprocess.return_value = MagicMock(returncode=0) + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = "test-branch" + + # Run the UpdateTask + os.chdir(self.temp_dir) + branch_name, org_name, package_name = self.update_task.run( + str(self.hub_dir.parent), mock_pr_strategy + ) + + # Verify the branch and basic info + self.assertEqual(branch_name, "test-branch") + self.assertEqual(org_name, "test_org") + self.assertEqual(package_name, "test_package") + + # Check that fusion compatibility was checked for each tag + self.assertEqual(mock_fusion_check.call_count, 2) # Called for both tags + + # Verify version-specific JSON files were created with correct fusion compatibility + version_dir = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "versions" + ) + + # Check 1.0.0.json + version_1_0_0_file = version_dir / "1.0.0.json" + self.assertTrue(version_1_0_0_file.exists()) + + with open(version_1_0_0_file, "r") as f: + version_1_0_0_spec = json.load(f) + + self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], True) + self.assertEqual(version_1_0_0_spec["name"], "test_package") + self.assertEqual(version_1_0_0_spec["version"], "1.0.0") + self.assertIn("downloads", version_1_0_0_spec) + self.assertEqual(version_1_0_0_spec["downloads"]["sha1"], "abc123def456") + + # Check 1.1.0.json + version_1_1_0_file = version_dir / "1.1.0.json" + self.assertTrue(version_1_1_0_file.exists()) + + with open(version_1_1_0_file, "r") as f: + version_1_1_0_spec = json.load(f) + + self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], True) + self.assertEqual(version_1_1_0_spec["name"], "test_package") + self.assertEqual(version_1_1_0_spec["version"], "1.1.0") + + # Check index.json + index_file = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "index.json" + ) + self.assertTrue(index_file.exists()) + + with open(index_file, "r") as f: + index_spec = json.load(f) + + # Index should have fusion compatibility of the latest version (1.1.0) + self.assertEqual(index_spec["fusion-schema-compat"], True) + self.assertEqual(index_spec["name"], "test_package") + self.assertEqual(index_spec["namespace"], "test_org") + self.assertEqual(index_spec["latest"], "1.1.0") + + @patch("hubcap.records.git_helper.run_cmd") + @patch("hubcap.records.subprocess.run") + @patch("hubcap.records.package.parse_pkgs") + @patch("hubcap.records.package.parse_require_dbt_version") + @patch("hubcap.records.check_fusion_schema_compatibility") + @patch("hubcap.records.UpdateTask.get_sha1") + def test_fusion_incompatible_package_json_generation( + self, + mock_sha1, + mock_fusion_check, + mock_parse_dbt_version, + mock_parse_pkgs, + mock_subprocess, + mock_git_cmd, + ): + """Test that fusion-incompatible packages generate correct JSON files""" + # Mock returns + mock_sha1.return_value = "xyz789uvw012" + mock_fusion_check.return_value = False # Fusion incompatible + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_parse_pkgs.return_value = [ + {"name": "dbt-labs/dbt_utils", "version": ">=1.0.0"} + ] + mock_subprocess.return_value = MagicMock(returncode=0) + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = "test-branch-incompatible" + + # Run the UpdateTask + os.chdir(self.temp_dir) + branch_name, org_name, package_name = self.update_task.run( + str(self.hub_dir.parent), mock_pr_strategy + ) + + # Verify basic info + self.assertEqual(branch_name, "test-branch-incompatible") + self.assertEqual(org_name, "test_org") + self.assertEqual(package_name, "test_package") + + # Check that fusion compatibility was checked for each tag + self.assertEqual(mock_fusion_check.call_count, 2) # Called for both tags + + # Verify version-specific JSON files were created with correct fusion compatibility + version_dir = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "versions" + ) + + # Check 1.0.0.json + version_1_0_0_file = version_dir / "1.0.0.json" + self.assertTrue(version_1_0_0_file.exists()) + + with open(version_1_0_0_file, "r") as f: + version_1_0_0_spec = json.load(f) + + self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], False) + self.assertEqual(version_1_0_0_spec["name"], "test_package") + self.assertEqual(version_1_0_0_spec["version"], "1.0.0") + + # Check 1.1.0.json + version_1_1_0_file = version_dir / "1.1.0.json" + self.assertTrue(version_1_1_0_file.exists()) + + with open(version_1_1_0_file, "r") as f: + version_1_1_0_spec = json.load(f) + + self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], False) + self.assertEqual(version_1_1_0_spec["name"], "test_package") + self.assertEqual(version_1_1_0_spec["version"], "1.1.0") + + # Check index.json + index_file = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "index.json" + ) + self.assertTrue(index_file.exists()) + + with open(index_file, "r") as f: + index_spec = json.load(f) + + # Index should have fusion compatibility of the latest version (1.1.0) + self.assertEqual(index_spec["fusion-schema-compat"], False) + self.assertEqual(index_spec["name"], "test_package") + self.assertEqual(index_spec["namespace"], "test_org") + self.assertEqual(index_spec["latest"], "1.1.0") + + @patch("hubcap.records.git_helper.run_cmd") + @patch("hubcap.records.subprocess.run") + @patch("hubcap.records.package.parse_pkgs") + @patch("hubcap.records.package.parse_require_dbt_version") + @patch("hubcap.records.check_fusion_schema_compatibility") + @patch("hubcap.records.UpdateTask.get_sha1") + def test_mixed_fusion_compatibility_versions( + self, + mock_sha1, + mock_fusion_check, + mock_parse_dbt_version, + mock_parse_pkgs, + mock_subprocess, + mock_git_cmd, + ): + """Test packages with mixed fusion compatibility across versions""" + # Mock returns - different compatibility for different versions + mock_sha1.return_value = "mixed123compat" + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_parse_pkgs.return_value = [] + mock_subprocess.return_value = MagicMock(returncode=0) + + # Mock fusion compatibility check to return different values for different calls + # First call (1.0.0): incompatible, Second call (1.1.0): compatible + mock_fusion_check.side_effect = [False, True] + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = "test-branch-mixed" + + # Run the UpdateTask + os.chdir(self.temp_dir) + branch_name, org_name, package_name = self.update_task.run( + str(self.hub_dir.parent), mock_pr_strategy + ) + + # Verify basic info + self.assertEqual(branch_name, "test-branch-mixed") + + # Check that fusion compatibility was checked for each tag + self.assertEqual(mock_fusion_check.call_count, 2) + + # Verify version-specific JSON files have correct individual compatibility + version_dir = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "versions" + ) + + # Check 1.0.0.json (should be incompatible) + with open(version_dir / "1.0.0.json", "r") as f: + version_1_0_0_spec = json.load(f) + self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], False) + + # Check 1.1.0.json (should be compatible) + with open(version_dir / "1.1.0.json", "r") as f: + version_1_1_0_spec = json.load(f) + self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], True) + + # Check index.json (should reflect latest version - 1.1.0 - which is compatible) + index_file = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "index.json" + ) + with open(index_file, "r") as f: + index_spec = json.load(f) + + self.assertEqual( + index_spec["fusion-schema-compat"], True + ) # Latest version (1.1.0) is compatible + self.assertEqual(index_spec["latest"], "1.1.0") + + @patch("hubcap.records.git_helper.run_cmd") + @patch("hubcap.records.subprocess.run") + @patch("hubcap.records.package.parse_pkgs") + @patch("hubcap.records.package.parse_require_dbt_version") + @patch("hubcap.records.check_fusion_schema_compatibility") + @patch("hubcap.records.UpdateTask.get_sha1") + def test_existing_index_file_fusion_compatibility_update( + self, + mock_sha1, + mock_fusion_check, + mock_parse_dbt_version, + mock_parse_pkgs, + mock_subprocess, + mock_git_cmd, + ): + """Test that existing index.json files are properly updated with new fusion compatibility""" + # Create existing index.json with old data + index_dir = self.hub_dir / "data" / "packages" / "test_org" / "test_package" + index_dir.mkdir(parents=True, exist_ok=True) + + existing_index = { + "name": "test_package", + "namespace": "test_org", + "description": "A test package for fusion compatibility", + "latest": "0.9.0", + "fusion-schema-compat": False, # Old version was incompatible + "assets": {"logo": "logos/custom.svg"}, + } + + with open(index_dir / "index.json", "w") as f: + json.dump(existing_index, f) + + # Mock returns for new version + mock_sha1.return_value = "update123test" + mock_fusion_check.return_value = True # New version is compatible + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_parse_pkgs.return_value = [] + mock_subprocess.return_value = MagicMock(returncode=0) + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = "test-branch-update" + + # Update existing tags to include the old version + self.update_task.existing_tags = ["0.9.0"] + + # Run the UpdateTask + os.chdir(self.temp_dir) + self.update_task.run(str(self.hub_dir.parent), mock_pr_strategy) + + # Check updated index.json + with open(index_dir / "index.json", "r") as f: + updated_index = json.load(f) + + # Should preserve existing description and assets + self.assertEqual( + updated_index["description"], "A test package for fusion compatibility" + ) + self.assertEqual(updated_index["assets"]["logo"], "logos/custom.svg") + + # Should update latest version and fusion compatibility + self.assertEqual(updated_index["latest"], "1.1.0") # Latest new tag + self.assertEqual( + updated_index["fusion-schema-compat"], True + ) # New latest version is compatible + self.assertEqual(updated_index["name"], "test_package") + self.assertEqual(updated_index["namespace"], "test_org") + + def test_fusion_compatibility_directory_bug(self): + """Test that fusion compatibility is checked on the correct directory""" + # This test is designed to catch the bug where fusion compatibility + # is checked on the wrong directory (hub dir instead of package dir) + + with patch( + "hubcap.records.check_fusion_schema_compatibility" + ) as mock_fusion_check: + with patch("hubcap.records.git_helper.run_cmd"): + with patch("hubcap.records.subprocess.run") as mock_subprocess: + with patch("hubcap.records.package.parse_pkgs") as mock_parse_pkgs: + with patch( + "hubcap.records.package.parse_require_dbt_version" + ) as mock_parse_dbt_version: + with patch( + "hubcap.records.UpdateTask.get_sha1" + ) as mock_sha1: + # Setup mocks + mock_fusion_check.return_value = True + mock_subprocess.return_value = MagicMock(returncode=0) + mock_parse_pkgs.return_value = [] + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_sha1.return_value = "test123" + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = ( + "test-branch" + ) + + # Set up a single tag for simpler testing + self.update_task.new_tags = ["1.0.0"] + + # Run the UpdateTask + os.chdir(self.temp_dir) + self.update_task.run( + str(self.hub_dir.parent), mock_pr_strategy + ) + + # Verify that fusion compatibility was called with the package directory + # NOT the hub directory + mock_fusion_check.assert_called_once() + + # Get the actual call argument + actual_call_path = mock_fusion_check.call_args[0][0] + + # Verify that fusion compatibility was called with the package directory + print( + f"Fusion compatibility called with path: {actual_call_path}" + ) + print( + f"Expected package path: {self.update_task.local_path_to_repo}" + ) + print(f"Current working directory: {os.getcwd()}") + + # This should now pass after fixing the bug + self.assertEqual( + str(actual_call_path), + str(self.update_task.local_path_to_repo), + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test_fusion_compatibility.py b/tests/test_fusion_compatibility.py similarity index 100% rename from test_fusion_compatibility.py rename to tests/test_fusion_compatibility.py diff --git a/test_fusion_integration.py b/tests/test_fusion_integration.py similarity index 100% rename from test_fusion_integration.py rename to tests/test_fusion_integration.py diff --git a/tests/test_update_task_fusion.py b/tests/test_update_task_fusion.py new file mode 100644 index 0000000..62cc7bf --- /dev/null +++ b/tests/test_update_task_fusion.py @@ -0,0 +1,500 @@ +"""Tests for UpdateTask fusion compatibility JSON generation""" + +import json +import os +import tempfile +import unittest +import shutil +from pathlib import Path +from unittest.mock import patch, MagicMock + +# Import the classes we need to test +from hubcap.records import UpdateTask + + +class TestUpdateTaskFusionCompatibility(unittest.TestCase): + """Test UpdateTask's generation of index.json and version.json with fusion compatibility""" + + def setUp(self): + """Set up test fixtures""" + # Create temporary directories + self.temp_dir = tempfile.mkdtemp() + self.hub_dir = Path(self.temp_dir) / "hub.getdbt.com" + self.package_dir = Path(self.temp_dir) / "test_org_test_package" + + # Create hub directory structure + self.hub_dir.mkdir(parents=True) + self.package_dir.mkdir(parents=True) + + # Create a basic dbt project in the package directory + self._create_test_package() + + # Create UpdateTask with test data + self.update_task = UpdateTask( + github_username="test_org", + github_repo_name="test_package", + local_path_to_repo=self.package_dir, + package_name="test_package", + existing_tags=[], + new_tags=["1.0.0", "1.1.0"], + hub_repo="hub.getdbt.com", + ) + + def tearDown(self): + """Clean up test fixtures""" + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _create_test_package(self): + """Create a test dbt package structure""" + # Create dbt_project.yml + dbt_project_content = """ +name: 'test_package' +version: '1.0.0' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +models: + test_package: + +schema: public + +materialized: table +""" + with open(self.package_dir / "dbt_project.yml", "w") as f: + f.write(dbt_project_content) + + # Create models directory and a simple model + models_dir = self.package_dir / "models" + models_dir.mkdir(exist_ok=True) + + model_content = """ +{{ config(materialized='table') }} + +select + 1 as id, + 'test' as name, + current_timestamp as created_at +""" + with open(models_dir / "test_model.sql", "w") as f: + f.write(model_content) + + # Create a packages.yml file (sometimes needed) + packages_content = """ +packages: + - package: dbt-labs/dbt_utils + version: ">=1.0.0" +""" + with open(self.package_dir / "packages.yml", "w") as f: + f.write(packages_content) + + @patch("hubcap.records.git_helper.run_cmd") + @patch("hubcap.records.subprocess.run") + @patch("hubcap.records.package.parse_pkgs") + @patch("hubcap.records.package.parse_require_dbt_version") + @patch("hubcap.records.check_fusion_schema_compatibility") + @patch("hubcap.records.UpdateTask.get_sha1") + def test_fusion_compatible_package_json_generation( + self, + mock_sha1, + mock_fusion_check, + mock_parse_dbt_version, + mock_parse_pkgs, + mock_subprocess, + mock_git_cmd, + ): + """Test that fusion-compatible packages generate correct JSON files""" + # Mock returns + mock_sha1.return_value = "abc123def456" + mock_fusion_check.return_value = True # Fusion compatible + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_parse_pkgs.return_value = [ + {"name": "dbt-labs/dbt_utils", "version": ">=1.0.0"} + ] + mock_subprocess.return_value = MagicMock(returncode=0) + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = "test-branch" + + # Run the UpdateTask + os.chdir(self.temp_dir) + branch_name, org_name, package_name = self.update_task.run( + str(self.hub_dir.parent), mock_pr_strategy + ) + + # Verify the branch and basic info + self.assertEqual(branch_name, "test-branch") + self.assertEqual(org_name, "test_org") + self.assertEqual(package_name, "test_package") + + # Check that fusion compatibility was checked for each tag + self.assertEqual(mock_fusion_check.call_count, 2) # Called for both tags + + # Verify version-specific JSON files were created with correct fusion compatibility + version_dir = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "versions" + ) + + # Check 1.0.0.json + version_1_0_0_file = version_dir / "1.0.0.json" + self.assertTrue(version_1_0_0_file.exists()) + + with open(version_1_0_0_file, "r") as f: + version_1_0_0_spec = json.load(f) + + self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], True) + self.assertEqual(version_1_0_0_spec["name"], "test_package") + self.assertEqual(version_1_0_0_spec["version"], "1.0.0") + self.assertIn("downloads", version_1_0_0_spec) + self.assertEqual(version_1_0_0_spec["downloads"]["sha1"], "abc123def456") + + # Check 1.1.0.json + version_1_1_0_file = version_dir / "1.1.0.json" + self.assertTrue(version_1_1_0_file.exists()) + + with open(version_1_1_0_file, "r") as f: + version_1_1_0_spec = json.load(f) + + self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], True) + self.assertEqual(version_1_1_0_spec["name"], "test_package") + self.assertEqual(version_1_1_0_spec["version"], "1.1.0") + + # Check index.json + index_file = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "index.json" + ) + self.assertTrue(index_file.exists()) + + with open(index_file, "r") as f: + index_spec = json.load(f) + + # Index should have fusion compatibility of the latest version (1.1.0) + self.assertEqual(index_spec["fusion-schema-compat"], True) + self.assertEqual(index_spec["name"], "test_package") + self.assertEqual(index_spec["namespace"], "test_org") + self.assertEqual(index_spec["latest"], "1.1.0") + + @patch("hubcap.records.git_helper.run_cmd") + @patch("hubcap.records.subprocess.run") + @patch("hubcap.records.package.parse_pkgs") + @patch("hubcap.records.package.parse_require_dbt_version") + @patch("hubcap.records.check_fusion_schema_compatibility") + @patch("hubcap.records.UpdateTask.get_sha1") + def test_fusion_incompatible_package_json_generation( + self, + mock_sha1, + mock_fusion_check, + mock_parse_dbt_version, + mock_parse_pkgs, + mock_subprocess, + mock_git_cmd, + ): + """Test that fusion-incompatible packages generate correct JSON files""" + # Mock returns + mock_sha1.return_value = "xyz789uvw012" + mock_fusion_check.return_value = False # Fusion incompatible + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_parse_pkgs.return_value = [ + {"name": "dbt-labs/dbt_utils", "version": ">=1.0.0"} + ] + mock_subprocess.return_value = MagicMock(returncode=0) + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = "test-branch-incompatible" + + # Run the UpdateTask + os.chdir(self.temp_dir) + branch_name, org_name, package_name = self.update_task.run( + str(self.hub_dir.parent), mock_pr_strategy + ) + + # Verify basic info + self.assertEqual(branch_name, "test-branch-incompatible") + self.assertEqual(org_name, "test_org") + self.assertEqual(package_name, "test_package") + + # Check that fusion compatibility was checked for each tag + self.assertEqual(mock_fusion_check.call_count, 2) # Called for both tags + + # Verify version-specific JSON files were created with correct fusion compatibility + version_dir = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "versions" + ) + + # Check 1.0.0.json + version_1_0_0_file = version_dir / "1.0.0.json" + self.assertTrue(version_1_0_0_file.exists()) + + with open(version_1_0_0_file, "r") as f: + version_1_0_0_spec = json.load(f) + + self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], False) + self.assertEqual(version_1_0_0_spec["name"], "test_package") + self.assertEqual(version_1_0_0_spec["version"], "1.0.0") + + # Check 1.1.0.json + version_1_1_0_file = version_dir / "1.1.0.json" + self.assertTrue(version_1_1_0_file.exists()) + + with open(version_1_1_0_file, "r") as f: + version_1_1_0_spec = json.load(f) + + self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], False) + self.assertEqual(version_1_1_0_spec["name"], "test_package") + self.assertEqual(version_1_1_0_spec["version"], "1.1.0") + + # Check index.json + index_file = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "index.json" + ) + self.assertTrue(index_file.exists()) + + with open(index_file, "r") as f: + index_spec = json.load(f) + + # Index should have fusion compatibility of the latest version (1.1.0) + self.assertEqual(index_spec["fusion-schema-compat"], False) + self.assertEqual(index_spec["name"], "test_package") + self.assertEqual(index_spec["namespace"], "test_org") + self.assertEqual(index_spec["latest"], "1.1.0") + + @patch("hubcap.records.git_helper.run_cmd") + @patch("hubcap.records.subprocess.run") + @patch("hubcap.records.package.parse_pkgs") + @patch("hubcap.records.package.parse_require_dbt_version") + @patch("hubcap.records.check_fusion_schema_compatibility") + @patch("hubcap.records.UpdateTask.get_sha1") + def test_mixed_fusion_compatibility_versions( + self, + mock_sha1, + mock_fusion_check, + mock_parse_dbt_version, + mock_parse_pkgs, + mock_subprocess, + mock_git_cmd, + ): + """Test packages with mixed fusion compatibility across versions""" + # Mock returns - different compatibility for different versions + mock_sha1.return_value = "mixed123compat" + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_parse_pkgs.return_value = [] + mock_subprocess.return_value = MagicMock(returncode=0) + + # Mock fusion compatibility check to return different values for different calls + # First call (1.0.0): incompatible, Second call (1.1.0): compatible + mock_fusion_check.side_effect = [False, True] + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = "test-branch-mixed" + + # Run the UpdateTask + os.chdir(self.temp_dir) + branch_name, org_name, package_name = self.update_task.run( + str(self.hub_dir.parent), mock_pr_strategy + ) + + # Verify basic info + self.assertEqual(branch_name, "test-branch-mixed") + + # Check that fusion compatibility was checked for each tag + self.assertEqual(mock_fusion_check.call_count, 2) + + # Verify version-specific JSON files have correct individual compatibility + version_dir = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "versions" + ) + + # Check 1.0.0.json (should be incompatible) + with open(version_dir / "1.0.0.json", "r") as f: + version_1_0_0_spec = json.load(f) + self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], False) + + # Check 1.1.0.json (should be compatible) + with open(version_dir / "1.1.0.json", "r") as f: + version_1_1_0_spec = json.load(f) + self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], True) + + # Check index.json (should reflect latest version - 1.1.0 - which is compatible) + index_file = ( + self.hub_dir + / "data" + / "packages" + / "test_org" + / "test_package" + / "index.json" + ) + with open(index_file, "r") as f: + index_spec = json.load(f) + + self.assertEqual( + index_spec["fusion-schema-compat"], True + ) # Latest version (1.1.0) is compatible + self.assertEqual(index_spec["latest"], "1.1.0") + + @patch("hubcap.records.git_helper.run_cmd") + @patch("hubcap.records.subprocess.run") + @patch("hubcap.records.package.parse_pkgs") + @patch("hubcap.records.package.parse_require_dbt_version") + @patch("hubcap.records.check_fusion_schema_compatibility") + @patch("hubcap.records.UpdateTask.get_sha1") + def test_existing_index_file_fusion_compatibility_update( + self, + mock_sha1, + mock_fusion_check, + mock_parse_dbt_version, + mock_parse_pkgs, + mock_subprocess, + mock_git_cmd, + ): + """Test that existing index.json files are properly updated with new fusion compatibility""" + # Create existing index.json with old data + index_dir = self.hub_dir / "data" / "packages" / "test_org" / "test_package" + index_dir.mkdir(parents=True, exist_ok=True) + + existing_index = { + "name": "test_package", + "namespace": "test_org", + "description": "A test package for fusion compatibility", + "latest": "0.9.0", + "fusion-schema-compat": False, # Old version was incompatible + "assets": {"logo": "logos/custom.svg"}, + } + + with open(index_dir / "index.json", "w") as f: + json.dump(existing_index, f) + + # Mock returns for new version + mock_sha1.return_value = "update123test" + mock_fusion_check.return_value = True # New version is compatible + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_parse_pkgs.return_value = [] + mock_subprocess.return_value = MagicMock(returncode=0) + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = "test-branch-update" + + # Update existing tags to include the old version + self.update_task.existing_tags = ["0.9.0"] + + # Run the UpdateTask + os.chdir(self.temp_dir) + self.update_task.run(str(self.hub_dir.parent), mock_pr_strategy) + + # Check updated index.json + with open(index_dir / "index.json", "r") as f: + updated_index = json.load(f) + + # Should preserve existing description and assets + self.assertEqual( + updated_index["description"], "A test package for fusion compatibility" + ) + self.assertEqual(updated_index["assets"]["logo"], "logos/custom.svg") + + # Should update latest version and fusion compatibility + self.assertEqual(updated_index["latest"], "1.1.0") # Latest new tag + self.assertEqual( + updated_index["fusion-schema-compat"], True + ) # New latest version is compatible + self.assertEqual(updated_index["name"], "test_package") + self.assertEqual(updated_index["namespace"], "test_org") + + def test_fusion_compatibility_directory_bug(self): + """Test that fusion compatibility is checked on the correct directory""" + # This test is designed to catch the bug where fusion compatibility + # is checked on the wrong directory (hub dir instead of package dir) + + with patch( + "hubcap.records.check_fusion_schema_compatibility" + ) as mock_fusion_check: + with patch("hubcap.records.git_helper.run_cmd"): + with patch("hubcap.records.subprocess.run") as mock_subprocess: + with patch("hubcap.records.package.parse_pkgs") as mock_parse_pkgs: + with patch( + "hubcap.records.package.parse_require_dbt_version" + ) as mock_parse_dbt_version: + with patch( + "hubcap.records.UpdateTask.get_sha1" + ) as mock_sha1: + # Setup mocks + mock_fusion_check.return_value = True + mock_subprocess.return_value = MagicMock(returncode=0) + mock_parse_pkgs.return_value = [] + mock_parse_dbt_version.return_value = ">=1.0.0" + mock_sha1.return_value = "test123" + + # Mock PR strategy + mock_pr_strategy = MagicMock() + mock_pr_strategy.branch_name.return_value = ( + "test-branch" + ) + + # Set up a single tag for simpler testing + self.update_task.new_tags = ["1.0.0"] + + # Run the UpdateTask + os.chdir(self.temp_dir) + self.update_task.run( + str(self.hub_dir.parent), mock_pr_strategy + ) + + # Verify that fusion compatibility was called with the package directory + # NOT the hub directory + mock_fusion_check.assert_called_once() + + # Get the actual call argument + actual_call_path = mock_fusion_check.call_args[0][0] + + # Verify that fusion compatibility was called with the package directory + print( + f"Fusion compatibility called with path: {actual_call_path}" + ) + print( + f"Expected package path: {self.update_task.local_path_to_repo}" + ) + print(f"Current working directory: {os.getcwd()}") + + # This should now pass after fixing the bug + self.assertEqual( + str(actual_call_path), + str(self.update_task.local_path_to_repo), + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From c8f6c099a7264cb2402e0d75d726ac1a5a69bd4f Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Tue, 5 Aug 2025 00:18:53 -0700 Subject: [PATCH 05/15] Fix uv deps --- .github/workflows/hubcap-scheduler.yml | 14 +++++++++----- .github/workflows/main.yml | 14 +++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/hubcap-scheduler.yml b/.github/workflows/hubcap-scheduler.yml index 4152f2d..428c587 100644 --- a/.github/workflows/hubcap-scheduler.yml +++ b/.github/workflows/hubcap-scheduler.yml @@ -31,17 +31,21 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.9' - - name: Install dependencies + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Install dependencies with uv run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + uv pip install --system -r requirements.txt - name: Install dbt fusion run: | @@ -77,7 +81,7 @@ jobs: - name: Run hubcap run: | echo "Starting hubcap execution..." - python3 hubcap.py 2>&1 | tee hubcap.log + uv run python hubcap.py 2>&1 | tee hubcap.log - name: Check execution results if: always() diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7d2d3f2..0d3fa1f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,8 +30,8 @@ jobs: - name: Install dependencies with uv run: | - uv pip install -r requirements.txt - uv pip install -r requirements-dev.txt + uv pip install --system -r requirements.txt + uv pip install --system -r requirements-dev.txt - name: Create test config run: | @@ -95,10 +95,10 @@ jobs: - name: Install python dependencies with uv run: | - uv pip install pre-commit + uv pip install --system pre-commit pre-commit --version - uv pip install -r requirements.txt - uv pip install -r requirements-dev.txt + uv pip install --system -r requirements.txt + uv pip install --system -r requirements-dev.txt - name: Run pre-commit hooks run: pre-commit run --all-files --show-diff-on-failure @@ -130,8 +130,8 @@ jobs: - name: Install dependencies with uv run: | - uv pip install -r requirements.txt - uv pip install -r requirements-dev.txt + uv pip install --system -r requirements.txt + uv pip install --system -r requirements-dev.txt - name: Install dbt fusion (for integration tests) run: | From aa5c2275ddf9a47320bfc0bb23e99f4d6034a484 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Tue, 5 Aug 2025 00:21:51 -0700 Subject: [PATCH 06/15] Ensure dbt fusion is available --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0d3fa1f..e8ec2db 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,6 +55,9 @@ jobs: cat config.json >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV + - name: Install dbt fusion (for integration tests) + run: | + curl -fsSL https://public.cdn.getdbt.com/fs/install/install.sh | sh -s -- --update - name: Run hubcap dry run run: | From bd19f0a22216ca69c59bf146d5acb2b07e6270bb Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Tue, 5 Aug 2025 00:26:12 -0700 Subject: [PATCH 07/15] remove duplicate test --- test_update_task_fusion.py | 500 ------------------------------------- 1 file changed, 500 deletions(-) delete mode 100644 test_update_task_fusion.py diff --git a/test_update_task_fusion.py b/test_update_task_fusion.py deleted file mode 100644 index 62cc7bf..0000000 --- a/test_update_task_fusion.py +++ /dev/null @@ -1,500 +0,0 @@ -"""Tests for UpdateTask fusion compatibility JSON generation""" - -import json -import os -import tempfile -import unittest -import shutil -from pathlib import Path -from unittest.mock import patch, MagicMock - -# Import the classes we need to test -from hubcap.records import UpdateTask - - -class TestUpdateTaskFusionCompatibility(unittest.TestCase): - """Test UpdateTask's generation of index.json and version.json with fusion compatibility""" - - def setUp(self): - """Set up test fixtures""" - # Create temporary directories - self.temp_dir = tempfile.mkdtemp() - self.hub_dir = Path(self.temp_dir) / "hub.getdbt.com" - self.package_dir = Path(self.temp_dir) / "test_org_test_package" - - # Create hub directory structure - self.hub_dir.mkdir(parents=True) - self.package_dir.mkdir(parents=True) - - # Create a basic dbt project in the package directory - self._create_test_package() - - # Create UpdateTask with test data - self.update_task = UpdateTask( - github_username="test_org", - github_repo_name="test_package", - local_path_to_repo=self.package_dir, - package_name="test_package", - existing_tags=[], - new_tags=["1.0.0", "1.1.0"], - hub_repo="hub.getdbt.com", - ) - - def tearDown(self): - """Clean up test fixtures""" - shutil.rmtree(self.temp_dir, ignore_errors=True) - - def _create_test_package(self): - """Create a test dbt package structure""" - # Create dbt_project.yml - dbt_project_content = """ -name: 'test_package' -version: '1.0.0' - -model-paths: ["models"] -analysis-paths: ["analyses"] -test-paths: ["tests"] -seed-paths: ["seeds"] -macro-paths: ["macros"] -snapshot-paths: ["snapshots"] - -target-path: "target" -clean-targets: - - "target" - - "dbt_packages" - -models: - test_package: - +schema: public - +materialized: table -""" - with open(self.package_dir / "dbt_project.yml", "w") as f: - f.write(dbt_project_content) - - # Create models directory and a simple model - models_dir = self.package_dir / "models" - models_dir.mkdir(exist_ok=True) - - model_content = """ -{{ config(materialized='table') }} - -select - 1 as id, - 'test' as name, - current_timestamp as created_at -""" - with open(models_dir / "test_model.sql", "w") as f: - f.write(model_content) - - # Create a packages.yml file (sometimes needed) - packages_content = """ -packages: - - package: dbt-labs/dbt_utils - version: ">=1.0.0" -""" - with open(self.package_dir / "packages.yml", "w") as f: - f.write(packages_content) - - @patch("hubcap.records.git_helper.run_cmd") - @patch("hubcap.records.subprocess.run") - @patch("hubcap.records.package.parse_pkgs") - @patch("hubcap.records.package.parse_require_dbt_version") - @patch("hubcap.records.check_fusion_schema_compatibility") - @patch("hubcap.records.UpdateTask.get_sha1") - def test_fusion_compatible_package_json_generation( - self, - mock_sha1, - mock_fusion_check, - mock_parse_dbt_version, - mock_parse_pkgs, - mock_subprocess, - mock_git_cmd, - ): - """Test that fusion-compatible packages generate correct JSON files""" - # Mock returns - mock_sha1.return_value = "abc123def456" - mock_fusion_check.return_value = True # Fusion compatible - mock_parse_dbt_version.return_value = ">=1.0.0" - mock_parse_pkgs.return_value = [ - {"name": "dbt-labs/dbt_utils", "version": ">=1.0.0"} - ] - mock_subprocess.return_value = MagicMock(returncode=0) - - # Mock PR strategy - mock_pr_strategy = MagicMock() - mock_pr_strategy.branch_name.return_value = "test-branch" - - # Run the UpdateTask - os.chdir(self.temp_dir) - branch_name, org_name, package_name = self.update_task.run( - str(self.hub_dir.parent), mock_pr_strategy - ) - - # Verify the branch and basic info - self.assertEqual(branch_name, "test-branch") - self.assertEqual(org_name, "test_org") - self.assertEqual(package_name, "test_package") - - # Check that fusion compatibility was checked for each tag - self.assertEqual(mock_fusion_check.call_count, 2) # Called for both tags - - # Verify version-specific JSON files were created with correct fusion compatibility - version_dir = ( - self.hub_dir - / "data" - / "packages" - / "test_org" - / "test_package" - / "versions" - ) - - # Check 1.0.0.json - version_1_0_0_file = version_dir / "1.0.0.json" - self.assertTrue(version_1_0_0_file.exists()) - - with open(version_1_0_0_file, "r") as f: - version_1_0_0_spec = json.load(f) - - self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], True) - self.assertEqual(version_1_0_0_spec["name"], "test_package") - self.assertEqual(version_1_0_0_spec["version"], "1.0.0") - self.assertIn("downloads", version_1_0_0_spec) - self.assertEqual(version_1_0_0_spec["downloads"]["sha1"], "abc123def456") - - # Check 1.1.0.json - version_1_1_0_file = version_dir / "1.1.0.json" - self.assertTrue(version_1_1_0_file.exists()) - - with open(version_1_1_0_file, "r") as f: - version_1_1_0_spec = json.load(f) - - self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], True) - self.assertEqual(version_1_1_0_spec["name"], "test_package") - self.assertEqual(version_1_1_0_spec["version"], "1.1.0") - - # Check index.json - index_file = ( - self.hub_dir - / "data" - / "packages" - / "test_org" - / "test_package" - / "index.json" - ) - self.assertTrue(index_file.exists()) - - with open(index_file, "r") as f: - index_spec = json.load(f) - - # Index should have fusion compatibility of the latest version (1.1.0) - self.assertEqual(index_spec["fusion-schema-compat"], True) - self.assertEqual(index_spec["name"], "test_package") - self.assertEqual(index_spec["namespace"], "test_org") - self.assertEqual(index_spec["latest"], "1.1.0") - - @patch("hubcap.records.git_helper.run_cmd") - @patch("hubcap.records.subprocess.run") - @patch("hubcap.records.package.parse_pkgs") - @patch("hubcap.records.package.parse_require_dbt_version") - @patch("hubcap.records.check_fusion_schema_compatibility") - @patch("hubcap.records.UpdateTask.get_sha1") - def test_fusion_incompatible_package_json_generation( - self, - mock_sha1, - mock_fusion_check, - mock_parse_dbt_version, - mock_parse_pkgs, - mock_subprocess, - mock_git_cmd, - ): - """Test that fusion-incompatible packages generate correct JSON files""" - # Mock returns - mock_sha1.return_value = "xyz789uvw012" - mock_fusion_check.return_value = False # Fusion incompatible - mock_parse_dbt_version.return_value = ">=1.0.0" - mock_parse_pkgs.return_value = [ - {"name": "dbt-labs/dbt_utils", "version": ">=1.0.0"} - ] - mock_subprocess.return_value = MagicMock(returncode=0) - - # Mock PR strategy - mock_pr_strategy = MagicMock() - mock_pr_strategy.branch_name.return_value = "test-branch-incompatible" - - # Run the UpdateTask - os.chdir(self.temp_dir) - branch_name, org_name, package_name = self.update_task.run( - str(self.hub_dir.parent), mock_pr_strategy - ) - - # Verify basic info - self.assertEqual(branch_name, "test-branch-incompatible") - self.assertEqual(org_name, "test_org") - self.assertEqual(package_name, "test_package") - - # Check that fusion compatibility was checked for each tag - self.assertEqual(mock_fusion_check.call_count, 2) # Called for both tags - - # Verify version-specific JSON files were created with correct fusion compatibility - version_dir = ( - self.hub_dir - / "data" - / "packages" - / "test_org" - / "test_package" - / "versions" - ) - - # Check 1.0.0.json - version_1_0_0_file = version_dir / "1.0.0.json" - self.assertTrue(version_1_0_0_file.exists()) - - with open(version_1_0_0_file, "r") as f: - version_1_0_0_spec = json.load(f) - - self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], False) - self.assertEqual(version_1_0_0_spec["name"], "test_package") - self.assertEqual(version_1_0_0_spec["version"], "1.0.0") - - # Check 1.1.0.json - version_1_1_0_file = version_dir / "1.1.0.json" - self.assertTrue(version_1_1_0_file.exists()) - - with open(version_1_1_0_file, "r") as f: - version_1_1_0_spec = json.load(f) - - self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], False) - self.assertEqual(version_1_1_0_spec["name"], "test_package") - self.assertEqual(version_1_1_0_spec["version"], "1.1.0") - - # Check index.json - index_file = ( - self.hub_dir - / "data" - / "packages" - / "test_org" - / "test_package" - / "index.json" - ) - self.assertTrue(index_file.exists()) - - with open(index_file, "r") as f: - index_spec = json.load(f) - - # Index should have fusion compatibility of the latest version (1.1.0) - self.assertEqual(index_spec["fusion-schema-compat"], False) - self.assertEqual(index_spec["name"], "test_package") - self.assertEqual(index_spec["namespace"], "test_org") - self.assertEqual(index_spec["latest"], "1.1.0") - - @patch("hubcap.records.git_helper.run_cmd") - @patch("hubcap.records.subprocess.run") - @patch("hubcap.records.package.parse_pkgs") - @patch("hubcap.records.package.parse_require_dbt_version") - @patch("hubcap.records.check_fusion_schema_compatibility") - @patch("hubcap.records.UpdateTask.get_sha1") - def test_mixed_fusion_compatibility_versions( - self, - mock_sha1, - mock_fusion_check, - mock_parse_dbt_version, - mock_parse_pkgs, - mock_subprocess, - mock_git_cmd, - ): - """Test packages with mixed fusion compatibility across versions""" - # Mock returns - different compatibility for different versions - mock_sha1.return_value = "mixed123compat" - mock_parse_dbt_version.return_value = ">=1.0.0" - mock_parse_pkgs.return_value = [] - mock_subprocess.return_value = MagicMock(returncode=0) - - # Mock fusion compatibility check to return different values for different calls - # First call (1.0.0): incompatible, Second call (1.1.0): compatible - mock_fusion_check.side_effect = [False, True] - - # Mock PR strategy - mock_pr_strategy = MagicMock() - mock_pr_strategy.branch_name.return_value = "test-branch-mixed" - - # Run the UpdateTask - os.chdir(self.temp_dir) - branch_name, org_name, package_name = self.update_task.run( - str(self.hub_dir.parent), mock_pr_strategy - ) - - # Verify basic info - self.assertEqual(branch_name, "test-branch-mixed") - - # Check that fusion compatibility was checked for each tag - self.assertEqual(mock_fusion_check.call_count, 2) - - # Verify version-specific JSON files have correct individual compatibility - version_dir = ( - self.hub_dir - / "data" - / "packages" - / "test_org" - / "test_package" - / "versions" - ) - - # Check 1.0.0.json (should be incompatible) - with open(version_dir / "1.0.0.json", "r") as f: - version_1_0_0_spec = json.load(f) - self.assertEqual(version_1_0_0_spec["fusion-schema-compat"], False) - - # Check 1.1.0.json (should be compatible) - with open(version_dir / "1.1.0.json", "r") as f: - version_1_1_0_spec = json.load(f) - self.assertEqual(version_1_1_0_spec["fusion-schema-compat"], True) - - # Check index.json (should reflect latest version - 1.1.0 - which is compatible) - index_file = ( - self.hub_dir - / "data" - / "packages" - / "test_org" - / "test_package" - / "index.json" - ) - with open(index_file, "r") as f: - index_spec = json.load(f) - - self.assertEqual( - index_spec["fusion-schema-compat"], True - ) # Latest version (1.1.0) is compatible - self.assertEqual(index_spec["latest"], "1.1.0") - - @patch("hubcap.records.git_helper.run_cmd") - @patch("hubcap.records.subprocess.run") - @patch("hubcap.records.package.parse_pkgs") - @patch("hubcap.records.package.parse_require_dbt_version") - @patch("hubcap.records.check_fusion_schema_compatibility") - @patch("hubcap.records.UpdateTask.get_sha1") - def test_existing_index_file_fusion_compatibility_update( - self, - mock_sha1, - mock_fusion_check, - mock_parse_dbt_version, - mock_parse_pkgs, - mock_subprocess, - mock_git_cmd, - ): - """Test that existing index.json files are properly updated with new fusion compatibility""" - # Create existing index.json with old data - index_dir = self.hub_dir / "data" / "packages" / "test_org" / "test_package" - index_dir.mkdir(parents=True, exist_ok=True) - - existing_index = { - "name": "test_package", - "namespace": "test_org", - "description": "A test package for fusion compatibility", - "latest": "0.9.0", - "fusion-schema-compat": False, # Old version was incompatible - "assets": {"logo": "logos/custom.svg"}, - } - - with open(index_dir / "index.json", "w") as f: - json.dump(existing_index, f) - - # Mock returns for new version - mock_sha1.return_value = "update123test" - mock_fusion_check.return_value = True # New version is compatible - mock_parse_dbt_version.return_value = ">=1.0.0" - mock_parse_pkgs.return_value = [] - mock_subprocess.return_value = MagicMock(returncode=0) - - # Mock PR strategy - mock_pr_strategy = MagicMock() - mock_pr_strategy.branch_name.return_value = "test-branch-update" - - # Update existing tags to include the old version - self.update_task.existing_tags = ["0.9.0"] - - # Run the UpdateTask - os.chdir(self.temp_dir) - self.update_task.run(str(self.hub_dir.parent), mock_pr_strategy) - - # Check updated index.json - with open(index_dir / "index.json", "r") as f: - updated_index = json.load(f) - - # Should preserve existing description and assets - self.assertEqual( - updated_index["description"], "A test package for fusion compatibility" - ) - self.assertEqual(updated_index["assets"]["logo"], "logos/custom.svg") - - # Should update latest version and fusion compatibility - self.assertEqual(updated_index["latest"], "1.1.0") # Latest new tag - self.assertEqual( - updated_index["fusion-schema-compat"], True - ) # New latest version is compatible - self.assertEqual(updated_index["name"], "test_package") - self.assertEqual(updated_index["namespace"], "test_org") - - def test_fusion_compatibility_directory_bug(self): - """Test that fusion compatibility is checked on the correct directory""" - # This test is designed to catch the bug where fusion compatibility - # is checked on the wrong directory (hub dir instead of package dir) - - with patch( - "hubcap.records.check_fusion_schema_compatibility" - ) as mock_fusion_check: - with patch("hubcap.records.git_helper.run_cmd"): - with patch("hubcap.records.subprocess.run") as mock_subprocess: - with patch("hubcap.records.package.parse_pkgs") as mock_parse_pkgs: - with patch( - "hubcap.records.package.parse_require_dbt_version" - ) as mock_parse_dbt_version: - with patch( - "hubcap.records.UpdateTask.get_sha1" - ) as mock_sha1: - # Setup mocks - mock_fusion_check.return_value = True - mock_subprocess.return_value = MagicMock(returncode=0) - mock_parse_pkgs.return_value = [] - mock_parse_dbt_version.return_value = ">=1.0.0" - mock_sha1.return_value = "test123" - - # Mock PR strategy - mock_pr_strategy = MagicMock() - mock_pr_strategy.branch_name.return_value = ( - "test-branch" - ) - - # Set up a single tag for simpler testing - self.update_task.new_tags = ["1.0.0"] - - # Run the UpdateTask - os.chdir(self.temp_dir) - self.update_task.run( - str(self.hub_dir.parent), mock_pr_strategy - ) - - # Verify that fusion compatibility was called with the package directory - # NOT the hub directory - mock_fusion_check.assert_called_once() - - # Get the actual call argument - actual_call_path = mock_fusion_check.call_args[0][0] - - # Verify that fusion compatibility was called with the package directory - print( - f"Fusion compatibility called with path: {actual_call_path}" - ) - print( - f"Expected package path: {self.update_task.local_path_to_repo}" - ) - print(f"Current working directory: {os.getcwd()}") - - # This should now pass after fixing the bug - self.assertEqual( - str(actual_call_path), - str(self.update_task.local_path_to_repo), - ) - - -if __name__ == "__main__": - unittest.main(verbosity=2) From cd8e11c208be108b0ce131d05c2c0d0ab97a034b Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Mon, 11 Aug 2025 14:09:19 -0700 Subject: [PATCH 08/15] Run dry-run scheduler on PR --- .github/workflows/hubcap-scheduler.yml | 32 +++++++++++-- .github/workflows/main.yml | 66 -------------------------- 2 files changed, 29 insertions(+), 69 deletions(-) diff --git a/.github/workflows/hubcap-scheduler.yml b/.github/workflows/hubcap-scheduler.yml index 428c587..e9d7cf7 100644 --- a/.github/workflows/hubcap-scheduler.yml +++ b/.github/workflows/hubcap-scheduler.yml @@ -4,6 +4,8 @@ on: schedule: # Run every hour at :00 for production - cron: '0 * * * *' + pull_request: + # Run on PRs in dry-run mode for testing workflow_dispatch: inputs: environment: @@ -46,6 +48,10 @@ jobs: - name: Install dependencies with uv run: | uv pip install --system -r requirements.txt + # Install dev requirements for PR testing + if [ "${{ github.event_name }}" = "pull_request" ]; then + uv pip install --system -r requirements-dev.txt + fi - name: Install dbt fusion run: | @@ -54,11 +60,31 @@ jobs: - name: Configure environment env: HUBCAP_CONFIG: ${{ secrets.HUBCAP_CONFIG }} - INPUT_DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} - ENVIRONMENT: ${{ github.event.inputs.environment || 'production' }} + INPUT_DRY_RUN: ${{ github.event.inputs.dry_run || (github.event_name == 'pull_request' && 'true') || 'false' }} + ENVIRONMENT: ${{ github.event.inputs.environment || (github.event_name == 'pull_request' && 'test') || 'production' }} run: | echo "Using $ENVIRONMENT environment configuration" - echo "$HUBCAP_CONFIG" > base_config.json + + # Create config based on event type + if [ "${{ github.event_name }}" = "pull_request" ]; then + # Use test config for PR events (secrets may not be available for forks) + cat > base_config.json << EOF + { + "org": "dbt-labs", + "repo": "hub.getdbt.com", + "push_branches": false, + "one_branch_per_repo": true, + "user": { + "name": "github-actions", + "email": "github-actions@github.com", + "token": "dummy-token" + } + } + EOF + else + # Use production config from secrets + echo "$HUBCAP_CONFIG" > base_config.json + fi # Modify config for dry run if requested if [ "$INPUT_DRY_RUN" = "true" ]; then diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e8ec2db..99710ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,72 +10,6 @@ jobs: - name: Validate JSON run: "cat hub.json | python3 -m json.tool" - hubcap-dry-run: - name: Hubcap Dry Run - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Check out the repository - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - version: "latest" - - - name: Install dependencies with uv - run: | - uv pip install --system -r requirements.txt - uv pip install --system -r requirements-dev.txt - - - name: Create test config - run: | - cat > config.json << EOF - { - "org": "dbt-labs", - "repo": "hub.getdbt.com", - "push_branches": false, - "one_branch_per_repo": true, - "user": { - "name": "github-actions", - "email": "github-actions@github.com", - "token": "dummy-token" - } - } - EOF - - - name: Set environment variables - run: | - echo "CONFIG<> $GITHUB_ENV - cat config.json >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Install dbt fusion (for integration tests) - run: | - curl -fsSL https://public.cdn.getdbt.com/fs/install/install.sh | sh -s -- --update - - - name: Run hubcap dry run - run: | - set -o pipefail - # tee to capture the full output (stdout+stderr) into hubcap.log - PYTHONPATH=${{ github.workspace }} python3 hubcap.py 2>&1 | tee hubcap.log - - - name: Check for any hubcap errors - if: always() - run: | - if grep -q 'ERROR' hubcap.log; then - echo "Found ERROR events in hubcap.log:" - grep 'ERROR' hubcap.log - exit 1 - else - echo "No ERROR-level log entries found." - fi - code-quality: name: code-quality From 407b96d162c66e2fb6cab25280e4170cf0e10b7a Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Mon, 11 Aug 2025 14:12:56 -0700 Subject: [PATCH 09/15] Update to checkout v4 --- .github/workflows/hubcap-scheduler.yml | 4 ++-- .github/workflows/main.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/hubcap-scheduler.yml b/.github/workflows/hubcap-scheduler.yml index e9d7cf7..1042dc7 100644 --- a/.github/workflows/hubcap-scheduler.yml +++ b/.github/workflows/hubcap-scheduler.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -143,7 +143,7 @@ jobs: - name: Upload execution artifacts if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: hubcap-execution-${{ github.event.inputs.environment || 'production' }}-${{ github.run_number }} path: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 99710ad..6be0a43 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: verify-json: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Validate JSON run: "cat hub.json | python3 -m json.tool" @@ -18,7 +18,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -53,7 +53,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From 347f89f34427ce8fceabe2252e7eb3e810aeea84 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Mon, 11 Aug 2025 14:20:51 -0700 Subject: [PATCH 10/15] artifacts --- .github/workflows/hubcap-scheduler.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/hubcap-scheduler.yml b/.github/workflows/hubcap-scheduler.yml index 1042dc7..dca9009 100644 --- a/.github/workflows/hubcap-scheduler.yml +++ b/.github/workflows/hubcap-scheduler.yml @@ -146,9 +146,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: hubcap-execution-${{ github.event.inputs.environment || 'production' }}-${{ github.run_number }} - path: | - hubcap.log - target/ + path: hubcap.log retention-days: 30 - name: Fail job if execution failed From eafc11f422906784cf7fb6307d0dcfd2e04fe130 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Mon, 18 Aug 2025 19:47:47 -0700 Subject: [PATCH 11/15] Update to run with worktree since fusion can modify workspace when checking for compat --- hubcap/records.py | 55 +++++++++++++++++++++++++------- tests/test_update_task_fusion.py | 10 +++--- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/hubcap/records.py b/hubcap/records.py index 7e0b6eb..baedca4 100644 --- a/hubcap/records.py +++ b/hubcap/records.py @@ -6,6 +6,8 @@ import os import requests import subprocess +import shutil +import tempfile from abc import ABC, abstractmethod from pathlib import Path @@ -224,17 +226,46 @@ def run(self, main_dir, pr_strategy): # create a version spec for each tag for tag in self.new_tags: - # go to repo dir to checkout tag and tag-commit specific package list - os.chdir(self.local_path_to_repo) - git_helper.run_cmd(f"git checkout tags/{tag}") - packages = package.parse_pkgs(Path(os.getcwd())) - require_dbt_version = package.parse_require_dbt_version(Path(os.getcwd())) - os.chdir(main_dir) - # check fusion compatibility - is_fusion_compatible = check_fusion_schema_compatibility( - self.local_path_to_repo - ) - self.fusion_compatibility[tag] = is_fusion_compatible + # Create an isolated git worktree for this tag + worktree_parent = tempfile.mkdtemp(prefix="hubcap-wt-") + worktree_path = Path(worktree_parent) / f"wt-{tag}" + try: + # Add the worktree for the specific tag + prev_cwd = os.getcwd() + try: + os.chdir(self.local_path_to_repo) + git_helper.run_cmd( + f'git worktree add --detach "{str(worktree_path)}" "tags/{tag}"' + ) + finally: + os.chdir(prev_cwd) + + # Parse package metadata from the isolated worktree + packages = package.parse_pkgs(worktree_path) + require_dbt_version = package.parse_require_dbt_version(worktree_path) + + # Check fusion compatibility within the isolated worktree + is_fusion_compatible = check_fusion_schema_compatibility(worktree_path) + self.fusion_compatibility[tag] = is_fusion_compatible + finally: + # Remove the worktree and its parent temp directory + prev_cwd = os.getcwd() + try: + os.chdir(self.local_path_to_repo) + try: + git_helper.run_cmd( + f'git worktree remove --force "{str(worktree_path)}"' + ) + except Exception: + # Best-effort cleanup; continue + pass + finally: + os.chdir(prev_cwd) + try: + shutil.rmtree(worktree_parent, ignore_errors=True) + except Exception: + pass + # return to hub and build spec package_spec = self.make_spec( self.github_username, @@ -305,7 +336,7 @@ def make_index( "namespace": org_name, "description": description, "latest": latest_version.replace("=", ""), # LOL - "fusion-schema-compat": fusion_compatibility[latest_version], + "latest-fusion-schema-compat": fusion_compatibility[latest_version], "assets": assets, } diff --git a/tests/test_update_task_fusion.py b/tests/test_update_task_fusion.py index 62cc7bf..6d22c5b 100644 --- a/tests/test_update_task_fusion.py +++ b/tests/test_update_task_fusion.py @@ -187,7 +187,7 @@ def test_fusion_compatible_package_json_generation( index_spec = json.load(f) # Index should have fusion compatibility of the latest version (1.1.0) - self.assertEqual(index_spec["fusion-schema-compat"], True) + self.assertEqual(index_spec["latest-fusion-schema-compat"], True) self.assertEqual(index_spec["name"], "test_package") self.assertEqual(index_spec["namespace"], "test_org") self.assertEqual(index_spec["latest"], "1.1.0") @@ -282,7 +282,7 @@ def test_fusion_incompatible_package_json_generation( index_spec = json.load(f) # Index should have fusion compatibility of the latest version (1.1.0) - self.assertEqual(index_spec["fusion-schema-compat"], False) + self.assertEqual(index_spec["latest-fusion-schema-compat"], False) self.assertEqual(index_spec["name"], "test_package") self.assertEqual(index_spec["namespace"], "test_org") self.assertEqual(index_spec["latest"], "1.1.0") @@ -362,7 +362,7 @@ def test_mixed_fusion_compatibility_versions( index_spec = json.load(f) self.assertEqual( - index_spec["fusion-schema-compat"], True + index_spec["latest-fusion-schema-compat"], True ) # Latest version (1.1.0) is compatible self.assertEqual(index_spec["latest"], "1.1.0") @@ -391,7 +391,7 @@ def test_existing_index_file_fusion_compatibility_update( "namespace": "test_org", "description": "A test package for fusion compatibility", "latest": "0.9.0", - "fusion-schema-compat": False, # Old version was incompatible + "latest-fusion-schema-compat": False, # Old version was incompatible "assets": {"logo": "logos/custom.svg"}, } @@ -429,7 +429,7 @@ def test_existing_index_file_fusion_compatibility_update( # Should update latest version and fusion compatibility self.assertEqual(updated_index["latest"], "1.1.0") # Latest new tag self.assertEqual( - updated_index["fusion-schema-compat"], True + updated_index["latest-fusion-schema-compat"], True ) # New latest version is compatible self.assertEqual(updated_index["name"], "test_package") self.assertEqual(updated_index["namespace"], "test_org") From a1a5f29c96a36d0b6d57e930aa244d3355250ea4 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Mon, 18 Aug 2025 21:30:09 -0700 Subject: [PATCH 12/15] Fix record cleanup after fusion --- hubcap/records.py | 59 +++++++++++++++-------------------------------- 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/hubcap/records.py b/hubcap/records.py index baedca4..b76c681 100644 --- a/hubcap/records.py +++ b/hubcap/records.py @@ -6,8 +6,6 @@ import os import requests import subprocess -import shutil -import tempfile from abc import ABC, abstractmethod from pathlib import Path @@ -226,45 +224,24 @@ def run(self, main_dir, pr_strategy): # create a version spec for each tag for tag in self.new_tags: - # Create an isolated git worktree for this tag - worktree_parent = tempfile.mkdtemp(prefix="hubcap-wt-") - worktree_path = Path(worktree_parent) / f"wt-{tag}" - try: - # Add the worktree for the specific tag - prev_cwd = os.getcwd() - try: - os.chdir(self.local_path_to_repo) - git_helper.run_cmd( - f'git worktree add --detach "{str(worktree_path)}" "tags/{tag}"' - ) - finally: - os.chdir(prev_cwd) - - # Parse package metadata from the isolated worktree - packages = package.parse_pkgs(worktree_path) - require_dbt_version = package.parse_require_dbt_version(worktree_path) - - # Check fusion compatibility within the isolated worktree - is_fusion_compatible = check_fusion_schema_compatibility(worktree_path) - self.fusion_compatibility[tag] = is_fusion_compatible - finally: - # Remove the worktree and its parent temp directory - prev_cwd = os.getcwd() - try: - os.chdir(self.local_path_to_repo) - try: - git_helper.run_cmd( - f'git worktree remove --force "{str(worktree_path)}"' - ) - except Exception: - # Best-effort cleanup; continue - pass - finally: - os.chdir(prev_cwd) - try: - shutil.rmtree(worktree_parent, ignore_errors=True) - except Exception: - pass + # go to repo dir to checkout tag and tag-commit specific package list + os.chdir(self.local_path_to_repo) + git_helper.run_cmd(f"git checkout tags/{tag}") + packages = package.parse_pkgs(Path(os.getcwd())) + require_dbt_version = package.parse_require_dbt_version(Path(os.getcwd())) + os.chdir(main_dir) + + # check fusion compatibility + is_fusion_compatible = check_fusion_schema_compatibility( + self.local_path_to_repo + ) + self.fusion_compatibility[tag] = is_fusion_compatible + + # Reset and clean the repo to ensure clean state after fusion check + os.chdir(self.local_path_to_repo) + git_helper.run_cmd("git reset --hard HEAD") + git_helper.run_cmd("git clean -fd") + os.chdir(main_dir) # return to hub and build spec package_spec = self.make_spec( From b7eb28d3f15747887aad4c815a558c3a57638cc8 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Tue, 19 Aug 2025 00:35:56 -0700 Subject: [PATCH 13/15] further cleanup --- hubcap/records.py | 166 +++++++++++++++++++++++++++++++--------------- 1 file changed, 112 insertions(+), 54 deletions(-) diff --git a/hubcap/records.py b/hubcap/records.py index b76c681..17f0a58 100644 --- a/hubcap/records.py +++ b/hubcap/records.py @@ -211,73 +211,117 @@ def __init__( self.fusion_compatibility = {} def run(self, main_dir, pr_strategy): - os.chdir(main_dir) - # Ensure versions directory for a hub package entry - Path.mkdir(self.hub_version_index_path, parents=True, exist_ok=True) + original_dir = os.getcwd() + branch_name = None - branch_name = self.cut_version_branch(pr_strategy) - - # create an updated version of the repo's index.json - index_filepath = ( - Path(os.path.dirname(self.hub_version_index_path)) / "index.json" - ) - - # create a version spec for each tag - for tag in self.new_tags: - # go to repo dir to checkout tag and tag-commit specific package list - os.chdir(self.local_path_to_repo) - git_helper.run_cmd(f"git checkout tags/{tag}") - packages = package.parse_pkgs(Path(os.getcwd())) - require_dbt_version = package.parse_require_dbt_version(Path(os.getcwd())) + try: os.chdir(main_dir) + # Ensure versions directory for a hub package entry + Path.mkdir(self.hub_version_index_path, parents=True, exist_ok=True) - # check fusion compatibility - is_fusion_compatible = check_fusion_schema_compatibility( - self.local_path_to_repo - ) - self.fusion_compatibility[tag] = is_fusion_compatible + branch_name = self.cut_version_branch(pr_strategy) - # Reset and clean the repo to ensure clean state after fusion check - os.chdir(self.local_path_to_repo) - git_helper.run_cmd("git reset --hard HEAD") - git_helper.run_cmd("git clean -fd") - os.chdir(main_dir) + # create an updated version of the repo's index.json + index_filepath = ( + Path(os.path.dirname(self.hub_version_index_path)) / "index.json" + ) - # return to hub and build spec - package_spec = self.make_spec( + # create a version spec for each tag + for tag in self.new_tags: + try: + # go to repo dir to checkout tag and tag-commit specific package list + os.chdir(self.local_path_to_repo) + git_helper.run_cmd(f"git checkout tags/{tag}") + packages = package.parse_pkgs(Path(os.getcwd())) + require_dbt_version = package.parse_require_dbt_version( + Path(os.getcwd()) + ) + os.chdir(main_dir) + + # check fusion compatibility + is_fusion_compatible = check_fusion_schema_compatibility( + self.local_path_to_repo + ) + self.fusion_compatibility[tag] = is_fusion_compatible + + # Reset and clean the repo to ensure clean state after fusion check + os.chdir(self.local_path_to_repo) + git_helper.run_cmd("git reset --hard HEAD") + git_helper.run_cmd("git clean -fd") + os.chdir(main_dir) + + # return to hub and build spec + package_spec = self.make_spec( + self.github_username, + self.github_repo_name, + self.package_name, + packages, + require_dbt_version, + tag, + is_fusion_compatible, + ) + + version_path = self.hub_version_index_path / Path(f"{tag}.json") + with open(version_path, "w") as f: + logging.info(f"writing spec to {version_path}") + f.write(str(json.dumps(package_spec, indent=4))) + + msg = f"hubcap: Adding tag {tag} for {self.github_username}/{self.github_repo_name}" + logging.info(msg) + git_helper.run_cmd("git add -A") + subprocess.run( + args=["git", "commit", "-am", f"{msg}"], capture_output=True + ) + + except Exception as e: + logging.error(f"Error processing tag {tag}: {str(e)}") + # Ensure we're back in main_dir before continuing + os.chdir(main_dir) + raise + + new_index_entry = self.make_index( self.github_username, self.github_repo_name, self.package_name, - packages, - require_dbt_version, - tag, - is_fusion_compatible, + self.fetch_index_file_contents(index_filepath), + set(self.new_tags) | set(self.existing_tags), + self.fusion_compatibility, ) + with open(index_filepath, "w") as f: + logging.info(f"writing index.json to {index_filepath}") + f.write(str(json.dumps(new_index_entry, indent=4))) - version_path = self.hub_version_index_path / Path(f"{tag}.json") - with open(version_path, "w") as f: - logging.info(f"writing spec to {version_path}") - f.write(str(json.dumps(package_spec, indent=4))) - - msg = f"hubcap: Adding tag {tag} for {self.github_username}/{self.github_repo_name}" + # Commit the updated index.json file + msg = f"hubcap: Update index.json for {self.github_username}/{self.github_repo_name}" logging.info(msg) git_helper.run_cmd("git add -A") subprocess.run(args=["git", "commit", "-am", f"{msg}"], capture_output=True) - new_index_entry = self.make_index( - self.github_username, - self.github_repo_name, - self.package_name, - self.fetch_index_file_contents(index_filepath), - set(self.new_tags) | set(self.existing_tags), - self.fusion_compatibility, - ) - with open(index_filepath, "w") as f: - logging.info(f"writing index.json to {index_filepath}") - f.write(str(json.dumps(new_index_entry, indent=4))) + # if successful return branchname + return branch_name, self.github_username, self.github_repo_name - # if succesful return branchname - return branch_name, self.github_username, self.github_repo_name + except Exception as e: + # Clean up on failure + logging.error(f"Error in UpdateTask.run(): {str(e)}") + os.chdir(main_dir) + + # If we created a branch but failed, try to clean it up + if branch_name: + try: + # Switch to main branch + git_helper.run_cmd("git checkout -q main") + # Delete the problematic branch + git_helper.run_cmd(f"git branch -D {branch_name}") + logging.info(f"Cleaned up failed branch: {branch_name}") + except Exception as cleanup_error: + logging.warning( + f"Failed to clean up branch {branch_name}: {str(cleanup_error)}" + ) + + raise + finally: + # Always restore original directory + os.chdir(original_dir) def cut_version_branch(self, pr_strategy): """designed to be run in a hub repo which is sibling to package code repos""" @@ -286,11 +330,25 @@ def cut_version_branch(self, pr_strategy): ) helper.logging.info(f"checking out branch {branch_name} in the hub repo") + # First, try to create a new branch completed_subprocess = subprocess.run( - ["git", "checkout", "-q", "-b", branch_name] + ["git", "checkout", "-q", "-b", branch_name], capture_output=True ) + if completed_subprocess.returncode == 128: - git_helper.run_cmd(f"git checkout -q {branch_name}") + # Branch already exists, delete it and recreate from main/master + logging.warning( + f"Branch {branch_name} already exists. Deleting and recreating from main branch." + ) + + # Switch to main branch first + git_helper.run_cmd("git checkout -q main") + + # Delete the existing branch + git_helper.run_cmd(f"git branch -D {branch_name}") + + # Create the branch again from main + git_helper.run_cmd(f"git checkout -q -b {branch_name}") return branch_name From c3c3825a9f99d5e0b7660384489fbcf3701cb636 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Tue, 19 Aug 2025 00:46:02 -0700 Subject: [PATCH 14/15] final fixes --- hubcap/records.py | 170 ++++++++++++++++------------------------------ 1 file changed, 59 insertions(+), 111 deletions(-) diff --git a/hubcap/records.py b/hubcap/records.py index 17f0a58..395e254 100644 --- a/hubcap/records.py +++ b/hubcap/records.py @@ -211,117 +211,79 @@ def __init__( self.fusion_compatibility = {} def run(self, main_dir, pr_strategy): - original_dir = os.getcwd() - branch_name = None + os.chdir(main_dir) + # Ensure versions directory for a hub package entry + Path.mkdir(self.hub_version_index_path, parents=True, exist_ok=True) - try: - os.chdir(main_dir) - # Ensure versions directory for a hub package entry - Path.mkdir(self.hub_version_index_path, parents=True, exist_ok=True) + branch_name = self.cut_version_branch(pr_strategy) - branch_name = self.cut_version_branch(pr_strategy) + # create an updated version of the repo's index.json + index_filepath = ( + Path(os.path.dirname(self.hub_version_index_path)) / "index.json" + ) + + # create a version spec for each tag + for tag in self.new_tags: + # go to repo dir to checkout tag and tag-commit specific package list + os.chdir(self.local_path_to_repo) + git_helper.run_cmd(f"git checkout tags/{tag}") + packages = package.parse_pkgs(Path(os.getcwd())) + require_dbt_version = package.parse_require_dbt_version(Path(os.getcwd())) + os.chdir(main_dir) - # create an updated version of the repo's index.json - index_filepath = ( - Path(os.path.dirname(self.hub_version_index_path)) / "index.json" + # check fusion compatibility + is_fusion_compatible = check_fusion_schema_compatibility( + self.local_path_to_repo ) + self.fusion_compatibility[tag] = is_fusion_compatible - # create a version spec for each tag - for tag in self.new_tags: - try: - # go to repo dir to checkout tag and tag-commit specific package list - os.chdir(self.local_path_to_repo) - git_helper.run_cmd(f"git checkout tags/{tag}") - packages = package.parse_pkgs(Path(os.getcwd())) - require_dbt_version = package.parse_require_dbt_version( - Path(os.getcwd()) - ) - os.chdir(main_dir) - - # check fusion compatibility - is_fusion_compatible = check_fusion_schema_compatibility( - self.local_path_to_repo - ) - self.fusion_compatibility[tag] = is_fusion_compatible - - # Reset and clean the repo to ensure clean state after fusion check - os.chdir(self.local_path_to_repo) - git_helper.run_cmd("git reset --hard HEAD") - git_helper.run_cmd("git clean -fd") - os.chdir(main_dir) - - # return to hub and build spec - package_spec = self.make_spec( - self.github_username, - self.github_repo_name, - self.package_name, - packages, - require_dbt_version, - tag, - is_fusion_compatible, - ) - - version_path = self.hub_version_index_path / Path(f"{tag}.json") - with open(version_path, "w") as f: - logging.info(f"writing spec to {version_path}") - f.write(str(json.dumps(package_spec, indent=4))) - - msg = f"hubcap: Adding tag {tag} for {self.github_username}/{self.github_repo_name}" - logging.info(msg) - git_helper.run_cmd("git add -A") - subprocess.run( - args=["git", "commit", "-am", f"{msg}"], capture_output=True - ) - - except Exception as e: - logging.error(f"Error processing tag {tag}: {str(e)}") - # Ensure we're back in main_dir before continuing - os.chdir(main_dir) - raise - - new_index_entry = self.make_index( + # Reset and clean the repo to ensure clean state after fusion check + os.chdir(self.local_path_to_repo) + git_helper.run_cmd("git reset --hard HEAD") + git_helper.run_cmd("git clean -fd") + os.chdir(main_dir) + + # return to hub and build spec + package_spec = self.make_spec( self.github_username, self.github_repo_name, self.package_name, - self.fetch_index_file_contents(index_filepath), - set(self.new_tags) | set(self.existing_tags), - self.fusion_compatibility, + packages, + require_dbt_version, + tag, + is_fusion_compatible, ) - with open(index_filepath, "w") as f: - logging.info(f"writing index.json to {index_filepath}") - f.write(str(json.dumps(new_index_entry, indent=4))) - # Commit the updated index.json file - msg = f"hubcap: Update index.json for {self.github_username}/{self.github_repo_name}" + version_path = self.hub_version_index_path / Path(f"{tag}.json") + with open(version_path, "w") as f: + logging.info(f"writing spec to {version_path}") + f.write(str(json.dumps(package_spec, indent=4))) + + msg = f"hubcap: Adding tag {tag} for {self.github_username}/{self.github_repo_name}" logging.info(msg) git_helper.run_cmd("git add -A") subprocess.run(args=["git", "commit", "-am", f"{msg}"], capture_output=True) - # if successful return branchname - return branch_name, self.github_username, self.github_repo_name + new_index_entry = self.make_index( + self.github_username, + self.github_repo_name, + self.package_name, + self.fetch_index_file_contents(index_filepath), + set(self.new_tags) | set(self.existing_tags), + self.fusion_compatibility, + ) + with open(index_filepath, "w") as f: + logging.info(f"writing index.json to {index_filepath}") + f.write(str(json.dumps(new_index_entry, indent=4))) - except Exception as e: - # Clean up on failure - logging.error(f"Error in UpdateTask.run(): {str(e)}") - os.chdir(main_dir) + # Commit the updated index.json file + msg = f"hubcap: Update index.json for {self.github_username}/{self.github_repo_name}" + logging.info(msg) + git_helper.run_cmd("git add -A") + subprocess.run(args=["git", "commit", "-am", f"{msg}"], capture_output=True) - # If we created a branch but failed, try to clean it up - if branch_name: - try: - # Switch to main branch - git_helper.run_cmd("git checkout -q main") - # Delete the problematic branch - git_helper.run_cmd(f"git branch -D {branch_name}") - logging.info(f"Cleaned up failed branch: {branch_name}") - except Exception as cleanup_error: - logging.warning( - f"Failed to clean up branch {branch_name}: {str(cleanup_error)}" - ) - - raise - finally: - # Always restore original directory - os.chdir(original_dir) + # if successful return branchname + return branch_name, self.github_username, self.github_repo_name def cut_version_branch(self, pr_strategy): """designed to be run in a hub repo which is sibling to package code repos""" @@ -330,25 +292,11 @@ def cut_version_branch(self, pr_strategy): ) helper.logging.info(f"checking out branch {branch_name} in the hub repo") - # First, try to create a new branch completed_subprocess = subprocess.run( - ["git", "checkout", "-q", "-b", branch_name], capture_output=True + ["git", "checkout", "-q", "-b", branch_name] ) - if completed_subprocess.returncode == 128: - # Branch already exists, delete it and recreate from main/master - logging.warning( - f"Branch {branch_name} already exists. Deleting and recreating from main branch." - ) - - # Switch to main branch first - git_helper.run_cmd("git checkout -q main") - - # Delete the existing branch - git_helper.run_cmd(f"git branch -D {branch_name}") - - # Create the branch again from main - git_helper.run_cmd(f"git checkout -q -b {branch_name}") + git_helper.run_cmd(f"git checkout -q {branch_name}") return branch_name From c455f1325608fa07495847b00bc8064b2b400492 Mon Sep 17 00:00:00 2001 From: Alexander Bogdanowicz Date: Tue, 19 Aug 2025 01:41:21 -0700 Subject: [PATCH 15/15] found root cause --- hubcap/records.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hubcap/records.py b/hubcap/records.py index 395e254..3ad4718 100644 --- a/hubcap/records.py +++ b/hubcap/records.py @@ -319,7 +319,9 @@ def make_index( "namespace": org_name, "description": description, "latest": latest_version.replace("=", ""), # LOL - "latest-fusion-schema-compat": fusion_compatibility[latest_version], + "latest-fusion-schema-compat": fusion_compatibility.get( + latest_version, False + ), "assets": assets, }