chore: add github automations (#1)

Add dependabot configuration to manage the Github Action versions.
Add CI/CD workflows to verify and publish the package using semantic
commits.
Add linting and fix typing issues

Signed-off-by: Lucas Roesler <[email protected]>

Author: LucasRoesler

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+count = true
+max-line-length = 127
+max-complexity = 10
+statistics = true
+# stop the build if there are Python syntax errors or undefined names
+select = E9,F63,F7,F82
+show-source = true

.github/dependabot.yaml

@@ -0,0 +1,11 @@
+# Set update schedule for GitHub Actions
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"
+    commit-message:
+      prefix: ci

.github/workflows/cicd.yaml

@@ -0,0 +1,104 @@
+name: "CI/CD"
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - synchronize
+      - opened
+      - reopened
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Test
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres_pass
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    env:
+      ENV: cicd
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Get Metadata
+        id: metadata
+        uses: contiamo/git-metadata-action@main
+      - name: Setup Task
+        uses: arduino/setup-task@v1
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: Gr1N/setup-poetry@v8
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+
+      - name: Setup package
+        run: task setup
+
+      - name: Setup database
+        run: task sample-view
+
+      - name: Test package
+        run: task run
+
+  publish:
+    # Only run if we're on main branch or it's a release:
+    if: github.event_name == 'release'
+    needs: [test]
+    runs-on: ubuntu-latest
+    name: Build and Publish package
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Get Metadata
+        id: metadata
+        uses: contiamo/git-metadata-action@main
+      - name: Setup Task
+        uses: arduino/setup-task@v1
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: Gr1N/setup-poetry@v8
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+
+      - name: Setup package
+        run: task setup
+
+      - name: Build package
+        run: task build
+
+      - name: Publish package
+        run: task publish

.github/workflows/commit-titles.yaml

@@ -0,0 +1,40 @@
+name: "Conventional commit titles"
+on:
+  pull_request:
+    types:
+      # Check title when opened.
+      - opened
+      # Check title when new commits are pushed.
+      # Required to use as a status check.
+      - synchronize
+      # When the title or description change
+      - edited
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v6
+        env:
+          # Ensure pull request titles match the Conventional Commits specification
+          # https://www.conventionalcommits.org/en/v1.0.0/
+          regex: '^(feat|fix|chore|ci|refactor|test|docs)(\(.*\))?!?:'
+        with:
+          script: |
+            if (context.eventName != "pull_request") {
+              core.setFailed("This action only works on pull_request events");
+              return;
+            }
+            core.info(`Checking pull request title with regex: ${process.env.regex}`);
+            const regex = RegExp(process.env.regex);
+            const {data: pullRequest} = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.payload.pull_request.number
+            });
+            const title = pullRequest.title;
+            core.info(`Pull Request title: "${title}"`);
+            if (!regex.test(title)) {
+                core.setFailed(`Pull Request title "${title}" failed to pass match regex - ${regex}`);
+                return;
+            }

.github/workflows/release-please.yaml

@@ -0,0 +1,31 @@
+on:
+  push:
+    branches:
+      - main
+
+name: release-please
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: GoogleCloudPlatform/[email protected]
+        id: release
+        with:
+          # use the CI token to pretend not to be a action
+          token: ${{ secrets.CONTIAMO_CI_TOKEN }}
+          release-type: python
+          package-name: ""
+          bump-minor-pre-major: true
+          changelog-types: |
+            [
+              {"type":"feat","section":"Features","hidden":false},
+              {"type":"fix","section":"Bug Fixes","hidden":false},
+              {"type":"chore","section":"Miscellaneous","hidden":false},
+              {"type":"docs","section":"Miscellaneous","hidden":false},
+              {"type":"refactor","section":"Miscellaneous","hidden":false}
+            ]
+
+      - name: Debug release output
+        env:
+          OUTPUTS: ${{ toJSON(steps.release.outputs) }}
+        run: echo '$OUTPUTS'

.gitignore

@@ -1,3 +1,6 @@
 __pycache__
 .venv
 .pytest_cache
+.env
+dist/
+.mypy_cache

Taskfile.yaml

@@ -23,6 +23,39 @@ tasks:
     cmds:
       - poetry install
 
+  build:
+    desc: build the python package
+    cmds:
+      - poetry build
+
+  publish:
+    desc: publish the python package
+    cmds:
+      - poetry publish
+
+  flake:
+    internal: true
+    cmds:
+      - poetry run flake8 datahub_postgres_lineage
+
+  format:
+    internal: true
+    cmds:
+      - poetry run black --check .
+      # - poetry run isort --check-only .
+
+  typings:
+    internal: true
+    cmds:
+      - poetry run mypy .
+
+  lint:
+    desc: lint the python package
+    deps:
+      - flake
+      - format
+      - typings
+
   wait:
     cmds:
       - sleep {{.SLEEP}}
@@ -71,7 +104,7 @@ tasks:
   sample-view:
     desc: create a sample view
     cmds:
-      - psql -f env/local/sample.sql
+      - psql -f {{.CONFIG_DIR}}/{{.ENV}}/sample.sql
     status:
       - psql -At -c "select 1 from information_schema.views where table_name = 'names'" | grep "1"
       - psql -At -c "select 1 from information_schema.views where table_name = 'emails'" | grep "1"

datahub_postgres_lineage/ingestion.py

@@ -1,34 +1,38 @@
 import logging
 from contextlib import contextmanager
 from dataclasses import dataclass, field
-from typing import Any, Dict, Iterable, List, Optional
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+)
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.emitter import mce_builder
 from datahub.emitter.mcp_builder import mcps_from_mce
+from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
     config_class,
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.common import PipelineContext
-from datahub.ingestion.api.source import TestableSource, SourceReport
+from datahub.ingestion.api.source import SourceReport, TestableSource
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.sql.postgres import PostgresConfig
-from datahub.ingestion.source.sql.sql_common import (
-    make_sqlalchemy_uri,
-)
+from datahub.ingestion.source.sql.sql_common import make_sqlalchemy_uri
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
     StatefulIngestionSourceBase,
 )
 from datahub.utilities.lossy_collections import LossyList
-
 from pydantic import BaseModel, Field, SecretStr
-
 from sqlalchemy import create_engine
-from sqlalchemy.engine import Connection, CursorResult
+from sqlalchemy.engine import Connection
+from sqlalchemy.engine.cursor import CursorResult
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -89,6 +93,8 @@ def report_dropped(self, ent_name: str) -> None:
 
 
 class PostgresLineageConfig(StatefulIngestionConfigBase):
+    options: dict = {}
+
     username: Optional[str] = Field(default=None, description="username")
     password: Optional[SecretStr] = Field(
         default=None, exclude=True, description="password"
@@ -140,6 +146,9 @@ def get_sql_alchemy_url(self, uri_opts: Optional[Dict[str, Any]] = None) -> str:
 @config_class(PostgresLineageConfig)
 @support_status(SupportStatus.TESTING)
 class PostgresLineageSource(StatefulIngestionSourceBase, TestableSource):
+    config: PostgresLineageConfig  # type: ignore
+    report: LineageSourceReport  # type: ignore
+
     def __init__(self, config: PostgresLineageConfig, ctx: PipelineContext):
         super().__init__(config, ctx)
         self.platform = "postgres"
@@ -149,7 +158,7 @@ def __init__(self, config: PostgresLineageConfig, ctx: PipelineContext):
     ### Start required abstract class methods
     @classmethod
     def create(cls, config_dict, ctx):
-        config = PostgresConfig.parse_obj(config_dict)
+        config = PostgresLineageConfig.parse_obj(config_dict)
         return cls(config, ctx)
 
     def get_platform_instance_id(self) -> str:
@@ -195,7 +204,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
         if len(data) == 0:
             return None
 
-        lineage_elements = {}
+        lineage_elements: Dict[str, List[str]] = {}
         # Loop over the lineages in the JSON data.
         for lineage in data:
 
@@ -221,12 +230,9 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
             lineage_elements[key].append(
                 mce_builder.make_dataset_urn(
                     "postgres",
-                    ".".join(
-                        [
-                            self.config.database_alias or self.config.database,
-                            lineage.source_schema,
-                            lineage.source_table,
-                        ]
+                    self.config.get_identifier(
+                        lineage.source_schema,
+                        lineage.source_table,
                     ),
                     self.config.env,
                 )
@@ -240,7 +246,10 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
             # Construct a lineage object.
             urn = mce_builder.make_dataset_urn(
                 "postgres",
-                ".".join([self.config.database, dependent_schema, dependent_view]),
+                self.config.get_identifier(
+                    lineage.dependent_schema,
+                    lineage.dependent_view,
+                ),
                 self.config.env,
             )
 
@@ -257,7 +266,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
                 yield wu
 
     @contextmanager
-    def _get_connection(self) -> Connection:
+    def _get_connection(self) -> Iterator[Connection]:
         # This method can be overridden in the case that you want to dynamically
         # run on multiple databases.
 

env/cicd/.env

@@ -0,0 +1,7 @@
+PGHOST=localhost
+PGPORT=5432
+PGDATABASE=postgres
+PGUSER=postgres
+PGPASSWORD=postgres_pass
+
+DATAHUB_TELEMETRY_ENABLED="false"