chore: update CI to use sharding (#581)

Author: erezrokah

.github/workflows/example.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         dbversion: ['postgres:latest']
-        cloudquery-version: ['5.12.1', 'v5.12.1']
+        cloudquery-version: ['6.8.0', 'v6.8.0']
         os: [ubuntu-latest]
       fail-fast: false
     services:
@@ -49,4 +49,4 @@ jobs:
       - name: Sync with CloudQuery
         env:
           CLOUDQUERY_API_KEY: ${{ secrets.CLOUDQUERY_API_KEY }}
-        run: cloudquery sync example_configs/serial.yml --log-console
+        run: cloudquery sync example_configs/spec.yml --log-console

.github/workflows/example_matrix.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        region: [us-east-1, us-east-2, us-west-1, us-west-2, eu-west-1, eu-west-2] # List of regions to sync in parallel
+        shard: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6]
     services:
       postgres:
         image: postgres:latest
@@ -31,12 +31,6 @@ jobs:
           --health-retries 5
     steps:
       - uses: actions/checkout@v4 # Checkout the code so we have access to the config file
-      - name: Set region in config file
-        uses: jacobtomlinson/gha-find-replace@099c88fbf2a7da26b083521a8bfa13e4f0886b97
-        with:
-          include: example_configs/parallel.yml
-          find: REGION_PLACEHOLDER
-          replace: ${{ matrix.region }}
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4
         with:
@@ -45,9 +39,9 @@ jobs:
       - name: Setup CloudQuery
         uses: ./.
         with:
-          version: 'v5.12.1'
+          version: 'v6.8.0'
       - name: Sync with CloudQuery
-        run: cloudquery sync example_configs/parallel.yml --log-console
+        run: cloudquery sync example_configs/spec.yml --log-console --shard ${{ matrix.shard }}
         env:
           CLOUDQUERY_API_KEY: ${{ secrets.CLOUDQUERY_API_KEY }}
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres?sslmode=disable

example_configs/parallel.yml

@@ -4,8 +4,8 @@ spec:
   path: 'cloudquery/aws'
   version: 'v27.20.0' # latest version of aws plugin
   destinations: ['postgresql']
-  tables: [aws_s3_buckets]
-  skip_dependent_tables: true
+  tables: ['*']
+  skip_tables: ['aws_cloudtrail_events', 'aws_iam_*']
 ---
 kind: destination
 spec: