Merge pull request #30 from MobileTeleSystems/develop

Release 0.1.1

Author: dmitry-pedchenko

.dockerignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+*.envx
+*.cover
+.coverage
+.coverage.*
+.cache
+*.log
+.git
+.mypy_cache
+.pytest_cache
+.idea
+.vscode
+.venv/
+venv/
+cached_jars/

.env.docker

@@ -0,0 +1,53 @@
+TZ=UTC
+
+# Postgres
+POSTGRES_HOST=db
+POSTGRES_PORT=5432
+POSTGRES_DB=syncmaster
+POSTGRES_USER=user
+POSTGRES_PASSWORD=secret
+
+# RabbitMQ
+RABBITMQ_HOST=rabbitmq
+RABBITMQ_PORT=5672
+RABBITMQ_USER=guest
+RABBITMQ_PASSWORD=guest
+
+# HDFS
+HDFS_HOST=test-hive
+HDFS_WEBHDFS_PORT=9870
+HDFS_IPC_PORT=9820
+
+# S3
+MINIO_ACCESS_KEY=syncmaster
+MINIO_ROOT_USER=syncmaster
+MINIO_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+MINIO_ROOT_PASSWORD=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+
+TEST_S3_HOST=test-s3
+TEST_S3_PORT=9000
+TEST_S3_ACCESS_KEY=syncmaster
+TEST_S3_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+TEST_S3_BUCKET=syncmaster
+
+TEST_POSTGRES_HOST=test-postgres
+TEST_POSTGRES_PORT=5432
+TEST_POSTGRES_USER=user
+TEST_POSTGRES_PASSWORD=secret
+TEST_POSTGRES_DB=syncmaster
+
+TEST_ORACLE_HOST=test-oracle
+TEST_ORACLE_PORT=1521
+TEST_ORACLE_USER=test_user
+TEST_ORACLE_PASSWORD=test_password
+TEST_ORACLE_SERVICE_NAME=XEPDB1
+
+TEST_HIVE_CLUSTER=test-hive
+TEST_HIVE_USER=hive_user
+TEST_HIVE_PASSWORD=hive_password
+
+ENV=LOCAL
+
+SPARK_CONF_DIR=/app/tests/spark/hive/conf/
+HADOOP_CONF_DIR=/app/tests/spark/hadoop/
+HIVE_CONF_DIR=/app/tests/spark/hive/conf/

.env.local

@@ -0,0 +1,53 @@
+export TZ=UTC
+
+# Postgres
+export POSTGRES_HOST=localhost
+export POSTGRES_PORT=5432
+export POSTGRES_DB=syncmaster
+export POSTGRES_USER=user
+export POSTGRES_PASSWORD=secret
+
+# RabbitMQ
+export RABBITMQ_HOST=localhost
+export RABBITMQ_PORT=5672
+export RABBITMQ_USER=guest
+export RABBITMQ_PASSWORD=guest
+
+# HDFS
+export HDFS_HOST=localhost
+export HDFS_WEBHDFS_PORT=9870
+export HDFS_IPC_PORT=9820
+
+# S3
+export MINIO_ACCESS_KEY=syncmaster
+export MINIO_ROOT_USER=syncmaster
+export MINIO_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+export MINIO_ROOT_PASSWORD=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+
+export TEST_S3_HOST=localhost
+export TEST_S3_PORT=9010
+export TEST_S3_ACCESS_KEY=syncmaster
+export TEST_S3_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+export TEST_S3_BUCKET=syncmaster
+
+export TEST_POSTGRES_HOST=localhost
+export TEST_POSTGRES_PORT=5433
+export TEST_POSTGRES_USER=user
+export TEST_POSTGRES_PASSWORD=secret
+export TEST_POSTGRES_DB=syncmaster
+
+export TEST_ORACLE_HOST=localhost
+export TEST_ORACLE_PORT=1522
+export TEST_ORACLE_USER=test_user
+export TEST_ORACLE_PASSWORD=test_password
+export TEST_ORACLE_SERVICE_NAME=XEPDB1
+
+export TEST_HIVE_CLUSTER=test-hive
+export TEST_HIVE_USER=hive_user
+export TEST_HIVE_PASSWORD=hive_password
+
+export ENV=LOCAL
+
+export SPARK_CONF_DIR=./app/tests/spark/hive/conf/
+export HADOOP_CONF_DIR=./app/tests/spark/hadoop/
+export HIVE_CONF_DIR=./app/tests/spark/hive/conf/

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,24 @@
+<!-- Thank you for your contribution! -->
+<!-- Unless your change is trivial, please create an issue to discuss the change before creating a PR -->
+<!-- See https://github.com/MobileTeleSystems/syncmaster/blob/develop/CONTRIBUTING.rst for help on Contributing -->
+<!-- PLEASE DO **NOT** put issue ids in the PR title! Instead, add a descriptive title and put ids in the body -->
+
+## Change Summary
+
+<!-- Please give a short summary of the changes. -->
+
+## Related issue number
+
+<!-- Are there any issues opened that will be resolved by merging this change? -->
+<!-- WARNING: please use "fix #123" style references so the issue is closed when this PR is merged. -->
+
+## Checklist
+
+* [ ] Commit message and PR title is comprehensive
+* [ ] Keep the change as small as possible
+* [ ] Unit and integration tests for the changes exist
+* [ ] Tests pass on CI and coverage does not decrease
+* [ ] Documentation reflects the changes where applicable
+* [ ] `docs/changelog/next_release/<pull request or issue id>.<change type>.rst` file added describing change
+  (see [CONTRIBUTING.rst](https://github.com/MobileTeleSystems/syncmaster/blob/develop/CONTRIBUTING.rst) for details.)
+* [ ] My PR is ready to review.

.github/dependabot.yml

@@ -0,0 +1,26 @@
+# Set update schedule for GitHub Actions
+
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    labels:
+      - type:ci
+    # https://til.simonwillison.net/github/dependabot-python-setup
+    groups:
+      github-actions:
+        patterns:
+          - '*'
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: weekly
+    labels:
+      - type:dependency
+    # https://til.simonwillison.net/github/dependabot-python-setup
+    groups:
+      python-packages:
+        patterns:
+          - '*'

.github/labels.yml

@@ -0,0 +1,69 @@
+# config for https://github.com/marketplace/actions/github-labeler
+
+  - name: attention:help wanted
+    description: Extra attention is needed
+    color: bfdadc
+    from_name: help wanted
+
+  - name: attention:invalid
+    description: This doesn't seem right
+    color: ea2357
+    from_name: invalid
+
+  - name: ci:skip-changelog
+    description: Add this label to skip changelog file check
+    color: 04990f
+
+  - name: component:backend
+    description: Backend-related changes
+    color: '5319e7'
+
+  - name: component:worker
+    description: Worker-related changes
+    color: 75f526
+
+  - name: kind:bug
+    description: Something isn't working
+    color: d73a4a
+    from_name: bug
+
+  - name: kind:feature
+    description: New feature or request
+    color: 389a3f
+
+  - name: kind:improvement
+    description: Improvement of some existing feature
+    color: 1a92c2
+    from_name: enhancement
+
+  - name: kind:question
+    description: Further information is requested
+    color: 0e857c
+    from_name: question
+
+  - name: resolution:duplicate
+    description: This issue or pull request already exists
+    color: cfd3d7
+    from_name: duplicate
+
+  - name: resolution:wontfix
+    description: This will not be worked on
+    color: ec103b
+    from_name: wontfix
+
+  - name: type:ci
+    description: CI-related changes
+    color: cdb0bd
+
+  - name: type:dependency
+    description: Dependency-related changes
+    color: 214efe
+
+  - name: type:documentation
+    description: Improvements or additions to documentation
+    color: 6b9f54
+    from_name: documentation
+
+  - name: type:tests
+    description: Tests-related changes
+    color: 5cca5b

.github/workflows/automerge.yml

@@ -0,0 +1,27 @@
+name: Automerge
+
+on:
+  pull_request_target:
+
+jobs:
+  automerge:
+    name: Enable pull request automerge
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.user.login == 'pre-commit-ci[bot]' || github.event.pull_request.user.login == 'dependabot[bot]'
+
+    steps:
+      - uses: alexwilson/[email protected]
+        with:
+          github-token: ${{ secrets.AUTOMERGE_TOKEN }}
+          merge-method: REBASE
+
+  autoapprove:
+    name: Automatically approve pull request
+    needs: [automerge]
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.user.login == 'pre-commit-ci[bot]' || github.event.pull_request.user.login == 'dependabot[bot]'
+
+    steps:
+      - uses: hmarr/auto-approve-action@v4
+        with:
+          github-token: ${{ secrets.AUTOMERGE_TOKEN }}

.github/workflows/backend_docker_image.yml

@@ -0,0 +1,80 @@
+name: Backend docker image
+
+on:
+  push:
+    branches:
+      - develop
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  release:
+    name: Build & push backend image to Dockerhub
+    runs-on: ubuntu-latest
+    if: github.repository == 'MobileTeleSystems/syncmaster'  # prevent running on forks
+
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set tag
+        id: set_tag
+        run: |
+          if [[ "${{ github.ref_type }}" == "branch" && "${{ github.ref_name }}" == "develop" ]]; then
+            echo "TAG=mtsrus/syncmaster-backend:develop" >> $GITHUB_ENV
+          elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+            echo "TAG=mtsrus/syncmaster-backend:latest,mtsrus/syncmaster-backend:${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
+      - name: Build Backend image
+        uses: docker/build-push-action@v5
+        with:
+          tags: ${{ env.TAG }}
+          context: .
+          file: docker/backend.dockerfile
+          pull: true
+          push: true
+          cache-to: type=gha,mode=max
+          cache-from: type=gha
+          platforms: |
+            linux/amd64
+            linux/arm64/v8
+          provenance: mode=max
+
+      - name: Convert README to Markdown
+        uses: docker://pandoc/core:2.9
+        with:
+          args: >-
+            --output=README.md
+            --from=rst
+            --to=gfm
+            --wrap=none
+            README.rst
+
+      - name: Update DockerHub Description
+        uses: peter-evans/dockerhub-description@v4
+        if: github.ref_type == 'tag'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          # this requires token with read+write+delete permissions. read+write is not enough!
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          repository: mtsrus/syncmaster-backend
+          short-description: ${{ github.event.repository.description }}
+          enable-url-completion: true