Matrix builds, interactive installer (#465)

Author: droserasprout

.dockerignore

@@ -6,8 +6,8 @@
 !pyproject.toml
 !poetry.lock
 !README.md
-!install_dependencies.sh
 !requirements**
+!scripts/install_dependencies.sh
 
 # Add code
 !src/dipdup/

.github/workflows/build.yml

@@ -23,6 +23,27 @@ jobs:
       DOCKER_REGISTRY: ghcr.io
       DOCKER_IMAGE_NAME: ${{ github.repository }}
 
+    strategy:
+      matrix:
+        image:
+          - name: default
+            suffix: ''
+            platforms: linux/amd64,linux/arm64
+            dockerfile: Dockerfile
+            build_args: 'PYTEZOS=0'
+
+          - name: pytezos
+            suffix: -pytezos
+            platforms: linux/amd64,linux/arm64
+            dockerfile: Dockerfile
+            build_args: 'PYTEZOS=1'
+
+          - name: slim
+            suffix: -slim
+            dockerfile: Dockerfile.slim
+            platforms: linux/amd64,linux/arm64
+            build_args: ''
+
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
@@ -44,75 +65,27 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up metadata (default)
-        id: meta-default
-        uses: docker/metadata-action@v3
-        with:
-          images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
-          flavor: |
-            latest=false
-          tags: |
-            type=ref,event=branch
-            type=ref,event=tag
-
-      - name: Set up metadata (pytezos)
-        id: meta-pytezos
+      - name: Set up metadata
+        id: meta
         uses: docker/metadata-action@v3
         with:
           images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
           flavor: |
             latest=false
-            suffix=-pytezos
+            suffix={{ matrix.image.suffix }}
           tags: |
             type=ref,event=branch
             type=ref,event=tag
 
-      - name: Set up metadata (slim)
-        id: meta-slim
-        uses: docker/metadata-action@v3
-        with:
-          images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
-          flavor: |
-            latest=false
-            suffix=-slim
-          tags: |
-            type=ref,event=branch
-            type=ref,event=tag
-
-      - name: Publish nightly image (default)
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          cache-from: type=gha,scope=default
-          cache-to: type=gha,scope=default,mode=max
-          tags: ${{ steps.meta-default.outputs.tags }}
-          labels: ${{ steps.meta-default.outputs.labels }}
-          build-args: PYTEZOS=0
-
-      - name: Publish nightly image (pytezos)
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          cache-from: type=gha,scope=pytezos
-          cache-to: type=gha,scope=pytezos,mode=max
-          tags: ${{ steps.meta-pytezos.outputs.tags }}
-          labels: ${{ steps.meta-pytezos.outputs.labels }}
-          build-args: PYTEZOS=1
-
-      - name: Publish nightly image (slim)
+      - name: Publish nightly image
         uses: docker/build-push-action@v3
         with:
           context: .
-          file: Dockerfile.slim
-          platforms: linux/amd64,linux/arm64
+          file: ${{ matrix.image.dockerfile }}
+          platforms: ${{ matrix.image.platforms }}
           push: true
-          cache-from: type=gha,scope=slim
-          cache-to: type=gha,scope=slim,mode=max
-          tags: ${{ steps.meta-slim.outputs.tags }}
-          labels: ${{ steps.meta-slim.outputs.labels }}
+          cache-from: type=gha,scope={{ matrix.image.name }}
+          cache-to: type=gha,scope={{ matrix.image.name }},mode=max
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: ${{ matrix.image.build_args }}

.github/workflows/docs.yml

@@ -8,6 +8,8 @@ on:
       - master
     paths:
       - 'docs/**'
+      - 'scripts/**'
+      - ".github/workflows/docs.yml"
 
 jobs:
   docs:
@@ -68,4 +70,12 @@ jobs:
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: ./docs/dist/docs
-        destination_dir: docs-next
+        destination_dir: docs-next
+
+    - name: Deploy scripts to GH Pages
+      if: startsWith(github.ref, 'refs/heads') || contains(github.ref, '-rc')
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./scripts
+        keep_files: true

CHANGELOG.md

@@ -9,9 +9,10 @@ and this project adheres to [Semantic Versioning].
 
 ### Added
 
-- ci: Introduced official MacOS support.
 - ci: Build `arm64` images for M1/M2 silicon.
 - ci: Build `-slim` images based on Alpine Linux.
+- ci: Introduced official MacOS support.
+- ci: Introduced interactive installer (dipdup.net/install.py).
 
 ## [6.0.1] - 2022-08-19
 

Dockerfile

@@ -52,7 +52,7 @@ WORKDIR /home/dipdup/
 ENTRYPOINT ["dipdup"]
 CMD ["run"]
 
-COPY --chown=dipdup --chmod=0755 install_dependencies.sh /opt/dipdup/.venv/bin/install_dependencies
-COPY --chown=dipdup --chmod=0755 install_dependencies.sh /opt/dipdup/.venv/bin/inject_pyproject
+COPY --chown=dipdup --chmod=0755 scripts/install_dependencies.sh /opt/dipdup/.venv/bin/install_dependencies
+COPY --chown=dipdup --chmod=0755 scripts/install_dependencies.sh /opt/dipdup/.venv/bin/inject_pyproject
 COPY --chown=dipdup --from=compile-image /opt/dipdup /opt/dipdup
 COPY --chown=dipdup . /opt/dipdup

Dockerfile.slim

@@ -20,7 +20,7 @@ WORKDIR /home/dipdup/
 ENTRYPOINT ["dipdup"]
 CMD ["run"]
 
-COPY --chown=dipdup --chmod=0755 install_dependencies.sh /opt/dipdup/bin/install_dependencies
-COPY --chown=dipdup --chmod=0755 install_dependencies.sh /opt/dipdup/bin/inject_pyproject
+COPY --chown=dipdup --chmod=0755 scripts/install_dependencies.sh /opt/dipdup/bin/install_dependencies
+COPY --chown=dipdup --chmod=0755 scripts/install_dependencies.sh /opt/dipdup/bin/inject_pyproject
 COPY --chown=dipdup --from=compile-image /opt/dipdup /opt/dipdup
 COPY --chown=dipdup . /opt/dipdup

Makefile

@@ -40,37 +40,39 @@ homepage:       ## Build homepage
 ##
 
 isort:          ## Format with isort
-	poetry run isort src tests
+	poetry run isort src tests scripts
 
 black:          ## Format with black
-	poetry run black src tests
+	poetry run black src tests scripts
 
 flake:          ## Lint with flake8
-	poetry run flakeheaven lint src tests
+	poetry run flakeheaven lint src tests scripts
 
 mypy:           ## Lint with mypy
-	poetry run mypy src tests
+	poetry run mypy src tests scripts
 
 cover:          ## Print coverage for the current branch
 	poetry run diff-cover --compare-branch `git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'` coverage.xml
 
 build:          ## Build Python wheel package
 	poetry build
 
-image:
+image:          ## Build all Docker images
 	make image-default
 	make image-pytezos
 	make image-slim
 
-image-default:          ## Build Docker image
+image-default:  ## Build default Docker image
 	docker buildx build . --progress plain -t dipdup:${TAG}
 
-image-pytezos:
+image-pytezos:  ## Build pytezos Docker image
 	docker buildx build . --progress plain -t dipdup:${TAG}-pytezos --build-arg PYTEZOS=1
 
-image-slim:
+image-slim:     ## Build slim Docker image
 	docker buildx build . --progress plain -t dipdup:${TAG}-slim -f Dockerfile.slim
 
+##
+
 release-patch:  ## Release patch version
 	bumpversion patch
 	git push --tags
@@ -86,21 +88,30 @@ release-major:  ## Release major version
 	git push --tags
 	git push
 
+##
+
 clean:          ## Remove all files from .gitignore except for `.venv`
 	git clean -xdf --exclude=".venv"
 
-##
-
-requirements:   ## Update dependencies, export requirements.txt
+update:         ## Update dependencies, export requirements.txt (wait an eternity)
 	make install
 	poetry update
+
 	cp pyproject.toml pyproject.toml.bak
 	cp poetry.lock poetry.lock.bak
+
 	poetry export -o requirements.txt
 	poetry export -o requirements.pytezos.txt -E pytezos
 	poetry export -o requirements.dev.txt --dev
+
 	poetry remove datamodel-code-generator
 	poetry export -o requirements.slim.txt
+
 	mv pyproject.toml.bak pyproject.toml
 	mv poetry.lock.bak poetry.lock
+
 	make install
+
+	scripts/update-demos.sh
+
+##

cookiecutter/cookiecutter.json

@@ -16,8 +16,8 @@
       "timescale/timescaledb:latest-pg13"
     ],
     "hasura_version": [
-      "hasura/graphql-engine:v2.10.1",
-      "hasura/graphql-engine:v2.9.0"
+      "hasura/graphql-engine:v2.11.0",
+      "hasura/graphql-engine:v2.10.1"
     ],
 
     "🧹": "Code quality (press enter)",

docs/Makefile

@@ -46,6 +46,14 @@ orphans:
 	cat SUMMARY.md | grep ".md" > mentioned.tmp
 	sed -i -E 's@^.*\[(.*)\]\((.*)\).*$$@\2@p' mentioned.tmp
 	cat mentioned.tmp | uniq | sort -o mentioned.tmp
-	find . | grep ".md" | grep -v -e "node_modules/" -e " book/" -e "placeholders.md" -e "SUMMARY.md" -e "mdbook-summary" -e "include" | cut -c "3-" | sort -o existing.tmp
+	find . | grep ".md" | grep -v \
+		-e "node_modules/" \
+		-e " book/" \
+		-e "placeholders.md" \
+		-e "SUMMARY.md" \
+		-e "mdbook-summary" \
+		-e "mdbook-cookiecutter" \
+		-e "include" \
+		| cut -c "3-" | sort -o existing.tmp
 	diff --color mentioned.tmp existing.tmp
 	rm mentioned.tmp existing.tmp

docs/getting-started/installation.md

@@ -26,7 +26,7 @@ We aim to improve cross-platform compatibility in future releases ([issue](https
 You can initialize a hello-world project interactively by choosing configuration options in the terminal. The following command will install [`cookiecutter`](https://cookiecutter.readthedocs.io/en/stable/README.html) and create a new project in the current directory.
 
 ```shell
-sh <(curl https://raw.githubusercontent.com/dipdup-net/dipdup-py/master/install.sh)
+python -c "$(curl -sSL https://dipdup.net/install.py)"
 ```
 
 ### Poetry