New installer, project templates and related CLI commands (#495)

Author: droserasprout

.github/workflows/build.yml

@@ -16,9 +16,8 @@ on:
 jobs:
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
-      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       DOCKER_REGISTRY: ghcr.io
       DOCKER_IMAGE_NAME: ${{ github.repository }}
 

.github/workflows/docs.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   docs:
     name: Docs
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - name: Check out the repo
       uses: actions/checkout@v3

.github/workflows/installer.yml

@@ -2,29 +2,34 @@ name: Installer
 on:
   push:
     paths:
-      - 'cookiecutter/**'
-      - 'scripts/**'
+      - 'src/**'
+      - 'scripts/install.py'
       - '.github/workflows/installer.yml'
 
 jobs:
   test:
     name: Installer
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
 
-      - name: Install poetry
-        run: pipx install poetry
-
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-          cache: 'poetry'
 
-      - name: Install poetry
-        run: pipx reinstall poetry --python python3.10
+      - name: Install DipDup
+        run: python src/dipdup/install.py --quiet --path .
+
+      - name: dipdup new
+        run: dipdup new --quiet
+
+      - name: dipdup init
+        run: cd dipdup-indexer; dipdup init
+
+      - name: make install
+        run: cd dipdup-indexer; make install
 
-      - name: Create new project and ensure it's valid
-        run: python scripts/install.py -q cookiecutter
+      - name: make lint
+        run: cd dipdup-indexer; make lint

.github/workflows/release.yml

@@ -7,14 +7,10 @@ on:
 jobs:
   release:
     name: Release
-    runs-on: ubuntu-latest
-    env:
-      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+    runs-on: ubuntu-22.04
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
+        uses: actions/checkout@v3
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -74,7 +70,7 @@ jobs:
       - name: Set up Poetry
         uses: snok/install-poetry@v1
         with:
-          version: '1.2.0'
+          version: '1.2.1'
 
       - name: Install project
         run: make install
@@ -84,7 +80,7 @@ jobs:
         run: make test
 
       - name: Publish stable image (default)
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: Dockerfile
@@ -95,7 +91,7 @@ jobs:
           build-args: PYTEZOS=0
 
       - name: Publish stable image (pytezos)
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: Dockerfile

.github/workflows/test.yml

@@ -40,7 +40,7 @@ jobs:
         uses: actions/cache@v2
         id: cache-replays
         with:
-          path: /tmp/dipdup/replays
+          path: ~/.cache/dipdup/replays
           key: ${{ matrix.os }}-${{ matrix.arch }}-replays
           restore-keys: |
             ${{ matrix.os }}-${{ matrix.arch }}-replays
@@ -49,5 +49,6 @@ jobs:
         run: make install
       - name: Run lint
         run: make lint
+        if: contains(matrix.os, 'ubuntu') && contains(matrix.arch, 'amd64')
       - name: Run tests
         run: make test

CHANGELOG.md

@@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog],
 and this project adheres to [Semantic Versioning].
 
+## [Unreleased]
+
+### Added
+
+- install: New install script based on pipx.
+- cli: `dipdup new` command to create a new project (ex. cookiecutter).
+- cli: `dipdup update` command to update local pipx and poetry installations.
+
+### Fixed
+
+- cli: Commands that only print help pages no longer require a valid config.
+- codegen: Fail lately when datamodel-codegen is not available.
+- config: Allow `dsn` field to be empty.
+- hooks: Raise a `FeatureAvailabilityHook` instead of a warning when trying to execute hooks on SQLite.
+
+### Removed
+
+- install: Cookiecutter template is no longer supported.
+
+### Performance
+
+- cli: Up to 5x decrease in startup time.
+
 ## [6.1.3] - 2022-09-21
 
 ### Added
@@ -48,7 +71,7 @@ and this project adheres to [Semantic Versioning].
 - ci: Build `arm64` images for M1/M2 silicon.
 - ci: Build `-slim` images based on Alpine Linux.
 - ci: Introduced official MacOS support.
-- ci: Introduced interactive installer (dipdup.net/install.py).
+- ci: Introduced interactive installer (dipdup.io/install.py).
 
 ## [6.0.1] - 2022-08-19
 

Dockerfile

@@ -8,7 +8,7 @@ RUN <<eot
     apt update
     apt install -y gcc make git `if [[ $PYTEZOS = "1" ]]; then echo build-essential pkg-config libsodium-dev libsecp256k1-dev libgmp-dev; fi`
 
-    pip install --no-cache-dir poetry==1.2.0
+    pip install --no-cache-dir poetry==1.2.1
 
     mkdir -p /opt/dipdup
 
@@ -37,7 +37,7 @@ SHELL ["/bin/bash", "-c"]
 
 RUN <<eot
     useradd -ms /bin/bash dipdup
-    pip install --no-cache-dir poetry==1.2.0 setuptools
+    pip install --no-cache-dir poetry==1.2.1 setuptools
 
     apt update
     apt install -y --no-install-recommends git `if [[ $PYTEZOS = "1" ]]; then echo libsodium-dev libsecp256k1-dev libgmp-dev; fi`

Makefile

@@ -15,7 +15,7 @@ TAG=latest
 help:           ## Show this help (default)
 	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//'
 
-all:            ## Run a whole CI pipeline: lint, run tests, build docs
+all:            ## Run a whole CI pipeline: formatters, linters and tests
 	make install lint test docs
 
 install:        ## Install project dependencies
@@ -30,6 +30,7 @@ test:           ## Run test suite
 	poetry run pytest --cov-report=term-missing --cov=dipdup --cov-report=xml -n auto --dist loadscope -s -v tests
 
 docs:           ## Build docs
+	scripts/update_cookiecutter.py
 	cd docs
 	make -s clean docs markdownlint orphans || true
 
@@ -70,16 +71,19 @@ image-slim:     ## Build slim Docker image
 ##
 
 release-patch:  ## Release patch version
+	make update all build image
 	bumpversion patch
 	git push --tags
 	git push
 
 release-minor:  ## Release minor version
+	make update all build image
 	bumpversion minor
 	git push --tags
 	git push
 
 release-major:  ## Release major version
+	make update all build image
 	bumpversion major
 	git push --tags
 	git push
@@ -89,14 +93,13 @@ release-major:  ## Release major version
 clean:          ## Remove all files from .gitignore except for `.venv`
 	git clean -xdf --exclude=".venv"
 
-update:         ## Update dependencies, export requirements.txt (wait an eternity)
+update:         ## Update dependencies, export requirements.txt
 	make install
 	poetry update
 
 	cp pyproject.toml pyproject.toml.bak
 	cp poetry.lock poetry.lock.bak
 
-	# NOTE: 1.2.0 spells
 	poetry export --without-hashes -o requirements.txt
 	poetry export --without-hashes -o requirements.pytezos.txt -E pytezos
 	poetry export --without-hashes -o requirements.dev.txt --with dev
@@ -108,6 +111,10 @@ update:         ## Update dependencies, export requirements.txt (wait an eternit
 
 	make install
 
-	scripts/update_demos.sh
+scripts:
+	python scripts/update_cookiecutter.py
+	python scripts/update_demos.py
+	make lint
 
-##
+##
+##

README.md

@@ -1,13 +1,12 @@
-[![Python](https://img.shields.io/badge/made%20with-python-blue.svg?)](https://www.python.org)
-[![GitHub stars](https://img.shields.io/github/stars/dipdup-net/dipdup)](https://github.com/dipdup-net/dipdup)
-[![Latest stable release](https://img.shields.io/github/v/release/dipdup-net/dipdup?label=stable)](https://github.com/dipdup-net/dipdup/releases)
-[![Latest pre-release)](https://img.shields.io/github/v/release/dipdup-net/dipdup?include_prereleases&label=latest)](https://github.com/dipdup-net/dipdup/releases)
-[![PyPI monthly downloads](https://img.shields.io/pypi/dm/dipdup)](https://pypi.org/project/dipdup/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/dipdup?color=2c2c2c)](https://www.python.org)
+[![Latest stable release](https://img.shields.io/github/v/release/dipdup-net/dipdup?label=stable&color=2c2c2c)](https://github.com/dipdup-net/dipdup/releases)
+[![Latest pre-release)](https://img.shields.io/github/v/release/dipdup-net/dipdup?include_prereleases&label=latest&color=2c2c2c)](https://github.com/dipdup-net/dipdup/releases)
+[![License: MIT](https://img.shields.io/github/license/dipdup-net/dipdup?color=2c2c2c)](https://github.com/dipdup-net/dipdup/blob/master/LICENSE)
 <br>
-[![GitHub tests](https://img.shields.io/github/workflow/status/dipdup-net/dipdup/Test)](https://github.com/dipdup-net/dipdup/actions)
-[![GitHub issues](https://img.shields.io/github/issues/dipdup-net/dipdup)](https://github.com/dipdup-net/dipdup/issues)
-[![GitHub pull requests](https://img.shields.io/github/issues-pr/dipdup-net/dipdup)](https://github.com/dipdup-net/dipdup/pulls)
-[![License: MIT](https://img.shields.io/github/license/dipdup-net/dipdup)](https://github.com/dipdup-net/dipdup/blob/master/LICENSE)
+[![GitHub stars](https://img.shields.io/github/stars/dipdup-net/dipdup?color=2c2c2c)](https://github.com/dipdup-net/dipdup)
+[![PyPI monthly downloads](https://img.shields.io/pypi/dm/dipdup?color=2c2c2c)](https://pypi.org/project/dipdup/)
+[![GitHub issues](https://img.shields.io/github/issues/dipdup-net/dipdup?color=2c2c2c)](https://github.com/dipdup-net/dipdup/issues)
+[![GitHub pull requests](https://img.shields.io/github/issues-pr/dipdup-net/dipdup?color=2c2c2c)](https://github.com/dipdup-net/dipdup/pulls)
 
 ```text
         ____   _         ____              
@@ -22,12 +21,12 @@ DipDup is a Python framework for building indexers of [Tezos](https://tezos.com/
 
 * **Ready to build your first indexer?** Head to [Quickstart](https://docs.dipdup.io/quickstart).
 
-* **Looking for examples?** Check out [Demo Projects](https://github.com/dipdup-net/dipdup/tree/master/src).
+* **Looking for examples?** Check out [Demo Projects](https://docs.dipdup.io/examples/demo-projects) and [Built with DipDup](https://docs.dipdup.io/examples/built-with-dipdup) pages.
 
-* **Want to contribute?** See [Contribution Guide](https://github.com/dipdup-net/dipdup/tree/master/CONTRIBUTING.md).
+* **Want to participate?** Vote for issues on [GitHub](https://github.com/dipdup-net/dipdup/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc) or [become a sponsor](https://github.com/sponsors/dipdup-net).
 
 * **Have a question?** Contact us on [Discord](https://discord.com/invite/RcPGSdcVSx), [Telegram](https://t.me/baking_bad_chat), or [Slack](https://tezos-dev.slack.com/archives/CV5NX7F2L)!
 
-This project is maintained by the [Baking Bad](https://baking-bad.org/) team.
+This project is maintained by the [Baking Bad](https://bakingbad.dev/) team.
 <br>
 Development is supported by [Tezos Foundation](https://tezos.foundation/).