feat(explorer): add ecto postgres db (#265)

Co-authored-by: Urix <[email protected]>
Co-authored-by: Mariano Nicolini <[email protected]>
Co-authored-by: Mauro Toscano <[email protected]>
Co-authored-by: Julian Arce <[email protected]>

Author: 5 people

Makefile

@@ -497,9 +497,6 @@ build_binaries:
 	@go build -o ./task_sender/build/aligned-task-sender ./task_sender/cmd/main.go
 	@echo "Task sender built into /task_sender/build/aligned-task-sender"
 
-run_local:
-	./scripts/run_local.sh
-
 __SP1_FFI__: ##
 build_sp1_macos:
 	@cd operator/sp1/lib && cargo build --release
@@ -671,13 +668,51 @@ build_all_ffi_linux: ## Build all FFIs for Linux
 
 
 __EXPLORER__:
-run_devnet_explorer:
+run_devnet_explorer: run_db ecto_setup_db
 	@cd explorer/ && \
 		mix setup && \
 		cp .env.dev .env && \
 		./start.sh
 
-run_explorer:
+run_explorer: run_db ecto_setup_db
 	@cd explorer/ && \
 		mix setup && \
 		./start.sh
+
+build_db:
+	@cd explorer && \
+		docker build -t explorer-postgres-image .
+
+run_db: remove_db_container
+	@cd explorer && \
+		docker run -d --name explorer-postgres-container -p 5432:5432 -v explorer-postgres-data:/var/lib/postgresql/data explorer-postgres-image
+
+ecto_setup_db:
+		@cd explorer/ && \
+		./ecto_setup_db.sh
+
+remove_db_container:
+	@cd explorer && \
+		docker stop explorer-postgres-container || true  && \
+		docker rm explorer-postgres-container || true
+
+clean_db: remove_db_container
+	@cd explorer && \
+		docker volume rm explorer-postgres-data || true
+
+dump_db:
+	@cd explorer && \
+		docker exec -t explorer-postgres-container pg_dumpall -c -U explorer_user > dump.$$(date +\%Y\%m\%d_\%H\%M\%S).sql
+	@echo "Dumped database successfully to /explorer"
+
+recover_db: run_db
+	@read -p $$'\e[32mEnter the dump file to recover (e.g., dump.20230607_123456.sql): \e[0m' DUMP_FILE && \
+	cd explorer && \
+	docker cp $$DUMP_FILE explorer-postgres-container:/dump.sql && \
+	docker exec -t explorer-postgres-container psql -U explorer_user -d explorer_db -f /dump.sql && \
+	echo "Recovered database successfully from $$DUMP_FILE"
+
+explorer_fetch_old_batches:
+	@cd explorer && \
+		./scripts/fetch_old_batches.sh 1600000 1716277 
+		

README.md

@@ -90,7 +90,7 @@ aligned \
 --proof_generator_addr [proof_generator_addr]
 ```
 
-**Examples**
+**Examples**:
 
 ```bash
 aligned \
@@ -866,12 +866,97 @@ To install Grafana, you can follow the instructions on the [official website](ht
 
 - [Erlang 26](https://github.com/asdf-vm/asdf-erlang)
 - [Elixir 1.16.2](https://elixir-ko.github.io/install.html), compiled with OTP 26
-- [Phoenix 1.7.12](https://hexdocs.pm/phoenix/installation.html)
-- [Ecto 3.11.2](https://hexdocs.pm/ecto/getting-started.html)
+- [Docker](https://docs.docker.com/get-docker/)
+
+### DB Setup
+
+To setup the explorer, an installation of the DB is needed.
+
+First you'll need to install docker if you don't have it already. You can follow the instructions [here](https://docs.docker.com/get-docker/).
+
+The explorer uses a PostgreSQL database. To build and start the DB using docker, just run:
+
+```bash
+make build_db
+```
+
+This will build the docker image to be used as our database.
+
+After this, both `make run_explorer` and `make run_devnet_explorer` (see [this](#running-for-local-devnet) for more details) will automatically start, setup and connect to the database, which will be available on `localhost:5432` and the data is persisted in a volume.
+
+<details>
+
+<summary>
+  (Optional) The steps to manually execute the database are as follows...
+</summary>
+
+- Run the database container, opening port `5432`:
+
+```bash
+make run_db
+```
+
+- Configure the database with ecto running `ecto.create` and `ecto.migrate`:
+
+```bash
+make ecto_setup_db
+```
+
+- Start the explorer:
+
+```bash
+make run_explorer # or make run_devnet_explorer
+```
+
+</details>
+
+<br>
+
+In order to clear the DB, you can run:
+
+```bash
+make clean_db
+```
+
+If you need to dumb the data from the DB, you can run:
+
+```bash
+make dump_db
+```
+
+This will create a `dump.$date.sql` SQL script on the `explorer` directory with all the existing data.
+
+Data can be recovered from a `dump.$date.sql` using the following command:
+
+```bash
+make recover_db
+```
+
+Then you'll be requested to enter the file name of the dump you want to recover already positioned in the `/explorer` directory.
+
+This will update your database with the dumped database data.
+
+### Extra scripts
+
+If you want to fetch past batches that for any reason were not inserted into the DB, you will first need to make sure you have the ELIXIR_HOSTNAME .env variable configured. You can get the hostname of your elixir by running `elixir -e 'IO.puts(:inet.gethostname() |> elem(1))'`
+
+Then you can run:
+
+```bash
+make explorer_fetch_old_batches
+```
+
+You can modify which blocks are being fetched by modify the parameters the `explorer_fetch_old_batches.sh` is being recieved
 
 ### Running for local devnet
 
-```make run_devnet_explorer```
+To run the explorer for the local devnet, you'll need to have the devnet running (see [local devnet setup](#local-devnet-setup)) and the DB already setup.
+
+To run the explorer, just run:
+
+```bash
+make run_devnet_explorer
+```
 
 Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.
 You can access to a tasks information by visiting `localhost:4000/batches/:merkle_root`.
@@ -885,6 +970,13 @@ Create a `.env` file in the `/explorer` directory of the project. The `.env` fil
 | `RPC_URL`     | The RPC URL of the network you want to connect to.                                              |
 | `ENVIRONMENT` | The environment you want to run the application in. It can be `devnet`, `holesky` or `mainnet`. |
 | `PHX_HOST`    | The host URL where the Phoenix server will be running.                                          |
+| `DB_NAME` | The name of the postgres database. |
+| `DB_USER` | The username of the postgres database. |
+| `DB_PASS` | The password of the postgres database. |
+| `DB_HOST` | The host URL where the postgres database will be running. |
+| `ELIXIR_HOSTNAME` |  The hostname of your running elixir. Read [Extra Scripts](#extra-scripts) section for more details |
+
+Then you can run the explorer with this env file config by entering the following command:
 
 ```make run_explorer```
 
@@ -935,11 +1027,14 @@ make test
 
 ### SP1
 
-#### Dependencies
+#### SP1 Dependencies
+
 This guide assumes that:
+
 - sp1 prover installed (instructions [here](https://succinctlabs.github.io/sp1/getting-started/install.html))
 - sp1 project to generate the proofs (instructions [here](https://succinctlabs.github.io/sp1/generating-proofs/setup.html))
 - aligned layer repository cloned:
+
     ```bash
     git clone https://github.com/yetanotherco/aligned_layer.git
     ```
@@ -953,13 +1048,15 @@ and check that the proof is generated with `client.prove_compressed` instead of
 First, open a terminal and navigate to the script folder in the sp1 project directory
 
 Then, run the following command to generate a proof:
+
 ```bash
 cargo run --release
 ```
 
 #### How to get the proof verified by AlignedLayer
 
 After generating the proof, you will have to find two different files:
+
 - proof file: usually found under `script` directory, with the name `proof.json` or similar
 - elf file: usually found under `program/elf/` directory
 

explorer/.env.dev

@@ -1,3 +1,13 @@
+# Network
 RPC_URL=http://localhost:8545
 ENVIRONMENT=devnet
+
+# Elixir
 PHX_HOST=http://localhost:4000
+ELIXIR_HOSTNAME=MAC
+
+# Database
+DB_NAME=explorer_db
+DB_USER=explorer_user
+DB_PASS=explorer_pass
+DB_HOST=localhost

explorer/.env.example

@@ -1,3 +1,13 @@
+# Network
 RPC_URL=<rpc_url>
 ENVIRONMENT=<devnet|holesky|mainnet>
+
+# Elixir
 PHX_HOST=<phx_host_url>
+ELIXIR_HOSTNAME=<iex_hostname>
+
+# Database
+DB_NAME=<db_name>
+DB_USER=<db_user>
+DB_PASS=<db_pass>
+DB_HOST=<db_host>

explorer/.gitignore

@@ -35,6 +35,7 @@ explorer-*.tar
 npm-debug.log
 /assets/node_modules/
 
-
 # Environment Variables
 /.env
+
+dump.*.sql

explorer/Dockerfile

@@ -0,0 +1,10 @@
+# https://hub.docker.com/_/postgres
+FROM postgres:16.3
+
+# Environment variables
+ENV POSTGRES_USER=explorer_user
+ENV POSTGRES_PASSWORD=explorer_pass
+ENV POSTGRES_DB=explorer_db
+
+# Expose the default PostgreSQL port
+EXPOSE 5432

explorer/assets/js/app.js

@@ -41,7 +41,7 @@ topbar.config({
 	shadowColor: "rgba(0, 0, 0, .3)"
 });
 window.addEventListener("phx:page-loading-start", (_info) =>
-	topbar.show(150)
+	topbar.show(50)
 );
 window.addEventListener("phx:page-loading-stop", (_info) =>
 	topbar.hide()

explorer/config/config.exs

@@ -21,14 +21,10 @@ config :explorer, ExplorerWeb.Endpoint,
   pubsub_server: Explorer.PubSub,
   live_view: [signing_salt: "XkOXIXZ0"]
 
-# Configures the mailer
-#
-# By default it uses the "Local" adapter which stores the emails
-# locally. You can see the emails in your browser, at "/dev/mailbox".
-#
-# For production it's recommended to configure a different adapter
-# at the `config/runtime.exs`.
-config :explorer, Explorer.Mailer, adapter: Swoosh.Adapters.Local
+# Configures the database
+config :explorer,
+  ecto_repos: [Explorer.Repo],
+  env: Mix.env()
 
 # Configure esbuild (the version is required)
 config :esbuild,

explorer/config/dev.exs

@@ -19,6 +19,7 @@ config :explorer, ExplorerWeb.Endpoint,
     tailwind: {Tailwind, :install_and_run, [:explorer, ~w(--watch)]}
   ]
 
+
 # ## SSL Support
 #
 # In order to use HTTPS in development, a self-signed
@@ -52,8 +53,8 @@ config :explorer, ExplorerWeb.Endpoint,
     ]
   ]
 
-# Enable dev routes for dashboard and mailbox
-config :explorer, dev_routes: true
+# Enable dev routes (for dashboard, etc). only enable behind auth
+# config :explorer, dev_routes: true
 
 # Do not include metadata nor timestamps in development logs
 config :logger, :console, format: "[$level] $message\n"
@@ -70,6 +71,3 @@ config :phoenix_live_view,
   debug_heex_annotations: true,
   # Enable helpful, but potentially expensive runtime checks
   enable_expensive_runtime_checks: true
-
-# Disable swoosh api client as it is only required for production adapters.
-config :swoosh, :api_client, false

explorer/config/prod.exs

@@ -7,12 +7,6 @@ import Config
 # before starting your production server.
 config :explorer, ExplorerWeb.Endpoint, cache_static_manifest: "priv/static/cache_manifest.json"
 
-# Configures Swoosh API Client
-config :swoosh, api_client: Swoosh.ApiClient.Finch, finch_name: Explorer.Finch
-
-# Disable Swoosh Local Memory Storage
-config :swoosh, local: false
-
 # Do not print debug messages in production
 config :logger, level: :info