Merge pull request #13 from zetroot/feature/persist-observes

Persist observations in database
+semver: breaking

Author: zetroot

.github/workflows/docker.yml

@@ -12,15 +12,14 @@ on:
 env:
   REGISTRY: ghcr.io
   # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
+  OBSERVER_IMAGE_NAME: "zetroot/ntorspectator-observer"
+  BUNDLE_IMAGE_NAME: "zetroot/ntorspectator-efbundle"
 
 jobs:
-  build:
-
+  tag-sources:
     runs-on: ubuntu-latest
     permissions:
       contents: write
-      packages: write
 
     steps:
       - name: Checkout repository
@@ -35,14 +34,78 @@ jobs:
 
       - name: Determine Version
         uses: gittools/actions/gitversion/execute@v0
-          
+
       - name: Tag commit
         if: github.event_name == 'push'
         uses: ydataai/create-tag@v1
         with:
           tag: "${{ env.GitVersion_SemVer }}"
           message: "Automated tagging of master commits"
 
+  build:
+    needs: tag-sources
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@master
+        with:
+          platforms: all
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata for observer image
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.OBSERVER_IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push observer image
+        id: build-and-push
+        uses: docker/build-push-action@master
+        with:
+          file: Observer.Dockerfile
+          context: .
+          platforms: linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+  
+
+  bundle:
+    needs: tag-sources
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@master
         with:
@@ -63,18 +126,19 @@ jobs:
 
       # Extract metadata (tags, labels) for Docker
       # https://github.com/docker/metadata-action
-      - name: Extract Docker metadata
+      - name: Extract Docker metadata for bundle image
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ env.BUNDLE_IMAGE_NAME }}
 
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
-      - name: Build and push Docker image
+      - name: Build and push bundle image
         id: build-and-push
         uses: docker/build-push-action@master
         with:
+          file: MigrationBundle.Dockerfile
           context: .
           platforms: linux/arm64
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/smoke.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  smoke-test:
     permissions: write-all
     runs-on: ubuntu-latest
     steps:

Dockerfile

@@ -0,0 +1,16 @@
+FROM --platform=linux/arm64 mcr.microsoft.com/dotnet/runtime:7.0 AS base
+#FROM mcr.microsoft.com/dotnet/runtime:7.0 AS base
+WORKDIR /app
+
+FROM mcr.microsoft.com/dotnet/sdk:7.0.102-jammy-amd64 AS build
+RUN dotnet tool install --global dotnet-ef
+ENV PATH="${PATH}:/root/.dotnet/tools"
+WORKDIR /src
+COPY ["src/", "./"]
+RUN mkdir "/publish"
+WORKDIR "/src/NTorSpectator.Database"
+RUN dotnet ef migrations bundle -r linux-arm64 -f -o /publish/bundle --verbose
+
+FROM base AS final
+WORKDIR /app
+COPY --from=build /publish .

MigrationBundle.Dockerfile

@@ -1,6 +1,13 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NTorSpectator", "NTorSpectator\NTorSpectator.csproj", "{1F4AA365-C2C1-4CEB-B84B-37AF167110F3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NTorSpectator.Observer", "src\NTorSpectator.Observer\NTorSpectator.Observer.csproj", "{8D938363-29D0-41C1-8C0F-FD89956F7A8F}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{0C068D08-CE4D-41B4-A27B-81B25C7AB8FE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NTorSpectator.Database", "src\NTorSpectator.Database\NTorSpectator.Database.csproj", "{816C35A4-1244-4940-93C6-AB440ADB6363}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NTorSpectator.Services", "src\NTorSpectator.Services\NTorSpectator.Services.csproj", "{BBDADD62-DE6E-4F3D-9E72-3B82FC7A8B3E}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -12,5 +19,22 @@ Global
 		{1F4AA365-C2C1-4CEB-B84B-37AF167110F3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1F4AA365-C2C1-4CEB-B84B-37AF167110F3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1F4AA365-C2C1-4CEB-B84B-37AF167110F3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8D938363-29D0-41C1-8C0F-FD89956F7A8F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8D938363-29D0-41C1-8C0F-FD89956F7A8F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8D938363-29D0-41C1-8C0F-FD89956F7A8F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8D938363-29D0-41C1-8C0F-FD89956F7A8F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{816C35A4-1244-4940-93C6-AB440ADB6363}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{816C35A4-1244-4940-93C6-AB440ADB6363}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{816C35A4-1244-4940-93C6-AB440ADB6363}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{816C35A4-1244-4940-93C6-AB440ADB6363}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BBDADD62-DE6E-4F3D-9E72-3B82FC7A8B3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BBDADD62-DE6E-4F3D-9E72-3B82FC7A8B3E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BBDADD62-DE6E-4F3D-9E72-3B82FC7A8B3E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BBDADD62-DE6E-4F3D-9E72-3B82FC7A8B3E}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{8D938363-29D0-41C1-8C0F-FD89956F7A8F} = {0C068D08-CE4D-41B4-A27B-81B25C7AB8FE}
+		{816C35A4-1244-4940-93C6-AB440ADB6363} = {0C068D08-CE4D-41B4-A27B-81B25C7AB8FE}
+		{BBDADD62-DE6E-4F3D-9E72-3B82FC7A8B3E} = {0C068D08-CE4D-41B4-A27B-81B25C7AB8FE}
 	EndGlobalSection
 EndGlobal

NTorSpectator.sln

@@ -0,0 +1,21 @@
+FROM --platform=linux/arm64 mcr.microsoft.com/dotnet/aspnet:7.0-jammy-arm64v8 AS base
+WORKDIR /app
+EXPOSE 8000
+
+FROM mcr.microsoft.com/dotnet/sdk:7.0.102-jammy-amd64 AS build
+WORKDIR /src
+COPY ["src/NTorSpectator.Database/NTorSpectator.Database.csproj", "NTorSpectator.Database/"]
+COPY ["src/NTorSpectator.Services/NTorSpectator.Services.csproj", "NTorSpectator.Services/"]
+COPY ["src/NTorSpectator.Observer/NTorSpectator.Observer.csproj", "NTorSpectator.Observer/"]
+RUN dotnet restore -r linux-arm64 "NTorSpectator.Observer/NTorSpectator.Observer.csproj"
+COPY /src .
+WORKDIR "/src/NTorSpectator.Observer"
+RUN dotnet build -r linux-arm64 --no-self-contained --no-restore "NTorSpectator.Observer.csproj" -c Release -o /app/build
+
+FROM build AS publish
+RUN dotnet publish "NTorSpectator.Observer.csproj" -c Release -o /app/publish -r linux-arm64 --no-self-contained
+
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "NTorSpectator.Observer.dll"]

NTorSpectator/Mastodon/IReporter.cs

@@ -4,15 +4,15 @@ This is a small bot to have fun with several things:
  - Tor daemon interaction
  - Mastodon posting
 
-This small thing starts and tries to get service descriptors for sites mentioned in `sites.txt`. If hidden service descriptor received - the site is marked as up. Else - down. That's simple.
+This small bot loads sites from `sites.txt` into database and tries to get descriptors for them. If hidden service descriptor received - the site is marked as up. Else - down. That's simple.
 
 ## Where is the bot
 
-Follow the bot on mastodon. Staging account is [devmacaca@botsin.space](https://botsin.space/@devmacaca)
+Follow the bot on mastodon [ntorspectator@botsin.space](https://botsin.space/@ntorspectator)
 
 ## Prerequisites 
 
-There are some prerequisites for set up. You need Tor deamon running to request hidden service descriptors, you will need a mastodon account to post collected statuses and you'd better have Seq to collect logs from application.
+There are some prerequisites for set up. You need Tor deamon running to request hidden service descriptors, you will need a mastodon account to post collected statuses and you'd better have Seq to collect logs from application. All collected data is stored in Postgresql, so you will need it too.
 
 ### Tor daemon
 
@@ -28,21 +28,31 @@ Also, save information on toot length on this instance. It will be used in confi
 
 Logging to console is great, but i prefer seq for pet projects. It is small, fast, reliable, painless log collector. The Seq Serilog sink is already built in NTorSpectator, pointing to `http://seq:5341`
 
+### PostgreSQL
+
+Database will run in container, for easier maintanence and setup, but you can run it wherever you want.
+
 ## Setup
 
-The best way to run applications is docker compose. So the sample file looks like this:
+The best way to run applications is docker compose. Try change my mind. So the sample file looks like this:
 
 ```yaml
 version: "3"
 
-services:
+services:    
   tor_spectator:
-    image: ghcr.io/zetroot/ntorspectator:latest
+    image: ghcr.io/zetroot/ntorspectator-observer:master
     platform: linux/arm64
     restart: always
     depends_on:
       seq:
         condition: service_started
+      database:
+        condition: service_healthy
+      migrator:
+        condition: service_completed_successfully
+    ports:
+      - 81:8000
     container_name: ntorspectator
     environment:
       - ASPNETCORE_ENVIRONMENT=Production # this is production environment, is not it?
@@ -51,7 +61,16 @@ services:
       - "./appsettings.Production.json:/app/appsettings.Production.json" # mount production config into app folder
       - "/run/tor/control:/run/tor/control" # also mount tor unix-socket
       - "/run/tor/control.authcookie:/run/tor/control.authcookie" # and not forget about auth cookie
-  
+
+  migrator:
+    image: ghcr.io/zetroot/ntorspectator-efbundle:master
+    depends_on:
+      database:
+        condition: service_healthy
+    platform: linux/arm64
+    restart: no
+    command: /app/bundle --connection "Server=pgdb;Port=5432;Database=ntorspectator;User Id=postgres;Password=password" # connection string to database to apply migrations
+
   seq:
     image: datalust/seq:latest
     container_name: seq_torspec
@@ -64,6 +83,26 @@ services:
       - "5342:5341"
     environment:
       - ACCEPT_EULA=Y
+
+  database:
+    image: 'postgres:15'
+    container_name: pg_torspec
+    hostname: pgdb
+    user: postgres
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 1s
+      timeout: 5s
+      retries: 10
+    ports:
+      - 5432:5432
+    environment:
+      - POSTGRES_USER=postgres # The PostgreSQL user (useful to connect to the database)
+      - POSTGRES_PASSWORD=password # The PostgreSQL password (useful to connect to the database)
+      - POSTGRES_DB=postgres # The PostgreSQL default database (automatically created at first launch)
+    volumes:
+      - "./db-data/:/var/lib/postgresql/data/"
+
 ```
 
 `Sites.txt` is a file with tor onion links, one at a line:
@@ -78,6 +117,9 @@ hzwjmjimhr7bdmfv.........ibt5ojjmpo3pbp5ctwcg37n3hyk7qzid.onion
 And provide production config in `appsettings.Production.json`:
 ```json
 {
+  "ConnectionStrings": {
+    "SpectatorDatabase":"Server=pgdb;Port=5432;Database=ntorspectator;User Id=postgres;Password=password;"
+  },
   "MastodonSettings" : {
     "Instance" : "https://your.cool.mastodon.instance.prod",
     "Token": "YourSuperSecretApiTokenGoesHere"

Observer.Dockerfile

@@ -0,0 +1,18 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+using NTorSpectator.Database.Models;
+
+namespace NTorSpectator.Database.Configuration;
+
+internal class ObservationEntityConfiguration : IEntityTypeConfiguration<Observation>
+{
+    public void Configure(EntityTypeBuilder<Observation> builder)
+    {
+        builder.ToTable("observations");
+        builder.HasKey(x => x.Id);
+        builder.HasIndex(x => x.ObservedAt);
+        builder.HasIndex(x => x.SiteId);
+        
+        builder.HasOne(x => x.Site).WithMany().HasForeignKey(x => x.SiteId).OnDelete(DeleteBehavior.Restrict);
+    }
+}

README.md

@@ -0,0 +1,15 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+using NTorSpectator.Database.Models;
+
+namespace NTorSpectator.Database.Configuration;
+
+internal class SiteEntityConfiguration : IEntityTypeConfiguration<Site>
+{
+    public void Configure(EntityTypeBuilder<Site> builder)
+    {
+        builder.ToTable("sites");
+        builder.HasKey(x => x.Id);
+        builder.HasIndex(x => x.SiteUri);
+    }
+}