✨ Windmill: Add Postgres-Backed Integration Tests for SQL Logic

.github/workflows/tests.yml

@@ -92,6 +92,56 @@ jobs:
           CARGO_BUILD_JOBS: ${{ matrix.build_jobs || 4 }}
         run: cd packages/${{ matrix.service }} && cargo test ${{ matrix.extra }}
 
+  windmill-sql-regression:
+    name: Windmill SQL regression
+    runs-on: ubuntu-24.04
+    timeout-minutes: 25
+    services:
+      postgres:
+        image: postgres:18-bookworm
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: windmill_test
+        ports:
+          - 5432/tcp
+        options: >-
+          --health-cmd "pg_isready -U postgres -d windmill_test"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 12
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: 1.90.0
+
+      - name: Cache Cargo registry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-registry-
+
+      - name: Install system dependencies
+        run: |
+            sudo apt-get update
+            sudo apt-get install -y build-essential pkg-config libssl-dev protobuf-compiler libprotobuf-dev
+
+      - name: Run ignored SQL regression
+        env:
+          PROTOC: "/usr/bin/protoc"
+          WINDMILL_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:${{ job.services.postgres.ports[5432] }}/windmill_test
+          CARGO_BUILD_JOBS: "2"
+        run: |
+          cd packages/windmill
+          cargo test find_area_ballots_returns_latest_vote_per_voter -- --ignored
+
   # ===========================================================================
   # JOB: notify-on-failure
   # ===========================================================================
@@ -100,7 +150,7 @@ jobs:
   # ===========================================================================
   notify-on-failure:
     name: Notify on failure
-    needs: run-tests
+    needs: [run-tests, windmill-sql-regression]
     if: failure()
     runs-on: gcp-selfhosted-ubuntu24
     steps:

.gitignore

@@ -60,4 +60,7 @@ packages/voting-portal/logs
 packages/ballot-verifier/logs
 
 # ignore artifacts for airgapped environments
-airgapped-artifacts
+airgapped-artifacts
+
+# nested private repo checkout used locally inside the codespace
+beyond/

packages/windmill/src/lib.rs

@@ -11,4 +11,6 @@ extern crate quick_error;
 pub mod postgres;
 pub mod services;
 pub mod tasks;
+#[cfg(test)]
+pub mod test_database;
 pub mod types;

packages/windmill/src/services/cast_votes.rs

@@ -62,30 +62,48 @@ impl TryFrom<Row> for CastVote {
     }
 }
 
-#[instrument(err)]
-pub async fn find_area_ballots(
-    hasura_transaction: &Transaction<'_>,
+fn build_area_ballots_select_sql(
     tenant_id: &str,
     election_event_id: &str,
     area_id: &str,
     election_id: &str,
-    output_file: &PathBuf,
-) -> Result<()> {
-    // COPY does not support parameters so we have to add them using format
-    let areas_statement = format!(
+) -> Result<String> {
+    let tenant_uuid = Uuid::parse_str(tenant_id)
+        .map_err(|err| anyhow!("Error parsing tenant_id as UUID: {}", err))?;
+    let election_event_uuid = Uuid::parse_str(election_event_id)
+        .map_err(|err| anyhow!("Error parsing election_event_id as UUID: {}", err))?;
+    let area_uuid = Uuid::parse_str(area_id)
+        .map_err(|err| anyhow!("Error parsing area_id as UUID: {}", err))?;
+    let election_uuid = Uuid::parse_str(election_id)
+        .map_err(|err| anyhow!("Error parsing election_id as UUID: {}", err))?;
+
+    Ok(format!(
         r#"
                     SELECT DISTINCT ON (election_id, voter_id_string)
                         voter_id_string,
                         content
                     FROM "sequent_backend".cast_vote
                     WHERE
-                        tenant_id = '{tenant_id}' AND
-                        election_event_id = '{election_event_id}' AND
-                        area_id = '{area_id}' AND
-                        election_id = '{election_id}'
+                        tenant_id = '{tenant_uuid}' AND
+                        election_event_id = '{election_event_uuid}' AND
+                        area_id = '{area_uuid}' AND
+                        election_id = '{election_uuid}'
                     ORDER BY election_id, voter_id_string, created_at DESC
                 "#
-    );
+    ))
+}
+
+#[instrument(err)]
+pub async fn find_area_ballots(
+    hasura_transaction: &Transaction<'_>,
+    tenant_id: &str,
+    election_event_id: &str,
+    area_id: &str,
+    election_id: &str,
+    output_file: &PathBuf,
+) -> Result<()> {
+    let areas_statement =
+        build_area_ballots_select_sql(tenant_id, election_event_id, area_id, election_id)?;
 
     let tokio_temp_file = File::create(output_file)
         .await
@@ -694,3 +712,199 @@ pub async fn count_cast_votes_election_event(
 
     Ok(count)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::build_area_ballots_select_sql;
+    use crate::test_database::TestDatabase;
+    use anyhow::{Context, Result};
+    use chrono::{Duration, Utc};
+    use serde_json::json;
+    use uuid::Uuid;
+
+    #[tokio::test]
+    #[ignore = "Requires Postgres bootstrap. Run with cargo test -p windmill find_area_ballots_returns_latest_vote_per_voter -- --ignored"]
+    async fn find_area_ballots_returns_latest_vote_per_voter() -> Result<()> {
+        let database = TestDatabase::bootstrap().await?;
+
+        let tenant_id = Uuid::new_v4();
+        let election_event_id = Uuid::new_v4();
+        let election_id = Uuid::new_v4();
+        let area_id = Uuid::new_v4();
+        let duplicated_voter_id = "voter-1";
+        let other_voter_id = "voter-2";
+        let older_vote_time = Utc::now() - Duration::minutes(2);
+        let newer_vote_time = older_vote_time + Duration::minutes(1);
+
+        database
+            .client
+            .execute(
+                r#"
+                INSERT INTO sequent_backend.tenant (id, slug)
+                VALUES ($1, $2)
+                "#,
+                &[&tenant_id, &"tenant"],
+            )
+            .await?;
+
+        database
+            .client
+            .execute(
+                r#"
+                INSERT INTO sequent_backend.election_event (
+                    id,
+                    tenant_id,
+                    presentation,
+                    encryption_protocol
+                )
+                VALUES ($1, $2, $3, $4)
+                "#,
+                &[
+                    &election_event_id,
+                    &tenant_id,
+                    &json!({"i18n": {"en": {"name": "Event"}}}),
+                    &"protocol-test",
+                ],
+            )
+            .await?;
+
+        database
+            .client
+            .execute(
+                r#"
+                INSERT INTO sequent_backend.election (id, tenant_id, election_event_id, presentation)
+                VALUES ($1, $2, $3, $4)
+                "#,
+                &[&election_id, &tenant_id, &election_event_id, &json!({"i18n": {"en": {"name": "Election"}}})],
+            )
+            .await?;
+
+        database
+            .client
+            .execute(
+                r#"
+                INSERT INTO sequent_backend.area (id, tenant_id, election_event_id, presentation)
+                VALUES ($1, $2, $3, $4)
+                "#,
+                &[
+                    &area_id,
+                    &tenant_id,
+                    &election_event_id,
+                    &json!({"i18n": {"en": {"name": "Area"}}}),
+                ],
+            )
+            .await?;
+
+        database
+            .client
+            .execute(
+                r#"
+                INSERT INTO sequent_backend.cast_vote (
+                    id,
+                    tenant_id,
+                    election_event_id,
+                    election_id,
+                    area_id,
+                    voter_id_string,
+                    content,
+                    created_at,
+                    last_updated_at
+                )
+                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $8)
+                "#,
+                &[
+                    &Uuid::new_v4(),
+                    &tenant_id,
+                    &election_event_id,
+                    &election_id,
+                    &area_id,
+                    &duplicated_voter_id,
+                    &"older-ballot",
+                    &older_vote_time,
+                ],
+            )
+            .await?;
+
+        database
+            .client
+            .execute(
+                r#"
+                INSERT INTO sequent_backend.cast_vote (
+                    id,
+                    tenant_id,
+                    election_event_id,
+                    election_id,
+                    area_id,
+                    voter_id_string,
+                    content,
+                    created_at,
+                    last_updated_at
+                )
+                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $8)
+                "#,
+                &[
+                    &Uuid::new_v4(),
+                    &tenant_id,
+                    &election_event_id,
+                    &election_id,
+                    &area_id,
+                    &duplicated_voter_id,
+                    &"newer-ballot",
+                    &newer_vote_time,
+                ],
+            )
+            .await?;
+
+        database
+            .client
+            .execute(
+                r#"
+                INSERT INTO sequent_backend.cast_vote (
+                    id,
+                    tenant_id,
+                    election_event_id,
+                    election_id,
+                    area_id,
+                    voter_id_string,
+                    content,
+                    created_at,
+                    last_updated_at
+                )
+                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $8)
+                "#,
+                &[
+                    &Uuid::new_v4(),
+                    &tenant_id,
+                    &election_event_id,
+                    &election_id,
+                    &area_id,
+                    &other_voter_id,
+                    &"other-ballot",
+                    &(newer_vote_time + Duration::minutes(1)),
+                ],
+            )
+            .await?;
+
+        let sql = build_area_ballots_select_sql(
+            tenant_id.to_string().as_str(),
+            election_event_id.to_string().as_str(),
+            area_id.to_string().as_str(),
+            election_id.to_string().as_str(),
+        )?;
+
+        let rows = database.client.query(sql.as_str(), &[]).await?;
+        assert_eq!(rows.len(), 2);
+
+        let latest_vote = rows
+            .iter()
+            .find(|row| row.get::<_, String>("voter_id_string") == duplicated_voter_id)
+            .context("Could not find duplicated voter in area ballots query result")?;
+
+        assert_eq!(
+            latest_vote.get::<_, Option<String>>("content"),
+            Some("newer-ballot".to_string())
+        );
+
+        Ok(())
+    }
+}