Merge pull request #1107 from kenkoooo/feature/docker-compose

まともに使える docker-compose

Author: kenkoooo

.github/workflows/ci.yml

@@ -9,68 +9,89 @@ env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
   RUST_TEST_THREADS: 1
+  SQL_URL: postgres://db_user:db_pass@localhost:5432/test_db
 
 jobs:
   backend_test:
     name: Run backend tests
     runs-on: ubuntu-latest
-    
+    defaults:
+      run:
+        working-directory: ./atcoder-problems-backend
+
     services:
-      postgres: 
+      postgres:
         image: postgres
         env:
-          POSTGRES_PASSWORD: pass
-          POSTGRES_USER: kenkoooo
-          POSTGRES_DB: test
-        # Set health checks to wait until postgres has started
+          POSTGRES_USER: db_user
+          POSTGRES_PASSWORD: db_pass
+          POSTGRES_DB: test_db
+          POSTGRES_INITDB_ARGS: "--encoding=UTF8"
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
         ports:
-          # Maps tcp port 5432 on service container to the host
           - 5432:5432
-    
+
     steps:
     - uses: actions/checkout@v2
+
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cargo/git
+          ~/.cargo/registry/cache
+          ~/.cargo/registry/index
+          ./atcoder-problems-backend/target
+        key: ${{ runner.os }}-cargo-${{ hashFiles('atcoder-problems-backend/Cargo.lock') }}
+
+    - name: Setup Postgresql
+      run: psql ${SQL_URL} < ../config/database-definition.sql
+
     - name: Setup
       run: rustup component add rustfmt
+
     - name: Check format
-      working-directory: ./atcoder-problems-backend
       run: cargo fmt --all -- --check
+
     - name: Build
-      working-directory: ./atcoder-problems-backend
       run: cargo build --verbose
+
     - name: Run tests
-      working-directory: ./atcoder-problems-backend
-      env:
-        SQL_URL: postgresql://kenkoooo:pass@localhost:5432/test
       run: cargo test --verbose --workspace -- --test-threads=1
 
   frontend_test:
     name: Run frontend tests
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./atcoder-problems-frontend
+
     steps:
-    - name: Setup
-      run: sudo apt install libgconf-2-4
     - uses: actions/checkout@v2
     - name: Use Node.js
       uses: actions/setup-node@v1
+
+    - name: Cache node_modules
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/Cypress
+          ./atcoder-problems-frontend/node_modules
+        key: ${{ runner.os }}-cargo-${{ hashFiles('atcoder-problems-frontend/yarn.lock') }}
+
     - name: Install dependencies
-      working-directory: ./atcoder-problems-frontend
       run: yarn
     - name: build
-      working-directory: ./atcoder-problems-frontend
       run: yarn build
     - name: test
-      working-directory: ./atcoder-problems-frontend
       run: yarn test
     - name: lint
-      working-directory: ./atcoder-problems-frontend
       run: yarn lint
     - name: Integration test
-      working-directory: ./atcoder-problems-frontend
       run: |
         yarn prepare-ci
         yarn start:ci &

atcoder-problems-backend/.rustfmt.toml

@@ -29,9 +29,9 @@ reqwest = { version = "0.11", features = ["json"] }
 
 async-trait = "0.1"
 
-anyhow = "1.0.32"
+anyhow = "1.0"
 
 [dev-dependencies]
 
 [workspace]
-members = ["sql-client", "atcoder-client"]
+members = ["sql-client", "atcoder-client"]

atcoder-problems-backend/Cargo.lock

@@ -1,3 +1,7 @@
+FROM rust:1.57.0 AS development
+RUN rustup component add rustfmt
+RUN rustup component add clippy
+
 FROM rust:1.57.0 AS builder
 
 WORKDIR /app
@@ -25,7 +29,7 @@ ADD ./atcoder-client ./atcoder-client
 RUN cargo clean --release -p atcoder-client -p sql-client
 RUN cargo build --release
 
-FROM rust:1.57.0
+FROM rust:1.57.0 AS production
 COPY --from=builder /app/target/release/batch_update                /usr/bin/batch_update
 COPY --from=builder /app/target/release/crawl_all_submissions       /usr/bin/crawl_all_submissions
 COPY --from=builder /app/target/release/crawl_for_virtual_contests  /usr/bin/crawl_for_virtual_contests
@@ -37,3 +41,7 @@ COPY --from=builder /app/target/release/delta_update                /usr/bin/del
 COPY --from=builder /app/target/release/dump_json                   /usr/bin/dump_json
 COPY --from=builder /app/target/release/fix_invalid_submissions     /usr/bin/fix_invalid_submissions
 COPY --from=builder /app/target/release/run_server                  /usr/bin/run_server
+
+
+RUN apt-get update && apt-get install -y awscli postgresql-client
+ADD ./scripts/sql-backup.sh /usr/bin/sql-backup.sh

atcoder-problems-backend/Cargo.toml

@@ -38,8 +38,8 @@ Below are the list of files you need to modify:
 ## Build
 
 ```bash
-cd atcoder-problems-backend/
-cargo build
+docker-compose up -d
+docker-compose exec backend-development cargo build
 ```
 
 ## Run
@@ -70,13 +70,8 @@ cargo run --bin fix_invalid_submissions
 ## Test
 
 ```bash
-# If you don't set up PostgreSQL in your local environment,
-# you need to run the following lines to run a PostgreSQL Docker container for testing.
-docker-compose up -d postgresql
-export SQL_URL=postgres://kenkoooo:pass@localhost:5432/test
-
-# Run all the tests
-cargo test --workspace -- --test-threads=1
+docker-compose up -d
+docker-compose exec backend-development cargo test --workspace -- --test-threads=1
 ```
 
 ## Format
@@ -86,6 +81,6 @@ Please make sure that your change is formatted before sending a pull request.
 You can format the code base with `cargo fmt` like the following:
 
 ```bash
-cd atcoder-problems-backend/
-cargo fmt
+docker-compose up -d
+docker-compose exec backend-development cargo fmt
 ```

atcoder-problems-backend/Dockerfile

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+psql ${SQL_URL} -c "\copy submissions TO './submissions.csv' WITH (FORMAT CSV, HEADER)"
+gzip submissions.csv
+aws s3 cp submissions.csv.gz s3://kenkoooo/submissions.csv.gz --acl public-read

atcoder-problems-backend/README.md

@@ -8,11 +8,11 @@ publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-sqlx = { version = "0.5.9", features = ["postgres", "runtime-tokio-rustls"] }
+sqlx = { version = "0.5.10", features = ["postgres", "runtime-tokio-rustls"] }
 async-trait = "0.1.52"
 serde = { version = "1.0", features = ["derive"] }
 uuid = { version = "0.8", features = ["serde", "v4"] }
-anyhow = "1.0.51"
+anyhow = "1.0"
 tokio = { version = "1.15", features = ["macros"] }
 regex = "1"
 chrono = "0.4"

atcoder-problems-backend/docker-compose.yml

@@ -13,6 +13,7 @@ pub(crate) mod virtual_contest;
 
 use actix_web::{http::header, web, App, HttpResponseBuilder, HttpServer};
 pub use auth::{Authentication, GitHubAuthentication, GitHubUserResponse};
+pub use services::config_services;
 
 const LOG_TEMPLATE: &str = r#"{"method":"%{method}xi", "url":"%U", "status":%s, "duration":%T}"#;
 
@@ -50,7 +51,7 @@ impl MakeCors for HttpResponseBuilder {
     }
 }
 
-pub(crate) struct AppData<A> {
+pub struct AppData<A> {
     pub(crate) authentication: A,
     pub(crate) pg_pool: sql_client::PgPool,
 }
@@ -65,7 +66,7 @@ impl<A: Clone> Clone for AppData<A> {
 }
 
 impl<A> AppData<A> {
-    fn new(pg_pool: sql_client::PgPool, authentication: A) -> Self {
+    pub fn new(pg_pool: sql_client::PgPool, authentication: A) -> Self {
         Self {
             authentication,
             pg_pool,