diff --git a/.gitignore b/.gitignore index 301c7d7ab..07e5c7ee7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# May contain sensitive information +user.csv +mysql.sql +kratos.sql +temp.sql +dump-* + # Local enmeshed connector /enmeshed/logs /enmeshed/config.json diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 000000000..5494e94cc --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,93 @@ +# Deployment Instructions with Docker Compose + +## Requirements + +- Docker +- Nginx +- Git +- GCloud CLI +- Gsutil +- unzip + +## Steps for the deployment + +1. `git clone https://github.com/serlo/api.serlo.org && cd api.serlo.org/deploy/` +2. `cd staging/` or `cd production/` depending on your enviroment. +3. Set up Nginx on the host machine using configuration file `nginx.default.conf`. + 1. First you need to set up SSL certificates (currently, self-signed ones are enough): + ```console + $ sudo mkdir -p /etc/nginx/ssl + $ sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout /etc/nginx/ssl/selfsigned.key \ + -out /etc/nginx/ssl/selfsigned.crt \ + # uncomment what apply + # -subj "/CN=*.serlo-staging.dev" + # -subj "/CN=*.serlo.org" + ``` + 2. Then configure the routes: + ```console + $ sudo cp nginx.default.conf /etc/nginx/sites-available/default # alternatively use the command `ln` + $ sudo systemctl restart nginx + ``` +4. Be sure the values at corresponding `.env` and `kratos/config.staging.yml` or `kratos/config.production.yml` (change the values with "PLACEHODER") are correct. +5. Deploy using Docker Compose. +6. Set the DNS accordingly. At the server, remember set the firewall rules to allow http and https. + +## Additional steps for STAGING + +### Serlo DB Setup + +You need to fill up the database with data and set the cronjob for that for every night. + +Set up the Gsutil. You need to authenticate and may use a key of the appropriate service account. + +1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one +2. Put the key in a file `staging_service_account_key.json` in the home directory +3. `gcloud auth activate-service-account --key-file ~/staging_service_account_key.json` +4. Run `./dbsetup.sh` in host +5. Set cron tab to run the dbsetup script every night at 2 am. + +### DB Migration Cronjob + +Add a crontab in host with the following command (replace the missing values) for 3 am. + +``` +docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER" --network staging_staging-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER +``` + +## Additional steps for PRODUCTION + +### Serlo DB Dump + +In your first deployment, you will need to import the existing data into mysql and postgres containers. +Manually dump the production databases. Take a look at `staging/dbsetup.sh` for some inspiration on how to +import the data. + +Afterwards, You need to set up the cronjob for dumping the database for staging. + +Set up the Gsutil. You need the credentials of a service account in order that the script runs correctly. + +1. Go to GC Console -> IAM -> Service Accounts -> choose the dbreader account -> generate a new one +2. Put the key in a file `production_service_account_key.json` in the home directory +3. Set cron tab to run the dbdump script every night at 1 am. + +### Rocket Chat DB Dump + +In your first deployment, you will need to import the existing data into mongodb container. + +1. Download the dump from the corresponding bucket in the GC project 'production' +2. Run + ``` + $ docker compose cp dump-????.gz mongodb:/dump.gz + $ docker compose exec mongodb mongorestore --archive=dump.gz --gzip + ``` + +Now, set up a crontab to upload a backup of the data to the bucket at midnight, using the script `mongodbdump.sh`. + +### DB Migration + +In case of db migration, run the following command in host (replace the missing values). + +``` +docker run --rm --name db-migration --env-file PATH/TO/.env -e SLACK_CHANNEL="PLACEHOLDER" -e SLACK_TOKEN="PLACEHOLDER" --network production-network ghcr.io/serlo/api.serlo.org/db-migration:PLACEHOLDER +``` diff --git a/deploy/kratos/config.production.yml b/deploy/kratos/config.production.yml new file mode 100644 index 000000000..ffa6e2a81 --- /dev/null +++ b/deploy/kratos/config.production.yml @@ -0,0 +1,191 @@ +dsn: postgres://serlo:secret@postgres:5432/kratos?sslmode=disable&max_conns=20&max_idle_conns=4 + +serve: + public: + base_url: https://kratos.serlo.org + request_log: + disable_for_health: true + admin: + request_log: + disable_for_health: true + +selfservice: + default_browser_return_url: https://serlo.org/ + allowed_return_urls: + # TODO: try with wildcard later + - https://fr.serlo.org/ + - https://hi.serlo.org/ + - https://de.serlo.org/ + - https://ta.serlo.org/ + - https://en.serlo.org/ + - https://es.serlo.org/ + methods: + password: + enabled: true + config: + haveibeenpwned_enabled: false + link: + enabled: true + config: + base_url: https://serlo.org/api/.ory/ + oidc: + enabled: true + config: + base_redirect_uri: https://serlo.org/api/.ory/ + providers: + - id: nbp + provider: generic + client_id: PLACEHOLDER + client_secret: PLACEHOLDER + issuer_url: https://aai.demo.meinbildungsraum.de/realms/nbp-aai + mapper_url: file:///etc/config/kratos/user_mapper.jsonnet + - id: vidis + provider: generic + client_id: PLACEHOLDER + client_secret: PLACEHOLDER + issuer_url: http://PLACEHOLDER + mapper_url: file:///etc/config/kratos/vidis_user_mapper.jsonnet + + flows: + error: + ui_url: https://serlo.org/auth/error + + settings: + ui_url: https://serlo.org/auth/settings + privileged_session_max_age: 15m + + recovery: + enabled: true + use: link + ui_url: https://serlo.org/auth/recovery + + verification: + enabled: true + use: link + ui_url: https://serlo.org/auth/verification + + logout: + after: + default_browser_return_url: https://serlo.org/auth/login + + login: + ui_url: https://serlo.org/auth/login + lifespan: 10m + after: + password: + hooks: + - hook: require_verified_address + - hook: web_hook + config: + url: https://api.serlo.org/kratos/updateLastLogin + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + response: + ignore: true + oidc: + default_browser_return_url: https://serlo.org/auth/login + + registration: + enable_legacy_one_step: true + lifespan: 10m + ui_url: https://serlo.org/auth/registration + after: + hooks: + - hook: web_hook + config: + url: https://api.serlo.org/kratos/register + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + auth: + type: api_key + config: + name: x-kratos-key + value: PLACEHOLDER + in: header + - hook: web_hook + config: + url: https://PLACEHOLDER + method: POST + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: PLACEHOLDER + oidc: + default_browser_return_url: https://serlo.org/auth/login + hooks: + - hook: web_hook + config: + url: https://api.serlo.org/kratos/register + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + auth: + type: api_key + config: + name: x-kratos-key + value: PLACEHOLDER + in: header + - hook: session + - hook: web_hook + config: + url: https://PLACEHOLDER + method: POST + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: PLACEHOLDER + +session: + lifespan: 720h + +secrets: + cookie: + - PLACEHOLDERPLACEHOLDER + +identity: + default_schema_id: default + schemas: + - id: default + url: file:///etc/config/kratos/identity.schema.json + +courier: + smtp: + connection_uri: smtp://PLACEHOLDER + from_name: Serlo + from_address: no-reply@mail.serlo.org + templates: + verification: + valid: + email: + subject: http://serlo.org/api/.ory/mail-templates/verification/valid/email.subject.gotmpl + body: + html: http://serlo.org/api/.ory/mail-templates/verification/valid/email.body.gotmpl + plaintext: http://serlo.org/api/.ory/mail-templates/verification/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.subject.gotmpl + body: + html: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.body.gotmpl + plaintext: http://serlo.org/api/.ory/mail-templates/verification/invalid/email.body.plaintext.gotmpl + recovery: + valid: + email: + subject: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.subject.gotmpl + body: + html: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.body.gotmpl + plaintext: http://serlo.org/api/.ory/mail-templates/recovery/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.subject.gotmpl + body: + html: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.body.gotmpl + plaintext: http://serlo.org/api/.ory/mail-templates/recovery/invalid/email.body.plaintext.gotmpl diff --git a/deploy/kratos/config.staging.yml b/deploy/kratos/config.staging.yml new file mode 100644 index 000000000..f581586f2 --- /dev/null +++ b/deploy/kratos/config.staging.yml @@ -0,0 +1,191 @@ +dsn: postgres://serlo:secret@postgres:5432/kratos?sslmode=disable&max_conns=20&max_idle_conns=4 + +serve: + public: + base_url: https://kratos.serlo-staging.dev + request_log: + disable_for_health: true + admin: + request_log: + disable_for_health: true + +selfservice: + default_browser_return_url: https://serlo-staging.dev/ + allowed_return_urls: + # TODO: try with wildcard later + - https://fr.serlo-staging.dev/ + - https://hi.serlo-staging.dev/ + - https://de.serlo-staging.dev/ + - https://ta.serlo-staging.dev/ + - https://en.serlo-staging.dev/ + - https://es.serlo-staging.dev/ + methods: + password: + enabled: true + config: + haveibeenpwned_enabled: false + link: + enabled: true + config: + base_url: https://serlo-staging.dev/api/.ory/ + oidc: + enabled: true + config: + base_redirect_uri: https://serlo-staging.dev/api/.ory/ + providers: + - id: nbp + provider: generic + client_id: PLACEHOLDER + client_secret: PLACEHOLDER + issuer_url: https://aai.demo.meinbildungsraum.de/realms/nbp-aai + mapper_url: file:///etc/config/kratos/user_mapper.jsonnet + - id: vidis + provider: generic + client_id: PLACEHOLDER + client_secret: PLACEHOLDER + issuer_url: http://PLACEHOLDER + mapper_url: file:///etc/config/kratos/vidis_user_mapper.jsonnet + + flows: + error: + ui_url: https://serlo-staging.dev/auth/error + + settings: + ui_url: https://serlo-staging.dev/auth/settings + privileged_session_max_age: 15m + + recovery: + enabled: true + use: link + ui_url: https://serlo-staging.dev/auth/recovery + + verification: + enabled: true + use: link + ui_url: https://serlo-staging.dev/auth/verification + + logout: + after: + default_browser_return_url: https://serlo-staging.dev/auth/login + + login: + ui_url: https://serlo-staging.dev/auth/login + lifespan: 10m + after: + password: + hooks: + - hook: require_verified_address + - hook: web_hook + config: + url: https://api.serlo-staging.dev/kratos/updateLastLogin + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + response: + ignore: true + oidc: + default_browser_return_url: https://serlo-staging.dev/auth/login + + registration: + enable_legacy_one_step: true + lifespan: 10m + ui_url: https://serlo-staging.dev/auth/registration + after: + hooks: + - hook: web_hook + config: + url: https://api.serlo-staging.dev/kratos/register + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + auth: + type: api_key + config: + name: x-kratos-key + value: PLACEHOLDER + in: header + - hook: web_hook + config: + url: https://PLACEHOLDER + method: POST + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: PLACEHOLDER + oidc: + default_browser_return_url: https://serlo-staging.dev/auth/login + hooks: + - hook: web_hook + config: + url: https://api.serlo-staging.dev/kratos/register + method: POST + body: file:///etc/config/kratos/identity_id.jsonnet + auth: + type: api_key + config: + name: x-kratos-key + value: PLACEHOLDER + in: header + - hook: session + - hook: web_hook + config: + url: https://PLACEHOLDER + method: POST + body: file:///etc/config/kratos/subscribe_newsletter_mapper.jsonnet + can_interrupt: false + response: + ignore: true + auth: + type: basic_auth + config: + user: serlo + password: PLACEHOLDER + +session: + lifespan: 720h + +secrets: + cookie: + - PLACEHOLDERPLACEHOLDER + +identity: + default_schema_id: default + schemas: + - id: default + url: file:///etc/config/kratos/identity.schema.json + +courier: + smtp: + connection_uri: smtp://PLACEHOLDER + from_name: Serlo + from_address: no-reply@mail.serlo.org + templates: + verification: + valid: + email: + subject: http://serlo-staging.dev/api/.ory/mail-templates/verification/valid/email.subject.gotmpl + body: + html: http://serlo-staging.dev/api/.ory/mail-templates/verification/valid/email.body.gotmpl + plaintext: http://serlo-staging.dev/api/.ory/mail-templates/verification/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://serlo-staging.dev/api/.ory/mail-templates/verification/invalid/email.subject.gotmpl + body: + html: http://serlo-staging.dev/api/.ory/mail-templates/verification/invalid/email.body.gotmpl + plaintext: http://serlo-staging.dev/api/.ory/mail-templates/verification/invalid/email.body.plaintext.gotmpl + recovery: + valid: + email: + subject: http://serlo-staging.dev/api/.ory/mail-templates/recovery/valid/email.subject.gotmpl + body: + html: http://serlo-staging.dev/api/.ory/mail-templates/recovery/valid/email.body.gotmpl + plaintext: http://serlo-staging.dev/api/.ory/mail-templates/recovery/valid/email.body.plaintext.gotmpl + invalid: + email: + subject: http://serlo-staging.dev/api/.ory/mail-templates/recovery/invalid/email.subject.gotmpl + body: + html: http://serlo-staging.dev/api/.ory/mail-templates/recovery/invalid/email.body.gotmpl + plaintext: http://serlo-staging.dev/api/.ory/mail-templates/recovery/invalid/email.body.plaintext.gotmpl diff --git a/deploy/kratos/identity.schema.json b/deploy/kratos/identity.schema.json new file mode 100644 index 000000000..3826a6d2d --- /dev/null +++ b/deploy/kratos/identity.schema.json @@ -0,0 +1,51 @@ +{ + "$id": "https://serlo.org/auth/kratos-identity.schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "User", + "type": "object", + "properties": { + "traits": { + "type": "object", + "properties": { + "email": { + "type": "string", + "format": "email", + "ory.sh/kratos": { + "credentials": { "password": { "identifier": true } }, + "verification": { "via": "email" }, + "recovery": { "via": "email" } + } + }, + "username": { + "type": "string", + "ory.sh/kratos": { + "credentials": { "password": { "identifier": true } } + }, + "pattern": "^[\\w\\-]+$", + "maxLength": 32 + }, + "subscribedNewsletter": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "motivation": { + "type": "string" + }, + "profileImage": { + "type": "string" + }, + "language": { + "type": "string" + }, + "interest": { + "type": "string", + "enum": ["parent", "teacher", "pupil", "student", "other", ""] + } + }, + "required": ["email", "username", "interest"], + "additionalProperties": false + } + } +} diff --git a/deploy/kratos/identity_id.jsonnet b/deploy/kratos/identity_id.jsonnet new file mode 100644 index 000000000..f380aac95 --- /dev/null +++ b/deploy/kratos/identity_id.jsonnet @@ -0,0 +1 @@ +function(ctx) { userId: ctx.identity.id } diff --git a/deploy/kratos/nbp_user_mapper.jsonnet b/deploy/kratos/nbp_user_mapper.jsonnet new file mode 100644 index 000000000..cac7eb7e8 --- /dev/null +++ b/deploy/kratos/nbp_user_mapper.jsonnet @@ -0,0 +1,13 @@ +local claims = std.extVar('claims'); + +local enshortenUuid(uuid) = std.split(uuid, '-')[0]; + +{ + identity: { + traits: { + email: enshortenUuid(claims.sub) + '@nbp', + username: enshortenUuid(claims.sub), + interest: '', + }, + }, +} diff --git a/deploy/kratos/subscribe_newsletter_mapper.jsonnet b/deploy/kratos/subscribe_newsletter_mapper.jsonnet new file mode 100644 index 000000000..a75c19130 --- /dev/null +++ b/deploy/kratos/subscribe_newsletter_mapper.jsonnet @@ -0,0 +1,18 @@ +local getInterestsKey = function(interest) + if interest == 'parent' then 'dec9a97288' + else if interest == 'teacher' then '05a5ab768a' + else if interest == 'pupil' then 'bbffc7a064' + else if interest == 'student' then 'ebff3b63f6' + else if interest == 'other' then 'd251aad97e'; + +function(ctx) { + email_address: if 'subscribedNewsletter' in ctx.identity.traits && ctx.identity.traits.subscribedNewsletter == true then + ctx.identity.traits.email + else + error 'User did not subscribe to newsletter. Aborting!', + merge_fields: { + UNAME: ctx.identity.traits.username, + }, + status: 'subscribed', + [if 'interest' in ctx.identity.traits then 'interests' else null]: { [getInterestsKey(ctx.identity.traits.interest)]: true }, +} diff --git a/deploy/kratos/user_mapper.jsonnet b/deploy/kratos/user_mapper.jsonnet new file mode 100644 index 000000000..3e8d7be3e --- /dev/null +++ b/deploy/kratos/user_mapper.jsonnet @@ -0,0 +1,11 @@ +local claims = std.extVar('claims'); + +{ + identity: { + traits: { + [if "email" in claims then "email" else null]: claims.email, + username: claims.preferred_username + "-1232kjl", + interest: "", + }, + }, +} \ No newline at end of file diff --git a/deploy/kratos/vidis_user_mapper.jsonnet b/deploy/kratos/vidis_user_mapper.jsonnet new file mode 100644 index 000000000..194581cad --- /dev/null +++ b/deploy/kratos/vidis_user_mapper.jsonnet @@ -0,0 +1,40 @@ +local claims = std.extVar('claims'); + +local enshortenUuid(uuid) = std.split(uuid, '-')[0]; + +local extractFromClaims = function(fieldName) + if fieldName in claims then claims[fieldName] else null; + +local uuid = extractFromClaims('sub'); + +local buildEmail = function() + local email = extractFromClaims('email'); + + if email != '' && email != null + then email + else enshortenUuid(uuid) + '@fakeemail.vidis'; + +local buildUsername = function() + local preferredUsername = extractFromClaims('preferred_username'); + local truncatedUsername = if std.length(preferredUsername) > 23 + then std.substr(preferredUsername, 0, 23) + else preferredUsername; + + if truncatedUsername != '' && truncatedUsername != null + then truncatedUsername + '-' + enshortenUuid(uuid) + else enshortenUuid(uuid); + +local checkIfIsTeacher = function() + local rawClaims = extractFromClaims('raw_claims'); + + if 'rolle' in rawClaims then rawClaims.rolle == 'LEHR' else false; + +if checkIfIsTeacher() then { + identity: { + traits: { + email: buildEmail(), + username: buildUsername(), + interest: 'other', + }, + }, +} else error 'ERR_BAD_ROLE' diff --git a/deploy/production/.env b/deploy/production/.env new file mode 100644 index 000000000..7c37a3a08 --- /dev/null +++ b/deploy/production/.env @@ -0,0 +1,36 @@ +# This a template. Replace the values with the actual ones in the environment. + +ENVIRONMENT=production +GOOGLE_SPREADSHEET_API_ACTIVE_DONORS=1qpyC0XzvTcKT6EISywvqESX3A0MwQoFDE8p-Bll4hps +GOOGLE_SPREADSHEET_API_MOTIVATION=142gKytX3rMT25DiIwQvhTLZrjZlojzZ4APIvwRuR4tU +GOOGLE_SPREADSHEET_API_SECRET=my-secret +LOG_LEVEL=ERROR +MAILCHIMP_API_KEY=secret-us5 +METADATA_API_VERSION=1.0.0 +MYSQL_URI=mysql://root:secret@mysql:3306/serlo?timezone=+00:00 +REDIS_URL=redis://redis:6379 +ROCKET_CHAT_API_USER_ID=an-user-id +ROCKET_CHAT_API_AUTH_TOKEN=an-auth-token +ROCKET_CHAT_URL=https://community.serlo.org/ +SERLO_ORG_DATABASE_LAYER_HOST=127.0.0.1:8080 +SERLO_ORG_SECRET=serlo.org-secret +# Set the following value to `empty` to disable caching +CACHE_TYPE=redis + +SERVER_HYDRA_HOST=http://hydra:4445 +SERVER_KRATOS_PUBLIC_HOST=http://kratos:4433 +SERVER_KRATOS_ADMIN_HOST=http://kratos:4434 +SERVER_KRATOS_SECRET=api.serlo.org-kratos-secret +SERVER_KRATOS_DB_URI=postgres://serlo:secret@postgres:5432/kratos?sslmode=disable\&max_conns=20\&max_idle_conns=4 +SERVER_SERLO_CLOUDFLARE_WORKER_SECRET=api.serlo.org-playground-secret +SERVER_SERLO_CACHE_WORKER_SECRET=api.serlo.org-cache-worker-secret +SERVER_SERLO_NOTIFICATION_EMAIL_SERVICE_SECRET=api.serlo.org-notification-email-service-secret +SERVER_SERLO_EDITOR_TESTING_SECRET=api.serlo.org-serlo-editor-testing-secret +SERVER_SWR_QUEUE_DASHBOARD_PASSWORD=secret +SERVER_SWR_QUEUE_DASHBOARD_USERNAME=secret + +SWR_QUEUE_WORKER_CONCURRENCY=2 +SWR_QUEUE_WORKER_DELAY=250 +CHECK_STALLED_JOBS_DELAY=600000 + +OPENAI_API_KEY="" diff --git a/deploy/production/dbdump.sh b/deploy/production/dbdump.sh new file mode 100755 index 000000000..972628259 --- /dev/null +++ b/deploy/production/dbdump.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +set -e + +mysql_cmd="docker compose exec -T mysql mysql --user=serlo --password=secret" + +echo "dump serlo.org database - start" + +echo "dump legacy serlo database schema" +mysql_dump_cmd="docker compose exec -T mysql mysqldump --user=serlo --password=secret --lock-tables=false" + +$mysql_dump_cmd --no-data --add-drop-database --databases serlo >mysql.sql + +$mysql_dump_cmd --no-create-info --lock-tables=false --add-locks --ignore-table=serlo.user serlo >>mysql.sql + +$mysql_cmd --batch -e "SELECT id, CONCAT(@rn:=@rn+1, '@localhost') AS email, username, '8a534960a8a4c8e348150a0ae3c7f4b857bfead4f02c8cbf0d' AS password, logins, date, CONCAT(@rn:=@rn+1, '') AS token, last_login, description FROM user, (select @rn:=2) r;" serlo >user.csv + +echo "dump kratos identities data" + +echo "dumping the data from the kratos database" +docker compose exec postgres pg_dump --user=serlo kratos >temp.sql + +echo "creating another container to manipulate and anonymize the data" +docker run --name temp_postgres -e POSTGRES_PASSWORD=secret -e POSTGRES_PASSWORD=password -e POSTGRES_DB=kratos -e PGPASSWORD=secret -v ./temp.sql:/temp.sql -d postgres:13 + +echo "waiting for the postgres container to be ready" +if ! timeout 60s sh -c 'until docker exec temp_postgres pg_isready -U postgres; do sleep 1; done'; then + echo "Postgres container did not become ready within 1 minute." + exit 1 +fi + +temp_postgres_cmd="docker exec temp_postgres psql -h localhost -U postgres --quiet -c" +$temp_postgres_cmd "CREATE user serlo;" +$temp_postgres_cmd "GRANT ALL PRIVILEGES ON DATABASE kratos TO serlo;" + +docker exec -i temp_postgres psql -h localhost -U serlo -d kratos > 'interest' != 'teacher';" +$temp_postgres_cmd "UPDATE identity_credentials SET config = '{\"hashed_password\": \"\$sha1\$pf=e1NBTFR9e1BBU1NXT1JEfQ==\$YTQwYzEwY2ZlNA==\$hTlqikjjSFoK43S4V7+t8CyMvw0=\"}';" +$temp_postgres_cmd "UPDATE identity_verifiable_addresses SET value = CONCAT(identity_id, '@localhost');" +$temp_postgres_cmd "UPDATE identity_recovery_addresses SET value = CONCAT(identity_id, '@localhost');" +$temp_postgres_cmd "UPDATE identity_credential_identifiers SET identifier = CONCAT(ic.identity_id, '@localhost') FROM (select id, identity_id FROM identity_credentials) AS ic where ic.id = identity_credential_id and identifier LIKE '%@%';" +$temp_postgres_cmd "TRUNCATE sessions, continuity_containers, courier_messages, identity_verification_codes, identity_recovery_codes, identity_recovery_tokens, identity_verification_tokens, selfservice_errors, selfservice_login_flows, selfservice_recovery_flows, selfservice_registration_flows, selfservice_settings_flows, selfservice_verification_flows, session_devices, session_token_exchanges, identity_login_codes CASCADE;" +docker exec temp_postgres pg_dump -h localhost -U serlo kratos >kratos.sql + +echo "compressing database dump" +day=$(date -I) +zip "dump-$day".zip mysql.sql user.csv kratos.sql + +gcloud auth activate-service-account --key-file=~/production_service-account.json +gsutil cp dump-$day gs://anonymous-dump +echo "latest dump uploaded" + +echo "removing temporary files and containers" +rm -f mysql.sql user.csv kratos.sql dump-$day.zip temp.sql +docker rm -f temp_postgres >/dev/null 2>&1 + +echo "dump of serlo.org database - end" diff --git a/deploy/production/docker-compose.yml b/deploy/production/docker-compose.yml new file mode 100644 index 000000000..8e57a7c62 --- /dev/null +++ b/deploy/production/docker-compose.yml @@ -0,0 +1,143 @@ +services: + api: + image: ghcr.io/serlo/api.serlo.org/server:production + pull_policy: always + env_file: + - .env + ports: + - '3001:3001' + # TODO best practice: healthcheck + depends_on: + - mysql + - redis + networks: + - production-network + swr-queue-worker: + image: ghcr.io/serlo/api.serlo.org/swr-queue-worker:production + pull_policy: always + env_file: + - .env + # TODO: ports needed? + ports: + - '3000:3000' + # TODO best practice: healthcheck + depends_on: + - mysql + - redis + networks: + - production-network + redis: + image: redis:7 + networks: + - production-network + mysql: + image: mysql:8 + environment: + - MYSQL_ROOT_PASSWORD=secret + - MYSQL_DATABASE=serlo + - MYSQL_USER=serlo + - MYSQL_PASSWORD=secret + volumes: + - mysql_data:/var/lib/mysql + networks: + - production-network + kratos: + image: oryd/kratos:v1.3.0 + ports: + - '4433:4433' # public + environment: + - DSN=postgres://serlo:secret@postgres:5432/kratos?sslmode=disable + depends_on: + - postgres + - kratos-migrate + command: serve -c /etc/config/kratos/config.production.yml --watch-courier + volumes: + - ../kratos:/etc/config/kratos + networks: + - production-network + kratos-migrate: + image: oryd/kratos:v1.3.0 + command: -c /etc/config/kratos/config.production.yml migrate sql -e --yes + depends_on: + - postgres + volumes: + - ../kratos:/etc/config/kratos + restart: on-failure + networks: + - production-network + postgres: + image: postgres:13 + environment: + - POSTGRES_USER=serlo + - POSTGRES_PASSWORD=secret + - POSTGRES_DB=kratos + - PGPASSWORD=secret + volumes: + - postgres_data:/var/lib/postgresql/data + networks: + - production-network + hydra: + image: oryd/hydra:v2.2.0 + ports: + # why both? + - '4444:4444' + - '4445:4445' + command: serve all --dev + volumes: + # using sqlite, maybe postgres would be better + - hydra_data:/var/lib/sqlite + environment: + # - LOG_LEVEL=debug + - LOG_LEAK_SENSITIVE_VALUES=true + - OAUTH2_EXPOSE_INTERNAL_ERRORS=1 + - URLS_SELF_ISSUER=http://localhost:4444 + - URLS_LOGIN=http://localhost:3000/auth/oauth/login + - URLS_LOGOUT=http://localhost:3000/auth/oauth/logout + - URLS_CONSENT=http://localhost:3000/auth/oauth/consent + - DSN=memory + - SECRETS_SYSTEM=youReallyNeedToChangeThis + - OIDC_SUBJECT_IDENTIFIERS_ENABLED=public,pairwise + - OIDC_SUBJECT_IDENTIFIERS_PAIRWISE_SALT=youReallyNeedToChangeThis + networks: + - production-network + rocketchat: + image: registry.rocket.chat/rocketchat/rocket.chat:7.4.0 + restart: on-failure + environment: + MONGO_URL: 'mongodb://mongodb:27017/rocketchat?replicaSet=rs0' + MONGO_OPLOG_URL: 'mongodb://mongodb:27017/local?replicaSet=rs0' + ROOT_URL: http://localhost:3030 + PORT: '3030' + DEPLOY_METHOD: docker + depends_on: + - mongodb + ports: + - '3030:3030' + networks: + - production-network + mongodb: + image: docker.io/bitnami/mongodb:6.0 + restart: on-failure + volumes: + - mongodb_data:/bitnami/mongodb + environment: + MONGODB_REPLICA_SET_MODE: primary + MONGODB_REPLICA_SET_NAME: rs0 + MONGODB_PORT_NUMBER: '27017' + MONGODB_INITIAL_PRIMARY_HOST: mongodb + MONGODB_INITIAL_PRIMARY_PORT_NUMBER: '27017' + MONGODB_ADVERTISED_HOSTNAME: mongodb + MONGODB_ENABLE_JOURNAL: 'true' + ALLOW_EMPTY_PASSWORD: 'yes' + networks: + - production-network + +networks: + production-network: + driver: bridge + +volumes: + mongodb_data: + hydra_data: + postgres_data: + mysql_data: diff --git a/deploy/production/mongodbdump.sh b/deploy/production/mongodbdump.sh new file mode 100755 index 000000000..d035074fc --- /dev/null +++ b/deploy/production/mongodbdump.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -e + +echo "rocket-chat mongodump - start" + +echo "cleaning up old dumps" +docker compose exec -u root mongodb rm /dump.gz || true + +echo "creating new dump" +docker compose exec -u root mongodb mongodump --archive=dump.gz --gzip +dumpname="dump-$(date -I).gz" +docker compose cp mongodb:/dump.gz "$dumpname" + +echo "uploading dump to GCS" +gsutil cp $dumpname gs://serlo-production-rocket-chat-mongodump + +echo "cleaning up" +docker compose exec -u root mongodb rm /dump.gz || true +rm $dumpname + +echo "rocket-chat mongodump - end" diff --git a/deploy/production/nginx.default.conf b/deploy/production/nginx.default.conf new file mode 100644 index 000000000..9b856d6e7 --- /dev/null +++ b/deploy/production/nginx.default.conf @@ -0,0 +1,61 @@ +server { + listen 80; + listen 443 ssl; + server_name api.serlo.org; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + location / { + proxy_pass http://localhost:3001; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80; + listen 443 ssl; + server_name kratos.serlo.org; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + location / { + proxy_pass http://localhost:4433; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80; + listen 443 ssl; + server_name community.serlo.org; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + location / { + proxy_pass http://localhost:3030; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +server { + listen 80 default_server; + listen 443 ssl default_server; + server_name _; + + ssl_certificate /etc/nginx/ssl/selfsigned.crt; + ssl_certificate_key /etc/nginx/ssl/selfsigned.key; + + return 444; +} diff --git a/deploy/staging/.env b/deploy/staging/.env new file mode 100644 index 000000000..b178eca95 --- /dev/null +++ b/deploy/staging/.env @@ -0,0 +1,40 @@ +# This a template. Replace the values with the actual ones in the environment. + +ENVIRONMENT=staging +GOOGLE_SPREADSHEET_API_ACTIVE_DONORS=1qpyC0XzvTcKT6EISywvqESX3A0MwQoFDE8p-Bll4hps +GOOGLE_SPREADSHEET_API_MOTIVATION=142gKytX3rMT25DiIwQvhTLZrjZlojzZ4APIvwRuR4tU +GOOGLE_SPREADSHEET_API_SECRET=my-secret +LOG_LEVEL=ERROR +MAILCHIMP_API_KEY=secret-us5 +METADATA_API_VERSION=1.0.0 +MYSQL_URI=mysql://root:secret@mysql:3306/serlo?timezone=+00:00 +REDIS_URL=redis://redis:6379 +ROCKET_CHAT_API_USER_ID=an-user-id +ROCKET_CHAT_API_AUTH_TOKEN=an-auth-token +ROCKET_CHAT_URL=https://community.serlo.org/ +SERLO_ORG_DATABASE_LAYER_HOST=127.0.0.1:8080 +SERLO_ORG_SECRET=serlo.org-secret +# Set the following value to `empty` to disable caching +CACHE_TYPE=redis + +SERVER_HYDRA_HOST=http://hydra:4445 +SERVER_KRATOS_PUBLIC_HOST=http://kratos:4433 +SERVER_KRATOS_ADMIN_HOST=http://kratos:4434 +SERVER_KRATOS_SECRET=api.serlo.org-kratos-secret +SERVER_KRATOS_DB_URI=postgres://serlo:secret@postgres:5432/kratos?sslmode=disable\&max_conns=20\&max_idle_conns=4 +SERVER_SERLO_CLOUDFLARE_WORKER_SECRET=api.serlo.org-playground-secret +SERVER_SERLO_CACHE_WORKER_SECRET=api.serlo.org-cache-worker-secret +SERVER_SERLO_NOTIFICATION_EMAIL_SERVICE_SECRET=api.serlo.org-notification-email-service-secret +SERVER_SERLO_EDITOR_TESTING_SECRET=api.serlo.org-serlo-editor-testing-secret +SERVER_SWR_QUEUE_DASHBOARD_PASSWORD=secret +SERVER_SWR_QUEUE_DASHBOARD_USERNAME=secret + +SWR_QUEUE_WORKER_CONCURRENCY=1 +SWR_QUEUE_WORKER_DELAY=250 +CHECK_STALLED_JOBS_DELAY=600000 + +ENMESHED_SERVER_HOST="http://enmeshed:8081/" +ENMESHED_SERVER_SECRET="apiKey" +ENMESHED_WEBHOOK_SECRET="webhookKey" + +OPENAI_API_KEY="" diff --git a/deploy/staging/dbsetup.sh b/deploy/staging/dbsetup.sh new file mode 100755 index 000000000..c444f5da0 --- /dev/null +++ b/deploy/staging/dbsetup.sh @@ -0,0 +1,63 @@ +#!/bin/sh + +set -e + +mysql_connect="docker compose exec -T mysql mysql --user=serlo --password=secret" + +echo "wait for mysql database to be ready" +until $mysql_connect -e "SHOW DATABASES" >/dev/null 2>/dev/null; do + echo "could not find mysql server - retry in 10 seconds" + sleep 10 +done + +newest_dump_uri=$(gsutil ls -l gs://anonymous-dump | grep dump | sort -rk 2 | head -n 1 | awk '{ print $3 }') +[ -z "$newest_dump_uri" ] && { + echo "no database dump available in anonymous db dump bucket" + exit 1 +} + +newest_dump=$(basename $newest_dump_uri) + +gsutil cp $newest_dump_uri "/tmp/$newest_dump" +echo "downloaded newest dump $newest_dump" +unzip -o "/tmp/$newest_dump" -d /tmp || { + echo "unzip of dump file failed" + exit 1 +} + +echo "Recreating serlo database" + +docker compose cp /tmp/mysql.sql mysql:/tmp/mysql.sql +docker compose cp /tmp/user.csv mysql:/tmp/user.csv + +$mysql_connect -e "DROP DATABASE serlo" +$mysql_connect -e "CREATE DATABASE serlo" +$mysql_connect serlo <"/tmp/mysql.sql" || { + echo "import of dump failed" + exit 1 +} + +docker compose exec -T mysql mysql --user=root --password=secret -e "SET GLOBAL local_infile = 1" +$mysql_connect --local-infile=1 -e "LOAD DATA LOCAL INFILE '/tmp/user.csv' INTO TABLE user FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n' IGNORE 1 ROWS;" serlo || { + echo "import of users failed" + exit 1 +} +$mysql_connect serlo -e "UPDATE user SET description = NULL WHERE description = 'NULL'" + +echo "imported serlo database dump $newest_dump" + +echo "Recreating kratos database" + +docker compose cp /tmp/kratos.sql postgres:/tmp/kratos.sql + +postgres_connect="docker compose exec -T postgres psql --user=serlo kratos " +$postgres_connect -c "DROP SCHEMA public CASCADE;" +$postgres_connect -c "CREATE SCHEMA public;" +$postgres_connect -c "GRANT ALL ON SCHEMA public TO serlo;" +$postgres_connect