diff --git a/.github/workflows/dockerfile.yaml b/.github/workflows/dockerfile.yaml index df2d02c96a..60aa7aa0b3 100644 --- a/.github/workflows/dockerfile.yaml +++ b/.github/workflows/dockerfile.yaml @@ -105,11 +105,9 @@ jobs: --lint-conf lintconf.yaml - name: Create kind cluster - if: steps.list-changed.outputs.changed == 'true' uses: helm/kind-action@v1 - name: Log in to GHCR - if: steps.list-changed.outputs.changed == 'true' uses: docker/login-action@v3 with: registry: ghcr.io @@ -117,23 +115,21 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Load image into kind - if: steps.list-changed.outputs.changed == 'true' run: | docker pull ghcr.io/${{ github.repository }}:pr-${{ github.event.pull_request.number }} kind load docker-image ghcr.io/${{ github.repository }}:pr-${{ github.event.pull_request.number }} --name chart-testing - name: Run chart-testing (install) id: chart-testing-install - if: steps.list-changed.outputs.changed == 'true' run: | ct install --github-groups \ --chart-dirs kubernetes/chart \ - --target-branch ${{ github.event.repository.default_branch }} \ + --all \ --helm-extra-set-args "--set image.tag=pr-${{ github.event.pull_request.number }}" \ --skip-clean-up - name: Fetch logs - if: steps.list-changed.outputs.changed == 'true' && ( success() || failure() ) + if: success() || failure() continue-on-error: true run: | namespace=$(helm list --all-namespaces --output json \ diff --git a/Dockerfile b/Dockerfile index 8b6bdb5fde..65cb53ef40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,8 +34,6 @@ RUN git clone "$ZULIP_GIT_URL" -b "$ZULIP_GIT_REF" WORKDIR /home/zulip/zulip -ARG CUSTOM_CA_CERTIFICATES - # Finally, we provision the development environment and build a release tarball RUN SKIP_VENV_SHELL_WARNING=1 ./tools/provision --build-release-tarball-only && \ uv run --no-sync ./tools/build-release-tarball docker && \ @@ -51,8 +49,6 @@ ENV DATA_DIR="/data" COPY --from=build /tmp/zulip-server-docker.tar.gz /root/ COPY custom_zulip_files/ /root/custom_zulip -ARG CUSTOM_CA_CERTIFICATES - WORKDIR /root RUN \ # Make sure Nginx is started by Supervisor. diff --git a/docker-compose.yml b/docker-compose.yml index 164beb95cf..a652aad46a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,8 +56,6 @@ services: ## Change these if you want to build zulip from a different repo/branch ZULIP_GIT_URL: https://github.com/zulip/zulip.git ZULIP_GIT_REF: "11.4" - ## Set this up if you plan to use your own CA certificate bundle for building - # CUSTOM_CA_CERTIFICATES: ports: - "25:25" - "80:80" diff --git a/entrypoint.sh b/entrypoint.sh index dcb3446220..9af05eb1c2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,49 +5,86 @@ if [ "$DEBUG" = "true" ] || [ "$DEBUG" = "True" ]; then set -o functrace fi set -e +set -u shopt -s extglob -# DB aka Database +normalize_bool() { + # Returns either "True" or "False" + local varname="$1" + local raw_value="${!varname:-}" + local value="${raw_value,,}" # Convert to lowercase + local default="${2:-False}" + + case "$value" in + true | enable | enabled | yes | y | 1 | on) + echo "True" + ;; + false | disable | disabled | no | n | 0 | off) + echo "False" + ;; + "") + echo "$default" + ;; + *) + echo "WARNING: Invalid boolean ('$raw_value') for '$varname'; defaulting to $default" >&2 + echo "$default" + ;; + esac +} + +## Settings + +# PostgreSQL DB_HOST="${DB_HOST:-127.0.0.1}" DB_HOST_PORT="${DB_HOST_PORT:-5432}" DB_NAME="${DB_NAME:-zulip}" DB_USER="${DB_USER:-zulip}" REMOTE_POSTGRES_SSLMODE="${REMOTE_POSTGRES_SSLMODE:-prefer}" + # RabbitMQ SETTING_RABBITMQ_HOST="${SETTING_RABBITMQ_HOST:-127.0.0.1}" SETTING_RABBITMQ_USER="${SETTING_RABBITMQ_USER:-zulip}" -export RABBITMQ_NODE="$SETTING_RABBITMQ_HOST" + # Redis -SETTING_RATE_LIMITING="${SETTING_RATE_LIMITING:-True}" SETTING_REDIS_HOST="${SETTING_REDIS_HOST:-127.0.0.1}" SETTING_REDIS_PORT="${SETTING_REDIS_PORT:-6379}" + # Memcached -if [ -z "$SETTING_MEMCACHED_LOCATION" ]; then - SETTING_MEMCACHED_LOCATION="127.0.0.1:11211" -fi -# Nginx settings -DISABLE_HTTPS="${DISABLE_HTTPS:-false}" +SETTING_MEMCACHED_LOCATION="${SETTING_MEMCACHED_LOCATION:-127.0.0.1:11211}" + +# Nginx and HTTP(S) settings +DISABLE_HTTPS="$(normalize_bool DISABLE_HTTPS)" NGINX_WORKERS="${NGINX_WORKERS:-2}" -NGINX_PROXY_BUFFERING="${NGINX_PROXY_BUFFERING:-off}" NGINX_MAX_UPLOAD_SIZE="${NGINX_MAX_UPLOAD_SIZE:-80m}" -TRUST_GATEWAY_IP="${TRUST_GATEWAY_IP:-False}" -# Zulip certificate parameters -SSL_CERTIFICATE_GENERATION="${SSL_CERTIFICATE_GENERATION:self-signed}" -# Zulip related settings +LOADBALANCER_IPS="${LOADBALANCER_IPS:-}" +TRUST_GATEWAY_IP="$(normalize_bool TRUST_GATEWAY_IP)" +SSL_CERTIFICATE_GENERATION="${SSL_CERTIFICATE_GENERATION:-self-signed}" + +# Outgoing proxy settings +PROXY_ALLOW_ADDRESSES="${PROXY_ALLOW_ADDRESSES:-}" +PROXY_ALLOW_RANGES="${PROXY_ALLOW_RANGES:-}" + +# Core Zulip settings ZULIP_AUTH_BACKENDS="${ZULIP_AUTH_BACKENDS:-EmailAuthBackend}" -ZULIP_RUN_POST_SETUP_SCRIPTS="${ZULIP_RUN_POST_SETUP_SCRIPTS:-True}" -# Zulip user setup -FORCE_FIRST_START_INIT="${FORCE_FIRST_START_INIT:-False}" +QUEUE_WORKERS_MULTIPROCESS="$(normalize_bool QUEUE_WORKERS_MULTIPROCESS)" + +# Configuration controls +FORCE_FIRST_START_INIT="$(normalize_bool FORCE_FIRST_START_INIT)" +ZULIP_RUN_POST_SETUP_SCRIPTS="$(normalize_bool ZULIP_RUN_POST_SETUP_SCRIPTS True)" +ZULIP_CUSTOM_SETTINGS="${ZULIP_CUSTOM_SETTINGS:-}" +MANUAL_CONFIGURATION="$(normalize_bool MANUAL_CONFIGURATION)" +LINK_SETTINGS_TO_DATA="$(normalize_bool LINK_SETTINGS_TO_DATA)" + # Auto backup settings -AUTO_BACKUP_ENABLED="${AUTO_BACKUP_ENABLED:-True}" +AUTO_BACKUP_ENABLED="$(normalize_bool AUTO_BACKUP_ENABLED True)" AUTO_BACKUP_INTERVAL="${AUTO_BACKUP_INTERVAL:-30 3 * * *}" -# Zulip configuration function specific variable(s) -SPECIAL_SETTING_DETECTION_MODE="${SPECIAL_SETTING_DETECTION_MODE:-}" -MANUAL_CONFIGURATION="${MANUAL_CONFIGURATION:-false}" -LINK_SETTINGS_TO_DATA="${LINK_SETTINGS_TO_DATA:-false}" -# entrypoint.sh specific variable(s) + +## Constants SETTINGS_PY="/etc/zulip/settings.py" +## Global state +GENERATE_CERTBOT_CERT_SCHEDULED="" + # BEGIN appRun functions # === initialConfiguration === prepareDirectories() { @@ -58,7 +95,7 @@ prepareDirectories() { ln -sfT "$DATA_DIR/uploads" /home/zulip/uploads chown zulip:zulip -R "$DATA_DIR/uploads" # Link settings folder - if [ "$LINK_SETTINGS_TO_DATA" = "True" ] || [ "$LINK_SETTINGS_TO_DATA" = "true" ]; then + if [ "$LINK_SETTINGS_TO_DATA" = "True" ]; then # Create settings directories if [ ! -d "$DATA_DIR/settings" ]; then mkdir -p "$DATA_DIR/settings" @@ -77,14 +114,9 @@ setConfigurationValue() { echo "No KEY given for setConfigurationValue." return 1 fi - if [ -z "$3" ]; then - echo "No FILE given for setConfigurationValue." - return 1 - fi local KEY="$1" local VALUE - local FILE="$3" - local TYPE="$4" + local TYPE="$3" if [ -z "$TYPE" ]; then case "$2" in [Tt][Rr][Uu][Ee] | [Ff][Aa][Ll][Ss][Ee] | [Nn]one) @@ -102,11 +134,6 @@ setConfigurationValue() { esac fi case "$TYPE" in - emptyreturn) - if [ -z "$2" ]; then - return 0 - fi - ;; literal) VALUE="$1" ;; @@ -117,32 +144,32 @@ setConfigurationValue() { VALUE="$KEY = '${2//\'/\'}'" ;; esac - echo "$VALUE" >>"$FILE" - echo "Setting key \"$KEY\", type \"$TYPE\" in file \"$FILE\"." + echo "$VALUE" >>"$SETTINGS_PY" + echo "Setting key \"$KEY\", type \"$TYPE\"." } nginxConfiguration() { echo "Executing nginx configuration ..." sed -i "s/worker_processes .*/worker_processes $NGINX_WORKERS;/g" /etc/nginx/nginx.conf sed -i "s/client_max_body_size .*/client_max_body_size $NGINX_MAX_UPLOAD_SIZE;/g" /etc/nginx/nginx.conf - sed -i "s/proxy_buffering .*/proxy_buffering $NGINX_PROXY_BUFFERING;/g" /etc/nginx/zulip-include/proxy_longpolling echo "Nginx configuration succeeded." } puppetConfiguration() { echo "Executing puppet configuration ..." - if [ "$DISABLE_HTTPS" == "True" ] || [ "$DISABLE_HTTPS" == "true" ]; then + if [ "$DISABLE_HTTPS" == "True" ]; then echo "Disabling https in nginx." crudini --set /etc/zulip/zulip.conf application_server http_only true fi - if [ "$QUEUE_WORKERS_MULTIPROCESS" == "True" ] || [ "$QUEUE_WORKERS_MULTIPROCESS" == "true" ]; then + if [ "$QUEUE_WORKERS_MULTIPROCESS" == "True" ]; then echo "Setting queue workers to run in multiprocess mode ..." crudini --set /etc/zulip/zulip.conf application_server queue_workers_multiprocess true - elif [ "$QUEUE_WORKERS_MULTIPROCESS" == "False" ] || [ "$QUEUE_WORKERS_MULTIPROCESS" == "false" ]; then + else echo "Setting queue workers to run in multithreaded mode ..." crudini --set /etc/zulip/zulip.conf application_server queue_workers_multiprocess false fi - if [ "$TRUST_GATEWAY_IP" == "True" ] || [ "$TRUST_GATEWAY_IP" == "true" ]; then + if [ "$TRUST_GATEWAY_IP" == "True" ]; then + local GATEWAY_IP GATEWAY_IP=$(ip route | grep default | awk '{print $3}') echo "Trusting local network gateway $GATEWAY_IP" LOADBALANCER_IPS="${LOADBALANCER_IPS:+$LOADBALANCER_IPS,}$GATEWAY_IP" @@ -174,6 +201,8 @@ puppetConfiguration() { /home/zulip/deployments/current/scripts/zulip-puppet-apply -f } configureCerts() { + local GENERATE_SELF_SIGNED_CERT + local GENERATE_CERTBOT_CERT case "$SSL_CERTIFICATE_GENERATION" in self-signed) GENERATE_SELF_SIGNED_CERT="True" @@ -256,9 +285,9 @@ secretsConfiguration() { } databaseConfiguration() { echo "Setting database configuration ..." - setConfigurationValue "REMOTE_POSTGRES_HOST" "$DB_HOST" "$SETTINGS_PY" "string" - setConfigurationValue "REMOTE_POSTGRES_PORT" "$DB_HOST_PORT" "$SETTINGS_PY" "string" - setConfigurationValue "REMOTE_POSTGRES_SSLMODE" "$REMOTE_POSTGRES_SSLMODE" "$SETTINGS_PY" "string" + setConfigurationValue "REMOTE_POSTGRES_HOST" "$DB_HOST" "string" + setConfigurationValue "REMOTE_POSTGRES_PORT" "$DB_HOST_PORT" "string" + setConfigurationValue "REMOTE_POSTGRES_SSLMODE" "$REMOTE_POSTGRES_SSLMODE" "string" # The password will be set in secretsConfiguration echo "Database configuration succeeded." } @@ -267,12 +296,13 @@ authenticationBackends() { local FIRST=true local auth_backends IFS=, read -r -a auth_backends <<<"$ZULIP_AUTH_BACKENDS" + local AUTH_BACKEND for AUTH_BACKEND in "${auth_backends[@]}"; do if [ "$FIRST" = true ]; then - setConfigurationValue "AUTHENTICATION_BACKENDS" "('zproject.backends.${AUTH_BACKEND//\'/\'}',)" "$SETTINGS_PY" "array" + setConfigurationValue "AUTHENTICATION_BACKENDS" "('zproject.backends.${AUTH_BACKEND//\'/\'}',)" "array" FIRST=false else - setConfigurationValue "AUTHENTICATION_BACKENDS += ('zproject.backends.${AUTH_BACKEND//\'/\'}',)" "" "$SETTINGS_PY" "literal" + setConfigurationValue "AUTHENTICATION_BACKENDS += ('zproject.backends.${AUTH_BACKEND//\'/\'}',)" "" "literal" fi echo "Adding authentication backend \"$AUTH_BACKEND\"." done @@ -288,7 +318,7 @@ zulipConfiguration() { [[ "$key" == SETTING_*([0-9A-Za-z_]) ]] || continue local setting_key="${key#SETTING_}" local setting_var="${!key}" - local type="string" + local type="" if [ -z "$setting_var" ]; then echo "Empty var for key \"$setting_key\"." continue @@ -316,16 +346,12 @@ zulipConfiguration() { || [ "$setting_key" = "ALLOWED_HOSTS" ]; then type="array" fi - if [ "$SPECIAL_SETTING_DETECTION_MODE" = "True" ] || [ "$SPECIAL_SETTING_DETECTION_MODE" = "true" ] \ - || [ "$type" = "string" ]; then - type="" - fi if [ "$setting_key" = "EMAIL_HOST_USER" ] \ || [ "$setting_key" = "EMAIL_HOST_PASSWORD" ] \ || [ "$setting_key" = "EXTERNAL_HOST" ]; then type="string" fi - setConfigurationValue "$setting_key" "$setting_var" "$SETTINGS_PY" "$type" + setConfigurationValue "$setting_key" "$setting_var" "$type" done if ! su zulip -c "/home/zulip/deployments/current/manage.py checkconfig"; then echo "Error in the Zulip configuration. Exiting." @@ -334,7 +360,7 @@ zulipConfiguration() { echo "Zulip configuration succeeded." } autoBackupConfiguration() { - if [ "$AUTO_BACKUP_ENABLED" != "True" ] && [ "$AUTO_BACKUP_ENABLED" != "true" ]; then + if [ "$AUTO_BACKUP_ENABLED" != "True" ]; then rm -f /etc/cron.d/autobackup echo "Auto backup is disabled. Continuing." return 0 @@ -348,7 +374,7 @@ initialConfiguration() { puppetConfiguration nginxConfiguration configureCerts - if [ "$MANUAL_CONFIGURATION" = "False" ] || [ "$MANUAL_CONFIGURATION" = "false" ]; then + if [ "$MANUAL_CONFIGURATION" = "False" ]; then # Start with the settings template file. cp -a /home/zulip/deployments/current/zproject/prod_settings_template.py "$SETTINGS_PY" databaseConfiguration @@ -357,11 +383,11 @@ initialConfiguration() { zulipConfiguration else # Check that the configuration will work - root_path="/etc/zulip" - if [ "$LINK_SETTINGS_TO_DATA" = "True" ] || [ "$LINK_SETTINGS_TO_DATA" = "true" ]; then + local root_path="/etc/zulip" + if [ "$LINK_SETTINGS_TO_DATA" = "True" ]; then root_path="/data/settings/etc-zulip" fi - failure=0 + local failure=0 for conf_file in zulip.conf zulip-secrets.conf settings.py; do if [ ! -f "/etc/zulip/$conf_file" ]; then echo "ERROR: $root_path/$conf_file does not exist!" @@ -401,7 +427,7 @@ waitingForDatabase() { } zulipFirstStartInit() { echo "Executing Zulip first start init ..." - if [ -e "$DATA_DIR/.initiated" ] && [ "$FORCE_FIRST_START_INIT" != "True" ] && [ "$FORCE_FIRST_START_INIT" != "true" ]; then + if [ -e "$DATA_DIR/.initiated" ] && [ "$FORCE_FIRST_START_INIT" != "True" ]; then echo "First Start Init not needed. Continuing." return 0 fi @@ -410,7 +436,7 @@ zulipFirstStartInit() { su zulip -c /home/zulip/deployments/current/scripts/setup/initialize-database RETURN_CODE=$? if [[ $RETURN_CODE != 0 ]]; then - echo "Zulip first start database initi failed in \"initialize-database\" exit code $RETURN_CODE. Exiting." + echo "Zulip first start database init failed in \"initialize-database\" exit code $RETURN_CODE. Exiting." exit $RETURN_CODE fi set -e @@ -420,20 +446,19 @@ zulipFirstStartInit() { zulipMigration() { echo "Running new database migrations..." set +e + local RETURN_CODE=0 su zulip -c "/home/zulip/deployments/current/manage.py migrate --noinput" - local RETURN_CODE=$? + RETURN_CODE=$? if [[ $RETURN_CODE != 0 ]]; then echo "Zulip migration failed with exit code $RETURN_CODE. Exiting." exit $RETURN_CODE fi set -e - rm -rf "$DATA_DIR/.zulip-*" - touch "$DATA_DIR/.zulip-$ZULIP_VERSION" echo "Database migrations completed." } runPostSetupScripts() { echo "Post setup scripts execution ..." - if [ "$ZULIP_RUN_POST_SETUP_SCRIPTS" != "True" ] && [ "$ZULIP_RUN_POST_SETUP_SCRIPTS" != "true" ]; then + if [ "$ZULIP_RUN_POST_SETUP_SCRIPTS" != "True" ]; then echo "Not running post setup scripts. ZULIP_RUN_POST_SETUP_SCRIPTS isn't true." return 0 fi @@ -516,7 +541,7 @@ appInit() { bootstrappingEnvironment } appManagePy() { - COMMAND="$1" + local COMMAND="$1" shift 1 if [ -z "$COMMAND" ]; then echo "No command given for manage.py. Defaulting to \"shell\"." @@ -535,8 +560,7 @@ appBackup() { echo "Backup process failed. Exiting." exit 1 fi - local BACKUP_FOLDER - BACKUP_FOLDER="/tmp/backup-$TIMESTAMP)" + local BACKUP_FOLDER="/tmp/backup-$TIMESTAMP" mkdir -p "$BACKUP_FOLDER" waitingForDatabase pg_dump -h "$DB_HOST" -p "$DB_HOST_PORT" -U "$DB_USER" "$DB_NAME" >"$BACKUP_FOLDER/database-postgres.sql" @@ -605,9 +629,9 @@ appHelp() { echo "> [COMMAND] - Run given command with arguments in shell" } appVersion() { - echo "This container contains:" - echo "> Zulip server $ZULIP_VERSION" - echo "> Checksum: $ZULIP_CHECKSUM" + local ZULIP_VERSION + ZULIP_VERSION="$(su zulip -c "cd ~/deployments/current && python3 -c 'import version; print(version.ZULIP_VERSION)'")" + echo "This container contains Zulip Server $ZULIP_VERSION" exit 0 } # END app functions