diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..ea2c44a --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,2 @@ +# Docs for this feature: https://help.github.com/articles/about-codeowners/ +/ @plangrid/DevOps diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index b8e940c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM debian:jessie -MAINTAINER Kotaimen - -ENV DEBIAN_FRONTEND noninteractive - -RUN set -x \ - && apt-get -qq update \ - && apt-get install -yq --no-install-recommends pgbouncer \ - && apt-get purge -y --auto-remove \ - && rm -rf /var/lib/apt/lists/* - -ADD entrypoint.sh ./ - -EXPOSE 6432 -ENTRYPOINT ["./entrypoint.sh"] diff --git a/Dockerfile.j2 b/Dockerfile.j2 new file mode 100644 index 0000000..c78fc5a --- /dev/null +++ b/Dockerfile.j2 @@ -0,0 +1,44 @@ +FROM ubuntu:16.04 + +ARG ARTIFACTORY_USER +ARG ARTIFACTORY_PASSWORD + +ENV PGBOUNCER_VERSION=1.7.2-2.pgdg16.04+1 \ + ARTIFACTORY_USER='read-only' \ + ARTIFACTORY_PASSWORD='UN5cDIG5EyJdDrc5EBwB' + +# Prep apt +# test +# Using /usr/bin/apt-get here because these are dependencies we need for our +# custom apt-get to work +RUN ln -sf /bin/bash /bin/sh \ + && /usr/bin/apt-get update \ + && /usr/bin/apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + locales \ + wget \ + gnupg \ + # make sure en_US.UTF-8 is installed + && locale-gen en_US.UTF-8 \ + # cleanup + && /usr/bin/apt-get clean \ + && /usr/bin/apt-get autoremove + +# Artifactory apt repo +RUN echo "deb https://${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD}@plangrid.jfrog.io/plangrid/debs-local xenial main" > /etc/apt/sources.list.d/artifactory.list \ + && curl -u ${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD} https://plangrid.jfrog.io/plangrid/api/gpg/key/public \ + | apt-key add - + +RUN set -x \ + && apt-get -qq update \ + && apt-get install -yq --no-install-recommends pgbouncer=$PGBOUNCER_VERSION \ + && apt-get purge -y --auto-remove \ + && rm -rf /var/lib/apt/lists/* + +ADD entrypoint.sh ./ + +EXPOSE 5432 + +CMD ["./entrypoint.sh"] diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..d74467c --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,4 @@ +#!groovy +node { + StandardBuild() +} diff --git a/entrypoint.sh b/entrypoint.sh index a1b2ad5..64ade6c 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,13 +1,77 @@ #!/bin/bash -set -e +set -xe PG_LOG=/var/log/postgresql/ -PG_CONFIG=/etc/pgbouncer/pgbouncer.ini +PG_CONFIG_DIR=/etc/pgbouncer PG_USER=postgres +if [ -f ${PG_CONFIG_DIR}/pgbouncer.ini ]; then + rm ${PG_CONFIG_DIR}/pgbouncer.ini +else + mkdir -p ${PG_CONFIG_DIR} +fi + +cp /mnt/private.pem ${PG_CONFIG_DIR}/private.pem +chmod 400 ${PG_CONFIG_DIR}/private.pem +cp /mnt/public.pem ${PG_CONFIG_DIR}/public.pem +chmod 400 ${PG_CONFIG_DIR}/public.pem + +echo "create pgbouncer config in ${PG_CONFIG_DIR}" + +printf "\ +[databases] +${DB_NAME} = host=${DB_HOST:?"Setup pgbouncer config error! You must set DB_HOST env"} port=${DB_PORT:-5432} \ +${DB_PASSWORD:+password=${DB_PASSWORD}} + +[pgbouncer] +logfile = /var/log/postgresql/pgbouncer.log +pidfile = /var/run/postgresql/pgbouncer.pid + +listen_addr = 0.0.0.0 +listen_port = 5432 + +unix_socket_dir = /var/run/postgresql + +auth_type = md5 +auth_file = /etc/pgbouncer/userlist.txt + +pool_mode = transaction + +server_reset_query = DISCARD ALL + +max_client_conn = 5000 +default_pool_size = 180 + +admin_users = ${PGBOUNCER_ADMIN_USERS} + +client_tls_sslmode = require +client_tls_key_file = /etc/pgbouncer/private.pem +client_tls_cert_file = /etc/pgbouncer/public.pem +client_tls_ca_file = /etc/pgbouncer/public.pem +client_tls_protocols = secure + +# fivetran connects with extra_float_digits. +# See ref for extra_float_digits: +# https://www.postgresql.org/docs/9.6/static/runtime-config-client.html +# +# DEVOPS-1640 +ignore_startup_parameters = extra_float_digits +" > ${PG_CONFIG_DIR}/pgbouncer.ini + +IFS=',' read -r -a pg_usernames <<< "$PG_USERS" +IFS=',' read -r -a pg_passwords <<< "$PG_PASSWORDS" + +for index in "${!pg_usernames[@]}" +do + printf "\"${pg_usernames[index]}\" \"${pg_passwords[index]}\"\n" >> ${PG_CONFIG_DIR}/userlist.txt +done + mkdir -p ${PG_LOG} chmod -R 755 ${PG_LOG} chown -R ${PG_USER}:${PG_USER} ${PG_LOG} echo "Starting pgbouncer..." -exec pgbouncer -q -u ${PG_USER} $PG_CONFIG +ls -al /etc/pgbouncer +cat ${PG_CONFIG_DIR}/pgbouncer.ini +cat ${PG_CONFIG_DIR}/userlist.txt +pgbouncer -u $PG_USER ${PG_CONFIG_DIR}/pgbouncer.ini diff --git a/etc/pgbouncer/pgbouncer.ini b/etc/pgbouncer/pgbouncer.ini deleted file mode 100644 index 379a9b9..0000000 --- a/etc/pgbouncer/pgbouncer.ini +++ /dev/null @@ -1,222 +0,0 @@ -;; database name = connect string -;; -;; connect string params: -;; dbname= host= port= user= password= -;; client_encoding= datestyle= timezone= -;; pool_size= connect_query= -[databases] - -; foodb over unix socket -;foodb = - -; redirect bardb to bazdb on localhost -;bardb = host=localhost dbname=bazdb - -; acceess to dest database will go with single user -;forcedb = host=127.0.0.1 port=300 user=baz password=foo client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1' - -; use custom pool sizes -;nondefaultdb = pool_size=50 reserve_pool=10 - -; fallback connect string -;* = host=testserver - -;; Configuration section -[pgbouncer] - -;;; -;;; Administrative settings -;;; - -logfile = /var/log/postgresql/pgbouncer.log -pidfile = /var/run/postgresql/pgbouncer.pid - -;;; -;;; Where to wait for clients -;;; - -; ip address or * which means all ip-s -listen_addr = 0.0.0.0 -listen_port = 6432 - -; unix socket is also used for -R. -; On debian it should be /var/run/postgresql -;unix_socket_dir = /tmp -;unix_socket_mode = 0777 -;unix_socket_group = -unix_socket_dir = /var/run/postgresql - -;;; -;;; Authentication settings -;;; - -; any, trust, plain, crypt, md5 -auth_type = trust -;auth_file = /8.0/main/global/pg_auth -auth_file = /etc/pgbouncer/userlist.txt - -;;; -;;; Users allowed into database 'pgbouncer' -;;; - -; comma-separated list of users, who are allowed to change settings -;admin_users = user2, someadmin, otheradmin - -; comma-separated list of users who are just allowed to use SHOW command -;stats_users = stats, root - -;;; -;;; Pooler personality questions -;;; - -; When server connection is released back to pool: -; session - after client disconnects -; transaction - after transaction finishes -; statement - after statement finishes -pool_mode = session - -; -; Query for cleaning connection immediately after releasing from client. -; No need to put ROLLBACK here, pgbouncer does not reuse connections -; where transaction is left open. -; -; Query for 8.3+: -; DISCARD ALL; -; -; Older versions: -; RESET ALL; SET SESSION AUTHORIZATION DEFAULT -; -; Empty if transaction pooling is in use. -; -server_reset_query = DISCARD ALL - -; -; Comma-separated list of parameters to ignore when given -; in startup packet. Newer JDBC versions require the -; extra_float_digits here. -; -;ignore_startup_parameters = extra_float_digits - -; -; When taking idle server into use, this query is ran first. -; SELECT 1 -; -;server_check_query = select 1 - -; If server was used more recently that this many seconds ago, -; skip the check query. Value 0 may or may not run in immediately. -;server_check_delay = 30 - -;;; -;;; Connection limits -;;; - -; total number of clients that can connect -max_client_conn = 100 - -; default pool size. 20 is good number when transaction pooling -; is in use, in session pooling it needs to be the number of -; max clients you want to handle at any moment -default_pool_size = 20 - -; how many additional connection to allow in case of trouble -;reserve_pool_size = 5 - -; if a clients needs to wait more than this many seconds, use reserve pool -;reserve_pool_timeout = 3 - -; log if client connects or server connection is made -;log_connections = 1 - -; log if and why connection was closed -;log_disconnections = 1 - -; log error messages pooler sends to clients -;log_pooler_errors = 1 - - -; If off, then server connections are reused in LIFO manner -;server_round_robin = 0 - -;;; -;;; Timeouts -;;; - -;; Close server connection if its been connected longer. -;server_lifetime = 1200 - -;; Close server connection if its not been used in this time. -;; Allows to clean unnecessary connections from pool after peak. -;server_idle_timeout = 60 - -;; Cancel connection attempt if server does not answer takes longer. -;server_connect_timeout = 15 - -;; If server login failed (server_connect_timeout or auth failure) -;; then wait this many second. -;server_login_retry = 15 - -;; Dangerous. Server connection is closed if query does not return -;; in this time. Should be used to survive network problems, -;; _not_ as statement_timeout. (default: 0) -;query_timeout = 0 - -;; Dangerous. Client connection is closed if the query is not assigned -;; to a server in this time. Should be used to limit the number of queued -;; queries in case of a database or network failure. (default: 0) -;query_wait_timeout = 0 - -;; Dangerous. Client connection is closed if no activity in this time. -;; Should be used to survive network problems. (default: 0) -;client_idle_timeout = 0 - -;; Disconnect clients who have not managed to log in after connecting -;; in this many seconds. -;client_login_timeout = 60 - -;; Clean automatically created database entries (via "*") if they -;; stay unused in this many seconds. -; autodb_idle_timeout = 3600 - -;;; -;;; Low-level tuning options -;;; - -;; buffer for streaming packets -;pkt_buf = 2048 - -;; man 2 listen -;listen_backlog = 128 - -;; networking options, for info: man 7 tcp - -;; Linux: notify program about new connection only if there -;; is also data received. (Seconds to wait.) -;; On Linux the default is 45, on other OS'es 0. -;tcp_defer_accept = 0 - -;; In-kernel buffer size (Linux default: 4096) -;tcp_socket_buffer = 0 - -;; whether tcp keepalive should be turned on (0/1) -;tcp_keepalive = 1 - -;; following options are Linux-specific. -;; they also require tcp_keepalive=1 - -;; count of keepaliva packets -;tcp_keepcnt = 0 - -;; how long the connection can be idle, -;; before sending keepalive packets -;tcp_keepidle = 0 - -;; The time between individual keepalive probes. -;tcp_keepintvl = 0 - -;; DNS lookup caching time -;dns_max_ttl = 15 - -;; DNS zone SOA lookup period -;dns_zone_check_period = 0 -