Merge branch 'tiredofit:main' into main

Author: oscarsiles

.github/workflows/main.yml

@@ -10,6 +10,5 @@ jobs:
   build:
     uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main
     #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main
-    #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main
     #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main
     secrets: inherit

.github/workflows/manual.yml

@@ -11,6 +11,5 @@ jobs:
   build:
     uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main
     #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main
-    #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main
     #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main
     secrets: inherit

CHANGELOG.md

@@ -1,3 +1,76 @@
+## 3.10.2 2023-09-14 <pimjansen@github>
+
+   ### Changed
+      - Update to wording when sending files to blobxfer
+      - Remove --databases flag when backing up a single mysql/mariadb backup which allows to omit the "USE <db_name>" statement in the backup allowing for better restores
+
+
+## 3.10.1 2023-09-13 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Bugfix to 3.10.0 with syntax error revolving around unbraced variable
+
+
+## 3.10.0 2023-09-13 <jcdirks@github>
+
+   ### Added
+      - Add EXTRA_DUMP_OPTS and EXTRA_ENUMERATION_OPTS to add different arguments when checking for databases, vs doing the actual backup
+
+
+## 3.9.12 2023-09-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Perform additional checks for ENABLE_CHECKSUM=FALSE and skip executing actions for S3/BlobXfer
+
+
+## 3.9.11 2023-08-24 <dave at tiredofit dot ca>
+
+   ### Changed
+      - AWS CLI 2.13.9
+
+
+## 3.9.10 2023-08-23 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Stop trying to move a non existent checksum file when ENABLE_CHECKSUM=FALSE
+
+
+## 3.9.9 2023-08-21 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Start compiling aws-cli instead of from packages to continue to support arm/v7
+
+
+## 3.9.8 2023-08-20 <ToshY@github>
+
+   ### Changed
+      - Restore armv7 and aarch64 builds
+
+
+## 3.9.7 2023-07-18 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Cleanup check_exit_code parameter and reduce duplicate output
+
+
+## 3.9.6 2023-06-16 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Resolve issues introduced with 3.9.3. Split exit codes to be specific for backing up and moving. Uses paremter $11 for post backup scripts
+
+
+## 3.9.5 2023-06-13 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Start building Influx DB v1 manually due to being removed from Alpine repositories
+
+
+## 3.9.4 2023-06-13 <dave at tiredofit dot ca>
+
+   ### Added
+      - Add abliity to use --rsyncable argument to zstd archives
+
+
 ## 3.9.3 2023-06-05 <dave at tiredofit dot ca>
 
    ### Added

Dockerfile

@@ -5,8 +5,10 @@ FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT}
 LABEL maintainer="Dave Conroy (github.com/tiredofit)"
 
 ### Set Environment Variables
-ENV INFLUX2_VERSION=2.4.0 \
+ENV INFLUX_VERSION=1.8.0 \
+    INFLUX2_VERSION=2.4.0 \
     MSSQL_VERSION=18.0.1.1-1 \
+    AWS_CLI_VERSION=1.25.97 \
     CONTAINER_ENABLE_MESSAGING=FALSE \
     CONTAINER_ENABLE_MONITORING=TRUE \
     CONTAINER_PROCESS_RUNAWAY_PROTECTOR=FALSE \
@@ -21,7 +23,9 @@ RUN source /assets/functions/00-container && \
     package install .db-backup-build-deps \
                build-base \
                bzip2-dev \
+               cargo \
                git \
+               go \
                libarchive-dev \
                openssl-dev \
                libffi-dev \
@@ -31,9 +35,8 @@ RUN source /assets/functions/00-container && \
                && \
     \
     package install .db-backup-run-deps \
-               aws-cli \
                bzip2 \
-               influxdb \
+               groff \
                libarchive \
                mariadb-client \
                mariadb-connector-c \
@@ -43,7 +46,16 @@ RUN source /assets/functions/00-container && \
                postgresql15 \
                postgresql15-client \
                pv \
+               py3-botocore \
+               py3-colorama \
                py3-cryptography \
+               py3-docutils \
+               py3-jmespath \
+               py3-rsa \
+               py3-setuptools \
+               py3-s3transfer \
+               py3-yaml \
+               python3 \
                redis \
                sqlite \
                xz \
@@ -60,7 +72,11 @@ RUN source /assets/functions/00-container && \
     \
     if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/msodbcsql18_${MSSQL_VERSION}_amd64.apk ; curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/mssql-tools18_${MSSQL_VERSION}_amd64.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSSQL_VERSION}_amd64.apk mssql-tools18_${MSSQL_VERSION}_amd64.apk ; else echo >&2 "Detected non x86_64 build variant, skipping MSSQL installation" ; fi; \
     if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \
-    \
+    clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \
+    python3 setup.py install --prefix=/usr && \
+    clone_git_repo https://github.com/influxdata/influxdb "${INFLUX_VERSION}" && \
+    go build -o /usr/sbin/influxd ./cmd/influxd && \
+    strip /usr/sbin/influxd && \
     mkdir -p /usr/src/pbzip2 && \
     curl -sSL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \
     cd /usr/src/pbzip2 && \
@@ -85,6 +101,7 @@ RUN source /assets/functions/00-container && \
             /*.apk \
             /etc/logrotate.d/* \
             /root/.cache \
+            /root/go \
             /tmp/* \
             /usr/src/*
 

README.md

@@ -179,15 +179,17 @@ Your Organization will be mapped to `DB_USER` and your root token will need to b
 
 ### Backup Options
 | Parameter                      | Description                                                                                                                  | Default                   | `_FILE` |
-| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------- | ------- |
+|--------------------------------|------------------------------------------------------------------------------------------------------------------------------|---------------------------|---------|
 | `COMPRESSION`                  | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE`                                                      | `ZSTD`                    |         |
 | `COMPRESSION_LEVEL`            | Numberical value of what level of compression to use, most allow `1` to `9` except for `ZSTD` which allows for `1` to `19` - | `3`                       |         |
 | `ENABLE_PARALLEL_COMPRESSION`  | Use multiple cores when compressing backups `TRUE` or `FALSE`                                                                | `TRUE`                    |         |
 | `PARALLEL_COMPRESSION_THREADS` | Maximum amount of threads to use when compressing - Integer value e.g. `8`                                                   | `autodetected`            |         |
 | `GZ_RSYNCABLE`                 | Use `--rsyncable` (gzip only) for faster rsync transfers and incremental backup deduplication. e.g. `TRUE`                   | `FALSE`                   |         |
 | `ENABLE_CHECKSUM`              | Generate either a MD5 or SHA1 in Directory, `TRUE` or `FALSE`                                                                | `TRUE`                    |         |
 | `CHECKSUM`                     | Either `MD5` or `SHA1`                                                                                                       | `MD5`                     |         |
-| `EXTRA_OPTS`                   | If you need to pass extra arguments to the backup command, add them here e.g. `--extra-command`                              |                           |         |
+| `EXTRA_OPTS`                   | If you need to pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command`     |                           |         |
+| `EXTRA_DUMP_OPTS`              | If you need to pass extra arguments to the backup command only, add them here e.g. `--extra-command`                         |                           |         |
+| `EXTRA_ENUMERATION_OPTS`       | If you need to pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command`           |                           |         |
 | `MYSQL_MAX_ALLOWED_PACKET`     | Max allowed packet if backing up MySQL / MariaDB                                                                             | `512M`                    |         |
 | `MYSQL_SINGLE_TRANSACTION`     | Backup in a single transaction with MySQL / MariaDB                                                                          | `TRUE`                    |         |
 | `MYSQL_STORED_PROCEDURES`      | Backup stored procedures with MySQL / MariaDB                                                                                | `TRUE`                    |         |
@@ -323,17 +325,18 @@ $ cat post-script.sh
 # #### $8=BACKUP FILENAME (Filename)
 # #### $9=BACKUP FILESIZE
 # #### $10=HASH (If CHECKSUM enabled)
+# #### $11=MOVE_EXIT_CODE
 
 echo "${1} ${2} Backup Completed on ${3} for ${4} on ${5} ending ${6} for a duration of ${7} seconds. Filename: ${8} Size: ${9} bytes MD5: ${10}"
 ````
 
       ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE
-      ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${backup_start_timme}" "${backup_finish_time}" "${backup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}"
+      ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${backup_start_timme}" "${backup_finish_time}" "${backup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}" "${move_exit_code}
 
 
 Outputs the following on the console:
 
-`0 mysql Backup Completed on example-db for example on 1647370800 ending 1647370920 for a duration of 120 seconds. Filename: mysql_example_example-db_202200315-000000.sql.bz2 Size: 7795 bytes Hash: 952fbaafa30437494fdf3989a662cd40`
+`0 mysql Backup Completed on example-db for example on 1647370800 ending 1647370920 for a duration of 120 seconds. Filename: mysql_example_example-db_202200315-000000.sql.bz2 Size: 7795 bytes Hash: 952fbaafa30437494fdf3989a662cd40 0`
 
 If you wish to change the size value from bytes to megabytes set environment variable `SIZE_VALUE=megabytes`
 

examples/mysql/docker-compose.yml

@@ -31,6 +31,8 @@ services:
       - ./backups:/backup
       #- ./post-script.sh:/assets/custom-scripts/post-script.sh
     environment:
+      - TIMEZONE=America/Vancouver
+      - CONTAINER_ENABLE_MONITORING=FALSE
       # - DEBUG_MODE=TRUE
       - DB_TYPE=mariadb
       - DB_HOST=example-db-host
@@ -43,7 +45,7 @@ services:
       - CHECKSUM=SHA1
       - COMPRESSION=GZ
       - SPLIT_DB=FALSE
-      - CONTAINER_ENABLE_MONITORING=FALSE
+
     restart: always
     networks:
       - example-db-network