diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6edc17c5..5afdde8f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,5 +1,5 @@ --- -name: Publish containers +name: Publish images on: push: @@ -7,8 +7,26 @@ on: - master jobs: - publish_base: + list_base_images: + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.set-versions.outputs.versions }} + base_has_changed: ${{ steps.changes.outputs.base }} + steps: + - uses: actions/checkout@v2 + - id: set-versions + run: echo "versions=$(ls base/images/ | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + base: + - 'base/**' + + list_prestashop_images: runs-on: ubuntu-latest + outputs: + versions: ${{ steps.set-versions.outputs.versions }} steps: # Fetch versions to work for images - uses: actions/checkout@v2 @@ -19,45 +37,48 @@ jobs: - id: set-versions run: echo "versions=$(./get_json_versions.py)" >> $GITHUB_OUTPUT - # Push image base + publish_base: + runs-on: ubuntu-latest + needs: list_base_images + strategy: + matrix: + version: ${{ fromJson(needs.list_base_images.outputs.versions) }} + steps: - name: Login to Docker Hub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - ## Check if there are modifications in the base/ directory - ## and store it in the variable `steps.changes.outputs.base` - ## The variable is built like: steps.{#id}.outputs.{#filter} - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - base: - - 'base/**' + + - name: Enable multi-platform builds + run: docker buildx create --name container --driver=docker-container + + - uses: actions/checkout@v2 - name: Base Images > Generate Tags run: ./generate_tags.sh working-directory: base - name: Base Images > Docker Build Tags - run: ./docker_tags.sh + run: DOCKER_REPOSITORY=${{ vars.DOCKER_BASE_REPOSITORY}} ./docker_tags.sh --version ${{ matrix.version }} if: ${{ github.event_name == 'pull_request' }} working-directory: base - name: Base Images > Docker Build & Force Push - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && steps.changes.outputs.base == 'true' }} - run: ./docker_tags.sh -p -f + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && needs.list_base_images.outputs.base_has_changed == 'true' }} + run: DOCKER_REPOSITORY=${{ vars.DOCKER_BASE_REPOSITORY}} ./docker_tags.sh -p -f --version ${{ matrix.version }} working-directory: base - outputs: - versions: ${{ steps.set-versions.outputs.versions }} - publish_images: + + publish_prestashop: runs-on: ubuntu-latest - needs: publish_base + needs: + - list_prestashop_images + - publish_base strategy: fail-fast: false matrix: - ps-version: ${{ fromJson(needs.publish_base.outputs.versions) }} + ps-version: ${{ fromJson(needs.list_prestashop_images.outputs.versions) }} steps: - name: Login to Docker Hub uses: docker/login-action@v2 @@ -65,6 +86,9 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Enable multi-platform builds + run: docker buildx create --name container --driver=docker-container + - uses: actions/checkout@v2 - name: Set up Python 3.8 uses: actions/setup-python@v2 @@ -73,9 +97,7 @@ jobs: - name: Install dependencies run: pip install -r requirements.txt - - name: Build Docker images - run: ./prestashop_docker.py --quiet tag build ${{ matrix.ps-version }} --force - - - name: Push Docker images + - name: Build & Push Docker images if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - run: ./prestashop_docker.py --quiet tag push ${{ matrix.ps-version }} --force + run: DOCKER_REPOSITORY=${{ vars.DOCKER_REPOSITORY}} ./prestashop_docker.py --quiet tag push ${{ matrix.ps-version }} --force + diff --git a/HOW-TO-USE.md b/HOW-TO-USE.md index b3cbc0ff..45f684a7 100644 --- a/HOW-TO-USE.md +++ b/HOW-TO-USE.md @@ -9,6 +9,12 @@ It requires Python 3.9+. $ pip install -r requirements.txt --break-system-packages ``` +If you plan to build the images locally, you'll need to create a builder instance. This command just needs to be run one time. + +```bash +$ docker buildx create --name container --driver=docker-container +``` + ## Usage Display the help: @@ -68,10 +74,10 @@ positional arguments: {exists,build,push,aliases} exists Check if tag exists on Docker Hub build Build container and create docker tag - push Push docker tags + push Build container and create docker tag then push docker tags aliases Get aliases -optional arguments: +options: -h, --help show this help message and exit ``` @@ -126,7 +132,7 @@ $ nosetests --with-id 7 This will also generate a `.nodeids` binary file, when you add new test methods you need to remove this file to re-generate the list of IDs. -## Building and running PrestaShop docker locally +## Building and running PrestaShop docker locally (local platform only, deprecated) First to make sure you will use the local docker containers and not the ones from Docker hub make sure you remove all existing PrestaShop images (including the base images) diff --git a/base/docker_tags.sh b/base/docker_tags.sh index 152a7f83..d172c23f 100755 --- a/base/docker_tags.sh +++ b/base/docker_tags.sh @@ -1,54 +1,72 @@ #!/bin/bash -cd $(cd "$( dirname "$0" )" && pwd) +: ${PLATFORM_ARGS:="linux/arm/v7,linux/arm64/v8,linux/amd64"} +: ${DOCKER_REPOSITORY:="prestashop/base"} -if [ -z "$1" ] || [ "$1" == "-p" ]; then - PS_VERSIONS_FILE="tags.txt"; -else - PS_VERSIONS_FILE="$1"; -fi +set -e +cd $(cd "$( dirname "$0" )" && pwd) +# Default values +PS_VERSIONS_FILE="tags.txt" +SINGLE_VERSION="" FORCE=false -while getopts ":fp" option; do - case $option in - p) - PUSH=true - ;; - f) - FORCE=true - ;; - esac +PUSH=false + +# Parse arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + SINGLE_VERSION="$2" + shift 2 + ;; + --file) + PS_VERSIONS_FILE="$2" + shift 2 + ;; + -f) + FORCE=true + shift + ;; + -p) + PUSH=true + shift + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 [--version ] [--file ] [-f] [-p]" + exit 1 + ;; + esac done docker_tag_exists() { - curl --silent -f -lSL https://hub.docker.com/v2/repositories/$1/tags/$2 > /dev/null + curl --silent -f -lSL https://hub.docker.com/v2/repositories/$1/tags/$2 > /dev/null 2>&1 } -docker_image() -{ - if ! $FORCE && docker_tag_exists prestashop/base ${version}; then - echo "Docker Image already pushed : prestashop/base:$version" +docker_image() { + version="$1" + if ! $FORCE && docker_tag_exists ${DOCKER_REPOSITORY} ${version}; then + echo "Docker Image already pushed : $DOCKER_REPOSITORY:$version" return else - echo "Docker build & tag : prestashop/base:$version" - id=$(echo $(docker build --quiet=true images/${version} 2>/dev/null) | awk '{print $NF}') - echo $id; - docker tag $id prestashop/base:${version} - - - if [ -z "$PUSH" ]; then - # Do not push - return - fi - echo "Docker Push : prestashop/base:$version" - - docker push prestashop/base:${version} + echo "Docker build & tag : $DOCKER_REPOSITORY:$version" + docker buildx build \ + --progress=plain \ + --platform ${PLATFORM_ARGS} \ + --builder container \ + --tag ${DOCKER_REPOSITORY}:${version} \ + $([ "$PUSH" == "true" ] && echo "--push") \ + images/${version} fi } - -# Generate base images for PHP tags -echo "Reading tags in ${PS_VERSIONS_FILE} ..." -while read version; do - docker_image $version -done < $PS_VERSIONS_FILE +if [ -n "$SINGLE_VERSION" ]; then + echo "Building single version: $SINGLE_VERSION" + docker_image "$SINGLE_VERSION" +else + echo "Reading tags in ${PS_VERSIONS_FILE} ..." + while read -r version; do + [ -z "$version" ] && continue + docker_image "$version" + done < "$PS_VERSIONS_FILE" +fi diff --git a/base/generate_tags.sh b/base/generate_tags.sh index c17579c8..d5b69604 100755 --- a/base/generate_tags.sh +++ b/base/generate_tags.sh @@ -31,7 +31,7 @@ generate_image() if [ -d images/$folder ] && [ -z "$FORCE" ]; then # Do not erase what we already defined in the directory - echo Already defined, skipping Use -f to forcre update + echo Already defined, skipping Use -f to force update return fi diff --git a/base/images/7.1-apache/Dockerfile b/base/images/7.1-apache/Dockerfile index e164c35e..28c6bfb3 100644 --- a/base/images/7.1-apache/Dockerfile +++ b/base/images/7.1-apache/Dockerfile @@ -24,6 +24,10 @@ PS_HANDLE_DYNAMIC_DOMAIN=0 \ PS_FOLDER_ADMIN=admin \ PS_FOLDER_INSTALL=install +RUN sed -ie "s/deb.debian.org/archive.debian.org/g" /etc/apt/sources.list \ + && sed -ie "s/security.debian.org/archive.debian.org/g" /etc/apt/sources.list \ + && echo 'Sources list updated' + RUN apt-get update \ && apt-get install -y libmcrypt-dev \ libjpeg62-turbo-dev \ diff --git a/base/images/7.1-fpm/Dockerfile b/base/images/7.1-fpm/Dockerfile index d2fedc44..6769516f 100644 --- a/base/images/7.1-fpm/Dockerfile +++ b/base/images/7.1-fpm/Dockerfile @@ -24,6 +24,10 @@ PS_HANDLE_DYNAMIC_DOMAIN=0 \ PS_FOLDER_ADMIN=admin \ PS_FOLDER_INSTALL=install +RUN sed -ie "s/deb.debian.org/archive.debian.org/g" /etc/apt/sources.list \ + && sed -ie "s/security.debian.org/archive.debian.org/g" /etc/apt/sources.list \ + && echo 'Sources list updated' + RUN apt-get update \ && apt-get install -y libmcrypt-dev \ libjpeg62-turbo-dev \ diff --git a/base/images/7.2-apache/Dockerfile b/base/images/7.2-apache/Dockerfile index 0abd4720..37982f11 100644 --- a/base/images/7.2-apache/Dockerfile +++ b/base/images/7.2-apache/Dockerfile @@ -24,6 +24,10 @@ PS_HANDLE_DYNAMIC_DOMAIN=0 \ PS_FOLDER_ADMIN=admin \ PS_FOLDER_INSTALL=install +RUN sed -ie "s/deb.debian.org/archive.debian.org/g" /etc/apt/sources.list \ + && sed -ie "s/security.debian.org/archive.debian.org/g" /etc/apt/sources.list \ + && echo 'Sources list updated' + RUN apt-get update \ && apt-get install -y libmcrypt-dev \ libjpeg62-turbo-dev \ diff --git a/base/images/7.2-fpm/Dockerfile b/base/images/7.2-fpm/Dockerfile index 6198df1f..fb5c0f49 100644 --- a/base/images/7.2-fpm/Dockerfile +++ b/base/images/7.2-fpm/Dockerfile @@ -24,6 +24,10 @@ PS_HANDLE_DYNAMIC_DOMAIN=0 \ PS_FOLDER_ADMIN=admin \ PS_FOLDER_INSTALL=install +RUN sed -ie "s/deb.debian.org/archive.debian.org/g" /etc/apt/sources.list \ + && sed -ie "s/security.debian.org/archive.debian.org/g" /etc/apt/sources.list \ + && echo 'Sources list updated' + RUN apt-get update \ && apt-get install -y libmcrypt-dev \ libjpeg62-turbo-dev \ diff --git a/base/tags.txt b/base/tags.txt index 6130b0d4..3413a6a1 100644 --- a/base/tags.txt +++ b/base/tags.txt @@ -1,5 +1,3 @@ -7.1-apache -7.2-apache 7.3-apache 7.4-apache 8.0-apache @@ -7,8 +5,6 @@ 8.2-apache 8.3-apache 8.4-apache -7.1-fpm -7.2-fpm 7.3-fpm 7.4-fpm 8.0-fpm diff --git a/prestashop_docker.py b/prestashop_docker.py index 4df5434c..11f3957f 100755 --- a/prestashop_docker.py +++ b/prestashop_docker.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import docker +import os from versions import VERSIONS from prestashop_docker.backlog import Backlog from prestashop_docker.generator import Generator @@ -12,6 +13,8 @@ import argparse import logging +docker_repository_name = os.getenv('DOCKER_REPOSITORY', 'prestashop/prestashop') + def get_parser(): parser = argparse.ArgumentParser(description='PrestaShop Docker manager.') @@ -53,7 +56,7 @@ def get_tag_parser(subparser): push_parser = tag_subparser.add_parser( 'push', - help='Push docker tags' + help='Build container and create docker tag then push docker tags' ) push_parser.add_argument('version', type=str, help='Version name', nargs='?') push_parser.add_argument('--force', action='store_const', const=True, help='Force build even if image already exists on Docker hub', default=False) @@ -125,7 +128,8 @@ def main(): docker.from_env(), VersionManager(path.join(path.dirname(path.realpath(__file__)), 'images')), args.cache, - args.quiet + args.quiet, + docker_repository_name, ) if args.tag_subcommand is None: tag_parser.print_help() @@ -138,7 +142,7 @@ def main(): elif args.tag_subcommand == 'build': tag_manager.build(args.version, args.force) elif args.tag_subcommand == 'push': - tag_manager.push(args.version, args.force) + tag_manager.build(args.version, args.force, True) elif args.tag_subcommand == 'aliases': tag_manager.get_aliases(args.version) else: diff --git a/prestashop_docker/docker_api.py b/prestashop_docker/docker_api.py index 3802dc8e..6ac081ff 100644 --- a/prestashop_docker/docker_api.py +++ b/prestashop_docker/docker_api.py @@ -28,7 +28,7 @@ def __init__(self, cache, debug): if self.cache: requests_cache.install_cache('cache') - def get_tags(self, image_name='prestashop/prestashop'): + def get_tags(self, image_name): """Generate return tags @return: The json content diff --git a/prestashop_docker/tag_manager.py b/prestashop_docker/tag_manager.py index 4b0420db..f270b993 100644 --- a/prestashop_docker/tag_manager.py +++ b/prestashop_docker/tag_manager.py @@ -1,11 +1,13 @@ import logging +import shutil +import subprocess from .stream import Stream logger = logging.getLogger(__name__) class TagManager(): - def __init__(self, docker_api, docker_client, version_manager, cache, quiet): + def __init__(self, docker_api, docker_client, version_manager, cache, quiet, docker_repository_name): ''' Constructor @@ -19,6 +21,8 @@ def __init__(self, docker_api, docker_client, version_manager, cache, quiet): @type cache: bool @param quiet: Quiet mode @type quiet: bool + @param docker_repository_name: Name of the Docker Hub repository + @type docker_repository_name: string ''' self.docker_api = docker_api self.docker_client = docker_client @@ -26,8 +30,9 @@ def __init__(self, docker_api, docker_client, version_manager, cache, quiet): self.version_manager = version_manager self.cache = cache self.tags = None + self.docker_repository_name = docker_repository_name - def build(self, version=None, force=False): + def build(self, version=None, force=False, push=False): ''' Build version on the current machine @@ -46,67 +51,36 @@ def build(self, version=None, force=False): # Do not build images that already exists on Docker Hub continue - log = self.docker_client.api.build( - path=str(version_path), - tag='prestashop/prestashop:' + version, - rm=True, - nocache=(not self.cache), - decode=True - ) - - self.stream.display(log) - - aliases = self.version_manager.get_aliases() - if version in aliases: - for alias in aliases[version]: - print( - 'Create tag {}'.format(alias) - ) - self.docker_client.api.tag( - 'prestashop/prestashop:' + version, - 'prestashop/prestashop', - alias - ) - - def push(self, version=None, force=False): - ''' - Push version on Docker Hub - - @param version: Optional version you want to build - @type version: str - ''' - versions = self.get_versions(version) - - for version in versions.keys(): - print( - 'Pushing {}'.format(version) - ) - - if not force and self.exists(version): - continue - - log = self.docker_client.api.push( - repository='prestashop/prestashop', - tag=version, - decode=True, - stream=True - ) - - self.stream.display(log) + if not shutil.which("docker"): + raise RuntimeError("The docker client must be installed") + tags = ["--tag", self.docker_repository_name + ":" + version] aliases = self.version_manager.get_aliases() if version in aliases: for alias in aliases[version]: - print( - 'Pushing tag {}'.format(alias) - ) - log = self.docker_client.api.push( - repository='prestashop/prestashop', - tag=alias, - decode=True, - stream=True - ) - self.stream.display(log) + print('Will be aliased as tag {}'.format(alias)) + tags = tags + ["--tag", self.docker_repository_name + ":" + alias] + + args = [] + if push: + args.append('--push') + if not self.cache: + args.append('--no-cache') + + cmd_args = [ + "docker", "buildx", "build", + "--platform", "linux/arm/v7,linux/arm64/v8,linux/amd64", + "--builder", "container", + ] + tags + args + [ + str(version_path) + ] + + process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE) + process.stdout.read() + process.wait() + + if process.returncode: + raise Exception('Command returned code {} while building version {}.'.format(process.returncode, version)) def exists(self, version): ''' @@ -119,7 +93,7 @@ def exists(self, version): ''' if self.tags is None: - self.tags = self.docker_api.get_tags() + self.tags = self.docker_api.get_tags(image_name=self.docker_repository_name) for tag in self.tags: if tag['name'] == version: