chore: enhance dockerized development environment (#977)

1. Add an option to control which PHP version is to be used in devenv
service via PHP env var passed at the time when dev-shell is started
(or any dev-* target is build).
2. Simplify `composer` and `go` install.
3. Fix `devenv` service image build for PHPs 7.2 and 7.3.
4. Apply `Dockerfile` best practices

Author: lavarou

Makefile

@@ -475,25 +475,28 @@ test-services-stop:
 # Docker Development Environment
 #
 
-dev-shell:
-	docker compose --profile dev up --build --remove-orphans -d
-	docker exec -it agent-devenv bash -c "sh files/set_path.sh ; bash"
+devenv-image:
+	@docker compose --profile dev build devenv
 
-dev-build:
-	docker compose --profile dev up --build --remove-orphans -d
-	docker exec -it agent-devenv bash -c "sh files/set_path.sh ; make -j4 all"
+dev-shell: devenv-image
+	docker compose --profile dev up --pull missing --remove-orphans -d
+	docker compose exec -it devenv bash -c "sh files/set_path.sh ; bash"
 
-dev-unit-tests:
-	docker compose --profile dev up --build --remove-orphans -d
-	docker exec -it agent-devenv bash -c "sh files/set_path.sh ; make -j4 valgrind"
+dev-build: devenv-image
+	docker compose --profile dev up --pull missing --remove-orphans -d
+	docker compose exec -it devenv bash -c "sh files/set_path.sh ; make -j4 all"
 
-dev-integration-tests:
-	docker compose --profile dev up --build --remove-orphans -d
-	docker exec -it agent-devenv bash -c "sh files/set_path.sh ; ./bin/integration_runner -agent ./agent/.libs/newrelic.so"
+dev-unit-tests: devenv-image
+	docker compose --profile dev up --pull missing --remove-orphans -d
+	docker compose exec -it devenv bash -c "sh files/set_path.sh ; make -j4 valgrind"
 
-dev-all:
-	docker compose --profile dev up --build --remove-orphans -d
-	docker exec -it agent-devenv bash -c "sh files/set_path.sh ; make -j4 all valgrind; ./bin/integration_runner -agent ./agent/.libs/newrelic.so"
+dev-integration-tests: devenv-image
+	docker compose --profile dev up --pull missing --remove-orphans -d
+	docker compose exec -it devenv bash -c "sh files/set_path.sh ; ./bin/integration_runner -agent ./agent/.libs/newrelic.so"
+
+dev-all: devenv-image
+	docker compose --profile dev up --pull missing --remove-orphans -d
+	docker compose exec -it devenv bash -c "sh files/set_path.sh ; make -j4 all valgrind; ./bin/integration_runner -agent ./agent/.libs/newrelic.so"
 
 dev-stop:
 	docker compose --profile dev stop

docker-compose.yaml

@@ -2,7 +2,6 @@
 # Copyright 2021 New Relic Corporation. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
-version: '3.8'
 services:
 # The Database
   mysqldb:
@@ -79,6 +78,8 @@ services:
     build:
       context: .
       dockerfile: files/Dockerfile
+      args:
+        PHP_VER: ${PHP:-8.3}
     user: ${UID}:${GID}
     environment:
       MEMCACHE_HOST: memcached

docs/dev_environment.md

@@ -6,19 +6,38 @@ The dockerized development environment prototype allows contributors to both dev
 
 docker-compose spins up `mysql` and `redis` and other databases in separate containers.
 
-Two environment variables to note:
-`NEWRELIC_LICENSE_KEY` is required to run the integration tests and should be set to your NR license key.
-If your collector isn’t the default (collector.newrelic.com), set the `NEWRELIC_COLLECTOR_HOST` to the appropriate value.
+## Prerequisites
 
-PHP_VER can also be set to vary the PHP version being used.
+### 1. Docker Compose
 
-Set all environment variables prior to running the development environment.  
+Dockerized development environment for the New Relic PHP Agent uses following Docker Compose services as a runtime platform. So you need `docker` with `docker compose` installed.
+
+### 2. Environment variables
+
+Dockerized development environment for the New Relic PHP Agent needs a valid license key available in `NEW_RELIC_LICENSE_KEY` environment variable.
+This environment variable must be set prior to starting Dockerized development environment for the New Relic PHP Agent. The easiest way to set
+`NEW_RELIC_LICENSE_KEY` environment variable is via `.env` file. Simply create `.env` file in the top level directory, and add definition
+of `NEW_RELIC_LICENSE_KEY` environment variable there, e.g.:
+```
+NEW_RELIC_LICENSE_KEY=...
+```
+
+The second, optional environment variable, that controls PHP version in Dockerized development environment for the New Relic PHP Agent is `PHP`. `PHP` defaults to latest PHP supported by the agent.
+This environment variable can be provided at the time when Dockerized development environment for the New Relic PHP Agent is started, e.g.:
+```
+make dev-shell PHP=8.2
+```
 
 ## Options for using the environment
 
-## With a shell environment
+### With a shell environment
+
+To start the dev environment type `make dev-shell`.  This will spin up `devenv` service in `agent-devenv` container, with:
+ - latest PHP supported by the agent (this can be overriden with `PHP` environment variable like this: `make dev-shell PHP=8.2`)
+ - all the tools needed to build the agent
+ - all the tools needed to run unit tests
+ - all the tools and supporting services to run integration tests
 
-To start the dev environment type `make dev-shell`.  This will create a set of docker containers.
 A prompt will open and you’ll be able to compile and run all `make` commands right away with no additional setup (for example: `make -j4 all` or `make -j4 valgrind` or `make -j4 run_tests`).
 
 After compiling the agent, the integration tests can be run using the `integration_runner`.
@@ -31,27 +50,27 @@ To end the session type `exit`.  You can run `make dev-stop` to stop the docker-
 
 In the shell, you can run all `make` commands as you normally would.
 
-## Build only
+### Build only
 
 `make dev-build`
 
-## Unit Tests only
+### Unit Tests only
 
 `make dev-unit tests`
 
-## Integration Tests only
+### Integration Tests only
 
 `make dev-integration-tests`
 
-## Build and test all
+### Build and test all
 
 `make dev-all`
 
-## Stop all containers
+### Stop all containers
 
 `make dev-stop`
 
-# Next steps and issues
+## Next steps and issues
 
-## There is possibly some incompatibility with mysql in the main build container as one of the mysql unit tests fails.  Unless this is resolved, It might make sense at a future point to have the integration tests run from a different container than the build container.
+### There is possibly some incompatibility with mysql in the main build container as one of the mysql unit tests fails.  Unless this is resolved, It might make sense at a future point to have the integration tests run from a different container than the build container.
 

files/Dockerfile

@@ -14,95 +14,65 @@ FROM php:${PHP_VER:-8.3}
 RUN docker-php-source extract
 
 ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update
-RUN apt-get install -y build-essential
-
-#
-# PHP dependencies
-#
-RUN apt-get update \
- && apt-get -y install gcc git netcat-openbsd \
- libpcre3 libpcre3-dev psmisc automake libtool \
- insserv procps vim ${PHP_USER_SPECIFIED_PACKAGES} \
- zlib1g-dev libmcrypt-dev
+RUN apt-get update && apt-get install -y --no-install-recommends \
+# Install build entrypoint - make:
+    make \
+# Makefile's default shell:
+    bash \
+# git; techincally not needed to build agent but is required for successfull
+# processing of top level Makefile:
+# - make/version.mk if GIT_COMMIT is not defined, git is used to compute it:
+    git \
+# The following are required to build PHP extension:
+    $PHPIZE_DEPS \
+# valgrind, awk; required for agent-valgrind and axiom-valgrind targets
+    valgrind gawk \
+# Other useful developer tools:
+    vim lcov gdb strace ccache procps psmisc curl wget bzip2 zip unzip perl sqlite3 openssl \
+# Other build dependencies:
+    argon2 \
+    automake \
+    autotools-dev \
+    dnsutils \
+    gyp \
+    insserv \
+    libc6 libc6-dev libc6-dbg \
+    libcurl4-openssl-dev \
+    libedit-dev \
+    libghc-argon2-dev \
+    libgtest-dev \
+    libmcrypt-dev \
+    libonig-dev \
+    libpcre3 libpcre3-dev \
+    libreadline-dev \
+    libssl-dev \
+    libsqlite3-dev \
+    libtool \
+    libxml2 libxml2-dev \
+    locales \
+    locales-all \
+    netcat-openbsd \
+    python3-yaml \
+    ${PHP_USER_SPECIFIED_PACKAGES} \
+    zlib1g-dev
 
 # pgsql extension
-RUN apt-get install -y libpq-dev
+RUN apt-get install -y --no-install-recommends libpq-dev
 RUN docker-php-ext-install pgsql
 
-#
-# Other tools
-#
-RUN apt-get install -y gdb valgrind libcurl4-openssl-dev pkg-config libpq-dev libedit-dev libreadline-dev git
-
-#
-# Install other packages.
-#
-RUN apt-get update && apt-get install -y \
-  autoconf \
-  autotools-dev \
-  build-essential \
-  bzip2 \
-  ccache \
-  curl \
-  dnsutils \
-  git \
-  gyp \
-  lcov \
-  libc6 \
-  libc6-dbg \
-  libc6-dev \
-  libgtest-dev \
-  libtool \
-  locales \
-  locales-all \
-  make \
-  perl \
-  strace \
-  python-dev-is-python3 \
-  python3-yaml \
-  sqlite3 \
-  libsqlite3-dev \
-  openssl \
-  libxml2 \
-  libxml2-dev \
-  libonig-dev \
-  libssl-dev \
-  unzip \
-  wget \
-  zip && apt-get clean
-
-#
-# Download and install Go
-#
-RUN arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \
-    wget https://go.dev/dl/go1.21.1.linux-${arch}.tar.gz -O- | tar -C /usr/local -zxvf -;
-
-RUN ln -s /usr/local/go/bin/go /usr/bin/go
+# install latest go to work with the daemon
+COPY --from=golang /usr/local/go /usr/local/go
+ENV PATH /usr/local/go/bin:$PATH
 
 #
-# If the debian version is jessie, don't install argon2
+# If the debian version is buster, don't install python-dev-is-python3
 #
-RUN if [ -z "$(grep '^8\.' /etc/debian_version)" ]; then \
-    apt-get install -y argon2 libghc-argon2-dev; \
+RUN if [ -z "$(grep '^10\.' /etc/debian_version)" ]; then \
+    apt-get install -y  --no-install-recommends python-dev-is-python3; \
     fi
 
 # install composer
-WORKDIR /usr/src
-
-# based on https://getcomposer.org/doc/faqs/how-to-install-composer-programmatically.md
-RUN \
- EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')" \
- && php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" \
- && ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")" \
- && if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]; \
-  then \
-    >&2 echo 'ERROR: Invalid installer checksum'; \
-    rm composer-setup.php; \
-    exit 1; \
-  fi \
- && php composer-setup.php \
- && php -r "unlink('composer-setup.php');"
+COPY --from=composer ["/usr/bin/composer", "/usr/bin/composer"]
 
 #
 # The explain plan in the sql tests contain partition/filtered properties
@@ -121,8 +91,16 @@ RUN \
 #   the mysql tests a separate machine running mysql server 5.6 is required.
 RUN docker-php-ext-install pdo pdo_mysql
 
-# redis
-RUN pecl install redis && docker-php-ext-enable redis
+# install redis extension required by test_redis:
+RUN \
+  php_cmp=$(php -r "echo version_compare(PHP_VERSION, '7.4.0', '>=');"); \
+  if [ "$php_cmp" = 1 ]; then \
+    # install latest redis for PHPs >= 7.4
+    echo 'no' | pecl install redis; \
+  else \
+    # install redis-4.3.0 - last one with support for php 7.2
+    echo 'no' | pecl install redis-4.3.0; \
+  fi && docker-php-ext-enable redis
 
 # memcache
 # Pre 8.0 requires 4.0.5.2
@@ -135,7 +113,7 @@ RUN \
   fi
 
 # memcached
-RUN apt-get install -y libmemcached-dev
+RUN apt-get install -y --no-install-recommends libmemcached-dev
 RUN pecl install memcached && docker-php-ext-enable memcached
 
 # uopz
@@ -152,13 +130,9 @@ RUN \
 # install predis
 # installation will be in /usr/src/vendor/predis/predis
 # which is value which should be used for PREDIS_HOME
-RUN php composer.phar require "predis/predis"
-RUN php composer.phar update
-
-#
-# install composer and make executable so it can be used in dev env
-#
-RUN cp composer.phar /usr/local/bin/composer && chmod +x /usr/local/bin/composer
+WORKDIR /usr/src
+RUN composer require "predis/predis"
+RUN composer update
 
 #
 # These args need to be repeated so we can propagate the VARS within this build context.

files/set_path.sh

@@ -17,11 +17,6 @@ case ":$PATH:" in
   *) PATH=/usr/local/bin:$PATH
 esac
 
-case ":$PATH:" in
-  *:/usr/local/go/bin:*) ;;
-  *) PATH=/usr/local/go/bin:$PATH
-esac
-
 export PATH