diff --git a/.buildkite/scripts/build.sh b/.buildkite/scripts/build.sh index d3cf3171599ac..a933c9db1fb72 100755 --- a/.buildkite/scripts/build.sh +++ b/.buildkite/scripts/build.sh @@ -9,7 +9,6 @@ export GIT_AUTHOR_EMAIL='docs-status+buildkite@elastic.co' export GIT_COMMITTER_NAME=$GIT_AUTHOR_NAME export GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL - build_args="" rebuild_opt="" broken_links_opt="" @@ -38,7 +37,8 @@ ssh-keyscan github.com >> "$HOME/.ssh/known_hosts" chmod 600 "$HOME/.ssh/id_rsa" ssh-agent bash -c " - ssh-add && + ssh-add && + eval \"\$(rbenv init -)\" && ./build_docs --all \ --target_repo git@github.com:elastic/built-docs \ --reference /opt/git-mirrors/ \ diff --git a/.buildkite/scripts/build_pr.sh b/.buildkite/scripts/build_pr.sh index 506a5a6d0ee11..65c087023e0b7 100755 --- a/.buildkite/scripts/build_pr.sh +++ b/.buildkite/scripts/build_pr.sh @@ -247,6 +247,12 @@ build_cmd="./build_docs --all \ echo "The following build command will be used" echo $build_cmd +diag_cmd="echo \"which ruby -> \$(which ruby)\" && \ +echo \"which asciidoctor -> \$(which asciidoctor)\" && \ +echo \"ruby -v -> \$(ruby -v || echo 'ruby not found')\" && \ +echo \"asciidoctor --version -> \$(asciidoctor --version || echo 'asciidoctor not found')\" && \ +echo \"GEM_PATH is: \$GEM_PATH\"" + # Temporary workaround until we can move to HTTPS auth vault read -field=private-key secret/ci/elastic-docs/elasticmachine-ssh-key > "$HOME/.ssh/id_rsa" vault read -field=public-key secret/ci/elastic-docs/elasticmachine-ssh-key > "$HOME/.ssh/id_rsa.pub" @@ -254,7 +260,7 @@ ssh-keyscan github.com >> "$HOME/.ssh/known_hosts" chmod 600 "$HOME/.ssh/id_rsa" # Kick off the build -ssh-agent bash -c "ssh-add && $build_cmd" +ssh-agent bash -c "ssh-add && eval \"\$(rbenv init -)\" && eval \"\$(export RBENV_ROOT /root/.rbenv)\" && eval \"\$(export PATH \$RBENV_ROOT/bin:\$RBENV_ROOT/shims:\$PATH)\" && eval \"\$(export GEM_PATH /var/lib/gems:/usr/local/bin:\$GEM_PATH)\" && $diag_cmd && $build_cmd" buildkite-agent annotate \ --style "success" \ diff --git a/.buildkite/scripts/test.sh b/.buildkite/scripts/test.sh index 3efcd03644434..137c07f4aeacb 100755 --- a/.buildkite/scripts/test.sh +++ b/.buildkite/scripts/test.sh @@ -2,6 +2,8 @@ set -euo pipefail +export GEM_PATH=/var/lib/gems${GEM_PATH:+:$GEM_PATH} + echo "Building images for docs test" for IMAGE in build py_test node_test ruby_test integ_test diff_tool; do echo Building $IMAGE diff --git a/.docker/ruby_openssl.patch b/.docker/ruby_openssl.patch new file mode 100644 index 0000000000000..9ce55606a9cb7 --- /dev/null +++ b/.docker/ruby_openssl.patch @@ -0,0 +1,11 @@ +--- a/ext/openssl/ossl_pkey_rsa.c ++++ b/ext/openssl/ossl_pkey_rsa.c +@@ -986,7 +986,9 @@ + rb_define_method(cRSA, "params", ossl_rsa_get_params, 0); + + DefRSAConst(PKCS1_PADDING); ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + DefRSAConst(SSLV23_PADDING); ++#endif + DefRSAConst(NO_PADDING); + DefRSAConst(PKCS1_OAEP_PADDING); \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ecc1e8cdf477a..6224ce286ccd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ # Debian builds the docs about 20% faster than alpine. The image is larger # and takes longer to build but that is worth it. -FROM bitnami/minideb:buster AS base +FROM bitnami/minideb:bookworm AS base # TODO install_packages calls apt-get update and then nukes the list files after. We should avoid multiple calls to apt-get update..... # We could probably fix this by running the update and installs ourself with `RUN --mount type=cache` but that is "experimental" @@ -18,9 +18,9 @@ COPY .docker/apt/keys/nodesource.gpg / RUN apt-key add /nodesource.gpg COPY .docker/apt/sources.list.d/nodesource.list /etc/apt/sources.list.d/ RUN install_packages \ - build-essential python2 \ + build-essential python-is-python3 \ # needed for compiling native modules on ARM - nodejs ruby \ + nodejs rbenv ruby-build \ # Used both to install dependencies and at run time bash less \ # Just in case you have to shell into the image @@ -32,22 +32,47 @@ RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && locale-gen ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 +ENV RBENV_ROOT /root/.rbenv +ENV PATH $RBENV_ROOT/bin:$RBENV_ROOT/shims:$PATH +ENV GEM_PATH /var/lib/gems:/usr/local/bin:$GEM_PATH +RUN rbenv init - FROM base AS ruby_deps +ENV RUBY_VERSION 2.5.8 +RUN eval "$(rbenv init -)" + RUN install_packages \ - bundler \ +# ruby-build bundler \ # Fetches ruby dependencies - ruby-dev make cmake gcc libc-dev patch +# ruby-dev \ + make cmake gcc libc-dev patch \ # Required to compile some of the native dependencies + libssl-dev libnss-wrapper + +ENV RUBY_CFLAGS="-O3 -fno-fast-math -g3 -Wall -Wno-error=deprecated-declarations" +RUN ls +COPY .docker/ruby_openssl.patch /tmp/ +RUN rbenv install $RUBY_VERSION --patch < /tmp/ruby_openssl.patch || (cat /tmp/ruby-build.*.log && exit 1) && \ + rbenv global $RUBY_VERSION && \ + gem install bundler -v 1.17.3 --no-document && \ + rbenv rehash + +ENV RUBY_CFLAGS="" + RUN bundle config --global silence_root_warning 1 COPY Gemfile* / # --frozen forces us to regenerate Gemfile.lock locally before using it in # docker which lets us lock the versions in place. RUN bundle install --binstubs --system --frozen --without test COPY .docker/asciidoctor_2_0_10.patch / -RUN cd /var/lib/gems/2.5.0/gems/asciidoctor-2.0.10 && patch -p1 < /asciidoctor_2_0_10.patch +RUN cd $RBENV_ROOT/versions/$RUBY_VERSION/lib/ruby/gems/2.5.0/gems/asciidoctor-2.0.10 && \ + patch -p1 < /asciidoctor_2_0_10.patch +RUN mkdir -p /var/lib/gems && \ + cp -R $RBENV_ROOT/versions/$RUBY_VERSION/lib/ruby/gems/* /var/lib/gems/ && \ + cp $RBENV_ROOT/versions/$RUBY_VERSION/bin/* /usr/local/bin/ +RUN find /usr/local/bin -type f -exec sed -i '1s|#!.*/bin/ruby|#!/usr/bin/env ruby|' {} + FROM base AS node_deps COPY .docker/apt/keys/yarn.gpg / @@ -59,6 +84,7 @@ COPY yarn.lock / ENV YARN_CACHE_FOLDER=/tmp/.yarn-cache # --frozen-lockfile forces us to regenerate yarn.lock locally before using it # in docker which lets us lock the versions in place. +RUN yarn global add node-gyp RUN yarn install --frozen-lockfile --production @@ -66,6 +92,9 @@ RUN yarn install --frozen-lockfile --production # Dockerfiles to make the images to serve previews and air gapped docs. FROM base AS build LABEL MAINTAINERS="Nik Everett " + +COPY --from=ruby_deps /root/.rbenv /root/.rbenv + RUN install_packages \ git \ # Clone source repositories and commit to destination repositories @@ -103,9 +132,9 @@ RUN rm -rf /var/log/nginx && rm -rf /run/nginx FROM base AS py_test # There's not a published wheel for yamale, so we need setuptools and wheel RUN install_packages python3 python3-pip python3-setuptools python3-wheel python3-dev libxml2-dev libxslt-dev zlib1g-dev -RUN pip3 install \ +RUN pip3 install --break-system-packages \ beautifulsoup4==4.8.1 \ - lxml==4.4.2 \ + lxml==4.9.4 \ pycodestyle==2.5.0 \ yamale==3.0.1 \ pyyaml==5.3.1 @@ -117,10 +146,12 @@ FROM ruby_deps AS ruby_test RUN bundle install --binstubs --system --frozen --with test FROM build AS integ_test +COPY --from=ruby_test /root/.rbenv /root/.rbenv COPY --from=node_test /node_modules /node_modules COPY --from=ruby_test /var/lib/gems /var/lib/gems COPY --from=ruby_test /usr/local/bin/rspec /usr/local/bin/rspec COPY --from=ruby_test /usr/local/bin/rubocop /usr/local/bin/rubocop +COPY --from=ruby_test /usr/local/bin/asciidoctor /usr/local/bin/asciidoctor FROM py_test AS diff_tool RUN install_packages git diff --git a/Dockerfile.dockerignore b/Dockerfile.dockerignore index 842dd25848cf4..e88b0ef590fc4 100644 --- a/Dockerfile.dockerignore +++ b/Dockerfile.dockerignore @@ -2,4 +2,4 @@ !.docker !Gemfile* !package.json -!yarn.lock +!yarn.lock \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 063ecacff5c27..8b059990d56c9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -52,7 +52,7 @@ DEPENDENCIES thread_safe (~> 0.3.6) RUBY VERSION - ruby 2.5.3p105 + ruby 2.5.8 BUNDLED WITH 1.17.3 diff --git a/build_docs.pl b/build_docs.pl index a972cd9b06537..ad570f60a7f1e 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -953,7 +953,7 @@ sub init_env { chomp($gid); print $override "docker:x:$uid:$gid:docker:/tmp:/bin/bash\n"; close $override; - $ENV{LD_PRELOAD} = '/usr/lib/libnss_wrapper.so'; + $ENV{LD_PRELOAD} = 'libnss_wrapper.so'; $ENV{NSS_WRAPPER_PASSWD} = '/tmp/passwd'; $ENV{NSS_WRAPPER_GROUP} = '/etc/group'; }