Merge pull request #386 from stackhpc/upstream/master-2025-02-24

Synchronise master with upstream

Author: priteau

.ansible-lint

@@ -0,0 +1,12 @@
+---
+exclude_paths:
+  - .cache/  # implicit unless exclude_paths is defined in config
+  - .zuul.d/
+offline: true
+parseable: true
+profile: basic
+skip_list:
+  - package-latest
+  - role-name
+strict: true
+use_default_rules: true

.yamllint

@@ -126,6 +126,8 @@
     run: tests/playbooks/run.yml
     post-run: tests/playbooks/post.yml
     attempts: 5
+    roles:
+      - zuul: openstack/ansible-collection-kolla
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*

.zuul.d/base.yaml

@@ -525,6 +525,98 @@ The template becomes now:
    RUN cp /additions/jenkins/jenkins.json /jenkins.json
    {% endblock %}
 
+Custom docker templates
+-----------------------
+
+In order to unify the process of managing OpenStack-related projects, Kolla
+provides a way of building images for external 'non-built-in' projects.
+
+If the template for a 'non-built-in' project meets Kolla template standards,
+an operator can provide a root directory with a template via the
+``--docker-dir`` CLI option (can be specified multiple times).
+
+All Kolla's jinja2 macros should be available the same as for built-in
+projects with some notes:
+
+- The ``configure_user`` macro. As the 'non-built-in' user is unknown to Kolla,
+  there are no default values for user ID and group ID to use.
+  To use this macro, an operator should specify "non-default" user details
+  with ``<custom_user_name>-user`` configuration section and include info
+  for ``uid`` and ``gid`` at least.
+
+Let's look into how an operator can build an image for an in-house project
+with Kolla using `openstack/releases <https://opendev.org/openstack/releases>`_
+project.
+
+First, create a ``Dockerfile.j2`` template for the project.
+
+.. path /home/kolla/custom-kolla-docker-templates/releaser/Dockerfile.j2
+.. code-block:: jinja
+
+   FROM {{ namespace }}/{{ image_prefix }}openstack-base:{{ tag }}
+
+   {% block labels %}
+   LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+   {% endblock %}
+
+   {% block releaser_header %}{% endblock %}
+
+   {% import "macros.j2" as macros with context %}
+
+   {{ macros.configure_user(name='releaser') }}
+
+   RUN ln -s releaser-source/* /releaser \
+       && {{ macros.install_pip(['/releaser-source']  | customizable("pip_packages")) }} \
+       && mkdir -p /etc/releaser \
+       && chown -R releaser: /etc/releaser \
+       && chmod 750 /etc/sudoers.d \
+       && touch /usr/local/bin/kolla_releaser_extend_start \
+       && chmod 644 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_releaser_extend_start
+
+   {% block footer %}{% endblock %}
+
+Suggested directory structure:
+
+.. code-block:: console
+
+   custom-kolla-docker-templates
+   |__ releaser
+       |__ Dockerfile.j2
+
+Then, modify Kolla's configuration so the engine can download sources and
+configure users.
+
+.. path /etc/kolla/kolla-build.conf
+.. code-block:: ini
+
+   [releaser]
+   type = git
+   location = https://opendev.org/openstack/releases
+   reference = master
+
+   [releaser-user]
+   uid = 53001
+   gid = 53001
+
+Last pre-check before building a new image - ensure that the new template
+is visible for Kolla:
+
+.. code-block:: console
+
+   $ kolla-build --list-images --docker-dir custom-kolla-docker-templates "^releaser$"
+   1 : base
+   2 : releaser
+   3 : openstack-base
+
+And finally, build the ``releaser`` image, passing the ``--docker-dir``
+argument:
+
+.. code-block:: console
+
+   kolla-build --docker-dir custom-kolla-docker-templates "^releaser$"
+
+Can I use the ``--template-override`` option for custom templates? Yes!
+
 Custom repos
 ------------
 

doc/source/admin/image-building.rst

@@ -43,6 +43,9 @@
 {% endmacro %}
 
 {% macro configure_user(name, groups=None, shell=None, homedir=None) %}
+{%- if name not in users %}
+{{ raise_error("Failed to find configuration for '" + name + "' user. Try specifying '" + name + "-user' config section.") }}
+{%- endif %}
 {% set user=users[name] %}
 {%- if not homedir %}
 {% set homedir='/var/lib/' + name %}

docker/macros.j2

@@ -271,6 +271,7 @@ def _get_methods(self):
         return {
             'debian_package_install': jinja_methods.debian_package_install,
             'handle_repos': jinja_methods.handle_repos,
+            'raise_error': jinja_methods.raise_error,
         }
 
     def get_users(self):
@@ -280,7 +281,17 @@ def get_users(self):
         for section in all_sections:
             match = re.search('^.*-user$', section)
             if match:
-                user = self.conf[match.group(0)]
+                cfg_group_name = match.group(0)
+
+                if cfg_group_name not in self.conf._groups:
+                    self.conf.register_opts(
+                        common_config.get_user_opts(
+                            None, None,
+                            # cut `-user` suffix
+                            group=cfg_group_name[:-5]),
+                        group=cfg_group_name
+                    )
+                user = self.conf[cfg_group_name]
                 ret[match.group(0)[:-5]] = {
                     'uid': user.uid,
                     'gid': user.gid,

kolla/image/kolla_worker.py

@@ -232,6 +232,7 @@ def reset_userinfo(tarinfo):
                 git.Git().clone(source['source'], clone_dir)
                 git.Git(clone_dir).checkout(source['reference'])
                 reference_sha = git.Git(clone_dir).rev_parse('HEAD')
+                git.Git(clone_dir).remote("remove", "origin")
                 self.logger.debug("Git checkout by reference %s (%s)",
                                   source['reference'], reference_sha)
             except Exception as e:

kolla/image/tasks.py

@@ -11,6 +11,8 @@
 # limitations under the License.
 
 import os
+import typing as t
+
 import yaml
 
 from jinja2 import pass_context
@@ -150,3 +152,7 @@ def handle_repos(context, reponames, mode):
         commands = "RUN %s" % commands
 
     return commands
+
+
+def raise_error(msg: str) -> t.NoReturn:
+    raise Exception(msg)

kolla/template/methods.py

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Extends the support of externally-managed projects provided by the
+    ``--docker-dir`` option with an ability to use ``configure_user``
+    jinja2 macros like Kolla built-in projects.
+    The operator should specify "non-default" user details with
+    ``<custom_user_name>-user`` configuration section and include info for
+    ``uid`` and ``gid`` at least.

releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Removes Git remote URLs after cloning to prevent credential exposure.
+    `LP#2098904 <https://bugs.launchpad.net/kolla/+bug/2098904>`__