Merge pull request ceph#57169 from jagombar/fixCpatch

cpatch: Search base image when patching files.

Reviewed-by: John Mulligan <[email protected]>
Reviewed-by: Josh Durgin <[email protected]>

Author: jdurgin

src/script/cpatch

@@ -123,7 +123,7 @@ if [ $py -eq 1 ] || [ $all -eq 1 ]; then
 	echo "py"
 	# Exclude node_modules because it's the huge sources in
 	# dashboard/frontend
-	exclude="--exclude=node_modules"
+	exclude="--exclude=node_modules --exclude=.tox --exclude=.angular"
     fi
     tar $exclude --exclude=tests --exclude-backups -cf $TMP/mgr_plugins.tar *
     popd > /dev/null
@@ -132,106 +132,100 @@ if [ $py -eq 1 ] || [ $all -eq 1 ]; then
     pushd ../src/python-common > /dev/null
     find ./ -name "*.pyc" -exec rm -f {} \;
     # Exclude node_modules because it's the huge sources in dashboard/frontend
-    tar --exclude=node_modules --exclude=tests --exclude-backups -cf $TMP/python_common.tar *
+    tar --exclude=node_modules --exclude=tests --exclude=.tox --exclude-backups -cf $TMP/python_common.tar *
     popd > /dev/null
-    dockerfile+=$'ADD python_common.tar /usr/lib/python3.8/site-packages\n'
+    dockerfile+=$'ADD python_common.tar tmp_python_common\n'
+    dockerfile+=$'RUN for i in tmp_python_common/*; do find /usr/lib/python*/site-packages -type d -name $(basename $i) -exec cp -frpv $i/* \'{}\' \;; done && rm -rf tmp_python_common\n'
 
     pushd lib/cython_modules/lib.3
     CYTHONLIBS="*.cpython-3*.so"
     mkdir -p $TMP/cythonlib
     for f in $CYTHONLIBS; do cp $f $TMP/cythonlib ; done
     [ $strip -eq 1 ] && strip $TMP/cythonlib/*
     popd > /dev/null
-    dockerfile+=$'ADD cythonlib /usr/lib64/python3.8/site-packages\n'
+    dockerfile+=$'ADD cythonlib tmp_python_common\n'
+    dockerfile+=$'RUN for i in tmp_python_common/*; do find /usr/lib/python*/site-packages -type d -name $(basename $i) -exec cp -frpv $i/* \'{}\' \;; done && rm -rf tmp_python_common\n'
 
     # cephadm
     pushd ../src/cephadm > /dev/null
-    build.sh $TMP/cephadm
+    ./build.sh $TMP/cephadm
     dockerfile+=$'ADD cephadm /usr/sbin/cephadm\n'
     popd > /dev/null
 fi
 
+# Create some temporary directories.  The binaries or libraries to patch are placed in these as the cli options are processed.
+# At the end the base container is searched for files in these directories and the original files are replaced.
+mkdir -p $TMP/bin
+mkdir -p $TMP/lib
+
 if [ $core -eq 1 ] || [ $all -eq 1 ]; then
     # binaries are annoying because the ceph version is embedded all over
     # the place, so we have to include everything but the kitchen sink.
-    echo "core"
-
     BINS="ceph-mgr ceph-mon ceph-osd rados"
-    mkdir -p $TMP/bin
-    for f in $BINS; do cp bin/$f $TMP/bin ; done
-    [ $strip -eq 1 ] && strip $TMP/bin/*
-    dockerfile+=$'ADD bin /usr/bin\n'
-
-    # We need ceph-common to support the binaries
-    # We need librados/rbd to support mgr modules
-    # that import the python bindings
-    LIBS="libceph-common.so.2 libceph-common.so librados.so.2 librados.so librados.so.2.0.0"
-    mkdir -p $TMP/lib
-    for f in $LIBS; do cp lib/$f $TMP/lib ; done
+    if [ $core -eq 1 ]; then
+        echo "core"
+        for f in $BINS; do cp bin/$f $TMP/bin ; done
+    else
+        # copy ALL locally built binaries (apart from test programs) over those that already exist in the image.
+        echo "all"
+        find bin -type f \! \( -name "ceph_test*" -o -name "test_*" -o -name "unittest_*" \) -exec cp {} $TMP/bin \; 
+        # Need to strip all binaries that are copied (except those in the core BINS list) otherwise the container will be huge
+        # Some of the files in the bins directory are actually scripts so ignore errors when strip fails for these files.
+        find $TMP/bin -type f $(printf "! -name %s " $BINS) -exec strip {} \; || true
+    fi
+    [ $strip -eq 1 ] && for f in $BINS; do strip $TMP/bin/$f; done
+
+    # Copy all locally built libraries over those that already exist in the image
+    cp -d lib/*.so lib/*.so.* $TMP/lib
     [ $strip -eq 1 ] && strip $TMP/lib/*
-    dockerfile+=$'ADD lib /usr/lib64\n'
-
-    ECLIBS="libec_*.so*"
-    mkdir -p $TMP/eclib
-    for f in lib/$ECLIBS; do cp $f $TMP/eclib ; done
-    [ $strip -eq 1 ] && strip $TMP/eclib/*
-    dockerfile+=$'ADD eclib /usr/lib64/ceph/erasure-code\n'
-
-    CLSLIBS="libcls_*.so*"
-    mkdir -p $TMP/clslib
-    for f in lib/$CLSLIBS; do cp $f $TMP/clslib ; done
-    [ $strip -eq 1 ] && strip $TMP/clslib/*
-    dockerfile+=$'ADD clslib /usr/lib64/rados-classes\n'
-
-    # by default locally built binaries assume /usr/local
-    dockerfile+=$'RUN rm -rf /usr/local/lib64 ; ln -s /usr/lib64 /usr/local ; ln -s /usr/share/ceph /usr/local/share\n'
 fi
 
 if [ $rgw -eq 1 ] || [ $all -eq 1 ]; then
     echo "rgw"
     RGW="radosgw radosgw-admin"
-    mkdir -p $TMP/rgw
-    for f in $RGW; do cp bin/$f $TMP/rgw ; done
-    [ $strip -eq 1 ] && strip $TMP/rgw/*
-    dockerfile+=$'ADD rgw /usr/bin\n'
-
-    RGWLIBS="libradosgw.so*"
-    mkdir -p $TMP/rgwlib
-    for f in lib/$RGWLIBS; do cp $f $TMP/rgwlib ; done
-    [ $strip -eq 1 ] && strip $TMP/rgwlib/*
-    dockerfile+=$'ADD rgwlib /usr/lib64\n'
+    for f in $RGW; do cp bin/$f $TMP/bin ; done
+    [ $strip -eq 1 ] && for f in $RGW; do strip $TMP/bin/$f; done
+
+    RGWLIBS="librados.so.* libceph-common.so.*"
+    for f in $RGWLIBS; do cp lib/$f $TMP/lib ; done
+    [ $strip -eq 1 ] && for f in $RGWLIBS; do strip $TMP/lib/$f; done
 fi
 
 if [ $cephfs -eq 1 ] || [ $all -eq 1 ]; then
     echo "cephfs"
     FS="ceph-mds"
-    mkdir -p $TMP/fs
-    for f in $FS; do cp bin/$f $TMP/fs ; done
-    [ $strip -eq 1 ] && strip $TMP/fs/*
-    dockerfile+=$'ADD fs /usr/bin\n'
+    for f in $FS; do cp bin/$f $TMP/bin ; done
+    [ $strip -eq 1 ] && for f in $FS; do strip $TMP/bin/$f; done
 
     FSLIBS="libcephfs.so*"
-    mkdir -p $TMP/fslib
     for f in lib/$FSLIBS; do cp $f $TMP/fslib ; done
-    [ $strip -eq 1 ] && strip $TMP/fslib/*
-    dockerfile+=$'ADD fslib /usr/lib64\n'
+    [ $strip -eq 1 ] && for f in $FSLIBS; do strip $TMP/lib/$f; done
 fi
 
 if [ $rbd -eq 1 ] || [ $all -eq 1 ]; then
     echo "rbd"
     RBD="rbd rbd-mirror"
-    mkdir -p $TMP/rbd
-    for f in $RBD; do cp bin/$f $TMP/rbd ; done
-    [ $strip -eq 1 ] && strip $TMP/rbd/*
-    dockerfile+=$'ADD rbd /usr/bin\n'
+    for f in $RBD; do cp bin/$f $TMP/bin ; done
+    [ $strip -eq 1 ] && for f in $RBD; do strip $TMP/bin/$f; done
 
     RBDLIBS="librbd.so*"
-    mkdir -p $TMP/rbdlib
-    for f in lib/$RBDLIBS; do cp $f $TMP/rbdlib ; done
-    [ $strip -eq 1 ] && strip $TMP/rbdlib/*
-    dockerfile+=$'ADD rbdlib /usr/lib64\n'
+    for f in lib/$RBDLIBS; do cp $f $TMP/lib ; done
+    [ $strip -eq 1 ] && for f in $RBDLIBS; do strip $TMP/lib/$f; done
 fi
 
+# For every binary file that was copied to the $TMP/bin directory by the steps above, search for the existing file in the container and replace it.
+dockerfile+=$'ADD bin /tmpbin\n'
+dockerfile+=$'RUN for i in tmpbin/*; do find /usr/bin /usr/sbin -name $(basename $i) -exec mv -f $i \'{}\' \;; echo $(basename $i); done && rm -rf tmpbin\n'
+
+# For every library file that was copied to the $TMP/lib directory by the steps above, search for the existing file in the container and replace it.
+dockerfile+=$'ADD lib /tmplib\n'
+dockerfile+=$'RUN for i in tmplib/*; do find /usr/lib64 -name $(basename $i) -exec mv -f $i \'{}\' \;; echo $(basename $i); done && rm -rf tmplib\n'
+
+# by default locally built binaries assume /usr/local
+dockerfile+=$'RUN rm -rf /usr/local/lib64 ; ln -sf /usr/lib64 /usr/local ; ln -sf /usr/share/ceph /usr/local/share\n'
+# locally built binaries assume libceph-common.so.2 is in /usr/lib64 - create link to library that was just copied
+dockerfile+=$'RUN ln -sf /usr/lib64/ceph/libceph-common.so.2 /usr/lib64/libceph-common.so.2\n'
+
 echo "build"
 pushd $TMP > /dev/null
 echo "$dockerfile" > Dockerfile

src/script/cpatch.py

@@ -459,6 +459,22 @@ def build(self):
             dresult = job(component)
             dlines.extend(dresult)
 
+        if os.path.isdir(self._workdir / "tmp_bin"):
+            # For every binary file that was copied to the tmp_bin directory by the jobs above, search for the existing file in the container and replace it.
+            dlines.append("ADD tmp_bin /tmpbin")
+            dlines.append("RUN for i in tmpbin/*; do find /usr/bin /usr/sbin -name $(basename $i) -exec mv -f $i '{}' \;; echo $(basename $i); done && rm -rf tmpbin")
+
+        if os.path.isdir(self._workdir / "tmp_lib"):
+            # For every library file that was copied to the tmp_lib directory by the jobs above, search for the existing file in the container and replace it.
+            dlines.append("ADD tmp_lib /tmplib")
+            dlines.append("RUN for i in tmplib/*; do find /usr/lib64 -name $(basename $i) -exec mv -f $i '{}' \;; echo $(basename $i); done && rm -rf tmplib")
+
+        if os.path.isdir(self._workdir / "tmp_bin"):
+            # by default locally built binaries assume /usr/local
+            dlines.append("RUN rm -rf /usr/local/lib64 && ln -sf /usr/lib64 /usr/local && ln -sf /usr/share/ceph /usr/local/share")
+            # locally built binaries assume libceph-common.so.2 is in /usr/lib64 - create link to library that was just copied
+            dlines.append("RUN ln -sf /usr/lib64/ceph/libceph-common.so.2 /usr/lib64/libceph-common.so.2")
+
         with open(self._workdir / "Dockerfile", "w") as fout:
             for line in dlines:
                 print(line, file=fout)
@@ -502,7 +518,9 @@ def _copy_binary(self, src_path, dst_path):
         if self._ctx.strip_binaries:
             log.debug("copy and strip: %s", dst_path)
             shutil.copy2(src_path, dst_path)
-            _run(["strip", str(dst_path)]).check_returncode()
+            output = _run(["file", str(dst_path)], capture_output=True, text=True)
+            if "ELF" in output.stdout:
+                _run(["strip", str(dst_path)]).check_returncode()
             return
         log.debug("hard linking: %s", dst_path)
         try:
@@ -511,39 +529,29 @@ def _copy_binary(self, src_path, dst_path):
             pass
         os.link(src_path, dst_path)
 
-    def _bins_and_libs(self, prefix, bin_patterns, lib_patterns):
+    def _bins_and_libs(self, prefix, bin_patterns, lib_patterns, exclude_prefixes=[]):
         out = []
 
         bin_src = self._ctx.build_dir / "bin"
         bin_dst = self._workdir / f"{prefix}_bin"
         bin_dst.mkdir(parents=True, exist_ok=True)
         for path in bin_src.iterdir():
             if any(path.match(m) for m in bin_patterns):
+                if any(path.match(f"{m}*") for m in exclude_prefixes):
+                    continue
                 self._copy_binary(path, bin_dst / path.name)
-        out.append(f"ADD {prefix}_bin /usr/bin")
 
         lib_src = self._ctx.build_dir / "lib"
         lib_dst = self._workdir / f"{prefix}_lib"
         lib_dst.mkdir(parents=True, exist_ok=True)
         for path in lib_src.iterdir():
             if any(path.match(m) for m in lib_patterns):
+                if any(path.match(f"{m}*") for m in exclude_prefixes):
+                    continue
                 self._copy_binary(path, lib_dst / path.name)
-        out.append(f"ADD {prefix}_lib /usr/lib64")
 
         return out
 
-    def _conditional_libs(self, src_dir, name, destination, lib_patterns):
-        lib_src = self._ctx.build_dir / src_dir
-        lib_dst = self._workdir / name
-        lib_dst.mkdir(parents=True, exist_ok=True)
-        try:
-            for path in lib_src.iterdir():
-                if any(path.match(m) for m in lib_patterns):
-                    self._copy_binary(path, lib_dst / path.name)
-        except FileNotFoundError as err:
-            log.warning("skipping lib %s: %s", name, err)
-        return f"ADD {name} {destination}"
-
     def _py_site_packages(self):
         """Return the correct python site packages dir for the image."""
         if self._cached_py_site_packages is not None:
@@ -583,7 +591,7 @@ def _py_mgr_job(self, component):
             exclude_dirs = ("tests",)
         else:
             log.debug("Excluding dashboard from mgr")
-            exclude_dirs = ("tests", "node_modules")
+            exclude_dirs = ("tests", "node_modules", ".tox", ".angular" )
         exclude_file_suffixes = (".pyc", ".pyo", ".tmp", "~")
         with tarfile.open(self._workdir / name, mode="w") as tar:
             with ChangeDir(self._ctx.source_dir / "src/pybind/mgr"):
@@ -596,7 +604,7 @@ def _py_mgr_job(self, component):
 
     def _py_common_job(self, component):
         name = "python_common.tar"
-        exclude_dirs = ("tests", "node_modules")
+        exclude_dirs = ("tests", "node_modules", ".tox" )
         exclude_file_suffixes = (".pyc", ".pyo", ".tmp", "~")
         with tarfile.open(self._workdir / name, mode="w") as tar:
             with ChangeDir(self._ctx.source_dir / "src/python-common"):
@@ -637,60 +645,39 @@ def _core_job(self, component):
         # [Quoth the original script]:
         # binaries are annoying because the ceph version is embedded all over
         # the place, so we have to include everything but the kitchen sink.
-        out = []
-
-        out.extend(
-            self._bins_and_libs(
-                prefix="core",
-                bin_patterns=["ceph-mgr", "ceph-mon", "ceph-osd", "rados"],
-                lib_patterns=["libceph-common.so*", "librados.so*"],
-            )
-        )
-
-        out.append(
-            self._conditional_libs(
-                src_dir="lib",
-                name="eclib",
-                destination="/usr/lib64/ceph/erasure-code",
-                lib_patterns=["libec_*.so*"],
-            )
-        )
-        out.append(
-            self._conditional_libs(
-                src_dir="lib",
-                name="clslib",
-                destination="/usr/lib64/rados-classes",
-                lib_patterns=["libcls_*.so*"],
-            )
-        )
+        if not self._ctx.components_selected:
+            log.warning("Copying ALL locally built binaries over those that already exist in the image.")
+            bins=['*']
+        else:
+            bins=["ceph-mgr", "ceph-mon", "ceph-osd", "rados"]           
 
-        # [Quoth the original script]:
-        # by default locally built binaries assume /usr/local
-        out.append(
-            "RUN rm -rf /usr/local/lib64 && ln -s /usr/lib64 /usr/local && ln -s /usr/share/ceph /usr/local/share"
+        return self._bins_and_libs(
+            prefix="tmp",
+            bin_patterns=bins,
+            lib_patterns=["*.so","*.so.*"],
+            exclude_prefixes=["ceph_test","test_","unittest_"],
         )
-
         return out
 
     def _rgw_job(self, component):
         return self._bins_and_libs(
-            prefix="rgw",
+            prefix="tmp",
             bin_patterns=["radosgw", "radosgw-admin"],
-            lib_patterns=["libradosgw.so*"],
+            lib_patterns=["librados.so*", "libceph-common.so*"],
         )
         return out
 
     def _cephfs_job(self, component):
         return self._bins_and_libs(
-            prefix="cephfs",
+            prefix="tmp",
             bin_patterns=["ceph-mds"],
             lib_patterns=["libcephfs.so*"],
         )
         return out
 
     def _rbd_job(self, component):
         return self._bins_and_libs(
-            prefix="rbd",
+            prefix="tmp",
             bin_patterns=["rbd", "rbd-mirror"],
             lib_patterns=["librbd.so*"],
         )