Fixed release pipeline. Set upstream branch for test CI.

Author: thijsmie

.azure/publish-release.yml

@@ -21,24 +21,25 @@ trigger:
 
 variables:
   # ---- COMMON ----
-  CIBW_VERSION: "2.1.*"
-  CIBW_SKIP: "pp*"  # No PyPy builds
+  CIBW_VERSION: "2.3.*"
+  CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *musllinux*"  # No PyPy/32bit/musl builds
   # Run testing with pytest after each wheel build
   CIBW_TEST_REQUIRES: "pytest pytest-cov pytest-mock"
-  CIBW_TEST_COMMAND: "pytest {package}/tests"
+  CIBW_TEST_COMMAND: "pytest -s --verbose {package}/tests"
   # We can crosscompile for Apple Silicon ARM-based macs
   # But they are not actually available to run on
   # So these wheels cannot be tested in CI
   CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64"
   CIBW_BUILD_VERBOSITY: 1
+  PY_VERSION: "3.10"
 
   # ---- BUILDER ----
   CLONE: >-
-    git clone https://github.com/eclipse-cyclonedds/cyclonedds.git main
+    git clone --branch releases/0.9.x https://github.com/eclipse-cyclonedds/cyclonedds.git main
   CONFIGURE: >-
     mkdir -p build &&
     cd build &&
-    cmake ../main -DCMAKE_INSTALL_PREFIX="$CYCLONEDDS_HOME" -DBUILD_SCHEMA=off -DENABLE_SHM=off -DENABLE_SSL=on -DENABLE_SECURITY=on -DENABLE_TOPIC_DISCOVERY=on -DENABLE_TYPE_DISCOVERY=on
+    cmake ../main -DCMAKE_INSTALL_PREFIX="$CYCLONEDDS_HOME" -DENABLE_SHM=off -DENABLE_SSL=off -DENABLE_SECURITY=off -DENABLE_TOPIC_DISCOVERY=on -DENABLE_TYPE_DISCOVERY=on
   BUILD: >-
     cmake --build . --target install
   CIBW_BEFORE_ALL: >-
@@ -84,13 +85,16 @@ jobs:
       'Windows_Server_2019_64_bit':
         image: windows-2019
         CIBW_ARCHS: auto64
+        CIBW_SKIP: "pp* *-win32 cp310*"
         CIBW_BEFORE_ALL_WINDOWS_PRE: >-
          $(CLONE) && $(CONFIGURE) -A x64 && $(BUILD)
-      'Windows_Server_2019_32_bit':
+      'Windows_Server_2019_64_bit_3_10_builds':
         image: windows-2019
-        CIBW_ARCHS: auto32
+        CIBW_ARCHS: auto64
+        CIBW_BUILD: "cp310*"
+        PY_VERSION: "3.9"
         CIBW_BEFORE_ALL_WINDOWS_PRE: >-
-         $(CLONE) && $(CONFIGURE) -A Win32 && $(BUILD)
+         $(CLONE) && $(CONFIGURE) -A x64 && $(BUILD)
   steps:
   - pwsh: |
       Write-Host "###vso[task.setvariable variable=STANDALONE_WHEELS;]1"

.azure/templates/build-cyclone.yml

@@ -16,7 +16,7 @@
 
 steps:
   - bash: |
-      git clone https://github.com/eclipse-cyclonedds/cyclonedds.git cyclonedds_c
+      git clone https://github.com/eclipse-cyclonedds/cyclonedds.git cyclonedds_c --branch releases/0.9.x
     name: clone_cyclone
     displayName: Clone the CycloneDDS repository
   - bash: |
@@ -26,7 +26,7 @@ steps:
     displayName: Check the build type.
   - task: Cache@2
     inputs:
-      key: cyclonedds_build | 4 | $(Agent.OS) | $(build_type) | cyclonedds_c/.git/refs/heads/master
+      key: cyclonedds_build | 4 | $(Agent.OS) | $(build_type) | cyclonedds_c/.git/refs/heads/releases/0.9.x
       path: cyclone-$(Agent.OS)-$(build_type)
       cacheHitVar: CACHE_RESTORED
     name: cyclonedds_cache

.azure/templates/build-wheel.yml

@@ -18,6 +18,8 @@
 steps:
   - task: UsePythonVersion@0
     displayName: Fetch Python
+    inputs:
+      versionSpec: $(PY_VERSION)
   - bash: |
       python -m pip install --user --upgrade pip
       pip install --user cibuildwheel==$(CIBW_VERSION)

.azure/templates/publish-package.yml

@@ -26,7 +26,7 @@ steps:
     name: install_twine
     displayName: Install Twine
   - bash: |
-      python setup.py sdist
+      BUILDING_SDIST=1 python setup.py sdist
     name: build_cyclonedds_src_dist
     displayName: Build CycloneDDS source distribution
   - task: DownloadSecureFile@1

buildhelp/bdist_wheel.py

@@ -22,18 +22,13 @@ def run(self):
             newlibdir = Path(self.bdist_dir) / 'cyclonedds' / '.libs'
 
             os.makedirs(newlibdir, exist_ok=True)
-            with open(Path(self.bdist_dir) / 'cyclonedds' / '__library__.py', "w", encoding='utf-8') as f:
-                f.write("from pathlib import Path\n\n")
-                f.write("in_wheel = True\n")
-                f.write(f"library_path = Path(__file__).parent / '.libs' / '{cyclone.ddsc_library.name}'")
+            (Path(self.bdist_dir) / 'cyclonedds' / '__library__.py').write_text(
+                (Path(__file__).parent / "wheel_library.py").read_text()
+            )
 
             shutil.copy(cyclone.ddsc_library, newlibdir / cyclone.ddsc_library.name)
 
             if cyclone.idlc_executable and cyclone.idlc_library:
                 shutil.copy(cyclone.idlc_executable, newlibdir / cyclone.idlc_executable.name)
-                shutil.copy(cyclone.idlc_library, newlibdir / cyclone.idlc_library.name)
-
-            for lib in cyclone.security_libs:
-                shutil.copy(lib, newlibdir / lib.name)
 
         super().run()

buildhelp/wheel_library.py

@@ -0,0 +1,14 @@
+# will be written to __library__.py
+from pathlib import Path
+import platform
+
+in_wheel = True
+
+dir = Path(__file__).parent.resolve()
+
+if platform.system() == "Darwin":
+    library_path = list((dir / ".dylibs").glob("libddsc*"))[0]
+elif platform.system() == "Windows":
+    library_path = list((dir / ".." / "cyclonedds.libs").glob("ddsc*"))[0]
+else:
+    library_path = list((dir / ".." / "cyclonedds.libs").glob("libddsc*"))[0]

clayer/pysertype.c

@@ -1638,11 +1638,17 @@ ddspy_read_participant(PyObject *self, PyObject *args)
     dds_sample_info_t* info = dds_alloc(sizeof(dds_sample_info_t) * Nu32);
     struct dds_builtintopic_participant** rcontainer = dds_alloc(sizeof(struct dds_builtintopic_participant*) * Nu32);
 
+    if (!info || !rcontainer) {
+        PyErr_SetString(PyExc_Exception, "Could not allocate memory");
+        return NULL;
+    }
+
     for(uint32_t i = 0; i < Nu32; ++i) {
         rcontainer[i] = NULL;
     }
 
     sts = dds_read(reader, (void**) rcontainer, info, Nu32, Nu32);
+
     if (sts < 0) {
         return PyLong_FromLong((long) sts);
     }

cyclonedds/tools/wheel_idlc/__init__.py

@@ -11,8 +11,7 @@
 
 
 This tool is used in wheels to perform the idlc command.
-The idlc binary is under cyclonedds/.libs/idlc, libs are under
-cyclonedds/.libs.
+The idlc binary is under cyclonedds/.libs/idlc, libs are under library_path.parent
 """
 
 import os
@@ -33,11 +32,13 @@ def command():
 
     environ = os.environ.copy()
 
+    from cyclonedds.__library__ import library_path
+
     if platform.system() == "Windows":
-        environ["PATH"] = ";".join([str(libdir)] + environ.get("PATH", "").split(";"))
+        environ["PATH"] = ";".join([str(library_path.parent)] + environ.get("PATH", "").split(";"))
     elif platform.system() == "Darwin":
-        environ["DYLD_LIBRARY_PATH"] = ":".join([str(libdir)] + environ.get("DYLD_LIBRARY_PATH", "").split(":"))
+        environ["DYLD_LIBRARY_PATH"] = ":".join([str(library_path.parent)] + environ.get("DYLD_LIBRARY_PATH", "").split(":"))
     else:
-        environ["LD_LIBRARY_PATH"] = ":".join([str(libdir)] + environ.get("LD_LIBRARY_PATH", "").split(":"))
+        environ["LD_LIBRARY_PATH"] = ":".join([str(library_path.parent)] + environ.get("LD_LIBRARY_PATH", "").split(":"))
 
     os.execvpe(idlc, sys.argv[1:], environ)

setup.py

@@ -29,59 +29,62 @@
     long_description = f.read()
 
 
-cyclone = find_cyclonedds()
+if "BUILDING_SDIST" not in os.environ:
+    cyclone = find_cyclonedds()
 
-if not cyclone:
-    print("Could not locate cyclonedds. Try to set CYCLONEDDS_HOME or CMAKE_PREFIX_PATH")
-    import sys
-    sys.exit(1)
+    if not cyclone:
+        print("Could not locate cyclonedds. Try to set CYCLONEDDS_HOME or CMAKE_PREFIX_PATH")
+        import sys
+        sys.exit(1)
 
 
-with open(this_directory / 'cyclonedds' / '__library__.py', "w", encoding='utf-8') as f:
-    f.write("in_wheel = False\n")
-    f.write(f"library_path = '{cyclone.ddsc_library}'")
+    with open(this_directory / 'cyclonedds' / '__library__.py', "w", encoding='utf-8') as f:
+        f.write("in_wheel = False\n")
+        f.write(f"library_path = '{cyclone.ddsc_library}'")
 
 
-ext_modules = [
-    Extension('cyclonedds._clayer', [
-            'clayer/cdrkeyvm.c',
-            'clayer/pysertype.c',
-            'clayer/typeser.c'
-        ],
-        include_dirs=[
-            str(cyclone.include_path),
-            str(this_directory / "clayer")
-        ],
-        libraries=['ddsc'],
-        library_dirs=[
-            str(cyclone.library_path),
-            str(cyclone.binary_path),
-        ]
-    )
-]
-
-if cyclone.idlc_library:
-    ext_modules += [
-        Library('cyclonedds._idlpy', [
-                'idlpy/src/context.c',
-                'idlpy/src/generator.c',
-                'idlpy/src/naming.c',
-                'idlpy/src/ssos.c',
-                'idlpy/src/types.c',
-                'idlpy/src/util.c'
+    ext_modules = [
+        Extension('cyclonedds._clayer', [
+                'clayer/cdrkeyvm.c',
+                'clayer/pysertype.c',
+                'clayer/typeser.c'
             ],
             include_dirs=[
                 str(cyclone.include_path),
-                str(this_directory / "idlpy" / "include")
+                str(this_directory / "clayer")
             ],
-            libraries=['ddsc', 'cycloneddsidl'],
+            libraries=['ddsc'],
             library_dirs=[
                 str(cyclone.library_path),
-                str(cyclone.binary_path)
+                str(cyclone.binary_path),
             ]
         )
     ]
 
+    if cyclone.idlc_library:
+        ext_modules += [
+            Library('cyclonedds._idlpy', [
+                    'idlpy/src/context.c',
+                    'idlpy/src/generator.c',
+                    'idlpy/src/naming.c',
+                    'idlpy/src/ssos.c',
+                    'idlpy/src/types.c',
+                    'idlpy/src/util.c'
+                ],
+                include_dirs=[
+                    str(cyclone.include_path),
+                    str(this_directory / "idlpy" / "include")
+                ],
+                libraries=['ddsc', 'cycloneddsidl'],
+                library_dirs=[
+                    str(cyclone.library_path),
+                    str(cyclone.binary_path)
+                ]
+            )
+        ]
+else:
+    ext_modules=[]
+
 
 setup(
     name='cyclonedds',

tests/test_loader.py

@@ -1,9 +1,14 @@
 from cyclonedds.internal import CycloneDDSLoaderException, load_cyclonedds
-from cyclonedds.__library__ import library_path
+from cyclonedds.__library__ import library_path, in_wheel
 from pytest_mock import MockerFixture
+import pytest
 import os
 
 
+if in_wheel:
+    pytest.skip("Loader tests are not needed in wheels: loadpath is static.", allow_module_level=True)
+
+
 def gen_test_loader(loadlist):
     def load(path):
         loadlist.append(path)