Package sdist properly (#86)

* Add timestamps to test logging
* Test sdist installs and works on all platforms
* Fix sdist included files
* Separate out sdist job

Author: coretl

.github/workflows/code.yml

@@ -30,6 +30,25 @@ jobs:
       - name: Lint
         run: flake8
 
+  sdist:
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v2
+        with:
+          # require history to get back to last tag for version number of branches
+          fetch-depth: 0
+          submodules: true
+
+      - name: Build Sdist
+        run: pipx run build --sdist .
+
+      - name: Upload Sdist
+        uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/*
+
   build:
     strategy:
       fail-fast: false
@@ -50,9 +69,6 @@ jobs:
           - os: ubuntu-latest
             cov_file: /output/coverage.xml
             results_file: /output/pytest-results.xml
-          # Build an sdist on linux so it has the right line endings
-          - os: ubuntu-latest
-            sdist: true
 
     name: build/${{ matrix.os }}/${{ matrix.python }}
     runs-on: ${{ matrix.os }}
@@ -74,10 +90,6 @@ jobs:
         # Pin cibuildwheel due to https://github.com/pypa/cibuildwheel/issues/962
         run: pip install build cibuildwheel>=2.3.1
 
-      - name: Build Sdist
-        if: matrix.sdist
-        run: python -m build --sdist .
-
       - name: Build Wheel
         run: cibuildwheel --output-dir dist
         env:
@@ -86,14 +98,16 @@ jobs:
           CIBW_BUILD: ${{ matrix.python }}*64
           CIBW_TEST_EXTRAS: dev
           CIBW_TEST_COMMAND: pytest {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }}
+          # Run with faulthandler and -s in the hope we get a stack trace on seg fault on windows...
+          CIBW_TEST_COMMAND_WINDOWS: python -X faulthandler -m pytest -s {project}/tests --cov-report xml:${{ matrix.cov_file }} --junit-xml=${{ matrix.results_file }}
           # Disable auditwheel as it isn't compatible with setuptools_dso approach
           # https://github.com/mdavidsaver/setuptools_dso/issues/17
           CIBW_REPAIR_WHEEL_COMMAND: ""
           CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
           CIBW_ENVIRONMENT_LINUX: SETUPTOOLS_DSO_PLAT_NAME=manylinux2014_x86_64
           CIBW_SKIP: "*-musllinux*" # epicscorelibs doesn't build on musllinux platforms
 
-      - name: Upload Wheel and Sdist
+      - name: Upload Wheel
         uses: actions/upload-artifact@v2
         with:
           name: dist
@@ -129,8 +143,28 @@ jobs:
         with:
           files: artifacts/**/*.xml
 
+  test-sdist:
+    needs: [sdist]
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python: [cp36, cp37, cp38, cp39, cp310]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: dist
+          path: dist
+
+      - name: Install sdist in a venv and check cli works
+        run: pipx run --spec dist/*.tar.gz pythonSoftIOC --version
+        shell: bash
+
   release:
-    needs: [build]
+    needs: [build, sdist]
     runs-on: ubuntu-latest
     # upload to PyPI and make a release on every tag
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')

setup.cfg

@@ -48,6 +48,15 @@ dev =
     cothread; sys_platform != "win32"
     p4p
 
+# Include the OS specific files in the sdist, even
+# if not packages by setup.py (as we only build sdist on one arch)
+[options.data_files]
+iocStats/devIocStats = iocStats/devIocStats/*.c, iocStats/devIocStats/*.h
+iocStats/devIocStats/os/default = iocStats/devIocStats/os/default/*
+iocStats/devIocStats/os/Darwin = iocStats/devIocStats/os/Darwin/*
+iocStats/devIocStats/os/WIN32 = iocStats/devIocStats/os/WIN32/*
+iocStats/devIocStats/os/Linux = iocStats/devIocStats/os/Linux/*
+
 [flake8]
 max-line-length = 80
 extend-ignore =

setup.py

@@ -23,7 +23,6 @@
     "devIocStatsWaveform.c",
     "devIocStatsSub.c",
     "devIocStatsTest.c",
-    "devIocStats.h",
 ]
 
 devIocStats_OSD = [
@@ -39,30 +38,19 @@
     "osdSystemInfo.c",
     "osdHostInfo.c",
     "osdPIDInfo.c",
-    "devIocStatsOSD.h",
 ]
 
 devIocStats_src = os.path.join("iocStats", "devIocStats")
 devIocStats_os = os.path.join(devIocStats_src, "os", get_config_var('OS_CLASS'))
 devIocStats_default = os.path.join(devIocStats_src, "os", "default")
 
-def _add_file(f):
-    if f.endswith(".h"):
-        # Only add header files if making an sdist
-        # https://github.com/pypa/packaging-problems/issues/84#issuecomment-383718492
-        should_add = "sdist" in sys.argv
-    else:
-        should_add = True
-    if should_add:
-        sources.append(f)
-
 for f in devIocStats_OSI:
-    _add_file(os.path.join(devIocStats_src, f))
+    sources.append(os.path.join(devIocStats_src, f))
 for f in devIocStats_OSD:
     if os.path.exists(os.path.join(devIocStats_os, f)):
-        _add_file(os.path.join(devIocStats_os, f))
+        sources.append(os.path.join(devIocStats_os, f))
     else:
-        _add_file(os.path.join(devIocStats_default, f))
+        sources.append(os.path.join(devIocStats_default, f))
 
 # Extension with all our C code
 ext = Extension(

tests/conftest.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import atexit
 import os
 import random
@@ -20,7 +21,7 @@
 WAVEFORM_LENGTH = 40
 
 # Default timeout for many operations across testing
-TIMEOUT = 5  # Seconds
+TIMEOUT = 10  # Seconds
 
 # Address for multiprocessing Listener/Client pair
 ADDRESS = ("localhost", 2345)
@@ -29,6 +30,12 @@ def create_random_prefix():
     """Create 12-character random string, for generating unique Device Names"""
     return "".join(random.choice(string.ascii_uppercase) for _ in range(12))
 
+# Can't use logging as it's not multiprocess safe, and
+# alteratives are overkill
+def log(*args):
+    print(datetime.now().strftime("%H:%M:%S"), *args)
+
+
 class SubprocessIOC:
     def __init__(self, ioc_py):
         self.pv_prefix = create_random_prefix()

tests/test_record_values.py

@@ -9,6 +9,7 @@
 from conftest import (
     requires_cothread,
     WAVEFORM_LENGTH,
+    log,
     select_and_recv,
     TIMEOUT
 )
@@ -30,7 +31,6 @@
     "string and so lets test it and prove that shall we?"
 
 
-
 def record_func_names(fixture_value):
     """Provide a nice name for the record_func fixture"""
     return fixture_value.__name__
@@ -461,7 +461,7 @@ def run_test_function(
                     creation_func in [builder.WaveformOut, builder.WaveformIn]
                     and expected_value.dtype in [numpy.float64, numpy.int32]
                 ):
-                    print(
+                    log(
                         "caget cannot distinguish between a waveform with 1 "
                         "element and a scalar value, and so always returns a "
                         "scalar. Therefore we skip this check.")
@@ -760,16 +760,16 @@ def none_value_test_func(self, record_func, queue):
         builder.LoadDatabase()
         softioc.iocInit(dispatcher)
 
-        print("CHILD: Soft IOC started, about to .set(None)")
+        log("CHILD: Soft IOC started, about to .set(None)")
 
         try:
             record.set(None)
-            print("CHILD: Uh-OH! No exception thrown when setting None!")
+            log("CHILD: Uh-OH! No exception thrown when setting None!")
         except Exception as e:
-            print("CHILD: Putting exception into queue", e)
+            log("CHILD: Putting exception into queue %s", e)
             queue.put(e)
         else:
-            print("CHILD: No exception raised when using None as value!")
+            log("CHILD: No exception raised when using None as value!")
             queue.put(Exception("FAIL: No exception raised during .set()"))
 
     @requires_cothread
@@ -785,14 +785,14 @@ def test_value_none_rejected_set_after_init(self, record_func_reject_none):
 
         process.start()
 
-        print("PARENT: Child process started, waiting for returned exception")
+        log("PARENT: Child process started, waiting for returned exception")
 
         try:
             exception = queue.get(timeout=TIMEOUT)
 
             assert isinstance(exception, self.expected_exceptions)
         finally:
-            print("PARENT: Issuing terminate to child process")
+            log("PARENT: Issuing terminate to child process")
             process.terminate()
             process.join(timeout=TIMEOUT)
             if process.exitcode is None:

tests/test_records.py

@@ -4,6 +4,7 @@
 import pytest
 
 from conftest import (
+    log,
     create_random_prefix,
     requires_cothread,
     _clear_records,
@@ -376,14 +377,14 @@ def on_update_done(_):
 
         conn.send("R")  # "Ready"
 
-        print("CHILD: Sent R over Connection to Parent")
+        log("CHILD: Sent R over Connection to Parent")
 
         # Keep process alive while main thread runs CAGET
         if conn.poll(TIMEOUT):
             val = conn.recv()
             assert val == "D", "Did not receive expected Done character"
 
-        print("CHILD: Received exit command, child exiting")
+        log("CHILD: Received exit command, child exiting")
 
     def on_update_runner(self, creation_func, always_update, put_same_value):
         parent_conn, child_conn = multiprocessing.Pipe()
@@ -397,15 +398,15 @@ def on_update_runner(self, creation_func, always_update, put_same_value):
 
         process.start()
 
-        print("PARENT: Child started, waiting for R command")
+        log("PARENT: Child started, waiting for R command")
 
         from cothread.catools import caget, caput, _channel_cache
 
         try:
             # Wait for message that IOC has started
             select_and_recv(parent_conn, "R")
 
-            print("PARENT: received R command")
+            log("PARENT: received R command")
 
 
             # Suppress potential spurious warnings
@@ -416,7 +417,7 @@ def on_update_runner(self, creation_func, always_update, put_same_value):
             # value to force processing to occur
             count = 1
 
-            print("PARENT: begining While loop")
+            log("PARENT: begining While loop")
 
             while count < 4:
                 put_ret = caput(
@@ -426,26 +427,26 @@ def on_update_runner(self, creation_func, always_update, put_same_value):
                 )
                 assert put_ret.ok, f"caput did not succeed: {put_ret.errorcode}"
 
-                print(f"PARENT: completed caput with count {count}")
+                log(f"PARENT: completed caput with count {count}")
 
                 count += 1
 
-            print("PARENT: Put'ing to DONE record")
+            log("PARENT: Put'ing to DONE record")
 
             caput(
                 device_name + ":ON-UPDATE-DONE",
                 1,
                 wait=True,
             )
 
-            print("PARENT: Waiting for C command")
+            log("PARENT: Waiting for C command")
 
             # Wait for action record to process, so we know all the callbacks
             # have finished processing (This assumes record callbacks are not
             # re-ordered, and will run in the same order as the caputs we sent)
             select_and_recv(parent_conn, "C")
 
-            print("PARENT: Received C command")
+            log("PARENT: Received C command")
 
             ret_val = caget(
                 device_name + ":ON-UPDATE-COUNTER-RECORD",
@@ -454,7 +455,7 @@ def on_update_runner(self, creation_func, always_update, put_same_value):
             assert ret_val.ok, \
                 f"caget did not succeed: {ret_val.errorcode}, {ret_val}"
 
-            print(f"PARENT: Received val from COUNTER: {ret_val}")
+            log(f"PARENT: Received val from COUNTER: {ret_val}")
 
 
             # Expected value is either 3 (incremented once per caput)
@@ -469,10 +470,10 @@ def on_update_runner(self, creation_func, always_update, put_same_value):
             # Suppress potential spurious warnings
             _channel_cache.purge()
 
-            print("PARENT:Sending Done command to child")
+            log("PARENT:Sending Done command to child")
             parent_conn.send("D")  # "Done"
             process.join(timeout=TIMEOUT)
-            print(f"PARENT: Join completed with exitcode {process.exitcode}")
+            log(f"PARENT: Join completed with exitcode {process.exitcode}")
             if process.exitcode is None:
                 pytest.fail("Process did not terminate")