Merge branch 'test_windows' into test_macos

Author: altendky

.github/workflows/ci.yml

@@ -20,6 +20,8 @@ jobs:
     name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
     runs-on: ${{ matrix.os.runs-on }}
     container: ${{ matrix.os.container[matrix.python.docker] }}
+    # present runtime seems to be about 1 minute 30 seconds
+    timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
@@ -38,10 +40,15 @@ jobs:
               3.7: docker://python:3.7-buster
               3.8: docker://python:3.8-buster
               3.9: docker://python:3.9-buster
-#          - name: Windows
-#            runs-on: windows-latest
-#            python_platform: win32
-#            matrix: windows
+              pypy2: docker://pypy:2-jessie
+              pypy3: docker://pypy:3-stretch
+          - name: Windows
+            runs-on: windows-latest
+            python_platform: win32
+            matrix: windows
+            openssl:
+              x86: win32
+              x64: win64
           - name: macOS
             runs-on: macos-latest
             python_platform: darwin
@@ -51,27 +58,53 @@ jobs:
             tox: py27
             action: 2.7
             docker: 2.7
+            matrix: 2.7
             implementation: cpython
+          - name: PyPy 2.7
+            tox: pypy27
+            action: pypy-2.7
+            docker: pypy2.7
+            matrix: 2.7
+            implementation: pypy
+            openssl_msvc_version: 2019
           - name: CPython 3.6
             tox: py36
             action: 3.6
             docker: 3.6
+            matrix: 3.6
             implementation: cpython
           - name: CPython 3.7
             tox: py37
             action: 3.7
             docker: 3.7
+            matrix: 3.7
             implementation: cpython
           - name: CPython 3.8
             tox: py38
             action: 3.8
             docker: 3.8
+            matrix: 3.8
             implementation: cpython
           - name: CPython 3.9
             tox: py39
             action: 3.9
             docker: 3.9
+            matrix: 3.9
             implementation: cpython
+          - name: PyPy 3.6
+            tox: pypy36
+            action: pypy-3.6
+            docker: pypy3.6
+            matrix: 3.6
+            implementation: pypy
+            openssl_msvc_version: 2019
+          - name: PyPy 3.7
+            tox: pypy37
+            action: pypy-3.7
+            docker: pypy3.7
+            matrix: 3.7
+            implementation: pypy
+            openssl_msvc_version: 2019
         arch:
           - name: x86
             action: x86
@@ -88,6 +121,12 @@ jobs:
               matrix: macos
             arch:
               matrix: x86
+          - os:
+              matrix: windows
+            python:
+              implementation: pypy
+            arch:
+              matrix: x64
     env:
       # Should match name above
       JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
@@ -106,7 +145,30 @@ jobs:
           pip install --upgrade pip setuptools wheel
           pip install --upgrade tox
       - uses: twisted/[email protected]
+      - name: Add PyPy Externals
+        if: ${{ matrix.os.matrix == 'windows' && matrix.python.implementation == 'pypy'}}
+        env:
+          PYPY_EXTERNALS_PATH: ${{ github.workspace }}/pypy_externals
+        shell: bash
+        run: |
+          echo $PYPY_EXTERNALS_PATH
+          mkdir --parents $(dirname $PYPY_EXTERNALS_PATH)
+          hg clone https://foss.heptapod.net/pypy/externals/ $PYPY_EXTERNALS_PATH
+          dir $PYPY_EXTERNALS_PATH
+          cd $PYPY_EXTERNALS_PATH && hg update win32_14x
+          echo "INCLUDE=$PYPY_EXTERNALS_PATH/include;$INCLUDE" >> $GITHUB_ENV
+          echo "LIB=$PYPY_EXTERNALS_PATH/lib;$LIB" >> $GITHUB_ENV
+#          echo "CL=${{ matrix.PYTHON.CL_FLAGS }}" >> $GITHUB_ENV
+      - name: rustup
+        if: ${{ matrix.os.matrix == 'windows' && matrix.python.implementation == 'pypy'}}
+        shell: bash
+        run: |
+          rustup target add i686-pc-windows-msvc
       - name: Test
+        env:
+          # When compiling Cryptography for PyPy on Windows there is a cleanup
+          # failure.  This is CI, it doesn't matter.
+          PIP_NO_CLEAN: 1
         run: |
           tox -vv -e ${{ matrix.python.tox }}
       - name: Coverage Processing

pymodbus/server/sync.py

@@ -327,16 +327,17 @@ def __init__(self, context, framer=None, identity=None,
         self.control = ModbusControlBlock()
         self.address = address or ("", Defaults.Port)
         self.handler = handler or ModbusConnectedRequestHandler
-        self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves',
+        self.ignore_missing_slaves = kwargs.pop('ignore_missing_slaves',
                                                 Defaults.IgnoreMissingSlaves)
-        self.broadcast_enable = kwargs.get('broadcast_enable', 
+        self.broadcast_enable = kwargs.pop('broadcast_enable',
                                            Defaults.broadcast_enable)
 
         if isinstance(identity, ModbusDeviceIdentification):
             self.control.Identity.update(identity)
 
         socketserver.ThreadingTCPServer.__init__(self, self.address,
-                                                 self.handler)
+                                                 self.handler,
+                                                 **kwargs)
 
     def process_request(self, request, client):
         """ Callback for connecting a new client thread
@@ -456,16 +457,16 @@ def __init__(self, context, framer=None, identity=None, address=None,
         self.control = ModbusControlBlock()
         self.address = address or ("", Defaults.Port)
         self.handler = handler or ModbusDisconnectedRequestHandler
-        self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves',
+        self.ignore_missing_slaves = kwargs.pop('ignore_missing_slaves',
                                                 Defaults.IgnoreMissingSlaves)
-        self.broadcast_enable = kwargs.get('broadcast_enable', 
+        self.broadcast_enable = kwargs.pop('broadcast_enable',
                                            Defaults.broadcast_enable)
 
         if isinstance(identity, ModbusDeviceIdentification):
             self.control.Identity.update(identity)
 
         socketserver.ThreadingUDPServer.__init__(self,
-            self.address, self.handler)
+            self.address, self.handler, **kwargs)
         # self._BaseServer__shutdown_request = True
 
     def process_request(self, request, client):

requirements-tests.txt

@@ -14,6 +14,11 @@ sqlalchemy>=1.1.15
 #wsgiref>=0.1.2
 verboselogs >= 1.5
 tornado==4.5.3
-Twisted[serial]>=20.3.0
+# using platform_python_implementation rather than
+# implementation_name for Python 2 support
+Twisted[conch,serial]>=20.3.0; platform_python_implementation != "PyPy" or sys_platform != "win32"
+# pywin32 isn't supported on pypy
+# https://github.com/mhammond/pywin32/issues/1289
+Twisted[conch]>=20.3.0; platform_python_implementation == "PyPy" and sys_platform == "win32"
 zope.interface>=4.4.0
 asynctest>=0.10.0

setup.py

@@ -97,8 +97,12 @@
                       'sphinx_rtd_theme',
                       'humanfriendly'],
         'twisted': [
-            'twisted[serial] >= 20.3.0',
-            'pyasn1 >= 0.1.4',
+            # using platform_python_implementation rather than
+            # implementation_name for Python 2 support
+            'Twisted[conch,serial]>=20.3.0; platform_python_implementation != "PyPy" or sys_platform != "win32"',
+            # pywin32 isn't supported on pypy
+            # https://github.com/mhammond/pywin32/issues/1289
+            'Twisted[conch]>=20.3.0; platform_python_implementation == "PyPy" and sys_platform == "win32"',
         ],
         'tornado': [
             'tornado == 4.5.3'

test/test_client_async.py

@@ -32,13 +32,18 @@
 import ssl
 
 IS_DARWIN = platform.system().lower() == "darwin"
+IS_WINDOWS = platform.system().lower() == "windows"
 OSX_SIERRA = LooseVersion("10.12")
 if IS_DARWIN:
     IS_HIGH_SIERRA_OR_ABOVE = LooseVersion(platform.mac_ver()[0])
     SERIAL_PORT = '/dev/ttyp0' if not IS_HIGH_SIERRA_OR_ABOVE else '/dev/ptyp0'
 else:
     IS_HIGH_SIERRA_OR_ABOVE = False
-    SERIAL_PORT = "/dev/ptmx"
+    if IS_WINDOWS:
+        # the use is mocked out
+        SERIAL_PORT = ""
+    else:
+        SERIAL_PORT = "/dev/ptmx"
 
 # ---------------------------------------------------------------------------#
 # Fixture
@@ -186,6 +191,10 @@ def testUdpAsycioClient(self, mock_gather, mock_event_loop):
     # Test Serial client
     # -----------------------------------------------------------------------#
 
+    @pytest.mark.skipif(
+        sys.platform == 'win32' and platform.python_implementation() == 'PyPy',
+        reason='Twisted serial requires pywin32 which is not compatible with PyPy',
+    )
     @pytest.mark.parametrize("method, framer", [("rtu", ModbusRtuFramer),
                                                 ("socket", ModbusSocketFramer),
                                                 ("binary", ModbusBinaryFramer),
@@ -196,30 +205,30 @@ def testSerialTwistedClient(self, method, framer):
         with patch("serial.Serial") as mock_sp:
             from twisted.internet import reactor
             from twisted.internet.serialport import SerialPort
+            with maybe_manage(sys.platform == 'win32', patch.object(SerialPort, "_finishPortSetup")):
+                with patch('twisted.internet.reactor') as mock_reactor:
 
-            with patch('twisted.internet.reactor') as mock_reactor:
-
-                protocol, client = AsyncModbusSerialClient(schedulers.REACTOR,
-                                                           method=method,
-                                                           port=SERIAL_PORT,
-                                                           proto_cls=ModbusSerClientProtocol)
+                    protocol, client = AsyncModbusSerialClient(schedulers.REACTOR,
+                                                               method=method,
+                                                               port=SERIAL_PORT,
+                                                               proto_cls=ModbusSerClientProtocol)
 
-                assert (isinstance(client, SerialPort))
-                assert (isinstance(client.protocol, ModbusSerClientProtocol))
-                assert (0 == len(list(client.protocol.transaction)))
-                assert (isinstance(client.protocol.framer, framer))
-                assert (client.protocol._connected)
+                    assert (isinstance(client, SerialPort))
+                    assert (isinstance(client.protocol, ModbusSerClientProtocol))
+                    assert (0 == len(list(client.protocol.transaction)))
+                    assert (isinstance(client.protocol.framer, framer))
+                    assert (client.protocol._connected)
 
-                def handle_failure(failure):
-                    assert (isinstance(failure.exception(), ConnectionException))
+                    def handle_failure(failure):
+                        assert (isinstance(failure.exception(), ConnectionException))
 
-                d = client.protocol._buildResponse(0x00)
-                d.addCallback(handle_failure)
+                    d = client.protocol._buildResponse(0x00)
+                    d.addCallback(handle_failure)
 
-                assert (client.protocol._connected)
-                client.protocol.close()
-                protocol.stop()
-                assert (not client.protocol._connected)
+                    assert (client.protocol._connected)
+                    client.protocol.close()
+                    protocol.stop()
+                    assert (not client.protocol._connected)
 
     @pytest.mark.parametrize("method, framer", [("rtu", ModbusRtuFramer),
                                         ("socket", ModbusSocketFramer),
@@ -228,7 +237,7 @@ def handle_failure(failure):
     def testSerialTornadoClient(self, method, framer):
         """ Test the serial tornado client client initialize """
         from serial import Serial
-        with maybe_manage(sys.platform == 'darwin', patch.object(Serial, "open")):
+        with maybe_manage(sys.platform in ('darwin', 'win32'), patch.object(Serial, "open")):
             protocol, future = AsyncModbusSerialClient(schedulers.IO_LOOP, method=method, port=SERIAL_PORT)
             client = future.result()
             assert(isinstance(client, AsyncTornadoModbusSerialClient))

test/test_client_sync.py

@@ -11,6 +11,9 @@
 import socket
 import serial
 import ssl
+import sys
+
+import pytest
 
 from pymodbus.client.sync import ModbusTcpClient, ModbusUdpClient
 from pymodbus.client.sync import ModbusSerialClient, BaseModbusClient
@@ -47,6 +50,15 @@ def setblocking(self, flag): return None
     def in_waiting(self): return None
 
 
+inet_pton_skipif = pytest.mark.skipif(
+    sys.platform == "win32" and sys.version_info < (3, 4),
+    reason=(
+        "Uses socket.inet_pton() which wasn't available on Windows until"
+        " 3.4.",
+    )
+)
+
+
 
 # ---------------------------------------------------------------------------#
 # Fixture
@@ -128,13 +140,15 @@ def testBasicSyncUdpClient(self):
 
         self.assertEqual("ModbusUdpClient(127.0.0.1:502)", str(client))
 
+    @inet_pton_skipif
     def testUdpClientAddressFamily(self):
         ''' Test the Udp client get address family method'''
         client = ModbusUdpClient()
         self.assertEqual(socket.AF_INET,
                          client._get_address_family('127.0.0.1'))
         self.assertEqual(socket.AF_INET6, client._get_address_family('::1'))
 
+    @inet_pton_skipif
     def testUdpClientConnect(self):
         ''' Test the Udp client connection method'''
         with patch.object(socket, 'socket') as mock_method:
@@ -151,6 +165,7 @@ def settimeout(self, *a, **kwa):
             client = ModbusUdpClient()
             self.assertFalse(client.connect())
 
+    @inet_pton_skipif
     def testUdpClientIsSocketOpen(self):
         ''' Test the udp client is_socket_open method'''
         client = ModbusUdpClient()

test/test_server_async.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from pymodbus.compat import IS_PYTHON3
 import unittest
+import pytest
 if IS_PYTHON3: # Python 3
     from unittest.mock import patch, Mock, MagicMock
 else: # Python 2
@@ -32,6 +33,11 @@
     IS_HIGH_SIERRA_OR_ABOVE = False
     SERIAL_PORT = "/dev/ptmx"
 
+no_twisted_serial_on_windows_with_pypy = pytest.mark.skipif(
+    sys.platform == 'win32' and platform.python_implementation() == 'PyPy',
+    reason='Twisted serial requires pywin32 which is not compatible with PyPy',
+)
+
 
 class AsynchronousServerTest(unittest.TestCase):
     '''
@@ -188,13 +194,15 @@ def testUdpServerStartup(self):
             self.assertEqual(mock_reactor.listenUDP.call_count, 1)
             self.assertEqual(mock_reactor.run.call_count, 1)
 
+    @no_twisted_serial_on_windows_with_pypy
     @patch("twisted.internet.serialport.SerialPort")
     def testSerialServerStartup(self, mock_sp):
         ''' Test that the modbus serial asynchronous server starts correctly '''
         with patch('twisted.internet.reactor') as mock_reactor:
             StartSerialServer(context=None, port=SERIAL_PORT)
             self.assertEqual(mock_reactor.run.call_count, 1)
 
+    @no_twisted_serial_on_windows_with_pypy
     @patch("twisted.internet.serialport.SerialPort")
     def testStopServerFromMainThread(self, mock_sp):
         """
@@ -207,6 +215,7 @@ def testStopServerFromMainThread(self, mock_sp):
             StopServer()
             self.assertEqual(mock_reactor.stop.call_count, 1)
 
+    @no_twisted_serial_on_windows_with_pypy
     @patch("twisted.internet.serialport.SerialPort")
     def testStopServerFromThread(self, mock_sp):
         """

test/test_server_asyncio.py

@@ -216,7 +216,9 @@ def connection_made(self, transport):
 
         transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1', port=random_port)
         yield from step1
-        # await asyncio.sleep(1)
+        # On Windows we seem to need to give this an extra chance to finish,
+        # otherwise there ends up being an active connection at the assert.
+        yield from asyncio.sleep(0.0)
         self.assertTrue(len(server.active_connections) == 1)
 
         protocol.transport.close()  # close isn't synchronous and there's no notification that it's done