Merge pull request #984 from netenglabs/develop

merge branch develop into master for 0.24.0 release

Author: LucaNicosia

.github/workflows/integration-tests.yml

@@ -20,31 +20,19 @@ jobs:
         python-version: [3.8, 3.9]
         group: [1, 2, 3, 4]
     steps:
-    - uses: actions/checkout@v2
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+    - uses: actions/checkout@v4
 
-    - name: Install Poetry
-      uses: snok/install-poetry@v1
-      with:
-        virtualenvs-create: true
-        virtualenvs-in-project: true
+    - name: Install poetry
+      run: pipx install poetry==1.*
 
-    - name: Load cached venv
-      id: cached-poetry-dependencies
-      uses: actions/cache@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
       with:
-        path: .venv
-        key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}
-
-    - name: Install dependencies
-      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      run: poetry install --no-interaction --no-root
+        python-version: ${{ matrix.python-version }}
+        cache: 'poetry'
 
-    - name: Install library
+    - name: Install dependencies and library
       run: poetry install --no-interaction
 
     - name: Set up suzieq config
@@ -57,7 +45,7 @@ jobs:
       run: mkdir test_durations
 
     - name: Get durations from cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: test_durations
         # artificial cache miss to always save at the end of the job and avoid out of sync test runs
@@ -69,7 +57,7 @@ jobs:
       run: poetry run pytest --splits 4 --group ${{ matrix.group }} --store-durations --durations-path test_durations/.test_durations_${{ matrix.python-version }}
 
     - name: Upload partial durations
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         retention-days: 1
         name: duration-chunk-${{ matrix.group }}-python-${{ matrix.python-version }}
@@ -81,10 +69,10 @@ jobs:
     needs: test
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Get durations from cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: test_durations
           # artificial cache miss to always save at the end of the job and avoid out of sync test runs
@@ -93,7 +81,7 @@ jobs:
             test-durations
 
       - name: Download artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
 
       - name: Merge test durations
         run: python3 tests/utilities/merge_test_durations_ci.py

.github/workflows/linting.yml

@@ -15,15 +15,21 @@ on:
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
-
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry==1.*
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: '3.8'
-          architecture: 'x64'
+          cache: 'poetry'
+
+      - name: Install dependencies and library
+        run: poetry install --no-interaction
 
       - name: Install pre-commit
         run: pip install pre-commit

.github/workflows/pre-commit.yml

@@ -8,3 +8,17 @@ repos:
     hooks:
       - id: end-of-file-fixer
       - id: mixed-line-ending
+  - repo: local
+    hooks:
+      - id: system
+        name: Pylint
+        entry: poetry run pylint -j 0 suzieq
+        pass_filenames: false
+        language: system
+  - repo: local
+    hooks:
+      - id: system
+        name: Flake8
+        entry: poetry run flake8 suzieq
+        pass_filenames: false
+        language: system

.pre-commit-config.yaml

@@ -22,8 +22,8 @@ RUN python3 -m pip install --no-cache-dir /tmp/suzieq-$version-py3-none-any.whl
     rm -rf /tmp/* /var/tmp/*
 
 VOLUME [ "/home/$username/parquet" ]
- 
-ENV PATH=/root/.local/bin:$PATH:/root/.local/lib/python3.7/site-packages/suzieq/cli/:/root/.local/lib/python3.7/site-packages/suzieq/poller/:/root/.local/lib/python3.7/site-packages/suzieq/restServer
+
+ENV PATH=/root/.local/bin:$PATH:/root/.local/lib/python3.9/site-packages/suzieq/cli/:/root/.local/lib/python3.9/site-packages/suzieq/poller/:/root/.local/lib/python3.9/site-packages/suzieq/restServer
 
 ENV SQENV=docker
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.14-slim-buster AS sqbase
+FROM python:3.9.21-slim-bullseye AS sqbase
 
 ENV PATH=/root/.local/bin:$PATH
 ENV PYTHONPATH=/root/.local/lib

Dockerfile-sqbase

@@ -17,7 +17,7 @@ If you answered yes to one or more of these questions, then SuzieQ is a tool tha
 
 SuzieQ does multiple things. It [collects](https://suzieq.readthedocs.io/en/latest/poller/) data from devices and systems across your network. It normalizes the data and then stores it in a vendor independent way. Then it allows analysis of that data. With the applications that we build on top of the framework we want to demonstrate a different and more systematic approach to thinking about networks. We want to show how useful it is to think of your network holistically.
 
-**An enterprise version of SuzieQ is also available**. It has been deployed in production by multiple customers, and the company behind SuzieQ, (Stardust Systems)[https://stardustsystems.net] was named a "Cool Vendor" by Gartner for making network automation easy for enterprises. 
+**An enterprise version of SuzieQ is also available**. It has been deployed in production by multiple customers, and the company behind SuzieQ, [Stardust Systems](https://stardustsystems.net) was named a "Cool Vendor" by Gartner for making network automation easy for enterprises. 
 
 ## Quick Start
 
@@ -42,7 +42,7 @@ To start collecting data for your network, create an inventory file to gather th
 
 ### Using Python Packaging
 
-If you don't want to use docker container or cannot use a docker container, an alternative approach is to install SuzieQ as a python package. It is **strongly** recommended to install suzieq inside a virtual environment. If you already use a tool to create and manage virtual environments, you can skip the step of creating a virtual envirobment below.
+If you don't want to use docker container or cannot use a docker container, an alternative approach is to install SuzieQ as a python package. It is **strongly** recommended to install suzieq inside a virtual environment. If you already use a tool to create and manage virtual environments, you can skip the step of creating a virtual environment below.
 
 SuzieQ requires python version 3.7.1 at least, and has been tested with python versions 3.7 and 3.8. It has not been tested to work on Windows. Use Linux (recommended) or macOS. To create a virtual environment, in case you haven't got a tool to create one, type:
 
@@ -86,7 +86,7 @@ The CLI supports the same kind of analysis as the explore page. ![CLI device](im
 
 ## Path
 
-SuzieQ has the ability to show the path between two IP addresses, including the ability to show the path through EVPN overlay. You can use this to see each of the paths from a source to a destination and to see if you have anything asymetrical in your paths. ![GUI PATH](images/path-gui.png)
+SuzieQ has the ability to show the path between two IP addresses, including the ability to show the path through EVPN overlay. You can use this to see each of the paths from a source to a destination and to see if you have anything asymmetrical in your paths. ![GUI PATH](images/path-gui.png)
 
 ## Asserts
 
@@ -132,4 +132,4 @@ We've also been adding screencasts on [Youtube](https://www.youtube.com/results?
 
 # SuzieQ Enterprise
 
-SuzieQ also has a commercial offering, SuzieQ Enterprise. To know more about this and contact us, please visit the Stardust Systems (website)[https://stardustsystems.net].
+SuzieQ also has a commercial offering, SuzieQ Enterprise. To know more about this and contact us, please visit the Stardust Systems [website](https://stardustsystems.net).

README.md

@@ -111,6 +111,7 @@ python-dateutil==2.8.2 ; python_full_version > "3.8.1" and python_version < "3.1
 python-dotenv==0.19.2 ; python_full_version > "3.8.1" and python_version < "3.10"
 python-nubia==0.2b5 ; python_full_version > "3.8.1" and python_version < "3.10"
 pytz==2023.3 ; python_full_version > "3.8.1" and python_version < "3.10"
+pyvmomi==8.0.2.0.1 ; python_full_version > "3.8.1" and python_version < "3.10"
 pywin32==306 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_full_version > "3.8.1" and python_version < "3.10"
 pywinpty==2.0.11 ; python_full_version > "3.8.1" and python_version < "3.10" and os_name == "nt"
 pyyaml==6.0 ; python_full_version > "3.8.1" and python_version < "3.10"
@@ -145,7 +146,7 @@ typing-extensions==4.7.1 ; python_full_version > "3.8.1" and python_version < "3
 typing-inspect==0.9.0 ; python_full_version > "3.8.1" and python_version < "3.10"
 tzlocal==2.1 ; python_full_version > "3.8.1" and python_version < "3.10"
 urllib3==1.26.16 ; python_full_version > "3.8.1" and python_version < "3.10"
-uvicorn==0.14.0 ; python_full_version > "3.8.1" and python_version < "3.10"
+uvicorn==0.15.0 ; python_full_version > "3.8.1" and python_version < "3.10"
 uvloop==0.17.0 ; python_full_version > "3.8.1" and python_version < "3.10"
 validators==0.20.0 ; python_full_version > "3.8.1" and python_version < "3.10"
 watchdog==3.0.0 ; python_full_version > "3.8.1" and python_version < "3.10" and platform_system != "Darwin"

build/requirements.txt

@@ -53,6 +53,7 @@ coalescer:
 | rest.logsize                 | maximum size of the REST logfile in bytes                                                                                                                                                                                                  | 10000000                         | no                  |
 | rest.log-stdout              | log everything on the standard output instead of a file                                                                                                                                                                                    | False                            | no                  |
 | rest.no-https                | if True, the REST server doesn't use SSL. Highly discouraged in production.                                                                                                                                                                | False                            | no                  |
+| rest.cert-verify             | if False the certificate will not be verified.<br>If True the certificate will be verified.<br>If a path, that will be checked as CA.                                                                                                                                                                                    | True                            | no                  |
 | poller.logging-level         | logging level for the poller.<br/> Choices: INFO, WARNING, ERROR                                                                                                                                                                           | WARNING                          | no                  |
 | poller.logfile               | log file for poller                                                                                                                                                                                                                        | /tmp/sq-poller.log               | no                  |
 | poller.log-stdout            | log on standard output instead of file                                                                                                                                                                                                     | False                            | no                  |

docs/config_file.md

@@ -1,5 +1,25 @@
 # Release Notes
 
+## 0.24.0 (May 7, 2025)
+
+This release adds several new capabilities and important bug fixes. Notably, it includes more flexible filtering options, improvements to REST engine parameter handling, support for certificate verification settings, and new integrations with the VMWare's vCenter and Junos IPv6 neighbor discovery. These changes are backward-compatible.
+
+* **Filter devices by address**  
+  You can now filter devices based on one or more IP addresses using the `address` argument in the CLI.
+
+* **Set certificate verification mode**  
+  Added support for the `cert-verify` parameter in configuration. You can set it to `True`, `False`, or a path to a CA certificate. This controls whether and how certificates are verified when connecting over HTTPS.
+
+* **REST engine parameter update fix**  
+  When switching to `engine='rest'`, the engine parameters (`rest-server-ip`, `rest-server-port`, `rest-api-key`, `rest-use-https`) are now correctly updated instead of keeping defaults. This avoids inconsistent behavior when switching between engines.
+
+* **Add VMWare's vCenter source for inventory**  
+  Introduced support for `vcenter` as an inventory source. This enables automatic discovery of VMs and their IPs based on custom attributes. Configuration includes support for authentication and SSL settings.  
+  Dependency added: `pyvnomi`.
+
+* **Collect IPv6 neighbor data on Junos**  
+  Extended `arpnd` data collection to include IPv6 neighbors for Junos devices. This complements the existing IPv4 support and provides a more complete view of network neighbors on Junos platforms.
+
 ## 0.23.0 (May 27, 2024)
 
 This has some useful features added, and a bunch of important bug fixes, most of them in the parsing logic, all of which are ports from the enterprise version. Some of these changes resulted in a  few breaking changes, all to fix inconsistencies in parsing output. Fixing these inconsistencies should make writing logic using this info much simpler. Existing scripts that used this logic need to be fixed.