Merge remote-tracking branch 'origin/develop'

Author: ddutt

build/requirements.txt

@@ -1,31 +1,32 @@
 aiofiles==0.8.0; python_version >= "3.6" and python_version < "4.0"
 aiohttp==3.7.4; python_version >= "3.6"
 altair==4.2.0; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
-appnope==0.1.3; sys_platform == "darwin" and python_version >= "3.6" and platform_system == "Darwin"
-argon2-cffi-bindings==21.2.0; python_version >= "3.6"
+appnope==0.1.3; sys_platform == "darwin" and python_version >= "3.7" and platform_system == "Darwin"
+argon2-cffi-bindings==21.2.0; python_version >= "3.7"
 argon2-cffi==21.3.0; python_version >= "3.7"
 asgiref==3.5.2; python_version >= "3.7"
 async-timeout==3.0.1; python_full_version >= "3.5.3"
 asyncssh==2.11.0; python_version >= "3.6"
-attrs==21.4.0; python_full_version >= "3.7.1" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7")
-backcall==0.2.0; python_version >= "3.6"
+attrs==21.4.0; python_full_version >= "3.7.1" and python_version >= "3.7" and python_version < "4.0" and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7")
+backcall==0.2.0; python_version >= "3.7"
 beautifulsoup4==4.11.1; python_full_version >= "3.6.0" and python_version >= "3.7"
 bidict==0.22.0; python_version >= "3.7"
 bleach==5.0.0; python_version >= "3.7"
-blinker==1.4; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.6"
+blinker==1.4; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 cachetools==5.2.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.7.1"
-certifi==2022.5.18.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-cffi==1.15.0; implementation_name == "pypy" and python_version >= "3.6"
+certifi==2022.5.18.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7"
+cffi==1.15.0; implementation_name == "pypy" and python_version >= "3.7"
 chardet==3.0.4; python_version >= "3.6"
-charset-normalizer==2.0.12; python_full_version >= "3.6.0" and python_version >= "3.6"
+charset-normalizer==2.0.12; python_full_version >= "3.6.0" and python_version >= "3.7"
 ciscoconfparse==1.6.40; python_version >= "3.6"
-click==8.0.4; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.6"
+click==8.0.4; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+commonmark==0.9.1; python_full_version >= "3.6.3" and python_full_version < "4.0.0" and python_version >= "3.7"
 configargparse==0.15.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
 cryptography==37.0.2; python_version >= "3.6"
 cycler==0.11.0; python_version >= "3.7"
 dateparser==1.1.1; python_version >= "3.5"
-decorator==5.1.1; python_version >= "3.6"
+decorator==5.1.1; python_version >= "3.7"
 defusedxml==0.7.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
 dnspython==2.2.1; python_version >= "3.6" and python_version < "4.0"
 entrypoints==0.4; python_version >= "3.7"
@@ -38,7 +39,7 @@ gitdb==4.0.9; python_version >= "3.7"
 gitpython==3.1.27; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 graphviz==0.15; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
 h11==0.13.0; python_version >= "3.6"
-idna==3.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+idna==3.3; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7"
 importlib-metadata==4.11.4; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 importlib-resources==5.7.1; python_version < "3.9" and python_version >= "3.7"
 ipaddress==1.0.23
@@ -69,42 +70,43 @@ nbformat==5.4.0; python_full_version >= "3.7.0" and python_version >= "3.7"
 nest-asyncio==1.5.5; python_full_version >= "3.7.0" and python_version >= "3.7"
 netconan==0.11.3
 networkx==2.8.3; python_version >= "3.8"
-notebook==6.4.10; python_version >= "3.6"
+notebook==6.4.12; python_version >= "3.7"
 numpy==1.22.4
 packaging==21.3; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 pandas==1.3.4; python_full_version >= "3.7.1"
 pandocfilters==1.5.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7"
 parso==0.7.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 passlib==1.7.4; python_version >= "3.6"
-pexpect==4.8.0; sys_platform != "win32" and python_version >= "3.6"
-pickleshare==0.7.5; python_version >= "3.6"
+pexpect==4.8.0; sys_platform != "win32" and python_version >= "3.7"
+pickleshare==0.7.5; python_version >= "3.7"
 pillow==9.1.1; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 ply==3.11
 prettytable==3.3.0; python_version >= "3.7"
 prometheus-client==0.14.1; python_version >= "3.7"
 prompt-toolkit==3.0.29; python_full_version >= "3.6.2"
 protobuf==3.20.1; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 ptyprocess==0.7.0; os_name != "nt" and python_version >= "3.7" and sys_platform != "win32"
-py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.5.0"
-pyarrow==5.0.0; python_version >= "3.6"
+py==1.11.0; python_version >= "3.7" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.7" and python_full_version >= "3.5.0"
+pyarrow==8.0.0; python_version >= "3.7"
 pycparser==2.21; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 pydantic==1.9.1; python_full_version >= "3.6.1" and python_version >= "3.6"
 pydeck==0.7.1; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
-pygments==2.12.0; python_version >= "3.7"
-pympler==1.0.1; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.6"
+pygments==2.12.0; python_full_version >= "3.6.3" and python_full_version < "4.0.0" and python_version >= "3.7"
+pympler==1.0.1; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 pyparsing==2.4.7; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
 pyrsistent==0.18.1; python_version >= "3.7"
-python-dateutil==2.8.2; python_full_version >= "3.7.1" and python_version >= "3.7" and (python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.5") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6") and python_version < "4.0" and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7")
+python-dateutil==2.8.2; python_full_version >= "3.7.1" and python_version >= "3.7" and (python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.5") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7") and python_version < "4.0"
 python-dotenv==0.19.2; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.5"
 python-nubia==0.2b5; python_version >= "3.6"
-pytz==2022.1; python_full_version >= "3.7.1" and python_version >= "3.6"
+pytz==2022.1; python_full_version >= "3.7.1" and python_version >= "3.7"
 pywin32==304; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.7"
 pywinpty==2.0.5; os_name == "nt" and python_version >= "3.7"
 pyyaml==6.0; python_version >= "3.6"
 pyzmq==23.1.0; python_version >= "3.7"
 regex==2022.3.2; python_version >= "3.6"
 requests==2.27.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
-semver==2.13.0; python_full_version >= "3.7.1" and python_version >= "3.6" and python_version < "4.0"
+rich==12.5.1; python_full_version >= "3.7.1" and python_full_version < "4.0.0" and python_version >= "3.7" and python_version < "4.0"
+semver==2.13.0; python_full_version >= "3.7.1" and python_version >= "3.7" and python_version < "4.0"
 send2trash==1.8.0; python_version >= "3.7"
 setuptools-scm==6.4.2; python_version >= "3.7"
 simplejson==3.17.6; python_full_version >= "3.7.1" and python_version < "4.0"
@@ -113,7 +115,7 @@ smmap==5.0.0; python_version >= "3.7"
 soupsieve==2.3.2.post1; python_full_version >= "3.6.0" and python_version >= "3.7"
 starlette==0.14.2; python_version >= "3.6"
 streamlit-aggrid==0.2.3.post2; python_full_version >= "3.7.1" and python_version < "4.0"
-streamlit==1.9.2; python_version >= "3.6"
+streamlit==1.11.1; python_version >= "3.7"
 tabulate==0.8.9
 termcolor==1.1.0; python_version >= "3.6"
 terminado==0.15.0; python_version >= "3.7"
@@ -125,14 +127,14 @@ tomli==2.0.1; python_version >= "3.7"
 toolz==0.11.2; python_version >= "3.7"
 tornado==6.1; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
 traitlets==5.2.2.post1; python_full_version >= "3.7.0" and python_version >= "3.7"
-typing-extensions==4.2.0; python_full_version >= "3.7.1" and python_version >= "3.7" and python_version < "4.0"
+typing-extensions==4.2.0; python_full_version >= "3.7.1" and python_version >= "3.7" and python_full_version < "4.0.0" and python_version < "3.9"
 typing-inspect==0.7.1; python_version >= "3.6"
 tzlocal==2.1
-urllib3==1.26.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
+urllib3==1.26.9; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7"
 uvicorn==0.14.0
 uvloop==0.16.0; python_version >= "3.7"
-validators==0.20.0; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.6"
-watchdog==2.1.8; platform_system != "Darwin" and python_version >= "3.6" and python_full_version >= "3.7.1" and python_version < "4.0"
+validators==0.20.0; python_full_version >= "3.7.1" and python_version < "4.0" and python_version >= "3.7"
+watchdog==2.1.8; platform_system != "Darwin" and python_version >= "3.7" and python_full_version >= "3.7.1" and python_version < "4.0"
 wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7"
 webencodings==0.5.1; python_version >= "3.7"
 widgetsnbextension==3.6.0; python_version >= "3.7"

docs/2020-priority.md

@@ -1,6 +1,6 @@
 # Suzieq Priorities
 
-(last updated August 2021)
+(last updated June 2022)
 
 This roadmap represents our best guess at broad priorities. 
 The point is to demonstrate what we think we should work on and the general
@@ -11,18 +11,6 @@ it more useful to you, please let us know. The best way is to
 We are trying to provide a mix of adding new collection, new analysis, and making Suzieq a better
 platform to build on.
 
-So far Suzieq is focused on datacenter, but we if we hear of 
-interest in the ISP/WAN space (or any other), then we can pivot
-towards that. There's nothing inherent in Suzieq that makes it just 
-for one part of the network than the others, we just started in the 
-datacenter. The current version will be useful anywhere, it's just 
-that we aren't collecting everything we'd need to do a great job
-in other places and we don't have asserts tuned towards other use cases.
-
-First release (0.1), was focused on good fundamentals and a good 
-representation of what Suzieq can be used for. Second release (0.2)
-was focused on NXOS and Junos support.
-
 ## Areas of Development
 
 There are six major areas that Suzieq development can be broken down into.
@@ -59,30 +47,17 @@ Given the categories, here is a rough list of tasks we will be tackling. We welc
 ~~    * maybe be able to just start with one IP address and then discover 
       everything that must be polled by suzieq
 * ~~support augmenting columns (like adding peerHostname in OSPF when all we have is peerIP)~~
-* change gen_schema to ignore _ fields
 * ~~make column manipulation at higher level than engine.py~~
 * ~~Schema evolution and versioning to make suzieq less brittle to changes in the schema~~
-* Network wide summarize to take advantage of data across all commands
 ~~* Web-based GUI--
 --  * ~Do you have a framework you'd like the GUI to use?~~
---  * For what functions would you use a GUI?~~
-  * caching and performance
-~~* suzieq as a daemon--
-  * do we need suzieq as a daemon -- what are the use cases
 * ~~REST API~~
-* Create tags or other ways to group  in a hierarchical way
-  * possibly reuse ansible grouping
 * Kubernetes
   * understand topology, pod and cluster
   * calico, cilium, vxlan
   * asserts
 * Better unit tests with mocking instead of just end-to-end with real data.
-* Integration with performance analysis
-  * integration with prometheus and influxdb
-  * what do we want to be able to do with this?
 * ~~Arista EVPN~~
-* Integration with systems for notification of events
-  * slack   * ???
 * ~~Users can do their own queries~~
   * ~~pandas or pandas sql query~~
 ~~* Great expectations or some other way of better verifying data output
@@ -93,8 +68,6 @@ Given the categories, here is a rough list of tasks we will be tackling. We welc
 * ~~Support for Cisco's IOS-XR~~
 * Temperature and power collection 
 * Cloud integration such as VPC from cloud providers
-* Kafka integration for streaming telemetry
-* Redundancy -- some way of making sure that one poller is always running
 * BMP to collect BGP data for realtime BGP analysis
 * Understand BGP routing policy and route maps / etc
 * ISIS

docs/inventory.md

@@ -19,7 +19,8 @@ sources:
 - name: netbox-instance-123
   token: af8717c89ec0ff420c19d89e6c20646ad55dd54e
   url: http://127.0.0.1:8000
-  tag: suzieq-demo
+  tag:
+  - suzieq-demo
   type: netbox
   period: 3600
 
@@ -98,12 +99,14 @@ Whenever a source has many fields in common with another, you don't have to rewr
   type: netbox
   token: your-api-token-here
   url: http://127.0.0.1:8000
-  tag: suzieq-demo
+  tag:
+  - suzieq-demo
   period: 3600
 
 - name: netbox-copy     # This source will use the same set of parameters of 'netbox-orig'
   copy: netbox-orig     # and only overrides the 'tag' field.
-  tag: suzieq-copy
+  tag:
+  - suzieq-copy
 
 ```
 ### <a name='source-host-list'></a>Host list
@@ -142,9 +145,9 @@ Now you can set the path of the ansible inventory in the source:
 
 ### <a name='source-netbox'></a>Netbox
 
-[Netbox](https://netbox.readthedocs.io/en/stable/) is often used to store devices type, management address and other useful information to be used in network automation. Suzieq can pull device data from Netbox selecting them by tag (currently only one per each source).
-To do so a token to access the netbox API is required as well as the netbox instance url.
-
+[Netbox](https://netbox.readthedocs.io/en/stable/) is often used to store devices type, management address and other useful information to be used in network automation.
+Suzieq can pull device data from Netbox selecting them by one or more tags.
+To grant access to netbox, a token and an url must be provided.
 The token is considered a [sensitive data](#sensitive-data), so it can be specified via an environment variable using the format `env:ENV_TOKEN`.
 
 Since Netbox is a _dynamic source_, the data are periodically pulled, the period can be set to any desired number in seconds (default is 3600).
@@ -160,12 +163,35 @@ Here is an example of the configuration of a netbox type source:
   type: netbox
   token: your-api-token-here
   url: https://127.0.0.1:8000
-  tag: suzieq-demo    # if not present, default is "suzieq"
+  tag:                # if not present, default is "suzieq"
+  - suzieq-demo
   period: 3600        # How frequently Netbox should be polled
   ssl-verify: false   # Netbox certificate validation will be skipped
 ```
+#### Selecting devices from Netbox
+
+Starting from 0.19, it's possible to specify more than one tag to be matched, defining a list of one or more rules.
+A single rule can contain a set of tags divided by the `,` separator, which should **ALL** be matched.
+A device is polled by SuzieQ if it matches at least one of the defined rules.
+
+```yaml
+- name: netbox-multi-tag
+  type: netbox
+  token: your-api-token-here
+  url: https://127.0.0.1:8000
+  tag:
+  - alpha
+  - bravo, charlie
+```
+For example, the source above tells SuzieQ to select from Netbox all the devices having the `alpha` OR `bravo & charlie` tags.
+
+!!!Warning
+    SuzieQ versions older than 0.19 supported one single tag.
+    The old syntax, following the pattern `tag: netbox-tag`, is deprecated and it might be removed in the future releases.
+
+#### Map Netbox sitenames to namespaces
 
-Moreover, netbox type source is capable to assign each device to a namespace which corresponds to the device's sitename.
+Netbox type source is capable to assign each device to a namespace which corresponds to the device's sitename.
 To obtain this behaviour, we need to declare a `namespace` object with `name: netbox-sitename`.
 
 Here is an example:
@@ -175,13 +201,16 @@ sources:
   type: netbox
   token: your-api-token-here
   url: http://127.0.0.1:8000
-  tag: suzieq-demo
+  tag:
+  - tag1
+  - tag2, tag3
 
 - name: netbox-dc-02
   type: netbox
   token: your-api-token-here
   url: http://127.0.0.1:9000
-  tag: suzieq
+  tag:
+  - suzieq
 
 auths:
 - name: auth-st