Corrected timestamps and reverted float casting. #294 #292 (#295)

* Pushed all times to UTC.  added debug flag. #292

* Additional corrections to timestamps.

- Corrected issue with comparing naive timestamps with aware timestamps.
- Added mapping DB cleanup to sync

* Reverting casting float to string #294

* Updated testing suite

Author: SteveMcGrath

pyproject.toml

@@ -55,3 +55,25 @@ readme = {file = ["README.md"], content-type = "text/markdown"}
 [tool.setuptools.packages.find]
 include         = ["tenb2jira*"]
 
+[tool.ruff]
+target-version = "py312"
+exclude = [
+  ".nova",
+  ".github",
+  ".git",
+  ".pytest_cache",
+  "__pycache__"
+]
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "B"]
+fixable = [ "ALL" ]
+unfixable = [ "B" ]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["E402"]
+"**/{tests,docs,tools}/*" = ["E402"]
+
+[tool.flake8]
+max-line-length = 88
+count = true

tenb2jira/cli.py

@@ -129,13 +129,26 @@ def sync(configfile: Path,
          verbose: bool = False,
          cleanup: bool = True,
          ignore_last_run: bool = False,
+         debug: bool = False,
          ):
     """
     Perform the sync between Tenable & Jira
     """
-    setup_logging(verbose)
     with configfile.open('r', encoding='utf-8') as fobj:
         config = tomlkit.load(fobj)
+
+    if debug:
+        verbose = True
+        cleanup = False
+        config['jira']['max_workers'] = 1
+
+    setup_logging(verbose)
+
+    dbfile = Path(config['mapping_database']['path'])
+    if dbfile.exists():
+        console.print('WARNING :: Mapping Cache discovered.  We will be removing it.')
+        dbfile.unlink()
+
     processor = Processor(config, ignore_last_run=ignore_last_run)
     console.print(Columns([tenable_table(config),
                            jira_table(config)

tenb2jira/jira/field.py

@@ -59,12 +59,15 @@ def __repr__(self):
 
     @property
     def attr(self):
-        if self.attribute:
-            return self.attribute
-        if self.static_value:
-            return self.attribute
+        """
+        Return the appropriate value (either platform_id, static_value, or
+        attribute) depending on how the field was configured.
+        """
         if self.platform_id:
             return self.platform_id
+        if self.static_value:
+            return self.static_value
+        return self.attribute
 
     def fetch_field_id(self, api) -> bool:
         """
@@ -155,7 +158,7 @@ def parse_value(self, finding: dict) -> Any:
 
             # float values should always be returned as a float.
             case 'float':
-                return str(float(value))
+                return float(value)
 
             # datetime values should be returned in a specific format.  Here
             # we attempt to normalize both timestamp and ISO formatted values

tenb2jira/jira/jira.py

@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Dict
 import time
 import logging
 from box import Box
@@ -24,12 +24,18 @@ class Jira:
     project: dict
 
     @property
-    def field_by_name_map(self):
-        return {f.name:f for f in self.fields}
+    def field_by_name_map(self) -> Dict[str, Field]:
+        """
+        Returns the fields in a dictionary with the field name as the key
+        """
+        return {f.name: f for f in self.fields}
 
     @property
-    def field_by_id_map(self):
-        return {f.id:f for f in self.fields}
+    def field_by_id_map(self) -> Dict[str, Field]:
+        """
+        Returns the fields in a dictionary with the field id as the key
+        """
+        return {f.id: f for f in self.fields}
 
     def __init__(self, config: dict):
         self.config = config

tenb2jira/jira/task.py

@@ -25,7 +25,7 @@ def __init__(self, issue_def: "Task", is_open: bool = True):
         self.is_open = is_open
 
     def __repr__(self):
-        return f'Task("{self.jql}", {len(self.fields)})'
+        return f'Task("{self.jql_stmt}", {len(self.fields)})'
 
     @property
     def jql_stmt(self):

tenb2jira/processor.py

@@ -1,3 +1,4 @@
+from typing import Optional
 import logging
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
@@ -29,8 +30,13 @@ class Processor:
     plugin_id: str
     closed_map: list[str]
 
-    def __init__(self, config: dict, ignore_last_run: bool = False):
-        dburi = f'sqlite:///{config["mapping_database"].get("path")}'
+    def __init__(self,
+                 config: dict,
+                 ignore_last_run: bool = False,
+                 dburi: Optional[str] = None,
+                 ):
+        if not dburi:
+            dburi = f'sqlite:///{config["mapping_database"].get("path")}'
 
         # For situations where we will need to ignore the last_run variable,
         # This will pull it from the config, forcing the integration to use
@@ -124,7 +130,7 @@ def build_mapping_db_model(self,
                             value = value[0]
                         item[fields[key]] = value
                     # item = {fields[k]: v for k, v in issue.fields.items()}
-                    item['updated'] = self.start_time
+                    item['updated'] = self.start_time.datetime
                     item['jira_id'] = issue.key
                     if not missing:
                         log.debug(f'Adding {issue.key} to cache.')
@@ -185,11 +191,11 @@ def upsert_task(self, s: Session, finding: dict) -> (int | None):
         # and return the jira issue id back to the caller.
         if sql:
             if finding.get('integration_pid_updated') > self.last_run:
-                if sql.updated <= self.start_time:
+                if arrow.get(sql.updated, 'UTC') <= self.start_time:
                     self.jira.api.issues.update(sql.jira_id,
                                                 fields=task.fields,
                                                 )
-                sql.updated = datetime.now()
+                sql.updated = datetime.utcnow()
                 s.commit()
                 log.info(f'Matched Task "{sql.jira_id}" to '
                          'SQL Cache and updated.')
@@ -213,7 +219,7 @@ def upsert_task(self, s: Session, finding: dict) -> (int | None):
         if len(page.issues) == 1:
             sql = TaskMap(plugin_id=task.fields[self.plugin_id],
                           jira_id=page.issues[0].key,
-                          updated=datetime.now(),
+                          updated=datetime.utcnow(),
                           )
             s.add(sql)
             s.commit()
@@ -232,7 +238,7 @@ def upsert_task(self, s: Session, finding: dict) -> (int | None):
             resp = self.jira.api.issues.create(fields=task.fields)
             sql = TaskMap(plugin_id=task.fields[self.plugin_id],
                           jira_id=resp.key,
-                          updated=datetime.now()
+                          updated=datetime.utcnow()
                           )
             s.add(sql)
             s.commit()
@@ -271,7 +277,7 @@ def upsert_subtask(self,
         if sql:
             if not task.is_open:
                 sql.is_open = task.is_open
-                sql.updated = datetime.now()
+                sql.updated = datetime.utcnow()
                 s.commit()
                 self.close_task(sql.jira_id)
                 action = 'closed subtask'
@@ -313,7 +319,7 @@ def upsert_subtask(self,
                                  finding_id=task.fields[self.finding_id],
                                  jira_id=page.issues[0].key,
                                  is_open=task.is_open,
-                                 updated=datetime.now(),
+                                 updated=datetime.utcnow(),
                                  )
                 s.add(sql)
                 s.commit()
@@ -341,7 +347,7 @@ def upsert_subtask(self,
                                      finding_id=task.fields[self.finding_id],
                                      jira_id=resp.key,
                                      is_open=task.is_open,
-                                     updated=datetime.now(),
+                                     updated=datetime.utcnow(),
                                      )
                     s.add(sql)
                     s.commit()
@@ -419,11 +425,10 @@ def sync(self, cleanup: bool = True):
         """
         Tenable to Jira Synchronization method.
         """
-        self.start_time = datetime.now()
-        ts = int(arrow.get(self.start_time).timestamp())
+        self.start_time = arrow.utcnow()
 
         # Get the findings and the asset cleanup generators.
-        findings = self.tenable.get_generator()
+        findings = self.tenable.get_generator(self.start_time)
         asset_cleanup = self.tenable.get_asset_cleanup()
 
         # build the db cache
@@ -469,8 +474,8 @@ def sync(self, cleanup: bool = True):
         self.close_empty_tasks()
 
         # update the last_run timestamp with the time that we started the sync.
-        self.config['tenable']['last_run'] = ts
-        self.finished_time = datetime.now()
+        self.config['tenable']['last_run'] = int(self.start_time.timestamp())
+        self.finished_time = arrow.utcnow()
 
         self.engine.dispose()
         # Delete the mapping database.

tenb2jira/tenable/tenable.py

@@ -9,8 +9,8 @@
 
 
 class Tenable:
-    tvm: (TenableIO | None) = None
-    tsc: (TenableSC | None) = None
+    tvm: TenableIO
+    tsc: TenableSC
     config: dict
     platform: str
     timestamp: int
@@ -82,11 +82,10 @@ def get_tvm_generator(self) -> Generator[Any, Any, Any]:
                                close_accepted=self.close_accepted,
                                )
 
-    def get_tsc_generator(self) -> Generator[Any, Any, Any]:
+    def get_tsc_generator(self, start_time: int) -> Generator[Any, Any, Any]:
         """
         Queries the Analysis API and returns the TSC Generator.
         """
-        self.last_run = int(arrow.now().timestamp())
 
         # The severity map to link the string severities to the integer values
         # that TSC expects.
@@ -99,7 +98,7 @@ def get_tsc_generator(self) -> Generator[Any, Any, Any]:
         }
 
         # Construct the TSC timestamp offsets.
-        tsc_ts = f'{self.timestamp}-{self.last_run}'
+        tsc_ts = f'{self.timestamp}-{start_time}'
 
         # The base parameters to pass to the API.
         params = {
@@ -136,14 +135,15 @@ def get_tsc_generator(self) -> Generator[Any, Any, Any]:
                                close_accepted=self.close_accepted,
                                )
 
-    def get_generator(self) -> (Generator[Any, Any, Any] | None):
+    def get_generator(self,
+                      start_time: arrow.Arrow
+                      ) -> Generator[Any, Any, Any]:
         """
         Retreives the appropriate generator based on the configured platform.
         """
         if self.platform == 'tvm':
             return self.get_tvm_generator()
-        if self.platform == 'tsc':
-            return self.get_tsc_generator()
+        return self.get_tsc_generator(int(start_time.timestamp()))
 
     def get_asset_cleanup(self) -> (Generator[Any, Any, Any] | list):
         if self.platform == 'tvm':
@@ -154,5 +154,4 @@ def get_asset_cleanup(self) -> (Generator[Any, Any, Any] | list):
                                               chunk_size=self.chunk_size
                                               )
             return tvm_asset_cleanup(dassets, tassets)
-        else:
-            return []
+        return []

tenb2jira/version.py

@@ -1 +1 @@
-version = '2.0.9'
+version = '2.0.10'

tests/jira/test_field.py

@@ -18,6 +18,43 @@ def field_config():
     }
 
 
+def test_field_repr(field_config):
+    f = Field(config=field_config,
+              platform='tvm',
+              platform_map={'tvm': 'Test Platform'}
+              )
+    assert repr(f) == 'Field(1: Test Field)'
+
+
+def test_field_attr(field_config):
+    f = Field(config=field_config,
+              platform='tvm',
+              platform_map={'tvm': 'Test Platform'}
+              )
+    assert f.attr == 'test'
+
+    field_config.pop('attr', None)
+    field_config['platform_id'] = True
+    f = Field(config=field_config,
+              platform='tvm',
+              platform_map={'tvm': 'Test Platform'}
+              )
+    assert f.attribute == None
+    assert f.platform_id == 'Test Platform'
+    assert f.attr == 'Test Platform'
+
+    field_config.pop('platform_id', None)
+    field_config['static_value'] = 'static'
+    f = Field(config=field_config,
+              platform='tvm',
+              platform_map={'tvm': 'Test Platform'}
+              )
+    assert f.attribute == None
+    assert f.platform_id == None
+    assert f.static_value == 'static'
+    assert f.attr == 'static'
+
+
 def test_field_noapi(field_config):
     f = Field(config=field_config,
               platform='tvm',
@@ -130,9 +167,9 @@ def test_field_parse_value_float(field_config):
               platform_map={'tvm': 'Test Platform'}
               )
     f.type = 'float'
-    assert f.parse_value({'test': 1}) == '1.0'
-    assert f.parse_value({'test': 1.0}) == '1.0'
-    assert f.parse_value({'test': '1'}) == '1.0'
+    assert f.parse_value({'test': 1}) == 1.0
+    assert f.parse_value({'test': 1.0}) == 1.0
+    assert f.parse_value({'test': '1'}) == 1.0
 
 
 def test_field_parse_value_datetime(field_config):