flaky, flaky

Author: rpiazza

stix2/datastore/relational_db/database_backends/database_backend_base.py

@@ -1,7 +1,8 @@
 from typing import Any
 
 from sqlalchemy import (
-    Boolean, CheckConstraint, Float, Integer, Sequence, String, Text, create_engine,
+    Boolean, CheckConstraint, Float, Integer, Sequence, String, Text,
+    create_engine,
 )
 from sqlalchemy_utils import create_database, database_exists, drop_database
 

stix2/datastore/relational_db/database_backends/sqlite_backend.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-from sqlalchemy import Table, Column, Text, event, insert, select, update
+from sqlalchemy import Column, Table, Text, event, insert, select, update
 from sqlalchemy.schema import CreateTable
 
 from .database_backend_base import DatabaseBackend
@@ -86,12 +86,13 @@ def next_id(self, data_sink):
 
     def create_sequence(self, metadata):
         # need id column, so update has something to work with (see above)
-        t = Table("my_general_seq",
-                  metadata,
-                  Column("id", SQLiteBackend.determine_sql_type_for_key_as_int()),
-                  Column("value", SQLiteBackend.determine_sql_type_for_integer_property()),
-                  schema=self.schema_for_core())
+        t = Table(
+            "my_general_seq",
+            metadata,
+            Column("id", SQLiteBackend.determine_sql_type_for_key_as_int()),
+            Column("value", SQLiteBackend.determine_sql_type_for_integer_property()),
+            schema=self.schema_for_core(),
+        )
         CreateTable(t).compile(self.database_connection)
         self.select_stmt_for_sequence = select(t.c.value)
         return t
-

stix2/datastore/relational_db/demo.py

@@ -1,11 +1,8 @@
 import json
 import sys
 
-from database_backends.postgres_backend import PostgresBackend   # noqa F401
+from database_backends.postgres_backend import PostgresBackend  # noqa F401
 from database_backends.sqlite_backend import SQLiteBackend  # noqa F401
-
-
-from database_backends.postgres_backend import PostgresBackend
 # needed so the relational db code knows to create tables for this
 from identity_contact_information import \
     identity_contact_information  # noqa F401
@@ -16,11 +13,6 @@
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
-# needed so the relational db code knows to create tables for this
-from incident import incident, event, task, impact
-from identity_contact_information import identity_contact_information
-from observed_string import observed_string
-
 
 def main():
     with open(sys.argv[1], "r") as f:

stix2/datastore/relational_db/input_creation.py

@@ -2,7 +2,8 @@
 
 from stix2.datastore.relational_db.add_method import add_method
 from stix2.datastore.relational_db.utils import (
-    SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name, shorten_extension_definition_id
+    SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
+    shorten_extension_definition_id,
 )
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
@@ -132,8 +133,6 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
         bindings["name"] = name
         bindings[value_binding] = data_sink.db_backend.process_value_for_insert(stix_type, value)
 
-
-
         insert_statements.append(insert(table).values(bindings))
 
     insert_statements.extend(child_table_inserts)

stix2/datastore/relational_db/query.py

@@ -6,8 +6,7 @@
 import stix2
 from stix2.datastore import DataSourceError
 from stix2.datastore.relational_db.utils import (
-    canonicalize_table_name, schema_for, table_name_for,
-    see_through_workbench,
+    canonicalize_table_name, see_through_workbench, table_name_for,
 )
 import stix2.properties
 import stix2.utils
@@ -50,13 +49,13 @@ def _tables_for(stix_class, metadata, db_backend):
     # Some fixed info about core tables
     if stix2.utils.is_sco(stix_class._type, stix2.DEFAULT_VERSION):
         canon_core_table_name = canonicalize_table_name(
-            "core_sco", db_backend.schema_for_core()
+            "core_sco", db_backend.schema_for_core(),
         )
 
     else:
         # for SROs and SMOs too?
         canon_core_table_name = canonicalize_table_name(
-            "core_sdo", db_backend.schema_for_core()
+            "core_sdo", db_backend.schema_for_core(),
         )
 
     core_table = metadata.tables[canon_core_table_name]
@@ -160,13 +159,13 @@ def _read_external_references(stix_id, metadata, conn, db_backend):
     ext_refs_table = metadata.tables[
         canonicalize_table_name(
             "external_references",
-            db_backend.schema_for_core()
+            db_backend.schema_for_core(),
         )
     ]
     ext_refs_hashes_table = metadata.tables[
         canonicalize_table_name(
             "external_references_hashes",
-            db_backend.schema_for_core()
+            db_backend.schema_for_core(),
         )
     ]
     ext_refs = []
@@ -206,7 +205,7 @@ def _read_object_marking_refs(stix_id, common_table_kind, metadata, conn, db_bac
 
     marking_table_name = canonicalize_table_name(
         "object_marking_refs_" + common_table_kind,
-        db_backend.schema_for_core()
+        db_backend.schema_for_core(),
     )
 
     # The SCO/SDO object_marking_refs tables are mostly identical; they just
@@ -237,7 +236,7 @@ def _read_granular_markings(stix_id, common_table_kind, metadata, conn, db_backe
 
     marking_table_name = canonicalize_table_name(
         "granular_marking_" + common_table_kind,
-        db_backend.schema_for_core()
+        db_backend.schema_for_core(),
     )
     marking_table = metadata.tables[marking_table_name]
 
@@ -310,7 +309,7 @@ def _read_dictionary_property(
     prop_instance,
     metadata,
     conn,
-    db_backend
+    db_backend,
 ):
     """
     Read a dictionary from a table.
@@ -350,11 +349,11 @@ def _read_dictionary_property(
                 list_table_name = f"{dict_table_name}_values"
                 list_table = metadata.tables[list_table_name]
                 stmt = sa.select(
-                    dict_table.c.name, list_table.c.value
+                    dict_table.c.name, list_table.c.value,
                 ).select_from(dict_table).join(
-                    list_table, list_table.c.id == dict_table.c["values"]
+                    list_table, list_table.c.id == dict_table.c["values"],
                 ).where(
-                    dict_table.c.id == stix_id
+                    dict_table.c.id == stix_id,
                 )
 
                 results = conn.execute(stmt)
@@ -509,7 +508,7 @@ def _read_complex_property_value(
     obj_table,
     metadata,
     conn,
-    db_backend
+    db_backend,
 ):
     """
     Read property values which require auxiliary tables to store.  These are
@@ -605,7 +604,7 @@ def _read_complex_property_value(
             prop_instance,
             metadata,
             conn,
-            db_backend
+            db_backend,
         )
 
     elif isinstance(prop_instance, stix2.properties.EmbeddedObjectProperty):
@@ -663,7 +662,7 @@ def _read_complex_top_level_property_value(
             stix_id,
             metadata,
             conn,
-            db_backend
+            db_backend,
         )
 
     elif prop_name == "object_marking_refs":
@@ -689,7 +688,7 @@ def _read_complex_top_level_property_value(
         label_table = metadata.tables[
             canonicalize_table_name(
                 f"core_{common_table_kind}_labels",
-                db_backend.schema_for_core()
+                db_backend.schema_for_core(),
             )
         ]
         prop_value = _read_simple_array(stix_id, "label", label_table, conn)
@@ -703,7 +702,7 @@ def _read_complex_top_level_property_value(
             type_table,
             metadata,
             conn,
-            db_backend
+            db_backend,
         )
 
     return prop_value
@@ -761,7 +760,7 @@ def read_object(stix_id, metadata, conn, db_backend):
     stix_obj = stix2.parse(
         obj_dict,
         allow_custom=True,
-        version=stix2.DEFAULT_VERSION
+        version=stix2.DEFAULT_VERSION,
     )
 
     return stix_obj

stix2/datastore/relational_db/relational_db_testing.py

@@ -3,7 +3,7 @@
 
 from database_backends.mariadb_backend import MariaDBBackend  # noqa: F401
 from database_backends.postgres_backend import PostgresBackend  # noqa: F401
-from database_backends.sqlite_backend import SQLiteBackend
+from database_backends.sqlite_backend import SQLiteBackend  # noqa: F401
 import pytz
 
 import stix2
@@ -290,7 +290,7 @@ def test_dictionary():
 
 def main():
     store = RelationalDBStore(
-        MariaDBBackend(f"mariadb+pymysql://admin:[email protected]:3306/rdb", force_recreate=True),
+        MariaDBBackend("mariadb+pymysql://admin:[email protected]:3306/rdb", force_recreate=True),
         # PostgresBackend("postgresql://localhost/stix-data-sink", force_recreate=True),
         # SQLiteBackend("sqlite:///stix-data-sink.db", force_recreate=True),
 

stix2/datastore/relational_db/table_creation.py

@@ -5,8 +5,9 @@
 
 from stix2.datastore.relational_db.add_method import add_method
 from stix2.datastore.relational_db.utils import (
-    canonicalize_table_name, determine_column_name, determine_core_properties, determine_sql_type_from_stix,
-    flat_classes, get_stix_object_classes, shorten_extension_definition_id
+    canonicalize_table_name, determine_column_name, determine_core_properties,
+    determine_sql_type_from_stix, flat_classes, get_stix_object_classes,
+    shorten_extension_definition_id,
 )
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
@@ -15,8 +16,7 @@
     ObjectReferenceProperty, Property, ReferenceProperty, StringProperty,
     TimestampProperty, TypeProperty,
 )
-
-from stix2.v21.base import (_Extension, _Observable)
+from stix2.v21.base import _Extension, _Observable
 from stix2.v21.common import KillChainPhase
 
 
@@ -275,9 +275,11 @@ def create_core_table(metadata, db_backend, stix_type_name):
             Column("created", db_backend.determine_sql_type_for_timestamp_property()),
             Column("modified", db_backend.determine_sql_type_for_timestamp_property()),
             Column("revoked", db_backend.determine_sql_type_for_boolean_property()),
-            Column("confidence",
-                   db_backend.determine_sql_type_for_integer_property(),
-                   db_backend.create_min_max_constraint_expression(IntegerProperty(min=0, max=100), "confidence")),
+            Column(
+                "confidence",
+                db_backend.determine_sql_type_for_integer_property(),
+                db_backend.create_min_max_constraint_expression(IntegerProperty(min=0, max=100), "confidence"),
+            ),
             Column("lang", db_backend.determine_sql_type_for_string_property()),
         ]
         columns.extend(sdo_columns)

stix2/datastore/relational_db/utils.py

@@ -51,6 +51,7 @@ def determine_core_properties(stix_object_class, is_embedded_object):
     else:
         raise ValueError(f"{stix_object_class} not a STIX object")
 
+
 def canonicalize_table_name(table_name, schema_name=None):
     if schema_name:
         full_name = schema_name + "." + table_name
@@ -112,6 +113,7 @@ def get_stix_object_classes():
         )
     )
 
+
 def schema_for(stix_class):
     if issubclass(stix_class, _DomainObject):
         schema_name = "sdo"

stix2/test/v21/test_datastore_relational_db.py

@@ -10,7 +10,9 @@
 from stix2.datastore.relational_db.database_backends.postgres_backend import (
     PostgresBackend,
 )
-from stix2.datastore.relational_db.database_backends.sqlite_backend import SQLiteBackend
+from stix2.datastore.relational_db.database_backends.sqlite_backend import (
+    SQLiteBackend,
+)
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 import stix2.registry
@@ -19,7 +21,7 @@
 
 @pytest.fixture(
     scope="module",
-    params=["postgresql", "sqlite"]
+    params=["postgresql", "sqlite"],
 )
 def db_backend(request):
     if request.param == "postgresql":