Python3 Upgrade

.github/workflows/ci.yml

@@ -8,8 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # Once we move to Python 3.x, this can be tested against multiple Python versions. eg. [3.6, 3.7, 3.8, 3.9]
-        python-version: [ 2.7 ]
+        # This allows the pipeline to be run against multiple Python versions. eg. [3.6, 3.7, 3.8, 3.9]. This results
+        # in linting and unit tests running for all listed versions as well as the creation of packages and wheels on
+        # creation of a tag in Git.
+        python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
 
     steps:
       # Get the code from the repository to be packaged
@@ -78,6 +80,8 @@ jobs:
       - name: Build Objects
         if: startsWith(github.ref, 'refs/tags')
         run: python setup.py sdist bdist_wheel
+        env:
+          TAG_VERSION: "${GITHUB_REF#refs/*/}"
 
       # Ensure the objects were packaged correctly and there wasn't an issue with
       # the compilation or packaging process.

.gitignore

@@ -30,6 +30,8 @@
 # Other
 /output
 /log
+
+# PyTest and Coverage
 /case
 /.pytest_cache
 /htmlcov

.gitlab-ci.yml

@@ -17,6 +17,8 @@ stages:
   - Test
   - Build
 
+image: python:3.9
+
 before_script:
   - pip config set global.index-url ${PIP_URL}
   - pip config set global.trusted-host ${ARTIFACTORY_HOST_NAME}

README.md

@@ -298,6 +298,5 @@ TODO:
 - [ ] Add additional logging messages to the master schema entries skipped in signature generation. 
 - [ ] Integrate in the SQLite Forensic Corpus into tests.
 - [ ] Look into updating terminology for versioning to timelining.
-- [ ] Update code for compatibility with Python 3.
 - [ ] Create PyUnit tests.
 - [ ] Create a GUI.

docs/source/conf.py

@@ -22,7 +22,7 @@
 author = 'Department of Defense Cyber Crime Center (DC3)'
 
 # The full version, including alpha/beta/rc tags
-release = '0.2.0'
+release = '1.0.0'
 
 # -- General configuration ---------------------------------------------------
 master_doc = 'index'

docs/source/index.rst

@@ -7,6 +7,12 @@ the associated journal files. If they are not in the same directory as the speci
 their location will need to be specified in the command. SQLite carving will not be done by default. See the full
 command line options to enable carving.
 
+SQLite Dissect is primarily developed and maintained by `Department of Defense Cyber Crime Center (DC3) <https://www.dc3.mil/>`_
+with support from open-source contributors. The tool is hosted on `GitHub <https://github.com/Defense-Cyber-Crime-Center/sqlite-dissect>`_.
+
+All released versions of the tool can be downloaded from the `GitHub releases page <https://github.com/Defense-Cyber-Crime-Center/sqlite-dissect/releases>`_.
+
+
 .. toctree::
    :maxdepth: 2
 

docs/source/sqlite_dissect/getting_started.rst

@@ -3,11 +3,14 @@ Getting Started
 
 System Requirements
 +++++++++++++++++++
-SQLite Dissect depends on Python 2.7, with support for Python 3.x expected soon. It has been tested on Windows, OSX, and
-Linux (Ubuntu) platforms.
+SQLite Dissect depends on Python 3.6+, with automated tests run against versions 3.6, 3.7, 3.8, 3.9, and 3.10. It has
+been tested on Windows, OSX, and Linux (Ubuntu) platforms.
 
-To try to limit the need for dependencies, only one package is required for SQLite Dissect, which the `openpxl <https://openpyxl.readthedocs.io/en/stable/>`_ package
-that is used for exporting the results into Excel format.
+SQLite Dissect versions up to and including 0.1.0 support Python 2.7 and can be downloaded from the
+`GitHub releases page <https://github.com/Defense-Cyber-Crime-Center/sqlite-dissect/releases>`_.
+
+To try to limit the need for dependencies, only one package is required for SQLite Dissect, which is the
+`openpxl <https://openpyxl.readthedocs.io/en/stable/>`_ package that is used for exporting the results into Excel format.
 
 Installation
 +++++++++++++++++++

setup.py

@@ -47,14 +47,18 @@
                 "sqlite_dissect.carving",
                 "sqlite_dissect.export"],
       classifiers=[
-          "Programming Language :: Python :: 2",
-          "Programming Language :: Python :: 2.7"
+          "Programming Language :: Python :: 3",
+          "Programming Language :: Python :: 3.6",
+          "Programming Language :: Python :: 3.7",
+          "Programming Language :: Python :: 3.8",
+          "Programming Language :: Python :: 3.9",
+          "Programming Language :: Python :: 3.10"
       ],
       entry_points={
           'console_scripts': ['sqlite_dissect=sqlite_dissect.entrypoint:cli'],
       },
       install_requires=[
-          "openpyxl==2.6.4",
+          "openpyxl==3.0.9",
           "ConfigArgParse"
       ],
       zip_safe=False

sqlite_dissect/_version.py

@@ -4,4 +4,4 @@
 This script identifies the version of the SQLite Dissect library.
 """
 
-__version__ = "0.2.0"
+__version__ = "1.0.0"

sqlite_dissect/carving/carved_cell.py

@@ -1,23 +1,18 @@
 from struct import unpack
 from warnings import warn
-from sqlite_dissect.carving.utilities import calculate_body_content_size
-from sqlite_dissect.carving.utilities import calculate_serial_type_definition_content_length_min_max
-from sqlite_dissect.carving.utilities import decode_varint_in_reverse
-from sqlite_dissect.carving.utilities import get_content_size
-from sqlite_dissect.constants import BLOB_SIGNATURE_IDENTIFIER
-from sqlite_dissect.constants import CELL_LOCATION
-from sqlite_dissect.constants import FILE_TYPE
-from sqlite_dissect.constants import TEXT_SIGNATURE_IDENTIFIER
-from sqlite_dissect.exception import CellCarvingError
-from sqlite_dissect.exception import InvalidVarIntError
+
+from sqlite_dissect.carving.utilities import (
+    calculate_body_content_size,
+    calculate_serial_type_definition_content_length_min_max,
+    decode_varint_in_reverse, get_content_size)
+from sqlite_dissect.constants import (BLOB_SIGNATURE_IDENTIFIER, CELL_LOCATION,
+                                      FILE_TYPE, TEXT_SIGNATURE_IDENTIFIER)
+from sqlite_dissect.exception import CellCarvingError, InvalidVarIntError
 from sqlite_dissect.file.database.page import BTreeCell
-from sqlite_dissect.file.database.payload import Payload
-from sqlite_dissect.file.database.payload import RecordColumn
-from sqlite_dissect.utilities import decode_varint
-from sqlite_dissect.utilities import encode_varint
-from sqlite_dissect.utilities import get_md5_hash
-from sqlite_dissect.utilities import get_record_content
-from sqlite_dissect.utilities import get_serial_type_signature
+from sqlite_dissect.file.database.payload import Payload, RecordColumn
+from sqlite_dissect.utilities import (decode_varint, encode_varint,
+                                      get_md5_hash, get_record_content,
+                                      get_serial_type_signature)
 
 """
 
@@ -221,7 +216,7 @@ def __init__(self, location, data, serial_type_definition_start_offset, serial_t
         self.truncated_beginning = False
         self.truncated_ending = False
 
-        record_column_md5_hash_strings = [""] * self.number_of_columns
+        record_column_md5_hash_strings = [b""] * self.number_of_columns
 
         column_index = 0
         body_byte_size = 0
@@ -273,7 +268,7 @@ def __init__(self, location, data, serial_type_definition_start_offset, serial_t
 
                     self.serial_type_signature += str(get_serial_type_signature(first_serial_type))
 
-                    record_column_md5_hash_strings[column_index] = ""
+                    record_column_md5_hash_strings[column_index] = b""
 
                     self.serial_type_definition_size += first_serial_type_varint_length
 
@@ -439,7 +434,7 @@ def __init__(self, location, data, serial_type_definition_start_offset, serial_t
 
                         self.serial_type_signature += str(get_serial_type_signature(first_serial_type))
 
-                        record_column_md5_hash_strings[column_index] = ""
+                        record_column_md5_hash_strings[column_index] = b""
 
                         self.serial_type_definition_size += first_serial_type_varint_length
 

sqlite_dissect/carving/carver.py

@@ -3,12 +3,8 @@
 from warnings import warn
 from sqlite_dissect.carving.carved_cell import CarvedBTreeCell
 from sqlite_dissect.carving.utilities import generate_signature_regex
-from sqlite_dissect.constants import BLOB_SIGNATURE_IDENTIFIER
-from sqlite_dissect.constants import CELL_LOCATION
-from sqlite_dissect.constants import LOGGER_NAME
-from sqlite_dissect.constants import TEXT_SIGNATURE_IDENTIFIER
-from sqlite_dissect.exception import CarvingError
-from sqlite_dissect.exception import CellCarvingError
+from sqlite_dissect.constants import BLOB_SIGNATURE_IDENTIFIER, CELL_LOCATION, LOGGER_NAME, TEXT_SIGNATURE_IDENTIFIER
+from sqlite_dissect.exception import CarvingError, CellCarvingError
 
 """
 
@@ -83,7 +79,7 @@ def carve_freeblocks(version, source, freeblocks, signature):
                           "carving freeblocks with signatures: {}.  Signatures starting with variable length serial " \
                           "types are not fully implemented and may result in carving false positives."
             log_message = log_message.format(first_column_serial_types, simplified_signature)
-            logger.warn(log_message)
+            logger.warning(log_message)
             warn(log_message, RuntimeWarning)
 
         # Retrieve and compile the serial type definition signature pattern
@@ -186,7 +182,7 @@ def carve_freeblocks(version, source, freeblocks, signature):
                                                      serial_type_definition_start_offset,
                                                      serial_type_definition_end_offset, cutoff_offset,
                                                      number_of_columns, signature.name, signature.table_name)
-                    logger.warn(log_message)
+                    logger.warning(log_message)
                     warn(log_message, RuntimeWarning)
 
         # Return the cells carved from the freeblocks
@@ -406,7 +402,7 @@ def carve_unallocated_space(version, source, page_number, unallocated_space_star
                                                  serial_type_definition_start_offset,
                                                  serial_type_definition_end_offset, cutoff_offset,
                                                  number_of_columns, signature.name, signature.table_name)
-                logger.warn(log_message)
+                logger.warning(log_message)
                 warn(log_message, RuntimeWarning)
 
         """
@@ -586,7 +582,7 @@ def carve_unallocated_space(version, source, page_number, unallocated_space_star
                                                              partial_serial_type_definition_end_offset,
                                                              partial_cutoff_offset, number_of_columns, signature.name,
                                                              signature.table_name)
-                            logger.warn(log_message)
+                            logger.warning(log_message)
                             warn(log_message, RuntimeWarning)
 
         # Return the cells carved from the freeblocks