Unified run.py file allowing execution of an excavator against a file.

Provide a central excavators module which re-exports the various named
configurations. Use this to implement a CLI recognizing a --excavator
argument which will call an exacavtion of the supplied filename.

Rework run_example as a wrapper around this new unified CLI. Expand the
README documenting both run_example and unified run entry points noting
that in the absence of arguments a list of supported excavators is output.

While here add smoke tests for basic validation of exports and scripts.

Author: alexjeffburke

Makefile

@@ -0,0 +1,3 @@
+test:
+	@python3 -m unittest discover tests "*_test.py"
+	@python3 -m unittest discover tests/autoarchaologist "*_test.py"

README.md

@@ -8,6 +8,35 @@ computers, in a convenient and user-friendly way.
 The AutoArchaeologist is a framework where plugins can be written to take
 apart and present old data media, in a browser-friendly fashion.
 
+## Getting started
+
+To get a feel for how this works and ensure you've got everything set up,
+we've included a sample file that can be excavated as an example:
+
+```sh
+python3 run_example.py
+```
+
+When you're ready to for further and process you own images use `run.py`.
+
+## Using run.py
+
+The various utilities that allow the extraction and processing of data are
+designed to be composed into an Excavation that can be run against binary
+images of disks and other media. Some standard excavations have been put
+together in bundles ad exposed via a single `run` command.
+
+Performing an exavation of a file is as simple as the following:
+
+```sh
+python3 run.py --excavator <excavator> <filename>
+```
+
+Usage information including the list of excavatos that are avalable as well as
+other options will be output when running without arguments: `python3 run.py`.
+
+## From the DDHF
+
 # First Example: Commodore CBM900 Harddisk
 
 Our first historic use of the AutoArchaeologist is now online:

autoarchaeologist/__init__.py

@@ -4,3 +4,4 @@
 from .base.excavation import Excavation, DuplicateArtifact
 from .record import Record
 from .base.namespace import NameSpace
+from . import excavators

autoarchaeologist/base/excavation.py

@@ -389,7 +389,7 @@ def summary(self, *args, **kwargs):
             font-family: "Inconsolata", "Courier New", mono-space;
         }
         td,th {
-            padding: 0 10px 0; 
+            padding: 0 10px 0;
         }
         th {
             border-bottom: 1px solid black;

autoarchaeologist/ddhf/cpm_exc.py autoarchaeologist/ddhf/cpm_excavator.pyautoarchaeologist/ddhf/cpm_exc.py renamed to autoarchaeologist/ddhf/cpm_excavator.py

@@ -4,6 +4,7 @@
 '''
 
 
+from ..base import excavation
 from ..base import type_case
 from ..generic import samesame
 from ..generic import textfiles
@@ -12,10 +13,12 @@
 cpm.cpm_filename_typecase.set_slug(0x5f, '_', '_')
 cpm.cpm_filename_typecase.set_slug(0x3b, ';', ';')
 
-def std_cpm_excavation(exc):
+class Cpm(excavation.Excavation):
+
     ''' Standard CP/M excavation '''
 
-    exc.type_case = type_case.DS2089Cpm()
-    exc.add_examiner(cpm.CpmFileSystem)
-    exc.add_examiner(textfiles.TextFile)
-    exc.add_examiner(samesame.SameSame)
+    def __init__(self, **kwargs):
+        self.type_case = type_case.DS2089Cpm()
+        self.add_examiner(cpm.CpmFileSystem)
+        self.add_examiner(textfiles.TextFile)
+        self.add_examiner(samesame.SameSame)

autoarchaeologist/ddhf/decorated_context.py

@@ -19,6 +19,7 @@ class DDHF_Excavation(excavation.Excavation):
 
     def __init__(
         self,
+        Excavation,
         ddhf_topic=None,
         ddhf_topic_link=None,
         ddhf_bitstore_cache=None,
@@ -31,6 +32,9 @@ def __init__(
         self.ddhf_bitstore_cache = ddhf_bitstore_cache
         super().__init__(**kwargs)
 
+        # apply the configuration of the chosen excavation
+        Excavation.__init__(self, **kwargs)
+
     def html_prefix_banner(self, file, _this):
         ''' Emit the banner for this excavation '''
         file.write('<table>\n')

autoarchaeologist/excavators.py

@@ -0,0 +1,55 @@
+import importlib.util
+import os
+import sys
+
+# XXX: importing excavations this way is a kludge to avoid multiple rounds
+#      of file splitting churn that would pollute diffs.. important as the
+#      the most critical thing (for now) is avoiding changes to behaviour
+def _import_name(import_name):
+    import_path = f"{import_name}.py"
+    spec = importlib.util.spec_from_file_location(import_name, import_path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+from .ddhf.cpm_excavator import Cpm as cpm
+cbm900 = _import_name("ddhf_cbm900").Cbm900
+cr80 = _import_name("ddhf_cr80").Cr80Floppy
+cr80_wang = _import_name("ddhf_cr80_wang").Cr80Wang
+dask = _import_name("ddhf_dask").Dask
+gier = _import_name("ddhf_gier").Gier
+intel_isis = _import_name("ddhf_intel_isis").IntelISIS
+r1k = _import_name("ddhf_r1k_tapes").R1k
+r1k_backup = _import_name("ddhf_r1k_backup").R1kBackup
+r1k_dfs = _import_name("ddhf_r1k_dfs").R1kDFS
+rc3600 = _import_name("ddhf_rc3600").Rc3600
+uug = _import_name("ddhf_dkuug").DkuugEuug
+zilog_mcz = _import_name("ddhf_zilog_mcz").ZilogMCZ
+
+# from .gier import configure_excavation as gier
+# from .intel_isis import configure_excavation as intel_isis
+# from .r1kdfs import configure_excavation as r1kdfs
+# from .uug import configure_excavation as uug
+
+__all__ = [
+    "cbm900",
+    "cpm",
+    "cr80",
+    "cr80_wang",
+    "dask",
+    "gier",
+    "intel_isis",
+    "r1k",
+    "r1k_backup",
+    "r1k_dfs",
+    "rc3600",
+    "uug",
+    "zilog_mcz",
+]
+
+EXCAVATORS = {name:getattr(sys.modules[__name__], name) for name in __all__}
+
+def excavator_by_name(excavator_name):
+    if excavator_name not in EXCAVATORS:
+        raise LookupError(f'no extractor named "{excavator_name}"')
+    return EXCAVATORS[excavator_name]

ddhf_butler.py

@@ -3,24 +3,22 @@
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 '''
 
-from autoarchaeologist import ddhf
-from autoarchaeologist.ddhf import cpm_exc
+from autoarchaeologist import ddhf, Excavation
+from autoarchaeologist.ddhf.cpm_excavator import Cpm
 
-class Butler(ddhf.DDHF_Excavation):
+class DDHF_Butler(ddhf.DDHF_Excavation):
     ''' All Butler artifacts '''
 
     def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-        cpm_exc.std_cpm_excavation(self)
+        super().__init__(Cpm, **kwargs)
 
         self.from_bitstore(
             "COMPANY/BOGIKA",
         )
 
 if __name__ == "__main__":
     ddhf.main(
-        Butler,
+        DDHF_Butler,
         html_subdir="butler",
         ddhf_topic = 'Bogika Butler',
         ddhf_topic_link = 'https://datamuseum.dk/wiki/BDS_Butler'

ddhf_cbm900.py

@@ -4,6 +4,7 @@
 '''
 
 from autoarchaeologist import ddhf
+from autoarchaeologist.base import excavation
 
 from autoarchaeologist.unix import cbm900_partition
 from autoarchaeologist.unix import v7_filesystem
@@ -12,7 +13,7 @@
 from autoarchaeologist.generic import textfiles
 from autoarchaeologist.generic import samesame
 
-class CBM900(ddhf.DDHF_Excavation):
+class Cbm900(excavation.Excavation):
 
     '''
     Two CBM900 hard-disk images, one also contains the four distribution
@@ -29,14 +30,18 @@ def __init__(self, **kwargs):
         self.add_examiner(textfiles.TextFile)
         self.add_examiner(samesame.SameSame)
 
+class DDHF_Cbm900(ddhf.DDHF_Excavation):
+    def __init__(self, **kwargs):
+        super().__init__(Cbm900, **kwargs)
+
         self.from_bitstore(
             "30001199",
             "30001972",
         )
 
 if __name__ == "__main__":
     ddhf.main(
-        CBM900,
+        DDHF_Cbm900,
         html_subdir="cbm900",
         ddhf_topic = "Commodore CBM-900",
         ddhf_topic_link = 'https://datamuseum.dk/wiki/Commodore/CBM900',

ddhf_comet.py

@@ -4,24 +4,22 @@
 '''
 
 from autoarchaeologist import ddhf
-from autoarchaeologist.ddhf import cpm_exc
+from autoarchaeologist.ddhf.cpm_excavator import Cpm
 
-class Comet(ddhf.DDHF_Excavation):
+class DDHF_Comet(ddhf.DDHF_Excavation):
 
     ''' All Comet artifacts '''
 
     def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-        cpm_exc.std_cpm_excavation(self)
+        super().__init__(Cpm, **kwargs)
 
         self.from_bitstore(
             "COMPANY/ICL/COMET",
         )
 
 if __name__ == "__main__":
     ddhf.main(
-        Comet,
+        DDHF_Comet,
         html_subdir="comet",
         ddhf_topic = 'ICL Comet',
         ddhf_topic_link = 'https://datamuseum.dk/wiki/ICL_Comet'