Merge pull request #25 from nasa/develop

Release 0.2

Author: bcoltin

.github/workflows/docs.yaml

@@ -0,0 +1,50 @@
+# Build documentation and commit to gh-pages branch.
+
+name: Build and Push Documentation to gh-pages Branch
+
+on:
+  push:
+    branches: [ 'master']
+
+jobs:
+  build_and_push_docs:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        path: repo/
+    - name: Checkout gh-pages
+      uses: actions/checkout@v2
+      with:
+        path: docs/
+        ref: gh-pages
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install pdoc3
+      run: |
+        python3 -m pip install pdoc3
+    - name: Install DELTA
+      run: |
+        cd repo
+        ./scripts/setup.sh
+        python3 -m pip install .
+    - name: Build Documentation
+      run: |
+        ./repo/scripts/docs.sh ./docs/
+    - name: Commit and Push
+      run: |
+        cd repo
+        EMAIL=`git show -s --format='%ae' HEAD`
+        NAME=`git show -s --format='%an' HEAD`
+        cd ..
+        cd docs/
+        git add .
+        git config user.email "$EMAIL"
+        git config user.name "$NAME"
+        git commit -m "Automatic update for $GITHUB_SHA."
+        git push origin gh-pages
+

bin/delta

@@ -18,30 +18,8 @@
 # limitations under the License.
 
 import sys
-import argparse
 
-from delta.config import config
-from delta.subcommands import commands
-
-def main(args):
-    parser = argparse.ArgumentParser(description='DELTA Machine Learning Toolkit')
-    subparsers = parser.add_subparsers()
-
-    for d in commands.SETUP_COMMANDS:
-        d(subparsers)
-
-    try:
-        options = parser.parse_args(args[1:])
-    except argparse.ArgumentError:
-        parser.print_help(sys.stderr)
-        sys.exit(1)
-
-    if not hasattr(options, 'function'):
-        parser.print_help(sys.stderr)
-        sys.exit(1)
-
-    config.initialize(options)
-    return options.function(options)
+from delta.subcommands import main
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv))
+    sys.exit(main.main(sys.argv))

delta/config/README.md

@@ -107,7 +107,7 @@ Used in the `delta train` and `delta mlflow_ui` commands to keep track of traini
    networks from different stages of training.
    * `frequency`: Frequency in batches to save a checkpoint. Networks can require a fair amount of disk space,
      so don't save too often.
-   * `save_latest`: If true, only keep the network file from the most recent checkpoint.
+   * `only_save_latest`: If true, only keep the network file from the most recent checkpoint.
 
 TensorBoard
 -----------

delta/config/config.py

@@ -35,6 +35,9 @@ def validate_positive(num, _):
         raise ValueError('%d is not positive' % (num))
     return num
 
+class _NotSpecified: #pylint:disable=too-few-public-methods
+    pass
+
 class DeltaConfigComponent:
     """
     DELTA configuration component.
@@ -78,7 +81,7 @@ def register_component(self, component, name : str, attr_name = None):
             attr_name = name
         setattr(self, attr_name, component)
 
-    def register_field(self, name : str, types, accessor = None, cmd_arg = None, validate_fn = None, desc = None):
+    def register_field(self, name : str, types, accessor = None, validate_fn = None, desc = None):
         """
         Register a field in this component of the configuration.
 
@@ -92,7 +95,6 @@ def register_field(self, name : str, types, accessor = None, cmd_arg = None, val
         self._fields.append(name)
         self._validate[name] = validate_fn
         self._types[name] = types
-        self._cmd_args[name] = cmd_arg
         self._descs[name] = desc
         if accessor:
             def access(self) -> types:
@@ -101,14 +103,44 @@ def access(self) -> types:
             access.__doc__ = desc
             setattr(self.__class__, accessor, access)
 
+    def register_arg(self, field, argname, **kwargs):
+        """
+        Registers a command line argument in this component.
+
+        field is the (registered) field this argument modifies.
+        argname is the name of the flag on the command line (i.e., '--flag')
+        **kwargs are arguments to ArgumentParser.add_argument.
+
+        If help and type are not specified, will use the ones for the field.
+        If default is not specified, will use the value from the config files.
+        """
+        assert field in self._fields, 'Field %s not registered.' % (field)
+        if 'help' not in kwargs:
+            kwargs['help'] = self._descs[field]
+        if 'type' not in kwargs:
+            kwargs['type'] = self._types[field]
+        elif kwargs['type'] is None:
+            del kwargs['type']
+        if 'default' not in kwargs:
+            kwargs['default'] = _NotSpecified
+        self._cmd_args[argname] = (field, kwargs)
+
+    def to_dict(self) -> dict:
+        """
+        Returns a dictionary representing the config object.
+        """
+        if isinstance(self._config_dict, dict):
+            exp = self._config_dict.copy()
+            for (name, c) in self._components.items():
+                exp[name] = c.to_dict()
+            return exp
+        return self._config_dict
+
     def export(self) -> str:
         """
-        Returns a YAML string of all configuration options.
+        Returns a YAML string of all configuration options, from to_dict.
         """
-        exp = self._config_dict.copy()
-        for (name, c) in self._components.items():
-            exp[name] = c.export()
-        return yaml.dump(exp)
+        return yaml.dump(self.to_dict())
 
     def _set_field(self, name : str, value : str, base_dir : str):
         if name not in self._fields:
@@ -139,12 +171,9 @@ def setup_arg_parser(self, parser, components = None) -> None:
         """
         if self._section_header is not None:
             parser = parser.add_argument_group(self._section_header)
-        for name in self._fields:
-            c = self._cmd_args[name]
-            if c is None:
-                continue
-            parser.add_argument(c, dest=c.replace('-', '_'), required=False,
-                                type=self._types[name], help=self._descs[name])
+        for (arg, value) in self._cmd_args.items():
+            (field, kwargs) = value
+            parser.add_argument(arg, dest=field, **kwargs)
 
         for (name, c) in self._components.items():
             if components is None or name in components:
@@ -157,14 +186,12 @@ def parse_args(self, options):
         configuration values.
         """
         d = {}
-        for name in self._fields:
-            c = self._cmd_args[name]
-            if c is None:
+        for (field, _) in self._cmd_args.values():
+            if not hasattr(options, field) or getattr(options, field) is None:
                 continue
-            n = c.replace('-', '_')
-            if not hasattr(options, n) or getattr(options, n) is None:
+            if getattr(options, field) is _NotSpecified:
                 continue
-            d[name] = getattr(options, n)
+            d[field] = getattr(options, field)
         self._load_dict(d, None)
 
         for c in self._components.values():
@@ -183,6 +210,7 @@ def load(self, yaml_file: str = None, yaml_str: str = None):
         """
         base_path = None
         if yaml_file:
+            #print("Loading config file: " + yaml_file)
             if not os.path.exists(yaml_file):
                 raise Exception('Config file does not exist: ' + yaml_file)
             with open(yaml_file, 'r') as f:

delta/config/delta.yaml

@@ -1,6 +1,7 @@
 general:
   # negative is all
   gpus:              -1
+  stop_on_input_error: true # If false skip past bad input files without halting training
 
 io:
   threads:           1
@@ -10,12 +11,16 @@ io:
   interleave_images: 5
   # ratio of tile width and height when loading images
   tile_ratio:        5.0
+  # When resuming training with a log_folder, skip input image where we have
+  # already loaded this many tiles.
+  resume_cutoff:     5000
   cache:
     # default is OS-specific, in Linux, ~/.cache/delta
     dir:              default
     limit:            8
 
 dataset:
+  log_folder:         ~ # Storage location for any record keeping files about the input dataset
   images:
     type:             tiff
     # preprocess the images when loading (i.e., scaling)
@@ -39,11 +44,33 @@ dataset:
     file_list:        ~
     files:            ~
 
+  # can either be a list of classes or the number of classes,
+  # if the labels are 0, 1, 2, 3
+  classes:            4
+  # labels are 1, 2, 3, 4; with names and display colors
+  # weight is optional, but required for either all or none
+  #classes:
+  #  - 1:
+  #      name: Water
+  #      color: 0x67a9cf
+  #      weight: 5.0
+  #  - 2:
+  #      name: No Water
+  #      color: 0xf6eff7
+  #      weight: 1.0
+  #  - 3:
+  #      name: Maybe Water
+  #      color: 0xbdc9e1
+  #      weight: 1.0
+  #  - 4:
+  #      name: Cloud
+  #      color: 0x02818a
+  #      weight: 1.0
+
 train:
   network:
     chunk_size:      16
     output_size:     8
-    classes:         4
     model:
       yaml_file:     networks/convpool.yaml
       params:        ~
@@ -91,8 +118,8 @@ mlflow:
   experiment_name: Default
   # rate in batches to save model checkpoints
   checkpoints:
-    frequency:     10000
-    save_latest:   true
+    frequency:          10000
+    only_save_latest:   true
 
 tensorboard:
   enabled:        false

delta/config/modules.py

@@ -0,0 +1,33 @@
+# Copyright © 2020, United States Government, as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All rights reserved.
+#
+# The DELTA (Deep Earth Learning, Tools, and Analysis) platform is
+# licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Registers all config modules.
+"""
+
+import delta.imagery.imagery_config
+import delta.ml.ml_config
+
+_config_initialized = False
+def register_all():
+    global _config_initialized #pylint: disable=global-statement
+    # needed to call twice when testing subcommands and when not
+    if _config_initialized:
+        return
+    delta.imagery.imagery_config.register()
+    delta.ml.ml_config.register()
+    _config_initialized = True

delta/config/networks/autoencoder_conv.yaml

@@ -2,28 +2,28 @@ layers:
   - Input:
       shape: in_shape
   - Conv2D:
-      filters: 300
+      filters: 50
       kernel_size: [3, 3]
       activation: relu
       padding: same
   - MaxPooling2D:
       pool_size: [2, 2]
   - Conv2D:
-      filters: 150
+      filters: 50 
       kernel_size: [3, 3]
       activation: relu
       padding: same
   - MaxPooling2D:
       pool_size: [2, 2]
   - Conv2D:
-      filters: 75
+      filters: 50 
       kernel_size: [3, 3]
       activation: relu
       padding: same
   - UpSampling2D:
       size: [2, 2]
   - Conv2D:
-      filters: 150
+      filters: 50
       kernel_size: [3, 3]
       activation: relu
       padding: same

delta/config/networks/autoencoder_conv_med_filters.yaml

@@ -0,0 +1,36 @@
+layers:
+  - Input:
+      shape: in_shape
+  - Conv2D:
+      filters: 50
+      kernel_size: [5, 5]
+      activation: relu
+      padding: same
+  - MaxPooling2D:
+      pool_size: [2, 2]
+  - Conv2D:
+      filters: 50 
+      kernel_size: [5, 5]
+      activation: relu
+      padding: same
+  - MaxPooling2D:
+      pool_size: [2, 2]
+  - Conv2D:
+      filters: 50 
+      kernel_size: [5, 5]
+      activation: relu
+      padding: same
+  - UpSampling2D:
+      size: [2, 2]
+  - Conv2D:
+      filters: 50
+      kernel_size: [5, 5]
+      activation: relu
+      padding: same
+  - UpSampling2D:
+      size: [2, 2]
+  - Conv2D:
+      filters: num_bands
+      kernel_size: [5, 5]
+      activation: relu
+      padding: same