Add the flake8 to be the default code formatter (#401)

wenbingl · web-flow · commit 57f8e150a146 · 2020-06-02T10:11:02.000-07:00
diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml
@@ -71,6 +71,10 @@ jobs:
       make lib
     displayName: 'Install dependencies'
 
+  - script: |
+      python -m flake8 ./onnxmltools
+    displayName: 'run flake8 check'
+
   - script: |
       export PYTHONPATH=$PYTHONPATH:libsvm/python
       python -c "import svmutil"
diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml
@@ -87,6 +87,11 @@ jobs:
       python -m pip show pytest
     displayName: 'Install dependencies'
 
+  - script: |
+      call activate py$(python.version)
+      python -m flake8 ./onnxmltools
+    displayName: 'run flake8 check'
+
   - script: |
       call activate py$(python.version)
       set PYTHONPATH=libsvm\python;%PYTHONPATH%
diff --git a/.flake8 b/.flake8
@@ -0,0 +1,10 @@
+[flake8]
+max-line-length = 120
+per-file-ignores =
+    __init__.py:F401
+exclude =
+    sparkml
+    coreml
+    libsvm
+    xgboost
+    utils
diff --git a/onnxmltools/convert/common/_apply_operation.py b/onnxmltools/convert/common/_apply_operation.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.onnx_ops import *
+from onnxconverter_common.onnx_ops import *  # noqa
diff --git a/onnxmltools/convert/common/_container.py b/onnxmltools/convert/common/_container.py
@@ -17,9 +17,11 @@ class LightGbmModelContainer(CommonSklearnModelContainer):
 class XGBoostModelContainer(CommonSklearnModelContainer):
     pass
 
+
 class H2OModelContainer(CommonSklearnModelContainer):
     pass
 
+
 class SparkmlModelContainer(RawModelContainer):
 
     def __init__(self, sparkml_model):
diff --git a/onnxmltools/convert/common/_registration.py b/onnxmltools/convert/common/_registration.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.registration import *
+from onnxconverter_common.registration import *  # noqa
diff --git a/onnxmltools/convert/common/_topology.py b/onnxmltools/convert/common/_topology.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.topology import *
+from onnxconverter_common.topology import *  # noqa
diff --git a/onnxmltools/convert/common/case_insensitive_dict.py b/onnxmltools/convert/common/case_insensitive_dict.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.case_insensitive_dict import *
+from onnxconverter_common.case_insensitive_dict import *  # noqa
diff --git a/onnxmltools/convert/common/data_types.py b/onnxmltools/convert/common/data_types.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.data_types import *
+from onnxconverter_common.data_types import *  # noqa
diff --git a/onnxmltools/convert/common/interface.py b/onnxmltools/convert/common/interface.py
@@ -6,4 +6,4 @@
 # This file defines the interface of the converter internal object for callback,
 # So the usage of the methods and properties list here will not be affected among the different versions.
 
-from onnxconverter_common.interface import *
+from onnxconverter_common.interface import *  # noqa
diff --git a/onnxmltools/convert/common/optimizer.py b/onnxmltools/convert/common/optimizer.py
diff --git a/onnxmltools/convert/common/shape_calculator.py b/onnxmltools/convert/common/shape_calculator.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.shape_calculator import *
+from onnxconverter_common.shape_calculator import *  # noqa
diff --git a/onnxmltools/convert/common/tree_ensemble.py b/onnxmltools/convert/common/tree_ensemble.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.tree_ensemble import *
+from onnxconverter_common.tree_ensemble import *  # noqa
diff --git a/onnxmltools/convert/common/utils.py b/onnxmltools/convert/common/utils.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.utils import *
+from onnxconverter_common.utils import *  # noqa
diff --git a/onnxmltools/convert/h2o/_parse.py b/onnxmltools/convert/h2o/_parse.py
@@ -7,6 +7,7 @@
 from ..common._container import H2OModelContainer
 from ..common._topology import Topology
 
+
 def _parse_h2o(scope, model, inputs):
     '''
     :param scope: Scope object
@@ -30,7 +31,6 @@ def _parse_h2o(scope, model, inputs):
 
 def parse_h2o(model, initial_types=None, target_opset=None,
               custom_conversion_functions=None, custom_shape_calculators=None):
-
     raw_model_container = H2OModelContainer(model)
     topology = Topology(raw_model_container, default_batch_size='None',
                         initial_types=initial_types, target_opset=target_opset,
diff --git a/onnxmltools/convert/h2o/convert.py b/onnxmltools/convert/h2o/convert.py
@@ -16,7 +16,7 @@
 from ._parse import parse_h2o
 
 # Invoke the registration of all our converters and shape calculators
-from . import operator_converters, shape_calculators
+from . import operator_converters, shape_calculators  # noqa
 
 
 def convert(model, name=None, initial_types=None, doc_string='', target_opset=None,
@@ -66,7 +66,8 @@ def convert(model, name=None, initial_types=None, doc_string='', target_opset=No
     mojo_str = h2o.print_mojo(model_path, format="json")
     mojo_model = json.loads(mojo_str)
     if mojo_model["params"]["algo"] != "gbm":
-        raise ValueError("Model type not supported (algo=%s). Only GBM Mojo supported for now." % mojo_model["params"]["algo"])
+        raise ValueError(
+            "Model type not supported (algo=%s). Only GBM Mojo supported for now." % mojo_model["params"]["algo"])
 
     target_opset = target_opset if target_opset else get_maximum_opset_supported()
     topology = parse_h2o(mojo_model, initial_types, target_opset, custom_conversion_functions, custom_shape_calculators)
diff --git a/onnxmltools/convert/lightgbm/_parse.py b/onnxmltools/convert/lightgbm/_parse.py
@@ -8,7 +8,7 @@
 from ..common._container import LightGbmModelContainer
 from ..common._topology import Topology
 from ..common.data_types import (FloatTensorType,
-    SequenceType, DictionaryType, StringType, Int64Type)
+                                 SequenceType, DictionaryType, StringType, Int64Type)
 
 from lightgbm import LGBMClassifier, LGBMRegressor
 
@@ -19,6 +19,7 @@
 lightgbm_operator_name_map = {LGBMClassifier: 'LgbmClassifier',
                               LGBMRegressor: 'LgbmRegressor'}
 
+
 class WrappedBooster:
 
     def __init__(self, booster):
@@ -44,7 +45,7 @@ def __init__(self, booster):
     def _generate_classes(self, model_dict):
         if model_dict['num_class'] == 1:
             return numpy.asarray([0, 1])
-        return numpy.arange(model_dict['num_class'])        
+        return numpy.arange(model_dict['num_class'])
 
 
 def _get_lightgbm_operator_name(model):
@@ -91,7 +92,7 @@ def _parse_lightgbm_simple_model(scope, model, inputs):
 
 def _parse_sklearn_classifier(scope, model, inputs):
     probability_tensor = _parse_lightgbm_simple_model(
-            scope, model, inputs)
+        scope, model, inputs)
     this_operator = scope.declare_local_operator('LgbmZipMap')
     this_operator.inputs = probability_tensor
     classes = model.classes_
@@ -145,7 +146,6 @@ def _parse_lightgbm(scope, model, inputs):
 
 def parse_lightgbm(model, initial_types=None, target_opset=None,
                    custom_conversion_functions=None, custom_shape_calculators=None):
-
     raw_model_container = LightGbmModelContainer(model)
     topology = Topology(raw_model_container, default_batch_size='None',
                         initial_types=initial_types, target_opset=target_opset,
diff --git a/onnxmltools/convert/lightgbm/convert.py b/onnxmltools/convert/lightgbm/convert.py
@@ -12,8 +12,7 @@
 from ._parse import parse_lightgbm, WrappedBooster
 
 # Invoke the registration of all our converters and shape calculators
-# from . import shape_calculators
-from . import operator_converters, shape_calculators
+from . import operator_converters, shape_calculators  # noqa
 
 
 def convert(model, name=None, initial_types=None, doc_string='', target_opset=None,
@@ -22,7 +21,7 @@ def convert(model, name=None, initial_types=None, doc_string='', target_opset=No
     '''
     This function produces an equivalent ONNX model of the given lightgbm model.
     The supported lightgbm modules are listed below.
-    
+
     * `LGBMClassifiers <https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMClassifier.html>`_
     * `LGBMRegressor <https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html>`_
     * `Booster <https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.Booster.html>`_
diff --git a/onnxmltools/convert/lightgbm/operator_converters/LightGbm.py b/onnxmltools/convert/lightgbm/operator_converters/LightGbm.py
@@ -8,7 +8,7 @@
 import numbers
 import numpy as np
 from collections import Counter
-from ...common._apply_operation import apply_div, apply_reshape, apply_sub, apply_cast
+from ...common._apply_operation import apply_div, apply_reshape, apply_sub, apply_cast, apply_identity
 from ...common._registration import register_converter
 from ...common.tree_ensemble import get_default_tree_classifier_attribute_pairs
 from ....proto import onnx_proto
diff --git a/onnxmltools/convert/main.py b/onnxmltools/convert/main.py
@@ -7,7 +7,6 @@
 import onnx
 from .common import utils
 import warnings
-import importlib
 
 
 def convert_coreml(model, name=None, initial_types=None, doc_string='', target_opset=None,
@@ -166,9 +165,8 @@ def convert_tensorflow(frozen_graph_def,
                        target_opset=None,
                        channel_first_inputs=None,
                        debug_mode=False, custom_op_conversions=None):
-    try:
-        importlib.import_module('tf2onnx')
-    except (ImportError, ModuleNotFoundError) as e:
+    import pkgutil
+    if not pkgutil.find_loader('tf2onnx'):
         raise RuntimeError('tf2onnx is not installed, please install it before calling this function.')
 
     return _convert_tf_wrapper(frozen_graph_def, name, input_names, output_names, doc_string,
diff --git a/onnxmltools/proto/__init__.py b/onnxmltools/proto/__init__.py
@@ -3,6 +3,16 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
+# Rather than using ONNX protobuf definition throughout our codebase, we import ONNX protobuf definition here so that
+# we can conduct quick fixes by overwriting ONNX functions without changing any lines elsewhere.
+from onnx import onnx_pb as onnx_proto  # noqa
+from onnx import helper
+
+# Overwrite the make_tensor defined in onnx.helper because of a bug (string tensor get assigned twice)
+from onnx import mapping
+from onnx.onnx_pb import TensorProto
+from onnx.helper import split_complex_to_pairs
+
 
 def _check_onnx_version():
     import pkg_resources
@@ -13,16 +23,6 @@ def _check_onnx_version():
 
 _check_onnx_version()
 
-# Rather than using ONNX protobuf definition throughout our codebase, we import ONNX protobuf definition here so that
-# we can conduct quick fixes by overwriting ONNX functions without changing any lines elsewhere.
-from onnx import onnx_pb as onnx_proto
-from onnx import helper
-
-# Overwrite the make_tensor defined in onnx.helper because of a bug (string tensor get assigned twice)
-from onnx import mapping
-from onnx.onnx_pb import TensorProto
-from onnx.helper import split_complex_to_pairs
-
 
 def _make_tensor_fixed(name, data_type, dims, vals, raw=False):
     '''
diff --git a/onnxmltools/utils/float16_converter.py b/onnxmltools/utils/float16_converter.py
@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from onnxconverter_common.float16 import *
+from onnxconverter_common.float16 import *  # noqa
diff --git a/onnxmltools/utils/main.py b/onnxmltools/utils/main.py
@@ -1,8 +1,8 @@
-#-------------------------------------------------------------------------
+# -------------------------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
-#--------------------------------------------------------------------------
+# --------------------------------------------------------------------------
 
 import os
 from ..proto import onnx_proto
@@ -116,5 +116,6 @@ def set_model_doc_string(model, doc, override=False):
     if not convert_utils.is_string_type(doc):
         raise ValueError("Doc must be a string type.")
     if model.doc_string and not doc and override is False:
-        raise ValueError("Failing to overwrite the doc string with a blank string, set override to True if intentional.")
+        raise ValueError(
+            "Failing to overwrite the doc string with a blank string, set override to True if intentional.")
     model.doc_string = doc
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -15,3 +15,4 @@ scipy
 svm
 wheel
 xgboost
+flake8
