make release-tag: Merge branch 'master' into stable

Author: csala

CONTRIBUTING.rst

@@ -195,3 +195,33 @@ Once this is done, run of the following commands:
 3. If you are releasing a major version::
 
     make release-major
+
+Release Candidates
+~~~~~~~~~~~~~~~~~~
+
+Sometimes it is necessary or convenient to upload a release candidate to PyPi as a pre-release,
+in order to make some of the new features available for testing on other projects before they
+are included in an actual full-blown release.
+
+In order to perform such an action, you can execute::
+
+    make release-candidate
+
+This will perform the following actions:
+
+1. Build and upload the current version to PyPi as a pre-release, with the format ``X.Y.Z.devN``
+
+2. Bump the current version to the next release candidate, ``X.Y.Z.dev(N+1)``
+
+After this is done, the new pre-release can be installed by including the ``dev`` section in the
+dependency specification, either in ``setup.py``::
+
+    install_requires = [
+        ...
+        'mlblocks>=X.Y.Z.dev',
+        ...
+    ]
+
+or in command line::
+
+    pip install 'mlblocks>=X.Y.Z.dev'

HISTORY.md

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+0.3.4 - 2019-11-01
+------------------
+
+* Ability to return intermediate context - [Issue #110](https://github.com/HDI-Project/MLBlocks/issues/110) by @csala
+* Support for static or class methods - [Issue #107](https://github.com/HDI-Project/MLBlocks/issues/107) by @csala
+
 0.3.3 - 2019-09-09
 ------------------
 

Makefile

@@ -155,7 +155,7 @@ publish: dist ## package and upload a release
 
 .PHONY: bumpversion-release
 bumpversion-release: ## Merge master to stable and bumpversion release
-	git checkout stable
+	git checkout stable || git checkout -b stable
 	git merge --no-ff master -m"make release-tag: Merge branch 'master' into stable"
 	bumpversion release
 	git push --tags origin stable
@@ -167,6 +167,10 @@ bumpversion-patch: ## Merge stable to master and bumpversion patch
 	bumpversion --no-tag patch
 	git push
 
+.PHONY: bumpversion-candidate
+bumpversion-candidate: ## Bump the version to the next candidate
+	bumpversion candidate --no-tag
+
 .PHONY: bumpversion-minor
 bumpversion-minor: ## Bump the version the next minor skipping the release
 	bumpversion --no-tag minor
@@ -175,23 +179,31 @@ bumpversion-minor: ## Bump the version the next minor skipping the release
 bumpversion-major: ## Bump the version the next major skipping the release
 	bumpversion --no-tag major
 
-CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
-CHANGELOG_LINES := $(shell git diff HEAD..stable HISTORY.md | wc -l)
+CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
+CHANGELOG_LINES := $(shell git diff HEAD..origin/stable HISTORY.md 2>&1 | wc -l)
 
-.PHONY: check-release
-check-release: ## Check if the release can be made
+.PHONY: check-master
+check-master: ## Check if we are in master branch
 ifneq ($(CURRENT_BRANCH),master)
 	$(error Please make the release from master branch\n)
 endif
+
+.PHONY: check-history
+check-history: ## Check if HISTORY.md has been modified
 ifeq ($(CHANGELOG_LINES),0)
 	$(error Please insert the release notes in HISTORY.md before releasing)
-else
-	@echo "A new release can be made"
 endif
 
+.PHONY: check-release
+check-release: check-master check-history ## Check if the release can be made
+	@echo "A new release can be made"
+
 .PHONY: release
 release: check-release bumpversion-release publish bumpversion-patch
 
+.PHONY: release-candidate
+release-candidate: check-master publish bumpversion-candidate
+
 .PHONY: release-minor
 release-minor: check-release bumpversion-minor release
 

mlblocks/__init__.py

@@ -20,7 +20,7 @@
 __copyright__ = 'Copyright (c) 2018, MIT Data To AI Lab'
 __email__ = '[email protected]'
 __license__ = 'MIT'
-__version__ = '0.3.3'
+__version__ = '0.3.4.dev2'
 
 __all__ = [
     'MLBlock', 'MLPipeline', 'add_pipelines_path', 'add_primitives_path',

mlblocks/mlblock.py

@@ -13,9 +13,17 @@
 
 def import_object(object_name):
     """Import an object from its Fully Qualified Name."""
+
     if isinstance(object_name, str):
-        package, name = object_name.rsplit('.', 1)
-        return getattr(importlib.import_module(package), name)
+        parent_name, attribute = object_name.rsplit('.', 1)
+        try:
+            parent = importlib.import_module(parent_name)
+        except ImportError:
+            grand_parent_name, parent_name = parent_name.rsplit('.', 1)
+            grand_parent = importlib.import_module(grand_parent_name)
+            parent = getattr(grand_parent, parent_name)
+
+        return getattr(parent, attribute)
 
     return object_name
 

mlblocks/mlpipeline.py

@@ -4,6 +4,7 @@
 
 import json
 import logging
+import re
 from collections import Counter, OrderedDict, defaultdict
 from copy import deepcopy
 
@@ -145,8 +146,11 @@ def _get_block_outputs(self, block_name):
         """Get the list of output variables for the given block."""
         block = self.blocks[block_name]
         outputs = deepcopy(block.produce_output)
+        output_names = self.output_names.get(block_name, dict())
         for output in outputs:
-            output['variable'] = '{}.{}'.format(block_name, output['name'])
+            name = output['name']
+            context_name = output_names.get(name, name)
+            output['variable'] = '{}.{}'.format(block_name, context_name)
 
         return outputs
 
@@ -195,12 +199,15 @@ def __init__(self, pipeline=None, primitives=None, init_params=None,
         if hyperparameters:
             self.set_hyperparameters(hyperparameters)
 
+        self._re_block_name = re.compile(r'(^[^#]+#\d+)(\..*)?')
+
     def _get_str_output(self, output):
         """Get the outputs that correspond to the str specification."""
         if output in self.outputs:
             return self.outputs[output]
         elif output in self.blocks:
-            return self._get_block_outputs(output)
+            return [{'name': output, 'variable': output}]
+            # return self._get_block_outputs(output)
         elif '.' in output:
             block_name, variable_name = output.rsplit('.', 1)
             block = self.blocks.get(block_name)
@@ -257,11 +264,11 @@ def get_outputs(self, outputs='default'):
 
         computed = list()
         for output in outputs:
+            if isinstance(output, int):
+                output = self._get_block_name(output)
+
             if isinstance(output, str):
                 computed.extend(self._get_str_output(output))
-            elif isinstance(output, int):
-                block_name = self._get_block_name(output)
-                computed.extend(self._get_block_outputs(block_name))
             else:
                 raise TypeError('Output Specification can only be str or int')
 
@@ -313,6 +320,18 @@ def get_output_variables(self, outputs='default'):
         outputs = self.get_outputs(outputs)
         return [output['variable'] for output in outputs]
 
+    def _extract_block_name(self, variable_name):
+        return self._re_block_name.search(variable_name).group(1)
+
+    def _prepare_outputs(self, outputs):
+        output_variables = self.get_output_variables(outputs)
+        outputs = output_variables.copy()
+        output_blocks = {
+            self._extract_block_name(variable)
+            for variable in output_variables
+        }
+        return output_variables, outputs, output_blocks
+
     @staticmethod
     def _flatten_dict(hyperparameters):
         return {
@@ -516,13 +535,11 @@ def _extract_outputs(self, block_name, outputs, block_outputs):
 
         return output_dict
 
-    def _update_outputs(self, block_name, output_variables, outputs, outputs_dict):
+    def _update_outputs(self, variable_name, output_variables, outputs, value):
         """Set the requested block outputs into the outputs list in the right place."""
-        for key, value in outputs_dict.items():
-            variable_name = '{}.{}'.format(block_name, key)
-            if variable_name in output_variables:
-                index = output_variables.index(variable_name)
-                outputs[index] = deepcopy(value)
+        if variable_name in output_variables:
+            index = output_variables.index(variable_name)
+            outputs[index] = deepcopy(value)
 
     def _fit_block(self, block, block_name, context):
         """Get the block args from the context and fit the block."""
@@ -551,7 +568,12 @@ def _produce_block(self, block, block_name, context, output_variables, outputs):
             context.update(outputs_dict)
 
             if output_variables:
-                self._update_outputs(block_name, output_variables, outputs, outputs_dict)
+                if block_name in output_variables:
+                    self._update_outputs(block_name, output_variables, outputs, context)
+                else:
+                    for key, value in outputs_dict.items():
+                        variable_name = '{}.{}'.format(block_name, key)
+                        self._update_outputs(variable_name, output_variables, outputs, value)
 
         except Exception:
             if self.verbose:
@@ -606,14 +628,12 @@ def fit(self, X=None, y=None, output_=None, start_=None, **kwargs):
         if y is not None:
             context['y'] = y
 
-        if output_ is not None:
-            output_variables = self.get_output_variables(output_)
-            outputs = output_variables.copy()
-            output_blocks = {variable.rsplit('.', 1)[0] for variable in output_variables}
-        else:
+        if output_ is None:
             output_variables = None
             outputs = None
             output_blocks = set()
+        else:
+            output_variables, outputs, output_blocks = self._prepare_outputs(output_)
 
         if isinstance(start_, int):
             start_ = self._get_block_name(start_)
@@ -637,7 +657,7 @@ def fit(self, X=None, y=None, output_=None, start_=None, **kwargs):
 
             # If there was an output_ but there are no pending
             # outputs we are done.
-            if output_ is not None and not output_blocks:
+            if output_variables is not None and not output_blocks:
                 if len(outputs) > 1:
                     return tuple(outputs)
                 else:
@@ -686,9 +706,7 @@ def predict(self, X=None, output_='default', start_=None, **kwargs):
         if X is not None:
             context['X'] = X
 
-        output_variables = self.get_output_variables(output_)
-        outputs = output_variables.copy()
-        output_blocks = {variable.rsplit('.', 1)[0] for variable in output_variables}
+        output_variables, outputs, output_blocks = self._prepare_outputs(output_)
 
         if isinstance(start_, int):
             start_ = self._get_block_name(start_)

setup.cfg

@@ -1,18 +1,21 @@
 [bumpversion]
-current_version = 0.3.3
+current_version = 0.3.4.dev2
 commit = True
 tag = True
-parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+))?
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?
 serialize = 
-	{major}.{minor}.{patch}-{release}
+	{major}.{minor}.{patch}.{release}{candidate}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:release]
 optional_value = release
+first_value = dev
 values = 
 	dev
 	release
 
+[bumpversion:part:candidate]
+
 [bumpversion:file:setup.py]
 search = version='{current_version}'
 replace = version='{new_version}'
@@ -34,8 +37,12 @@ include_trailing_comment = True
 line_length = 99
 lines_between_types = 0
 multi_line_output = 4
-not_skip = __init__.py
 use_parentheses = True
+not_skip = __init__.py
+skip_glob = *.bak
+
+[metadata]
+description-file = README.md
 
 [aliases]
 test = pytest

setup.py

@@ -100,6 +100,6 @@
     test_suite='tests',
     tests_require=tests_require,
     url='https://github.com/HDI-Project/MLBlocks',
-    version='0.3.3',
+    version='0.3.4.dev2',
     zip_safe=False,
 )

tests/features/test_partial_outputs.py

@@ -70,13 +70,14 @@ def test_fit_output(self):
         y = np.array([
             0, 0, 0, 0, 1
         ])
+        context = {'X': X, 'y': y}
 
         almost_equal(named_out, y)
         assert len(list_out) == 2
         almost_equal(list_out[0], y)
-        almost_equal(list_out[1], X)
-        almost_equal(X, int_out)
-        almost_equal(X, str_out)
+        almost_equal(list_out[1], context)
+        almost_equal(context, int_out)
+        almost_equal(context, str_out)
         almost_equal(X, str_out_variable)
         assert no_output is None
 

tests/test_mlblock.py

@@ -3,19 +3,44 @@
 from unittest import TestCase
 from unittest.mock import MagicMock, Mock, patch
 
-from mlblocks.mlblock import MLBlock, import_object
+import pytest
 
-# import pytest
+from mlblocks.mlblock import MLBlock, import_object
 
 
 class DummyClass:
+    def a_method(self):
+        pass
+
+
+def dummy_function():
     pass
 
 
-def test_import_object():
-    dummy_class = import_object(__name__ + '.DummyClass')
+class TestImportObject(TestCase):
+
+    def test_class(self):
+        imported = import_object(__name__ + '.DummyClass')
+
+        assert imported is DummyClass
+
+    def test_class_method(self):
+        imported = import_object(__name__ + '.DummyClass.a_method')
+
+        assert imported is DummyClass.a_method
+
+    def test_function(self):
+        imported = import_object(__name__ + '.dummy_function')
+
+        assert imported is dummy_function
+
+    def test_bad_object_name(self):
+        with pytest.raises(AttributeError):
+            import_object(__name__ + '.InvalidName')
 
-    assert dummy_class is DummyClass
+    def test_bad_module(self):
+        with pytest.raises(ImportError):
+            import_object('an.invalid.module')
 
 
 class TestMLBlock(TestCase):