Implement dynamic inputs and outputs. (#135)

* Implement dynamic inputs and outputs.

* Recover block_outputs if it's a string from the block's instance.

* Update tests

Author: pvk-developer

mlblocks/mlblock.py

@@ -111,8 +111,15 @@ def _extract_params(self, kwargs, hyperparameters):
             if name in kwargs:
                 init_params[name] = kwargs.pop(name)
 
-        fit_args = [arg['name'] for arg in self.fit_args]
-        produce_args = [arg['name'] for arg in self.produce_args]
+        if not isinstance(self.fit_args, str):
+            fit_args = [arg['name'] for arg in self.fit_args]
+        else:
+            fit_args = []
+
+        if not isinstance(self.produce_args, str):
+            produce_args = [arg['name'] for arg in self.produce_args]
+        else:
+            produce_args = []
 
         for name in list(kwargs.keys()):
             if name in fit_args:
@@ -257,6 +264,8 @@ def _get_method_kwargs(self, kwargs, method_args):
                 A dictionary containing the argument names and values to pass
                 to the primitive method.
         """
+        if isinstance(method_args, str):
+            method_args = getattr(self.instance, method_args)()
 
         method_kwargs = dict()
         for arg in method_args:

mlblocks/mlpipeline.py

@@ -177,6 +177,9 @@ def _get_block_variables(self, block_name, variables_attr, names):
         """
         block = self.blocks[block_name]
         variables = deepcopy(getattr(block, variables_attr))
+        if isinstance(variables, str):
+            variables = getattr(block.instance, variables)()
+
         variable_dict = {}
         for variable in variables:
             name = variable['name']
@@ -300,6 +303,12 @@ def get_inputs(self, fit=True):
 
         return inputs
 
+    def get_fit_args(self):
+        return list(self.get_inputs(fit=True).values())
+
+    def get_predict_args(self):
+        return list(self.get_inputs(fit=False).values())
+
     def get_outputs(self, outputs='default'):
         """Get the list of output variables that correspond to the specified outputs.
 
@@ -578,6 +587,10 @@ def _get_block_args(self, block_name, block_args, context):
 
         input_names = self.input_names.get(block_name, dict())
 
+        if isinstance(block_args, str):
+            block = self.blocks[block_name]
+            block_args = getattr(block.instance, block_args)()
+
         kwargs = dict()
         for arg in block_args:
             name = arg['name']
@@ -591,6 +604,9 @@ def _get_block_args(self, block_name, block_args, context):
     def _extract_outputs(self, block_name, outputs, block_outputs):
         """Extract the outputs of the method as a dict to be set into the context."""
         # TODO: type validation and/or transformation should be done here
+        if isinstance(block_outputs, str):
+            block = self.blocks[block_name]
+            block_outputs = getattr(block.instance, block_outputs)()
 
         if not isinstance(outputs, tuple):
             outputs = (outputs, )

tests/test_mlpipeline.py

@@ -381,6 +381,7 @@ def test_get_outputs_str_named(self):
             ]
         }
         pipeline = MLPipeline(['a_primitive', 'another_primitive'], outputs=outputs)
+
         returned = pipeline.get_outputs('debug')
 
         expected = [
@@ -389,13 +390,11 @@ def test_get_outputs_str_named(self):
                 'variable': 'another_variable',
             }
         ]
-
         assert returned == expected
 
     @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
     def test_get_outputs_str_variable(self):
         pipeline = MLPipeline(['a_primitive', 'another_primitive'])
-
         pipeline.blocks['a_primitive#1'].produce_output = [
             {
                 'name': 'output',
@@ -412,7 +411,6 @@ def test_get_outputs_str_variable(self):
                 'variable': 'a_primitive#1.output'
             }
         ]
-
         assert returned == expected
 
     @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
@@ -427,7 +425,6 @@ def test_get_outputs_str_block(self):
                 'variable': 'a_primitive#1',
             }
         ]
-
         assert returned == expected
 
     @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
@@ -442,7 +439,6 @@ def test_get_outputs_int(self):
                 'variable': 'another_primitive#1',
             }
         ]
-
         assert returned == expected
 
     @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
@@ -463,7 +459,6 @@ def test_get_outputs_combination(self):
             ]
         }
         pipeline = MLPipeline(['a_primitive', 'another_primitive'], outputs=outputs)
-
         pipeline.blocks['a_primitive#1'].produce_output = [
             {
                 'name': 'output',
@@ -498,7 +493,6 @@ def test_get_outputs_combination(self):
                 'variable': 'a_primitive#1.output'
             }
         ]
-
         assert returned == expected
 
     @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
@@ -550,32 +544,90 @@ def test_get_output_variables(self):
         assert names == ['a_variable']
 
     @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
-    def test__get_block_variables(self):
+    def test__get_block_variables_is_dict(self):
+        pipeline = MLPipeline(['a_primitive'])
+        pipeline.blocks['a_primitive#1'].produce_outputs = [
+            {
+                'name': 'output',
+                'type': 'whatever'
+            }
+        ]
+
+        outputs = pipeline._get_block_variables(
+            'a_primitive#1',
+            'produce_outputs',
+            {'output': 'name_output'}
+        )
+
         expected = {
             'name_output': {
                 'name': 'output',
                 'type': 'whatever',
             }
         }
+        assert outputs == expected
 
+    @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
+    def test__get_block_variables_is_str(self):
         pipeline = MLPipeline(['a_primitive'])
-
-        pipeline.blocks['a_primitive#1'].produce_outputs = [
+        pipeline.blocks['a_primitive#1'].produce_outputs = 'get_produce_outputs'
+        pipeline.blocks['a_primitive#1'].instance.get_produce_outputs.return_value = [
             {
-                'name': 'output',
-                'type': 'whatever'
+                'name': 'output_from_function',
+                'type': 'test'
             }
+
         ]
 
         outputs = pipeline._get_block_variables(
             'a_primitive#1',
             'produce_outputs',
             {'output': 'name_output'}
         )
+
+        expected = {
+            'output_from_function': {
+                'name': 'output_from_function',
+                'type': 'test',
+            }
+        }
         assert outputs == expected
+        pipeline.blocks['a_primitive#1'].instance.get_produce_outputs.assert_called_once_with()
 
     @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
     def test_get_inputs_fit(self):
+        pipeline = MLPipeline(['a_primitive', 'another_primitive'])
+        pipeline.blocks['a_primitive#1'].produce_args = [
+            {
+                'name': 'input',
+                'type': 'whatever'
+            }
+        ]
+        pipeline.blocks['a_primitive#1'].fit_args = [
+            {
+                'name': 'fit_input',
+                'type': 'whatever'
+            }
+        ]
+        pipeline.blocks['a_primitive#1'].produce_output = [
+            {
+                'name': 'output',
+                'type': 'another_whatever'
+            }
+        ]
+        pipeline.blocks['another_primitive#1'].produce_args = [
+            {
+                'name': 'output',
+                'type': 'another_whatever'
+            },
+            {
+                'name': 'another_input',
+                'type': 'another_whatever'
+            }
+        ]
+
+        inputs = pipeline.get_inputs()
+
         expected = {
             'input': {
                 'name': 'input',
@@ -589,32 +641,30 @@ def test_get_inputs_fit(self):
                 'name': 'another_input',
                 'type': 'another_whatever',
             }
-
         }
+        assert inputs == expected
 
+    @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
+    def test_get_inputs_no_fit(self):
         pipeline = MLPipeline(['a_primitive', 'another_primitive'])
-
         pipeline.blocks['a_primitive#1'].produce_args = [
             {
                 'name': 'input',
                 'type': 'whatever'
             }
         ]
-
         pipeline.blocks['a_primitive#1'].fit_args = [
             {
                 'name': 'fit_input',
                 'type': 'whatever'
             }
         ]
-
         pipeline.blocks['a_primitive#1'].produce_output = [
             {
                 'name': 'output',
                 'type': 'another_whatever'
             }
         ]
-
         pipeline.blocks['another_primitive#1'].produce_args = [
             {
                 'name': 'output',
@@ -626,11 +676,8 @@ def test_get_inputs_fit(self):
             }
         ]
 
-        inputs = pipeline.get_inputs()
-        assert inputs == expected
+        inputs = pipeline.get_inputs(fit=False)
 
-    @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
-    def test_get_inputs_no_fit(self):
         expected = {
             'input': {
                 'name': 'input',
@@ -640,46 +687,75 @@ def test_get_inputs_no_fit(self):
                 'name': 'another_input',
                 'type': 'another_whatever',
             }
-
         }
+        assert inputs == expected
 
-        pipeline = MLPipeline(['a_primitive', 'another_primitive'])
-
+    @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
+    def test_get_fit_args(self):
+        pipeline = MLPipeline(['a_primitive'])
         pipeline.blocks['a_primitive#1'].produce_args = [
             {
                 'name': 'input',
                 'type': 'whatever'
             }
         ]
-
         pipeline.blocks['a_primitive#1'].fit_args = [
             {
                 'name': 'fit_input',
                 'type': 'whatever'
             }
         ]
-
         pipeline.blocks['a_primitive#1'].produce_output = [
             {
                 'name': 'output',
                 'type': 'another_whatever'
             }
         ]
 
-        pipeline.blocks['another_primitive#1'].produce_args = [
+        outputs = pipeline.get_fit_args()
+
+        expected = [
             {
-                'name': 'output',
-                'type': 'another_whatever'
+                'name': 'input',
+                'type': 'whatever'
             },
             {
-                'name': 'another_input',
-                'type': 'another_whatever'
+                'name': 'fit_input',
+                'type': 'whatever',
             }
         ]
+        assert outputs == expected
 
-        inputs = pipeline.get_inputs(fit=False)
+    @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
+    def test_get_predict_args(self):
+        pipeline = MLPipeline(['a_primitive'])
+        pipeline.blocks['a_primitive#1'].produce_args = [
+            {
+                'name': 'input',
+                'type': 'whatever'
+            }
+        ]
+        pipeline.blocks['a_primitive#1'].fit_args = [
+            {
+                'name': 'fit_input',
+                'type': 'whatever'
+            }
+        ]
+        pipeline.blocks['a_primitive#1'].produce_output = [
+            {
+                'name': 'output',
+                'type': 'another_whatever'
+            }
+        ]
+        outputs = pipeline.get_predict_args()
 
-        assert inputs == expected
+        expected = [
+            {
+                'name': 'input',
+                'type': 'whatever'
+            }
+        ]
+        assert outputs == expected
 
     @patch('mlblocks.mlpipeline.MLBlock', new=get_mlblock_mock)
     def test_fit_pending_all_primitives(self):