Skip to content
This repository was archived by the owner on Aug 25, 2024. It is now read-only.

Commit 7f877c3

Browse files
0dust0dust
authored
operation: io: Add input and print
Fixes: #520
1 parent 9a6bfe8 commit 7f877c3

File tree

6 files changed

+260
-1
lines changed

6 files changed

+260
-1
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88
### Added
9+
- Operations for taking input from the user(AcceptUserInput) and for printing the output(print_output)
910
- Hugging Face Transformers tensorflow based NER models.
1011
- PNG ConfigLoader for reading images as arrays to predict using MNIST trained models
1112
- Docstrings and doctestable examples to `record.py`.

‎dffml/operation/io.py‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import asyncio
2+
import concurrent.futures
3+
from typing import Dict, Any
4+
5+
from dffml.df.types import Operation, Definition
6+
from dffml.df.base import (
7+
op,
8+
OperationImplementationContext,
9+
OperationImplementation,
10+
)
11+
12+
13+
# Definitions
14+
UserInput = Definition(name="UserInput", primitive="str")
15+
DataToPrint = Definition(name="DataToPrint", primitive="str")
16+
17+
AcceptUserInput = Operation(
18+
name="AcceptUserInput",
19+
inputs={},
20+
outputs={"InputData": UserInput},
21+
conditions=[],
22+
)
23+
24+
25+
class AcceptUserInputContext(OperationImplementationContext):
26+
@staticmethod
27+
def receive_input():
28+
return input()
29+
30+
async def run(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
31+
user_input = await self.parent.loop.run_in_executor(
32+
self.parent.pool, self.receive_input
33+
)
34+
return {"InputData": {"data": user_input}}
35+
36+
37+
class AcceptUserInput(OperationImplementation):
38+
"""
39+
Accept input from stdin using python input()
40+
41+
Parameters
42+
++++++++++
43+
inputs : dict
44+
A dictionary with a key and empty list as value.
45+
46+
Returns
47+
+++++++
48+
dict
49+
A dictionary containing user input.
50+
51+
Examples
52+
++++++++
53+
54+
The following example shows how to use AcceptUserInput.
55+
>>> dataflow = DataFlow.auto(AcceptUserInput, GetSingle)
56+
>>> dataflow.seed.append(
57+
... Input(
58+
... value=[AcceptUserInput.op.outputs["InputData"].name],
59+
... definition=GetSingle.op.inputs["spec"]
60+
... )
61+
... )
62+
>>>
63+
>>> async def main():
64+
... async for ctx, results in MemoryOrchestrator.run(dataflow, {"input":[]}):
65+
... print(results)
66+
>>>
67+
>>> asyncio.run(main())
68+
{'UserInput': {'data': 'Data flow is awesome'}}
69+
"""
70+
71+
op = AcceptUserInput
72+
CONTEXT = AcceptUserInputContext
73+
74+
def __init__(self, *args, **kwargs):
75+
super().__init__(*args, **kwargs)
76+
self.loop = None
77+
self.pool = None
78+
self.__pool = None
79+
80+
async def __aenter__(self) -> "OperationImplementationContext":
81+
self.loop = asyncio.get_event_loop()
82+
self.pool = concurrent.futures.ThreadPoolExecutor()
83+
self.__pool = self.pool.__enter__()
84+
return self
85+
86+
async def __aexit__(self, exc_type, exc_value, traceback):
87+
self.__pool.__exit__(exc_type, exc_value, traceback)
88+
self.__pool = None
89+
self.pool = None
90+
self.loop = None
91+
92+
93+
@op(
94+
inputs={"data": DataToPrint}, outputs={}, conditions=[],
95+
)
96+
async def print_output(data: str):
97+
"""
98+
Print the output on stdout using python print()
99+
100+
Parameters
101+
++++++++++
102+
inputs : list
103+
A list of Inputs whose value is to be printed.
104+
105+
Examples
106+
++++++++
107+
108+
The following example shows how to use print_output.
109+
>>> dataflow = DataFlow.auto(print_output, GetSingle)
110+
>>> inputs = [
111+
... Input(
112+
... value="print_output example",
113+
... definition=dataflow.definitions["DataToPrint"],
114+
... parents=None,)]
115+
>>>
116+
>>> async def main():
117+
... async for ctx, results in MemoryOrchestrator.run(dataflow, inputs):
118+
... print("String to be printed is 'print_output example'")
119+
>>>
120+
>>> asyncio.run(main())
121+
print_output example
122+
String to be printed is 'print_output example'
123+
"""
124+
print("\n" + data)

‎model/tensorflow_hub/tests/test_tfhub_integration.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88
import pathlib
99
import contextlib
1010

11-
1211
from dffml.cli.cli import CLI
12+
from dffml.df.types import DataFlow
13+
from dffml.service.dev import Develop
1314
from dffml.util.asynctestcase import IntegrationCLITestCase
1415

1516

@@ -118,3 +119,31 @@ async def test_run(self):
118119
self.assertIn("value", results)
119120
results = results["value"]
120121
self.assertIn(results, [0, 1])
122+
123+
# Make prediction using dffml.operations.predict
124+
results = await Develop.cli(
125+
"run",
126+
"-log",
127+
"debug",
128+
"dffml.operation.model:model_predict",
129+
"-features",
130+
json.dumps({"text": "My dog is awesome"}),
131+
"-config-model",
132+
"text_classifier",
133+
"-config-model-features",
134+
"text:str:1",
135+
"-config-model-predict",
136+
"sentiment:int:1",
137+
"-config-model-classifications",
138+
"0",
139+
"1",
140+
"-config-model-clstype",
141+
"int",
142+
)
143+
self.assertIn("model_predictions", results)
144+
results = results["model_predictions"]
145+
self.assertIn("sentiment", results)
146+
results = results["sentiment"]
147+
self.assertIn("value", results)
148+
results = results["value"]
149+
self.assertIn(results, [0, 1])

‎model/transformers/tests/test_ner_integration.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import contextlib
66

77
from dffml.cli.cli import CLI
8+
from dffml.df.types import DataFlow
9+
from dffml.service.dev import Develop
810
from dffml.util.asynctestcase import IntegrationCLITestCase
911

1012

@@ -143,3 +145,33 @@ async def test_run(self):
143145
self.assertIn("value", results)
144146
results = results["value"]
145147
self.assertTrue(results is not None)
148+
149+
# Make prediction using dffml.operations.predict
150+
results = await Develop.cli(
151+
"run",
152+
"-log",
153+
"debug",
154+
"dffml.operation.model:model_predict",
155+
"-features",
156+
json.dumps({"A": 1, "B": "Jack works at Yahoo"}),
157+
"-config-model",
158+
"ner_tagger",
159+
"-config-model-sid",
160+
"A:int:1",
161+
"-config-model-words",
162+
"B:str:1",
163+
"-config-model-predict",
164+
"target:str:1",
165+
"-config-model-model_architecture_type",
166+
"bert",
167+
"-config-model-model_name_or_path",
168+
"bert-base-cased",
169+
"-config-model-no_cuda",
170+
)
171+
self.assertIn("model_predictions", results)
172+
results = results["model_predictions"]
173+
self.assertIn("target", results)
174+
results = results["target"]
175+
self.assertIn("value", results)
176+
results = results["value"]
177+
self.assertTrue(results is not None)

‎setup.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@
9494
"dffml.dataflow.run = dffml.operation.dataflow:run_dataflow",
9595
# Model
9696
"dffml.model.predict = dffml.operation.model:model_predict",
97+
# io
98+
"AcceptUserInput = dffml.operation.io:AcceptUserInput",
99+
"print_output = dffml.operation.io:print_output",
97100
],
98101
"dffml.kvstore": ["memory = dffml.df.memory:MemoryKeyValueStore"],
99102
"dffml.input.network": ["memory = dffml.df.memory:MemoryInputNetwork"],

‎tests/operation/test_io.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import io
2+
import builtins
3+
import contextlib
4+
from unittest import mock
5+
6+
from dffml.df.base import op
7+
from dffml.operation.output import GetSingle
8+
from dffml.df.memory import MemoryOrchestrator
9+
from dffml.util.asynctestcase import AsyncTestCase
10+
from dffml.df.types import DataFlow, Input, Definition
11+
from dffml.operation.io import AcceptUserInput, print_output
12+
13+
14+
class TestInputOutput(AsyncTestCase):
15+
async def setUp(self):
16+
super().setUp()
17+
self.stdout = io.StringIO()
18+
InputDataflow = DataFlow(
19+
operations={
20+
"AcceptUserInput": AcceptUserInput.op,
21+
"get_single": GetSingle.imp.op,
22+
},
23+
seed=[
24+
Input(
25+
value=[AcceptUserInput.op.outputs["InputData"].name],
26+
definition=GetSingle.op.inputs["spec"],
27+
)
28+
],
29+
implementations={AcceptUserInput.op.name: AcceptUserInput},
30+
)
31+
32+
OutputDataflow = DataFlow(
33+
operations={
34+
"print_output": print_output.op,
35+
"get_single": GetSingle.imp.op,
36+
},
37+
implementations={print_output.op.name: print_output.imp},
38+
)
39+
40+
self.InputDataflow = InputDataflow
41+
self.OutputDataflow = OutputDataflow
42+
43+
async def test_AcceptUserInput(self):
44+
test_inputs = {"testInput": []}
45+
async with MemoryOrchestrator.withconfig({}) as orchestrator:
46+
async with orchestrator(self.InputDataflow) as octx:
47+
with mock.patch(
48+
"builtins.input", return_value="Testing AcceptUserInput"
49+
):
50+
async for ctx_str, results in octx.run(test_inputs):
51+
self.assertIn("UserInput", results)
52+
self.assertEqual(
53+
"Testing AcceptUserInput",
54+
results["UserInput"]["data"],
55+
)
56+
57+
async def test_print_output(self):
58+
test_inputs = [
59+
Input(
60+
value="Testing print_output",
61+
definition=self.OutputDataflow.definitions["DataToPrint"],
62+
parents=None,
63+
)
64+
]
65+
async with MemoryOrchestrator.withconfig({}) as orchestrator:
66+
async with orchestrator(self.OutputDataflow) as octx:
67+
with contextlib.redirect_stdout(self.stdout):
68+
async for ctx_str, _ in octx.run(test_inputs):
69+
results = self.stdout.getvalue()
70+
self.assertIn("Testing print_output", results)

0 commit comments

Comments
 (0)