Fn-powered truly-serverless apps with functions (#7)

Fn-powered truly-serverless apps with functions

Author: denismakogon

README.md

@@ -130,6 +130,8 @@ FDK is not only about developing functions, but providing necessary API to build
 that look like nothing but classes with methods powered by Fn.
 
 ```python
+import requests
+
 from fdk.application import decorators
 
 
@@ -139,15 +141,28 @@ class Application(object):
     def __init__(self, *args, **kwargs):
         pass
 
-    @decorators.fn_route(fn_image="denismakogon/os.environ:latest")
+    @decorators.with_fn(fn_image="denismakogon/os.environ:latest")
     def env(self, fn_data=None):
         return fn_data
 
-    @decorators.fn_route(fn_image="denismakogon/py-traceback-test:0.0.1",
-                         fn_format="http")
+    @decorators.with_fn(fn_image="denismakogon/py-traceback-test:0.0.1",
+                        fn_format="http")
     def traceback(self, fn_data=None):
         return fn_data
 
+    @decorators.fn(fn_type="sync")
+    def square(self, x, y, *args, **kwargs):
+        return x * y
+
+    @decorators.fn(fn_type="sync", dependencies={
+        "requests_get": requests.get
+    })
+    def request(self, *args, **kwargs):
+        requests_get = kwargs["dependencies"].get("requests_get")
+        r = requests_get('https://api.github.com/events')
+        r.raise_for_status()
+        return r.text
+
 if __name__ == "__main__":
     app = Application(config={})
 
@@ -161,6 +176,16 @@ if __name__ == "__main__":
         raise err
     print(res)
 
+    res, err = app.square(10, 20)
+    if err:
+        raise err
+    print(res)
+
+    res, err = app.request()
+    if err:
+        raise err
+    print(res)
+
 ```
 In order to identify to which Fn instance code needs to talk set following env var:
 
@@ -191,12 +216,45 @@ Applications powered by Fn: working with function's result
 
 In order to work with result from function you just need to read key-value argument `fn_data`:
 ```python
-    @decorators.fn_route(fn_image="denismakogon/py-traceback-test:0.0.1",
-                         fn_format="http")
+    @decorators.with_fn(fn_image="denismakogon/py-traceback-test:0.0.1",
+                        fn_format="http")
     def traceback(self, fn_data=None):
         return fn_data
 ```
 
+Applications powered by Fn: advanced serverless functions
+---------------------------------------------------------
+
+Since release v0.0.3 developer can consume new API to build truly serverless functions 
+without taking care of Docker images, application, etc.
+
+```python
+    @decorators.fn(fn_type="sync")
+    def square(self, x, y, *args, **kwargs):
+        return x * y
+
+    @decorators.fn(fn_type="sync", dependencies={
+        "requests_get": requests.get
+    })
+    def request(self, *args, **kwargs):
+        requests_get = kwargs["dependencies"].get("requests_get")
+        r = requests_get('https://api.github.com/events')
+        r.raise_for_status()
+        return r.text
+```
+
+Each function decorated with `@decorator.fn` will become truly serverless and distributed.
+So, how it works?
+
+    * A developer writes function
+    * FDK (Fn-powered app) creates a recursive Pickle v4.0 with 3rd-party dependencies
+    * FDK (Fn-powered app) transfers pickled object to a function based on Python3 GPI (general purpose image)
+    * FDK unpickles function and its 3rd-party dependencies and runs it
+    * Function sends response back to Fn-powered application function caller
+
+So, each CPU-intensive functions can be sent to Fn with the only load on networking (given example creates 7kB of traffic between app's host and Fn).
+
+
 Applications powered by Fn: exceptions
 --------------------------------------
 

fdk/application/decorators.py

@@ -12,10 +12,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import dill
 import functools
 import os
 import requests
-
+import types
 
 from fdk.application import errors
 
@@ -69,10 +70,124 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def fn_route(fn_image=None, fn_type=None,
-             fn_memory=256, fn_format=None,
-             fn_timeout=60, fn_idle_timeout=200,
-             fn_method="GET"):
+def fn(fn_type=None, fn_timeout=60, fn_idle_timeout=200, fn_memory=256,
+       dependencies=None):
+    """
+    Runs Python's function on general purpose Fn function
+
+
+    What it does?
+
+     Decorator does following:
+       - collects dat for Fn route and creates it
+       - when function called, that function transforms
+         into byte array (Pickle) then gets sent to general
+         purpose Fn Python3 function
+       - each external dependency (3rd-party libs)
+         that are required for func gets transformed
+         into byte array (Pickle)
+
+     It means that functions does't run locally but on Fn.
+
+    How is it different from other Python FDK functions?
+
+     - This function works with serialized Python callable objects via wire.
+       Each function supplied with set of external dependencies that are
+       represented as serialized functions, no matter if they are module-level,
+       class-level Python objects
+
+    :param fn_type: Fn function call type
+    :type fn_type: str
+    :param fn_timeout: Fn function call timeout
+    :type fn_timeout: int
+    :param fn_idle_timeout: Fn function call idle timeout
+    :type fn_idle_timeout: int
+    :param fn_memory: Fn function memory limit
+    :type fn_memory: int
+    :param dependencies: Python's function 3rd-party callable dependencies
+    :type dependencies: dict
+    :return:
+    """
+    fn_method = "POST"
+    fn_image = "denismakogon/python3-fn-gpi:0.0.1"
+    fn_format = "http"
+    dependencies = dependencies if dependencies else {}
+
+    def ext_wrapper(action):
+        @functools.wraps(action)
+        def inner_wrapper(*f_args, **f_kwargs):
+            fn_api_url = os.environ.get("API_URL")
+            requests.get(fn_api_url).raise_for_status()
+            self = f_args[0]
+            fn_path = action.__name__.lower()
+            if not hasattr(action, "__path_created"):
+                fn_routes_url = "{}/v1/apps/{}/routes".format(
+                    fn_api_url, self.__class__.__name__.lower())
+                resp = requests.post(fn_routes_url, json={
+                    "route": {
+                        "path": "/{}".format(fn_path),
+                        "image": fn_image,
+                        "memory": fn_memory if fn_memory else 256,
+                        "type": fn_type if fn_type else "sync",
+                        "format": fn_format if fn_format else "default",
+                        "timeout": fn_timeout if fn_timeout else 60,
+                        "idle_timeout": (fn_idle_timeout if
+                                         fn_idle_timeout else 120),
+                    },
+                })
+
+                try:
+                    resp.raise_for_status()
+                except requests.HTTPError:
+                    resp.close()
+                    return Exception(resp.content)
+
+                setattr(action, "__path_created", True)
+
+            fn_path = action.__name__.lower()
+            fn_exec_url = "{}/r/{}/{}".format(
+                fn_api_url, self.__class__.__name__.lower(), fn_path)
+
+            action_in_bytes = dill.dumps(action, recurse=True)
+            self_in_bytes = dill.dumps(self, recurse=True)
+
+            for name, method in dependencies.items():
+                dependencies[name] = list(dill.dumps(method, recurse=True))
+
+            f_kwargs.update(dependencies=dependencies)
+            req = requests.Request(
+                method=fn_method, url=fn_exec_url,
+                json={
+                    "is_coroutine": isinstance(action, types.CoroutineType),
+                    "action": list(action_in_bytes),
+                    "self": list(self_in_bytes),
+                    "args": f_args[1:],
+                    "kwargs": f_kwargs,
+                }
+            )
+            session = requests.Session()
+            resp = session.send(req.prepare())
+
+            try:
+                resp.raise_for_status()
+            except requests.HTTPError:
+                resp.close()
+                return None, errors.FnError(
+                    "{}/{}".format(self.__class__.__name__.lower(), fn_path),
+                    resp.content)
+
+            resp.close()
+            return dill.loads(resp.content), None
+
+        return inner_wrapper
+
+    return ext_wrapper
+
+
+def with_fn(fn_image=None, fn_type=None,
+            fn_memory=256, fn_format=None,
+            fn_timeout=60, fn_idle_timeout=200,
+            fn_method="GET"):
     """
     Sets up Fn app route based on parameters given above
     :param fn_image: Docker image
@@ -99,6 +214,9 @@ def inner_wrapper(*f_args, **f_kwargs):
             requests.get(fn_api_url).raise_for_status()
             self = f_args[0]
             fn_path = action.__name__.lower()
+            # TODO(xxx): hate code duplicates but extracting common function
+            # to create a route breaks dill routine somehow
+            # need to figure out how and fix that!
             if not hasattr(action, "__path_created"):
                 fn_routes_url = "{}/v1/apps/{}/routes".format(
                     fn_api_url, self.__class__.__name__.lower())
@@ -119,10 +237,11 @@ def inner_wrapper(*f_args, **f_kwargs):
                     resp.raise_for_status()
                 except requests.HTTPError:
                     resp.close()
-                    return None, Exception(resp.content)
+                    return Exception(resp.content)
 
                 setattr(action, "__path_created", True)
 
+            fn_path = action.__name__.lower()
             fn_exec_url = "{}/r/{}/{}".format(
                 fn_api_url, self.__class__.__name__.lower(), fn_path)
             req = requests.Request(method=fn_method,

fdk/tests/test_application.py

@@ -12,7 +12,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
+import requests
+import testtools
+
 from fdk.application import decorators
+from fdk.application import errors
 
 
 @decorators.fn_app
@@ -21,25 +26,82 @@ class Application(object):
     def __init__(self, *args, **kwargs):
         pass
 
-    @decorators.fn_route(fn_image="denismakogon/os.environ:latest")
+    @decorators.with_fn(fn_image="denismakogon/os.environ:latest")
     def env(self, fn_data=None):
         return fn_data
 
-    @decorators.fn_route(fn_image="denismakogon/py-traceback-test:0.0.1",
-                         fn_format="http")
+    @decorators.with_fn(fn_image="denismakogon/py-traceback-test:0.0.1",
+                        fn_format="http")
     def traceback(self, fn_data=None):
         return fn_data
 
+    @decorators.fn(fn_type="sync")
+    def square(self, x, y, *args, **kwargs):
+        return x * y
+
+    @decorators.fn(fn_type="sync", dependencies={
+        "requests_get": requests.get
+    })
+    def request(self, *args, **kwargs):
+        requests_get = kwargs["dependencies"].get("requests_get")
+        r = requests_get('https://api.github.com/events')
+        r.raise_for_status()
+        return r.text
+
+
+class TestApplication(testtools.TestCase):
+
+    @testtools.skipIf(os.getenv("API_URL") is None,
+                      "API_URL is not set")
+    def test_can_call_with_fn(self):
+        app = Application(config={})
+        res, err = app.env()
+        self.assertIsNone(err, message=str(err))
 
+    @testtools.skipIf(os.getenv("API_URL") is None,
+                      "API_URL is not set")
+    def test_can_get_traceback(self):
+        app = Application(config={})
+        res, err = app.traceback()
+        self.assertIsNotNone(err, message=str(err))
+        self.assertIsInstance(err, errors.FnError)
+
+    # @testtools.skipIf(os.getenv("API_URL") is None,
+    #                   "API_URL is not set")
+    # def test_can_call_fn_simple(self):
+    #     app = Application(config={})
+    #     res, err = app.square(10, 20)
+    #     self.assertIsNone(err, message=str(err))
+    #     self.assertEqual(int(res), 10 * 20)
+    #
+    # @testtools.skipIf(os.getenv("API_URL") is None,
+    #                   "API_URL is not set")
+    # def test_can_call_fn_with_deps(self):
+    #     app = Application(config={})
+    #     res, err = app.request()
+    #     self.assertIsNone(err, message=str(err))
+    #     self.assertNotNone(res)
+
+
+# NOTE: somehow dill pickler doesn't work fine in tests,
+# it gives pretty weird error:
+# _pickle.PicklingError: Can't pickle
+# <class 'fdk.tests.test_application.Application'>:
+# it's not the same object as fdk.tests.test_application.Application
+#
+# That why this module should be tested separately as CLI script
 if __name__ == "__main__":
-    app = Application(config={})
+    if os.getenv("API_URL") is not None:
+        app = Application(config={})
 
-    res, err = app.env()
-    if err:
-        raise err
-    print(res)
+        res, err = app.square(10, 20)
+        if err:
+            raise err
+        print(res)
 
-    res, err = app.traceback()
-    if err:
-        raise err
-    print(res)
+        res, err = app.request()
+        if err:
+            raise err
+        print(res)
+    else:
+        print("API_URL not test, skipping...")

requirements.txt

@@ -1,3 +1,4 @@
 pbr!=2.1.0,>=2.0.0 # Apache-2.0
 ujson==1.35
 requests==2.18.4
+dill==0.2.7.1

samples/python3-fn-gpi/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.6.2
+
+RUN mkdir /code
+ADD . /code/
+WORKDIR /code/
+RUN pip install -r requirements.txt
+
+ENTRYPOINT ["python3", "func.py"]