Merge pull request #75 from JosepSampe/pywren-cf

Install script to use latest release and not master branch

Author: gilv

README.md

@@ -287,7 +287,7 @@ result = pw.get_result()
 | method | method signature |
 |---| ---| 
 | `pw.map_reduce`(`my_map_function`, `iterdata`, `my_reduce_function`, `chunk_size`)| `iterdata` contains list of objects in the format of `bucket_name/object_name` |
-| `my_map_function(key, data_stream)` | `key` is an entry from `iterdata` that is assigned to the invocation|
+| `my_map_function`(`key`, `data_stream`) | `key` is an entry from `iterdata` that is assigned to the invocation|
 
 #### `map_reduce` where partitioner gets entire bucket
 
@@ -315,13 +315,12 @@ pw.map_reduce(my_map_function, bucket_name, my_reduce_function, chunk_size)
 result = pw.get_result()
 ```
 
-* If `chunk_size=None` then partitioner's granularity is a single object . 
-* `ibm_cos` is ibm_boto3 client instance. Can be used to access COS for aditional operations. See [boto3_client](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client) for allowed operations
+* If `chunk_size=None` then partitioner's granularity is a single object. 
 
 | method | method signature |
 |---| ---| 
-| `pw.map_reduce`(`my_map_function`, `bucket_name `, `my_reduce_function`, `chunk_size`, `ibm_cos`)| `bucket_name ` contains the name of the bucket |
-| `my_map_function(bucket, key, data_stream)` | `key` is a data object from bucket `bucket` that is assigned to the invocation|
+| `pw.map_reduce`(`my_map_function`, `bucket_name`, `my_reduce_function`, `chunk_size`)| `bucket_name` contains the name of the bucket |
+| `my_map_function`(`bucket`, `key`, `data_stream`, `ibm_cos`) | `key` is a data object from `bucket` that is assigned to the invocation. `ibm_cos` is an optional parameter which provides a `boto3_client` (see [here](#geting-boto3-client-from-any-map-function))|
 
 
 
@@ -352,7 +351,7 @@ result = pw.get_result()
 | method | method signature |
 |---| ---| 
 | `pw.map_reduce`(`my_map_function`, `iterdata`, `my_reduce_function`, `chunk_size`)| `iterdata` contains list of objects in the format of `http://myurl/myobject.data` |
-| `my_map_function(url, data_stream)` | `url` is an entry from `iterdata` that is assigned to the invocation|
+| `my_map_function`(`url`, `data_stream`) | `url` is an entry from `iterdata` that is assigned to the invocation|
 
 ### Reducer granularity			
 By default there will be one reducer for all the objects. If you need one reducer for each object, you must set the parameter
@@ -364,25 +363,22 @@ pw.map_reduce(my_map_function, bucket_name, my_reduce_function,
 ```
 
 ### Geting boto3 client from any map function
-Any map function can get `ibm_cos` which is [boto3_client](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client). This allows you to access your IBM COS account from any map function
-
-For example
-
-	import pywren_ibm_cloud as pywren
-	import os
-
-	iterdata = [1, 2, 3, 4]
+Any map function can get `ibm_cos` parameter which is [boto3_client](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client). This allows you to access your IBM COS account from any map function, for example:
+    
+```python
+import pywren_ibm_cloud as pywren
 
-	def my_map_function(x, ibm_cos):
-	    res = ibm_cos.get_object(Bucket = ‘mybucket’, Key = ‘mydata.data’)
-	    return x + 7
+iterdata = [1, 2, 3, 4]
 
-	if __name__ == '__main__':
-	    pw = pywren.ibm_cf_executor()
-	    futures = pw.map(my_map_function, iterdata)
-	    results = pw.get_result()
-	    print (results)
+def my_map_function(x, ibm_cos):
+    data_object = ibm_cos.get_object(Bucket='mybucket', Key='mydata.data')
+    # Do some process over the object
+    return x + 7
 
+pw = pywren.ibm_cf_executor()
+pw.map(my_map_function, iterdata)
+result = pw.get_result()
+```
 
 ## PyWren and IBM Watson Studio
 You can use PyWren inside an **IBM Watson Studio** notebook in order to execute parallel data analytics by using **IBM Cloud functions**.
@@ -400,11 +396,12 @@ except:
 ```
 
 ### Deploy PyWren runtime to your IBM Cloud Functions
-You can create PyWren runtime from the notebook itself
-
-	from pywren_ibm_cloud.deployutil import clone_runtime
-	clone_runtime('<dockerhub_space>/<name>:<version>', config, 'pywren-ibm-cloud-master')
+You can create PyWren runtime from the notebook itself:
 
+```python
+from pywren_ibm_cloud.deployutil import clone_runtime
+clone_runtime('<dockerhub_space>/<name>:<version>', config, 'pywren-ibm-cloud')
+```
 
 ## Additional resources
 

install_pywren.sh

@@ -1,7 +1,9 @@
 #!/bin/sh
-echo "Installing PyWren for IBM Cloud Functions..."
-rm -rf pywren-ibm-cloud.zip > /dev/null
-wget --no-check-certificate 'https://github.com/pywren/pywren-ibm-cloud/archive/master.zip' -O pywren-ibm-cloud.zip 2> /dev/null
-rm -rf pywren-ibm-cloud-master; unzip  pywren-ibm-cloud.zip > /dev/null
-cd pywren-ibm-cloud-master/pywren; pip install -U . > /dev/null
+LAST_RELEASE=`wget -q --no-check-certificate https://api.github.com/repos/pywren/pywren-ibm-cloud/releases/latest -O - | grep '"tag_name":' |  sed -E 's/.*"([^"]+)".*/\1/'`
+echo "Installing PyWren for IBM Cloud Functions - Release $LAST_RELEASE ..."
+rm -rf pywren-ibm-cloud* > /dev/null
+wget --no-check-certificate 'https://github.com/pywren/pywren-ibm-cloud/archive/'$LAST_RELEASE'.zip' -O pywren-ibm-cloud.zip 2> /dev/null
+unzip  pywren-ibm-cloud.zip > /dev/null
+mv pywren-ibm-cloud-$LAST_RELEASE pywren-ibm-cloud
+cd pywren-ibm-cloud/pywren; pip install -U . > /dev/null
 echo "done!"

pywren/pywren_ibm_cloud/deployutil.py

@@ -15,16 +15,16 @@
 #
 
 import os
-import sys
 from shutil import copyfile
 from pywren_ibm_cloud import wrenconfig
 from pywren_ibm_cloud.storage import storage
 from pywren_ibm_cloud.cf_connector import CloudFunctions
 from pywren_ibm_cloud.wrenconfig import CF_ACTION_NAME_DEFAULT
 
-def create_zip_action(pywren_location = None):
-    # starts from pywren-ibm-cloud-master/runtime
-    # we can start from pywren-ibm-cloud-master
+
+def create_zip_action(pywren_location=None):
+    # starts from pywren-ibm-cloud/runtime
+    # we can start from pywren-ibm-cloud
     if pywren_location is None:
         prefix = ".."
     else:
@@ -42,7 +42,7 @@ def create_zip_action(pywren_location = None):
     os.remove(prefix + '/pywren/__main__.py')
 
 
-def extract_modules(image_name, config = None, pywren_location = None):
+def extract_modules(image_name, config=None, pywren_location=None):
     # Extract installed Python modules from docker image
     # And store them into storage
 
@@ -79,7 +79,7 @@ def extract_modules(image_name, config = None, pywren_location = None):
     #sys.stdout = sys.__stdout__
 
 
-def create_blackbox_runtime(image_name, config = None, pywren_location = None):
+def create_blackbox_runtime(image_name, config=None, pywren_location=None):
     # Create runtime_name from image_name
     username, appname = image_name.split('/')
     runtime_name = appname.replace(':', '_')
@@ -102,7 +102,8 @@ def create_blackbox_runtime(image_name, config = None, pywren_location = None):
         cf_client.create_action(runtime_name, memory=512, timeout=600000,
                                 code=action_bin, kind='blackbox', image=image_name)
 
-def clone_runtime(image_name, config = None, pywren_location = None):
+
+def clone_runtime(image_name, config=None, pywren_location=None):
 
     print('Cloning docker image {}'.format(image_name))
     create_zip_action(pywren_location)
@@ -111,7 +112,8 @@ def clone_runtime(image_name, config = None, pywren_location = None):
 
     print('All done!')
 
-def default(config = None, pywren_location = None):
+
+def default(config=None, pywren_location=None):
     print('Updating runtime {}'.format(CF_ACTION_NAME_DEFAULT))
     if config is None:
         config = wrenconfig.default()
@@ -133,4 +135,3 @@ def default(config = None, pywren_location = None):
         cf_client = CloudFunctions(config['ibm_cf'])
         runtime_name = CF_ACTION_NAME_DEFAULT
         cf_client.create_action(runtime_name, memory=512, timeout=600000, code=action_bin)
-