Skip to content

Commit 96f394f

Browse files
committed
Get last release from installation script
1 parent 8b682df commit 96f394f

File tree

3 files changed

+34
-32
lines changed

3 files changed

+34
-32
lines changed

README.md

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ result = pw.get_result()
287287
| method | method signature |
288288
|---| ---|
289289
| `pw.map_reduce`(`my_map_function`, `iterdata`, `my_reduce_function`, `chunk_size`)| `iterdata` contains list of objects in the format of `bucket_name/object_name` |
290-
| `my_map_function(key, data_stream)` | `key` is an entry from `iterdata` that is assigned to the invocation|
290+
| `my_map_function`(`key`, `data_stream`) | `key` is an entry from `iterdata` that is assigned to the invocation|
291291

292292
#### `map_reduce` where partitioner gets entire bucket
293293

@@ -298,7 +298,7 @@ import pywren_ibm_cloud as pywren
298298

299299
bucket_name = 'my_data_bucket'
300300

301-
def my_map_function(bucket, key, data_stream, ibm_cos):
301+
def my_map_function(bucket, key, data_stream):
302302
for line in data_stream:
303303
# Do some process
304304
return partial_intersting_data
@@ -315,13 +315,12 @@ pw.map_reduce(my_map_function, bucket_name, my_reduce_function, chunk_size)
315315
result = pw.get_result()
316316
```
317317

318-
* If `chunk_size=None` then partitioner's granularity is a single object .
319-
* `ibm_cos` is ibm_boto3 client instance. Can be used to access COS for aditional operations. See [boto3_client](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client) for allowed operations
318+
* If `chunk_size=None` then partitioner's granularity is a single object.
320319

321320
| method | method signature |
322321
|---| ---|
323-
| `pw.map_reduce`(`my_map_function`, `bucket_name `, `my_reduce_function`, `chunk_size`, `ibm_cos`)| `bucket_name ` contains the name of the bucket |
324-
| `my_map_function(bucket, key, data_stream)` | `key` is a data object from bucket `bucket` that is assigned to the invocation|
322+
| `pw.map_reduce`(`my_map_function`, `bucket_name`, `my_reduce_function`, `chunk_size`)| `bucket_name` contains the name of the bucket |
323+
| `my_map_function`(`bucket`, `key`, `data_stream`) | `key` is a data object from bucket `bucket` that is assigned to the invocation|
325324

326325

327326

@@ -352,7 +351,7 @@ result = pw.get_result()
352351
| method | method signature |
353352
|---| ---|
354353
| `pw.map_reduce`(`my_map_function`, `iterdata`, `my_reduce_function`, `chunk_size`)| `iterdata` contains list of objects in the format of `http://myurl/myobject.data` |
355-
| `my_map_function(url, data_stream)` | `url` is an entry from `iterdata` that is assigned to the invocation|
354+
| `my_map_function`(`url`, `data_stream`) | `url` is an entry from `iterdata` that is assigned to the invocation|
356355

357356
### Reducer granularity
358357
By default there will be one reducer for all the objects. If you need one reducer for each object, you must set the parameter
@@ -366,20 +365,20 @@ pw.map_reduce(my_map_function, bucket_name, my_reduce_function,
366365
### Geting boto3 client from any map function
367366
Any map function can get `ibm_cos` which is [boto3_client](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client). This allows you to access your IBM COS account from any map function, for example:
368367

369-
```python
370-
import pywren_ibm_cloud as pywren
368+
```python
369+
import pywren_ibm_cloud as pywren
371370

372-
iterdata = [1, 2, 3, 4]
371+
iterdata = [1, 2, 3, 4]
373372

374-
def my_map_function(x, ibm_cos):
375-
data_object = ibm_cos.get_object(Bucket='mybucket', Key='mydata.data')
376-
# Do some process over the object
377-
return x + 7
373+
def my_map_function(x, ibm_cos):
374+
data_object = ibm_cos.get_object(Bucket='mybucket', Key='mydata.data')
375+
# Do some process over the object
376+
return x + 7
378377

379-
pw = pywren.ibm_cf_executor()
380-
pw.map(my_map_function, iterdata)
381-
result = pw.get_result()
382-
```
378+
pw = pywren.ibm_cf_executor()
379+
pw.map(my_map_function, iterdata)
380+
result = pw.get_result()
381+
```
383382

384383
## PyWren and IBM Watson Studio
385384
You can use PyWren inside an **IBM Watson Studio** notebook in order to execute parallel data analytics by using **IBM Cloud functions**.
@@ -397,11 +396,12 @@ except:
397396
```
398397

399398
### Deploy PyWren runtime to your IBM Cloud Functions
400-
You can create PyWren runtime from the notebook itself
401-
402-
from pywren_ibm_cloud.deployutil import clone_runtime
403-
clone_runtime('<dockerhub_space>/<name>:<version>', config, 'pywren-ibm-cloud-master')
399+
You can create PyWren runtime from the notebook itself:
404400

401+
```python
402+
from pywren_ibm_cloud.deployutil import clone_runtime
403+
clone_runtime('<dockerhub_space>/<name>:<version>', config, 'pywren-ibm-cloud')
404+
```
405405

406406
## Additional resources
407407

install_pywren.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,6 @@ echo "Installing PyWren for IBM Cloud Functions - Release $LAST_RELEASE ..."
44
rm -rf pywren-ibm-cloud* > /dev/null
55
wget --no-check-certificate 'https://github.com/pywren/pywren-ibm-cloud/archive/'$LAST_RELEASE'.zip' -O pywren-ibm-cloud.zip 2> /dev/null
66
unzip pywren-ibm-cloud.zip > /dev/null
7-
cd pywren-ibm-cloud-$LAST_RELEASE/pywren; pip install -U . > /dev/null
7+
mv pywren-ibm-cloud-$LAST_RELEASE pywren-ibm-cloud
8+
cd pywren-ibm-cloud/pywren; pip install -U . > /dev/null
89
echo "done!"

pywren/pywren_ibm_cloud/deployutil.py

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,16 @@
1515
#
1616

1717
import os
18-
import sys
1918
from shutil import copyfile
2019
from pywren_ibm_cloud import wrenconfig
2120
from pywren_ibm_cloud.storage import storage
2221
from pywren_ibm_cloud.cf_connector import CloudFunctions
2322
from pywren_ibm_cloud.wrenconfig import CF_ACTION_NAME_DEFAULT
2423

25-
def create_zip_action(pywren_location = None):
26-
# starts from pywren-ibm-cloud-master/runtime
27-
# we can start from pywren-ibm-cloud-master
24+
25+
def create_zip_action(pywren_location=None):
26+
# starts from pywren-ibm-cloud/runtime
27+
# we can start from pywren-ibm-cloud
2828
if pywren_location is None:
2929
prefix = ".."
3030
else:
@@ -42,7 +42,7 @@ def create_zip_action(pywren_location = None):
4242
os.remove(prefix + '/pywren/__main__.py')
4343

4444

45-
def extract_modules(image_name, config = None, pywren_location = None):
45+
def extract_modules(image_name, config=None, pywren_location=None):
4646
# Extract installed Python modules from docker image
4747
# And store them into storage
4848

@@ -79,7 +79,7 @@ def extract_modules(image_name, config = None, pywren_location = None):
7979
#sys.stdout = sys.__stdout__
8080

8181

82-
def create_blackbox_runtime(image_name, config = None, pywren_location = None):
82+
def create_blackbox_runtime(image_name, config=None, pywren_location=None):
8383
# Create runtime_name from image_name
8484
username, appname = image_name.split('/')
8585
runtime_name = appname.replace(':', '_')
@@ -102,7 +102,8 @@ def create_blackbox_runtime(image_name, config = None, pywren_location = None):
102102
cf_client.create_action(runtime_name, memory=512, timeout=600000,
103103
code=action_bin, kind='blackbox', image=image_name)
104104

105-
def clone_runtime(image_name, config = None, pywren_location = None):
105+
106+
def clone_runtime(image_name, config=None, pywren_location=None):
106107

107108
print('Cloning docker image {}'.format(image_name))
108109
create_zip_action(pywren_location)
@@ -111,7 +112,8 @@ def clone_runtime(image_name, config = None, pywren_location = None):
111112

112113
print('All done!')
113114

114-
def default(config = None, pywren_location = None):
115+
116+
def default(config=None, pywren_location=None):
115117
print('Updating runtime {}'.format(CF_ACTION_NAME_DEFAULT))
116118
if config is None:
117119
config = wrenconfig.default()
@@ -133,4 +135,3 @@ def default(config = None, pywren_location = None):
133135
cf_client = CloudFunctions(config['ibm_cf'])
134136
runtime_name = CF_ACTION_NAME_DEFAULT
135137
cf_client.create_action(runtime_name, memory=512, timeout=600000, code=action_bin)
136-

0 commit comments

Comments
 (0)