Merge pull request #8 from ibm-cds-labs/adding_r_scala_platform

Create sparklyr based R package

Author: G Adam Cox

r/sparklyr/DESCRIPTION

@@ -0,0 +1,13 @@
+Package: ibmos2sparklyr
+Title: Loads Object Store data into Softlayer and Bluemix
+Version: 0.0.7
+Authors@R: person("Jim", "Crozier", email = "[email protected]",
+                  role = c("aut", "cre"))
+Description: Loads data from Object Store in Softlayer and Bluemix with spark context built on sparklyr. 
+Depends:
+    R (>= 3.1.0)
+License: Apache
+LazyData: true
+RoxygenNote: 5.0.1
+Imports: 
+  sparklyr

r/LICENSE renamed to r/sparklyr/LICENSE

@@ -0,0 +1 @@
+exportPattern("^[[:alpha:]]+")

r/sparklyr/NAMESPACE

@@ -0,0 +1,139 @@
+setOldClass("jobj")
+
+
+swifturl = function(name, container_name, object_name){
+  return(paste0('swift2d://',container_name,'.',name,'/',object_name))
+}
+
+
+
+#' sparkcontext is a SparkContext object.
+#'
+#' name is a string that identifies this configuration. You can
+#'     use any string you like. This allows you to create
+#'     multiple configurations to different Object Storage accounts.
+#' auth_url, username and password are string credentials for your
+#' Softlayer Object Store
+#' @export softlayer
+#' @exportClass softlayer
+
+softlayer <- setRefClass("softlayer",
+  fields=list(name="character", container_name="character", object_name="character",
+             sparkcontext='jobj', auth_url="character",
+              tenant = "character", username="character", password="character"),
+  methods=list(initialize =
+    function( sparkcontext, name, auth_url, tenant, username, password,public=FALSE,
+              swift2d_driver='com.ibm.stocator.fs.ObjectStoreFileSystem'){
+
+        .self$name = name
+        # get spark_context
+        ctx <- spark_context(sparkcontext)
+
+        #set the java spark context
+        jsc <- invoke_static(
+          sparkcontext,
+          "org.apache.spark.api.java.JavaSparkContext",
+          "fromSparkContext",
+          ctx
+        )
+
+        #set the swift configs:
+        prefix = paste("fs.swift2d.service" , name, sep =".")
+        hconf = jsc %>% invoke("hadoopConfiguration")
+        hconf %>% invoke("set", "fs.swift2d.impl", swift2d_driver)
+        hconf %>% invoke("set", paste(prefix, "auth.url", sep='.'), auth_url)
+        hconf %>% invoke("set", paste(prefix, "username", sep='.'), username)
+        hconf %>% invoke("set", paste(prefix, "tenant", sep='.'), tenant)
+        hconf %>% invoke("set", paste(prefix, "auth.endpoint.prefix", sep='.'), "endpoints")
+        hconf %>% invoke("set", paste(prefix, "auth.method", sep='.'), "swiftauth")
+        hconf %>% invoke("set", paste(prefix, "http.port", sep='.'), "8080")
+        hconf %>% invoke("set", paste(prefix, "apikey", sep='.'), password)
+        invisible(hconf %>% invoke("setBoolean", paste(prefix, "public", sep='.'), public))
+        hconf %>% invoke("set", paste(prefix, "use.get.auth", sep='.'), "true")
+        invisible(hconf %>% invoke("setBoolean", paste(prefix, "location-aware", sep='.'), FALSE))
+        hconf %>% invoke("set", paste(prefix, "password", sep='.'), password)
+
+
+
+    },
+
+        url = function(container_name, object_name){
+        return(swifturl(name, container_name, object_name))}
+    )
+)
+
+
+
+#' sparkcontext:  a SparkContext object.
+#'
+#' credentials:  a dictionary with the following required keys:
+#'
+#'   auth_url
+#'   project_id (or projectId)
+#'   user_id (or userId)
+#'   password
+#'   region
+#' and optional key:
+#'   name  #[to be deprecated] The name of the configuration.
+#' name:  string that identifies this configuration. You can
+#'     use any string you like. This allows you to create
+#'     multiple configurations to different Object Storage accounts.
+#'     This is not required at the moment, since credentials['name']
+#'     is still supported.
+#' When using this from a IBM Spark service instance that
+#' is configured to connect to particular Bluemix object store
+#' instances, the values for these credentials can be obtained
+#' by clicking on the 'insert to code' link just below a data
+#' source.
+#' @export bluemix
+#' @exportClass bluemix
+
+
+bluemix <- setRefClass("bluemix",
+  fields=list(name="character", credentials = "list",
+             sparkcontext='jobj', public = "character"),
+  methods=list(initialize =
+    function(..., sparkcontext, name=NULL, credentials,
+             public=FALSE,swift2d_driver='com.ibm.stocator.fs.ObjectStoreFileSystem'){
+
+      callSuper(...,credentials=credentials)
+
+      if ( is.null(name)) name <<- credentials["name"][[1]]
+
+      user_id = try( credentials['user_id'][[1]])
+      if(class(user_id)=="try-error")  user_id = credentials['userId'][[1]]
+
+      tenant = try( credentials['project_id'][[1]])
+      if(class(tenant)=="try-error")  tenant = credentials['projectId'][[1]]
+
+        .self$name = name
+        # get spark_context
+        ctx <- spark_context(sparkcontext)
+
+        #set the java spark context
+        jsc <- invoke_static(
+          sparkcontext,
+          "org.apache.spark.api.java.JavaSparkContext",
+          "fromSparkContext",
+          ctx
+        )
+
+        #set the swift configs:
+        prefix = paste("fs.swift2d.service" , name, sep =".")
+        hconf = jsc %>% invoke("hadoopConfiguration")
+        hconf %>% invoke("set", "fs.swift2d.impl", swift2d_driver)
+        hconf %>% invoke("set", paste(prefix, "auth.url", sep='.'), paste(credentials['auth_url'][[1]],"/v3/auth/tokens",sep=""))
+        hconf %>% invoke("set", paste(prefix, "auth.endpoint.prefix", sep='.'), "endpoints")
+        hconf %>% invoke("set", paste(prefix, "tenant", sep='.'), tenant)
+        hconf %>% invoke("set", paste(prefix, "username", sep='.'), user_id)
+        hconf %>% invoke("set", paste(prefix, "password", sep='.'), credentials['password'][[1]])
+        hconf %>% invoke("set", paste(prefix, "auth.method", sep='.'), "keystoneV3")
+        hconf %>% invoke("set", paste(prefix, "region", sep='.'), credentials['region'][[1]])
+        invisible(hconf %>% invoke("setBoolean", paste(prefix, "public", sep='.'), public))
+        #invisible(SparkR:::callJMethod(hConf, "setInt", paste(prefix, "http.port", sep='.'), 8080))
+          },
+
+        url = function( container_name, object_name){
+        return(swifturl(name, container_name, object_name))}
+    )
+)

r/sparklyr/R/osconfig.R

@@ -0,0 +1,89 @@
+# ibmos2sparklyr
+
+The package sets Spark Hadoop configurations for connecting to 
+IBM Bluemix Object Storage and Softlayer Account Object Storage instances
+with the swift protocol. This packages uses the new 
+[swift2d/stocator](https://github.com/SparkTC/stocator) protocol, availble
+on the latest IBM Spark Service instances and through IBM Data Science Experience (DSX). 
+
+This package expects a SparkContext instantiated by sparklyr. It has been tested
+to work with IBM RStudio from DSX, though it should work with other Spark
+installations that utilize the [swift2d/stocator](https://github.com/SparkTC/stocator).
+
+## Installation 
+
+    library(devtools)
+    devtools::install_url("https://github.com/ibm-cds-labs/ibmos2spark/archive/<version>.zip", subdir= "r/sparklyr/",dependencies = FALSE)
+
+where `version` should be a tagged release, such as `0.0.7`. (If you're daring, you can use `master`.)
+
+###### WARNING
+
+In IBM Data Science Experience, please be sure to include the `dependencies = FALSE` flag when 
+calling `devtools::install_url`. If you forget to do this, you will most likely overwrite DSX's 
+special flavor of sparklyr, which will break your connection to IBM Spark Services. To repair this,
+go into your local R repo store ("/home/rstudio/R/x86_64-redhat-linux-gnu-library/RVERSION/") 
+where RVERSION is the newest install of R (currently 3.3) and delete the `sparklyr` folder. 
+After deleting, choose File->Quit Session to refresh your R kernel. These steps will refresh your 
+sparklyr package to the special DSX version. 
+
+## Usage
+
+### Bluemix
+
+    library(ibmos2sparklyr)
+    configurationname = "bluemixOScon" #can be any any name you like (allows for multiple configurations)
+
+    # In DSX notebooks, the "insert to code" will insert this credentials list for you
+    creds = list(
+            auth_url="https://identity.open.softlayer.com",
+            region="dallas", 
+            project_id = "XXXXX", 
+            user_id="XXXXX", 
+            password="XXXXX")
+            
+    bmconfig = bluemix(sparkcontext=sc, name=configurationname, credentials = creds)
+           
+    container = "my_container" # name of your object store container
+    object = "my_data.csv" # name of object that you want to retrieve in the container
+    spark_object_name = "dataFromSwift" # name to assign to the new spark object
+    
+    data = sparklyr::spark_read_csv(sc, spark_object_name,bmconfig$url(container,object))
+
+
+### Softlayer
+
+    library(ibmos2sparklyr)
+    configurationname = "softlayerOScon" #can be any any name you like (allows for multiple configurations)
+
+    slconfig = softlayer(sparkcontext=sc, 
+                 name=configurationname, 
+                 auth_url="https://identity.open.softlayer.com",
+                 tenant = "XXXXX", 
+                 username="XXXXX", 
+                 password="XXXXX"
+           )
+           
+    container = "my_container" # name of your object store container
+    object = "my_data.csv" # name of object that you want to retrieve in the container
+    spark_object_name = "dataFromSwift" # name to assign to the new spark object
+
+    data = sparklyr::spark_read_csv(sc, spark_object_name,slconfig$url(container,object))
+
+
+
+## License 
+
+Copyright 2016 IBM Cloud Data Services
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

r/sparklyr/README.md

@@ -1,4 +1,4 @@
-Package: ibmos2spark
+Package: ibmos2sparkR
 Title: Loads Object Store data into Softlayer and Bluemix
 Version: 0.0.7
 Authors@R: person("Jim", "Crozier", email = "[email protected]",