dynaml-notebook: WIP to zeppelin interpreter

 - Needs to be continued

Author: mandar2812

conf/shiro.ini

@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[users]
+# List of users with their password allowed to access Zeppelin.
+# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
+admin = password1, admin
+
+
+# Sample LDAP configuration, for user Authentication, currently tested for single Realm
+[main]
+### A sample for configuring Active Directory Realm
+#activeDirectoryRealm = org.apache.zeppelin.realm.ActiveDirectoryGroupRealm
+#activeDirectoryRealm.systemUsername = userNameA
+
+#use either systemPassword or hadoopSecurityCredentialPath, more details in http://zeppelin.apache.org/docs/latest/security/shiroauthentication.html
+#activeDirectoryRealm.systemPassword = passwordA
+#activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/zeppelin.jceks
+#activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
+#activeDirectoryRealm.url = ldap://ldap.test.com:389
+#activeDirectoryRealm.groupRolesMap = "CN=admin,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"admin","CN=finance,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"finance","CN=hr,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"hr"
+#activeDirectoryRealm.authorizationCachingEnabled = false
+
+### A sample for configuring LDAP Directory Realm
+#ldapRealm = org.apache.zeppelin.realm.LdapGroupRealm
+## search base for ldap groups (only relevant for LdapGroupRealm):
+#ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
+#ldapRealm.contextFactory.url = ldap://ldap.test.com:389
+#ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
+#ldapRealm.contextFactory.authenticationMechanism = simple
+
+### A sample PAM configuration
+#pamRealm=org.apache.zeppelin.realm.PamRealm
+#pamRealm.service=sshd
+
+### A sample for configuring ZeppelinHub Realm
+#zeppelinHubRealm = org.apache.zeppelin.realm.ZeppelinHubRealm
+## Url of ZeppelinHub
+#zeppelinHubRealm.zeppelinhubUrl = https://www.zeppelinhub.com
+#securityManager.realms = $zeppelinHubRealm
+
+sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
+
+### If caching of user is required then uncomment below lines
+#cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager
+#securityManager.cacheManager = $cacheManager
+
+securityManager.sessionManager = $sessionManager
+# 86,400,000 milliseconds = 24 hour
+securityManager.sessionManager.globalSessionTimeout = 86400000
+shiro.loginUrl = /api/login
+
+[roles]
+role1 = *
+role2 = *
+role3 = *
+admin = *
+
+[urls]
+# This section is used for url-based security.
+# You can secure interpreter, configuration and credential information by urls. Comment or uncomment the below urls that you want to hide.
+# anon means the access is anonymous.
+# authc means Form based Auth Security
+# To enfore security, comment the line below and uncomment the next one
+/api/version = anon
+#/api/interpreter/** = authc, roles[admin]
+#/api/configurations/** = authc, roles[admin]
+#/api/credential/** = authc, roles[admin]
+#/** = anon
+/** = authc

conf/zeppelin-site.xml

@@ -99,7 +99,7 @@
 
     <property>
         <name>zeppelin.interpreter.dir</name>
-        <value>lib</value>
+        <value>interpreter</value>
         <description>Interpreter implementation base directory</description>
     </property>
 
@@ -111,7 +111,7 @@
 
     <property>
         <name>zeppelin.interpreters</name>
-        <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,io.github.mandar2812.dynaml.zeppelin.DynaMLZeppelinInterpreter</value>
+        <value>org.apache.zeppelin.spark.SparkInterpreter,io.github.mandar2812.dynaml.zeppelin.DynaMLZeppelinInterpreter</value>
     </property>
 
     <property>

dynaml-notebook/src/main/scala-2.11/io/github/mandar2812/dynaml/zeppelin/DynaMLZeppelinInterpreter.scala

@@ -6,8 +6,8 @@ import java.util.Properties
 import ammonite.interp.{Parsers, Preprocessor}
 import ammonite.repl.Repl
 import ammonite.runtime.{History, Storage}
-import ammonite.util.{Name, Res}
-import ammonite.util.Util.CodeSource
+import ammonite.util._
+import ammonite.util.Util.{CodeSource, VersionedWrapperId}
 import fastparse.core.Parsed
 import io.github.mandar2812.dynaml.DynaZeppelin
 import io.github.mandar2812.dynaml.repl.{Defaults, DynaMLInterpreter}
@@ -22,7 +22,9 @@ class DynaMLZeppelinInterpreter(properties: Properties) extends Interpreter(prop
 
   protected val outputBuffer = new ByteArrayOutputStream()
 
-  protected val dynaml_instance = new DynaZeppelin(outputStream = outputBuffer)
+  protected val errorBuffer = new ByteArrayOutputStream()
+
+  protected val dynaml_instance = new DynaZeppelin(outputStream = outputBuffer, errorStream = errorBuffer)
 
   protected var CURRENT_LINE : Int = 0
 
@@ -66,7 +68,7 @@ class DynaMLZeppelinInterpreter(properties: Properties) extends Interpreter(prop
   override def interpret(s: String, interpreterContext: InterpreterContext) = {
     addHistory(s)
 
-    val wrapperName = Name("cmd" + CURRENT_LINE)
+    /*val wrapperName = Name("cmd" + CURRENT_LINE)
     val fileName = wrapperName.encoded + ".sc"
     val result = for {
       blocks <- Preprocessor.splitScript(ammonite.interp.Interpreter.skipSheBangLine(s), fileName)
@@ -106,14 +108,66 @@ class DynaMLZeppelinInterpreter(properties: Properties) extends Interpreter(prop
       new InterpreterResult(InterpreterResult.Code.SUCCESS, output)
     } else  {
       new InterpreterResult(InterpreterResult.Code.ERROR, result.toString)
+    }*/
+
+    Parsers.Splitter.parse(s) match {
+      case Parsed.Success(value, idx) =>
+        val computation_output = dynaml_interp.processLine(s, value, CURRENT_LINE, false, () => CURRENT_LINE += 1)
+        val output = outputBuffer.toString(Charset.defaultCharset())
+        val error = errorBuffer.toString(Charset.defaultCharset())
+
+        if(computation_output.isSuccess) {
+          outputBuffer.reset()
+          new InterpreterResult(InterpreterResult.Code.SUCCESS, output)
+        } else {
+          errorBuffer.reset()
+          new InterpreterResult(InterpreterResult.Code.ERROR, "Syntax Error Mofo!")
+        }
+
+      case Parsed.Failure(_, index, extra) =>
+        new InterpreterResult(InterpreterResult.Code.ERROR, fastparse.core.ParseError.msg(extra.input, extra.traced.expected, index))
     }
+
+
   }
 
   def evaluate(s: String) = {
     addHistory(s)
 
     val wrapperName = Name("cmd" + CURRENT_LINE)
     val fileName = wrapperName.encoded + ".sc"
+
+    def compileRunBlock(
+      leadingSpaces: String, hookInfo: ImportHookInfo,
+      codeSource: CodeSource,
+      eval: (Preprocessor.Output, Name) => Res[(Evaluated, Tag)],
+      indexedWrapperName: Name,
+      wrapperIndex: Int = 1) = {
+
+      val printSuffix = if (wrapperIndex == 1) "" else  " #" + wrapperIndex
+      dynaml_interp.printer.info("Compiling " + codeSource.printablePath + printSuffix)
+      for{
+        processed <- dynaml_interp.compilerManager.preprocess(codeSource.fileName).transform(
+          hookInfo.stmts,
+          "",
+          leadingSpaces,
+          codeSource.pkgName,
+          indexedWrapperName,
+          dynaml_interp.predefImports ++ dynaml_interp.frameImports ++ hookInfo.imports,
+          _ => "scala.Iterator[String]()",
+          extraCode = "",
+          skipEmpty = false
+        )
+
+        (ev, tag) <- eval(processed, indexedWrapperName)
+      } yield ScriptOutput.BlockMetadata(
+        VersionedWrapperId(ev.wrapper.map(_.encoded).mkString("."), tag),
+        leadingSpaces,
+        hookInfo,
+        ev.imports
+      )
+    }
+
     val result = for {
       blocks <- Preprocessor.splitScript(ammonite.interp.Interpreter.skipSheBangLine(s), fileName)
 
@@ -124,7 +178,26 @@ class DynaMLZeppelinInterpreter(properties: Properties) extends Interpreter(prop
         Some(dynaml_interp.wd/"(console)")
       )
 
-      metadata <- dynaml_interp.processAllScriptBlocks(
+      indexedWrapperName = ammonite.interp.Interpreter.indexWrapperName(codeSource.wrapperName, 1)
+
+
+      allSplittedChunks <- Res.Success(blocks)
+      (leadingSpaces, stmts) = allSplittedChunks(1 - 1)
+      (hookStmts, importTrees) = dynaml_interp.parseImportHooks(codeSource, stmts)
+      hookInfo <- dynaml_interp.resolveImportHooks(importTrees, hookStmts, codeSource)
+
+      res <- compileRunBlock(leadingSpaces, hookInfo, codeSource,
+        (processed, indexedWrapperName) =>
+          dynaml_interp.evaluateLine(
+            processed, dynaml_interp.printer, fileName,
+            indexedWrapperName, silent = false, () => CURRENT_LINE += 1),
+        indexedWrapperName
+      )
+
+
+
+
+      /*metadata <- dynaml_interp.processAllScriptBlocks(
         blocks.map(_ => None),
         Res.Success(blocks),
         dynaml_interp.predefImports ++ dynaml_interp.frameImports,
@@ -135,19 +208,19 @@ class DynaMLZeppelinInterpreter(properties: Properties) extends Interpreter(prop
             indexedWrapperName, silent = false, () => CURRENT_LINE += 1),
         autoImport = true,
         ""
-      )
+      )*/
     } yield {
-      metadata
+      res
     }
 
     if(result.isSuccess) {
 
       val output = outputBuffer.toString(Charset.defaultCharset())
 
       outputBuffer.reset()
-      val resStr = result.flatMap(d => {
+      /*val resStr = result.flatMap(d => {
         Res(Some(d.blockInfo.map(blockm => blockm.finalImports.value.map(d => d.fromName.raw).mkString("\n")).mkString("\n")), "")
-      })
+      })*/
 
       output
     } else  {