Merge branch 'PR303'

Author: holgerbrandl

build.gradle.kts

@@ -18,6 +18,7 @@ dependencies {
     compile("com.offbytwo:docopt:0.6.0.20150202")
 
     implementation("org.jetbrains.kotlin:kotlin-scripting-common:$kotlinVersion")
+    implementation("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion")
     implementation("org.jetbrains.kotlin:kotlin-scripting-jvm:$kotlinVersion")
     implementation("org.jetbrains.kotlin:kotlin-scripting-dependencies:$kotlinVersion")
     implementation("org.jetbrains.kotlin:kotlin-scripting-dependencies-maven:$kotlinVersion")

src/main/kotlin/kscript/app/AppHelpers.kt

@@ -2,6 +2,7 @@ package kscript.app
 
 import kscript.app.ShellUtils.requireInPath
 import java.io.*
+import java.net.URI
 import java.net.URL
 import java.nio.file.Files
 import java.nio.file.Paths
@@ -135,6 +136,10 @@ fun createTmpScript(scriptText: String, extension: String = "kts"): File {
     }
 }
 
+fun isFile(uri: URI) = uri.scheme.startsWith("file")
+fun isUrl(uri: URI) = uri.scheme.startsWith("http") || uri.scheme.startsWith("https")
+
+fun isUrl(scriptResource: String) = scriptResource.startsWith("http://") || scriptResource.startsWith("https://")
 
 fun fetchFromURL(scriptURL: String): File {
     val urlHash = md5(scriptURL)
@@ -157,6 +162,24 @@ fun fetchFromURL(scriptURL: String): File {
     return urlCache
 }
 
+fun fetchFromURI(scriptURI: URI): List<String> {
+    if (isFile(scriptURI)) {
+        return scriptURI.toURL().readText().lines()
+    }
+
+    val urlHash = md5(scriptURI.toString())
+    val urlCache = File(KSCRIPT_CACHE_DIR, "/url_cache_${urlHash}")
+
+    if (urlCache.exists()) {
+        return urlCache.readText().lines()
+    }
+
+    val urlContent = scriptURI.toURL().readText()
+    urlCache.writeText(urlContent)
+
+    return urlContent.lines()
+}
+
 
 fun md5(byteProvider: () -> ByteArray): String {
     // from https://stackoverflow.com/questions/304268/getting-a-files-md5-checksum-in-java

src/main/kotlin/kscript/app/Kscript.kt

@@ -85,40 +85,46 @@ fun main(args: Array<String>) {
         quit(0)
     }
 
-    // note: with current impt we still don't support `kscript -1` where "-1" is a valid kotlin expression
+    // note: with current implementation we still don't support `kscript -1` where "-1" is a valid kotlin expression
     val userArgs = args.dropWhile { it.startsWith("-") && it != "-" }.drop(1)
     val kscriptArgs = args.take(args.size - userArgs.size)
 
     val docopt = DocOptWrapper(kscriptArgs, USAGE)
     val loggingEnabled = !docopt.getBoolean("silent")
 
 
-    // create cache dir if it does not yet exist
-    if (!KSCRIPT_CACHE_DIR.isDirectory) {
-        KSCRIPT_CACHE_DIR.mkdir()
-    }
-
     // optionally clear up the jar cache
     if (docopt.getBoolean("clear-cache")) {
         info("Cleaning up cache...")
-        KSCRIPT_CACHE_DIR.listFiles().forEach { it.delete() }
-        //        evalBash("rm -f ${KSCRIPT_CACHE_DIR}/*")
+        KSCRIPT_CACHE_DIR.listFiles()?.forEach { it.delete() }
         quit(0)
     }
 
+    // create cache dir if it does not yet exist
+    if (!KSCRIPT_CACHE_DIR.isDirectory) {
+        KSCRIPT_CACHE_DIR.mkdir()
+    }
+
     // Resolve the script resource argument into an actual file
     val scriptResource = docopt.getString("script")
 
-    val enableSupportApi = docopt.getBoolean("text")
-    val (rawScript, includeContext) = prepareScript(scriptResource)
+    val (rawUri, includeContext) = prepareScript(scriptResource)
 
     if (docopt.getBoolean("add-bootstrap-header")) {
+        errorIf(!isFile(rawUri)) {
+            "Can not add bootstrap header to URL resources: $rawUri"
+        }
+
+        val rawScript = File(rawUri)
+
         errorIf(!rawScript.canWrite()) {
             "Script file not writable: $rawScript"
         }
+
         errorIf(rawScript.parentFile == SCRIPT_TEMP_DIR) {
             "Temporary script file detected: $rawScript, created from $scriptResource"
         }
+
         val scriptLines = rawScript.readLines().dropWhile {
             it.startsWith("#!/") && it != "#!/bin/bash"
         }
@@ -140,25 +146,25 @@ fun main(args: Array<String>) {
         quit(0)
     }
 
+    val enableSupportApi = docopt.getBoolean("text")
+
     // post process script (text-processing mode, custom dsl preamble, resolve includes)
     // and finally resolve all includes (see https://github.com/holgerbrandl/kscript/issues/34)
-    val (scriptFile, includeURLs) = resolveIncludes(resolvePreambles(rawScript, enableSupportApi), includeContext)
-
+    val (scriptFile, includeURLs) = resolveIncludes(resolvePreambles(rawUri, enableSupportApi), includeContext)
 
     val script = Script(scriptFile)
 
-
     // Find all //DEPS directives and concatenate their values
-    val dependencies = (script.collectDependencies() + Script(rawScript).collectDependencies()).distinct()
-    val customRepos = (script.collectRepos() + Script(rawScript).collectRepos()).distinct()
+    val dependencies = script.collectDependencies().distinct()
+    val customRepos = script.collectRepos().distinct()
 
     // Extract kotlin arguments
     val kotlinOpts = script.collectRuntimeOptions()
     val compilerOpts = script.collectCompilerOptions()
 
     //  Create temporary dev environment
     if (docopt.getBoolean("idea")) {
-        println(launchIdeaWithKscriptlet(rawScript, userArgs, dependencies, customRepos, includeURLs, compilerOpts))
+        println(launchIdeaWithKscriptlet(scriptFile, userArgs, dependencies, customRepos, includeURLs, compilerOpts))
         exitProcess(0)
     }
 
@@ -283,6 +289,7 @@ fun main(args: Array<String>) {
     }
 
     var extClassPath = "${jarFile}${CP_SEPARATOR_CHAR}${KOTLIN_HOME}${File.separatorChar}lib${File.separatorChar}kotlin-script-runtime.jar"
+
     if (classpath.isNotEmpty())
         extClassPath += CP_SEPARATOR_CHAR + classpath
 
@@ -321,8 +328,7 @@ private fun versionCheck() {
     }
 }
 
-
-fun prepareScript(scriptResource: String): Pair<File, URI> {
+fun prepareScript(scriptResource: String): Pair<URI, URI> {
     var scriptFile: File?
 
     // we need to keep track of the scripts dir or the working dir in case of stdin script to correctly resolve includes
@@ -382,13 +388,16 @@ fun prepareScript(scriptResource: String): Pair<File, URI> {
 
     // note script file must be not null at this point
 
-    return Pair(scriptFile!!, includeContext)
+    return Pair(scriptFile!!.toURI(), includeContext)
 }
 
+private fun resolvePreambles(rawUri: URI, enableSupportApi: Boolean): URI {
+    if (!isFile(rawUri)) {
+        return rawUri
+    }
 
-private fun resolvePreambles(rawScript: File, enableSupportApi: Boolean): File {
     // include preamble for custom interpreters (see https://github.com/holgerbrandl/kscript/issues/67)
-    var scriptFile = rawScript
+    var scriptFile = File(rawUri)
 
     System.getenv("CUSTOM_KSCRIPT_PREAMBLE")?.let { interpPreamble ->
         //        rawScript = Script(rawScript!!).prependWith(interpPreamble).createTmpScript()
@@ -416,9 +425,6 @@ private fun resolvePreambles(rawScript: File, enableSupportApi: Boolean): File {
 
         scriptFile = Script(scriptFile).prependWith("//INCLUDE ${preambleFile.absolutePath}").createTmpScript()
     }
-    return scriptFile
-}
-
 
-private fun postProcessScript(inputFile: File?, includeContext: URI): IncludeResult =
-    resolveIncludes(inputFile!!, includeContext)
+    return scriptFile.toURI()
+}

src/main/kotlin/kscript/app/ResolveIncludes.kt

@@ -1,13 +1,13 @@
 package kscript.app
 
 import java.io.File
-import java.io.FileNotFoundException
 import java.net.URI
 import java.net.URL
 
 /**
  * @author Holger Brandl
  * @author Ilan Pillemer
+ * @author Marcin Kuszczak
  */
 
 const val PACKAGE_STATEMENT_PREFIX = "package "
@@ -16,67 +16,88 @@ const val IMPORT_STATEMENT_PREFIX = "import " // todo make more solid by using o
 data class IncludeResult(val scriptFile: File, val includes: List<URL> = emptyList())
 
 /** Resolve include declarations in a script file. Resolved script will be put into another temporary script */
-fun resolveIncludes(template: File, includeContext: URI = template.parentFile.toURI()): IncludeResult {
-    var script = Script(template)
-
-    // just rewrite user scripts if includes a
-    if (!script.any { isIncludeDirective(it) }) {
-        return IncludeResult(template)
+fun resolveIncludes(uri: URI, includeContext: URI = uri.resolve(".")): IncludeResult {
+    val scriptText = uri.toURL().readText()
+
+    val urlExtension = when {
+        uri.toString().endsWith(".kt") -> "kt"
+        uri.toString().endsWith(".kts") -> "kts"
+        else -> if (scriptText.contains("fun main")) {
+            "kt"
+        } else {
+            "kts"
+        }
     }
 
-    val includes = emptyList<URL>().toMutableList()
-
-    // resolve as long as it takes. YAGNI but we do because we can!
-    while (script.any { isIncludeDirective(it) }) {
-        script = script.flatMap { line ->
-            if (isIncludeDirective(line)) {
-                val include = extractIncludeTarget(line)
-
-                val includeURL = when {
-                    isUrl(include) -> URL(include)
-                    include.startsWith("/") -> File(include).toURI().toURL()
-                    include.startsWith("~/") -> File(System.getenv("HOME")!! + include.substring(1)).toURI().toURL()
-                    else -> includeContext.resolve(URI(include.removePrefix("./"))).toURL()
-                }
-
-                // test if include was processed already (aka include duplication, see #151)
-                if (includes.map { it.path }.contains(includeURL.path)) {
-                    // include was already resolved, so we return an emtpy result here to avoid duplication errors
-                    emptyList()
-                } else {
-                    includes.add(includeURL)
-
-                    try {
-                        includeURL.readText().lines()
-                    } catch (e: FileNotFoundException) {
-                        errorMsg("Failed to resolve //INCLUDE '${include}'")
-                        System.err.println(e.message?.lines()!!.map { it.prependIndent("[kscript] [ERROR] ") })
-                        quit(1)
-                    }
-                }
-            } else {
-                listOf(line)
+    val includes = mutableListOf<URI>()
+    val lines = resolve(isFile(uri), uri, includeContext, includes)
+    val script = Script(lines, urlExtension)
+
+    return IncludeResult(script.consolidateStructure().createTmpScript(), includes.map { it.toURL() })
+}
+
+private fun resolve(allowFileReferences: Boolean, scriptUri: URI, includeContext: URI, includes: MutableList<URI>): List<String> {
+    val lines = readLines(scriptUri)
+    val result = mutableListOf<String>()
+
+    for (line in lines) {
+        if (isIncludeDirective(line)) {
+            val include = extractTarget(line)
+            val includeUri = resolveUri(includeContext, include)
+
+            if (!allowFileReferences && isFile(includeUri)) {
+                errorMsg("References to local filesystem from remote scripts are not allowed.\nIn script: $scriptUri; Reference: $includeUri")
+                quit(1)
+            }
+
+            // test if include was processed already (aka include duplication, see #151)
+            if (includes.map { it.path }.contains(includeUri.path)) {
+                // include was already resolved, so we just continue
+                continue
             }
-        }.let { script.copy(it) }
+
+            includes.add(includeUri)
+
+            val resolvedLines = resolve(allowFileReferences && isFile(includeUri), includeUri, includeUri.resolve("."), includes)
+            result.addAll(resolvedLines)
+            continue
+        }
+
+        result.add(line)
     }
 
-    return IncludeResult(script.consolidateStructure().createTmpScript(), includes)
+    return result
 }
 
-internal fun isUrl(s: String) = s.startsWith("http://") || s.startsWith("https://")
+private fun readLines(uri: URI): List<String> {
+    try {
+        return fetchFromURI(uri)
+    } catch (e: Exception) {
+        errorMsg("Failed to resolve include with URI: '${uri}'")
+        System.err.println(e.message?.lines()!!.map { it.prependIndent("[kscript] [ERROR] ") })
+        quit(1)
+    }
+}
 
-private const val INCLUDE_ANNOT_PREFIX = "@file:Include("
+private fun resolveUri(scriptPath: URI, include: String): URI {
+    val result = when {
+        include.startsWith("/") -> File(include).toURI()
+        include.startsWith("~/") -> File(System.getenv("HOME")!! + include.substring(1)).toURI()
+        else -> scriptPath.resolve(URI(include.removePrefix("./")))
+    }
 
-internal fun isIncludeDirective(line: String) = line.startsWith("//INCLUDE") || line.startsWith(INCLUDE_ANNOT_PREFIX)
+    return result.normalize()
+}
 
+internal fun isIncludeDirective(line: String) = line.startsWith("//INCLUDE") || line.startsWith(INCLUDE_ANNOT_PREFIX)
 
-internal fun extractIncludeTarget(incDirective: String) = when {
-    incDirective.startsWith(INCLUDE_ANNOT_PREFIX) -> incDirective
-        .replaceFirst(INCLUDE_ANNOT_PREFIX, "")
+internal fun extractTarget(incDirective: String) = when {
+    incDirective.startsWith(INCLUDE_ANNOT_PREFIX) -> incDirective.replaceFirst(INCLUDE_ANNOT_PREFIX, "")
         .split(")")[0].trim(' ', '"')
     else -> incDirective.split("[ ]+".toRegex()).last()
 }
 
+private const val INCLUDE_ANNOT_PREFIX = "@file:Include("
 
 /**
  * Basic launcher used for testing
@@ -94,6 +115,6 @@ internal fun extractIncludeTarget(incDirective: String) = when {
 object ResolveIncludes {
     @JvmStatic
     fun main(args: Array<String>) {
-        System.err.println(resolveIncludes(File(args[0])).scriptFile.readText())
+        System.err.println(resolveIncludes(File(args[0]).toURI()).scriptFile.readText())
     }
 }

src/main/kotlin/kscript/app/Script.kt

@@ -7,7 +7,7 @@ data class Script(val lines: List<String>, val extension: String = "kts") : Iter
 
     constructor(scriptFile: File) : this(scriptFile.readLines(), scriptFile.extension)
 
-    /** Returns a the namespace/package of the script (if declared). */
+    /** Returns a namespace/package of the script (if declared). */
     val pckg by lazy {
         lines.find { it.startsWith("package ") }?.split("[ ]+".toRegex())?.get(1)?.run { this + "." }
     }

src/test/kotlin/Tests.kt

@@ -196,7 +196,7 @@ class Tests {
         val file = File("test/resources/consolidate_includes/template.kts")
         val expected = File("test/resources/consolidate_includes/expected.kts")
 
-        val result = resolveIncludes(file)
+        val result = resolveIncludes(file.toURI())
 
         result.scriptFile.readText() shouldBe (expected.readText())
     }
@@ -205,24 +205,24 @@ class Tests {
     @Test
     fun test_include_annotations() {
         val file = File("test/resources/includes/include_variations.kts")
-        val expected = File("test/resources/includes/expexcted_variations.kts")
+        val expected = File("test/resources/includes/expected_variations.kts")
 
-        val result = resolveIncludes(file)
+        val result = resolveIncludes(file.toURI())
 
         result.scriptFile.readText() shouldBe (expected.readText())
     }
 
     @Test
     fun test_include_detection() {
-        val result = resolveIncludes(File("test/resources/includes/include_variations.kts"))
+        val result = resolveIncludes(File("test/resources/includes/include_variations.kts").toURI())
 
-        result.includes.filter { it.protocol == "file" }.map { File(it.toURI()).name } shouldBe List(4) { "include_${it + 1}.kt" }
-        result.includes.filter { it.protocol != "file" }.size shouldBe 1
+        result.includes.filter { it.protocol == "file" }.map { File(it.toURI()).name } shouldBe List(7) { "include_${it + 1}.kt" }
+        result.includes.filter { it.protocol != "file" }.size shouldBe 2
     }
 
     @Test
     fun `test include detection - should not include dependency twice`() {
-      val result = resolveIncludes(File("test/resources/includes/dup_include/dup_include.kts"))
+      val result = resolveIncludes(File("test/resources/includes/dup_include/dup_include.kts").toURI())
 
       result.includes.map { File(it.toURI()).name } shouldBe listOf(
             "dup_include_1.kt",