Skip to content

Commit 0d09ebf

Browse files
alexarchambaultalexarchambault
authored
Add --python to run command (#1295)
1 parent 639aacf commit 0d09ebf

File tree

16 files changed

+261
-30
lines changed

16 files changed

+261
-30
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,10 @@ jobs:
325325
with:
326326
fetch-depth: 0
327327
submodules: true
328+
- name: Set up Python
329+
uses: actions/setup-python@v4
330+
with:
331+
python-version: "3.10"
328332
- uses: VirtusLab/scala-cli-setup@a173ac6b0b5252a6cf506cfafdd304066bf09da8
329333
with:
330334
jvm: "temurin:17"
@@ -353,6 +357,10 @@ jobs:
353357
with:
354358
fetch-depth: 0
355359
submodules: true
360+
- name: Set up Python
361+
uses: actions/setup-python@v4
362+
with:
363+
python-version: "3.10"
356364
- uses: VirtusLab/scala-cli-setup@a173ac6b0b5252a6cf506cfafdd304066bf09da8
357365
with:
358366
jvm: "temurin:17"
@@ -381,6 +389,10 @@ jobs:
381389
with:
382390
fetch-depth: 0
383391
submodules: true
392+
- name: Set up Python
393+
uses: actions/setup-python@v4
394+
with:
395+
python-version: "3.10"
384396
- uses: VirtusLab/scala-cli-setup@a173ac6b0b5252a6cf506cfafdd304066bf09da8
385397
with:
386398
jvm: "temurin:17"

‎build.sc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,8 @@ trait Core extends ScalaCliSbtModule with ScalaCliPublishModule with HasTests
377377
|
378378
| def libsodiumVersion = "${deps.libsodiumVersion}"
379379
| def libsodiumjniVersion = "${Deps.libsodiumjni.dep.version}"
380+
|
381+
| def scalaPyVersion = "${Deps.scalaPy.dep.version}"
380382
|}
381383
|""".stripMargin
382384
if (!os.isFile(dest) || os.read(dest) != code)
@@ -680,6 +682,7 @@ trait Cli extends SbtModule with ProtoBuildModule with CliLaunchers
680682
Deps.jsoniterCore,
681683
Deps.libsodiumjni,
682684
Deps.metaconfigTypesafe,
685+
Deps.pythonNativeLibs,
683686
Deps.scalaPackager,
684687
Deps.signingCli,
685688
Deps.slf4jNop, // to silence jgit

‎modules/build/src/main/scala/scala/build/Build.scala‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -814,13 +814,22 @@ object Build {
814814
.map(v => List("-release", v).map(ScalacOpt(_)))
815815
.getOrElse(Nil)
816816

817+
val scalapyOptions =
818+
if (
819+
params.scalaVersion.startsWith("2.13.") &&
820+
options.notForBloopOptions.python.getOrElse(false)
821+
)
822+
Seq(ScalacOpt("-Yimports:java.lang,scala,scala.Predef,me.shadaj.scalapy"))
823+
else Nil
824+
817825
val scalacOptions =
818826
options.scalaOptions.scalacOptions.map(_.value) ++
819827
pluginScalacOptions ++
820828
semanticDbScalacOptions ++
821829
sourceRootScalacOptions ++
822830
scalaJsScalacOptions ++
823-
scalacReleaseV
831+
scalacReleaseV ++
832+
scalapyOptions
824833

825834
val compilerParams = ScalaCompilerParams(
826835
scalaVersion = params.scalaVersion,

‎modules/build/src/main/scala/scala/build/internal/Runner.scala‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,8 @@ object Runner {
300300
launcher: File,
301301
args: Seq[String],
302302
logger: Logger,
303-
allowExecve: Boolean = false
303+
allowExecve: Boolean = false,
304+
extraEnv: Map[String, String] = Map.empty
304305
): Process = {
305306

306307
import logger.{log, debug}
@@ -318,15 +319,17 @@ object Runner {
318319
Execve.execve(
319320
command.head,
320321
launcher.getName +: command.tail.toArray,
321-
sys.env.toArray.sorted.map { case (k, v) => s"$k=$v" }
322+
(sys.env ++ extraEnv).toArray.sorted.map { case (k, v) => s"$k=$v" }
322323
)
323324
sys.error("should not happen")
324325
}
325326
else {
326-
val process = new ProcessBuilder(command: _*)
327+
val builder = new ProcessBuilder(command: _*)
327328
.inheritIO()
328-
.start()
329-
process
329+
val env = builder.environment()
330+
for ((k, v) <- extraEnv)
331+
env.put(k, v)
332+
builder.start()
330333
}
331334
}
332335

‎modules/cli-options/src/main/scala/scala/cli/commands/SharedPythonOptions.scala‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package scala.cli.commands
2+
3+
import caseapp._
4+
5+
// format: off
6+
final case class SharedPythonOptions(
7+
@HelpMessage("Set Java options so that Python can be loaded")
8+
pythonSetup: Option[Boolean] = None,
9+
@HelpMessage("Enable Python support via ScalaPy")
10+
@ExtraName("py")
11+
python: Option[Boolean] = None
12+
)
13+
// format: on
14+
15+
object SharedPythonOptions {
16+
lazy val parser: Parser[SharedPythonOptions] = Parser.derive
17+
implicit lazy val parserAux: Parser.Aux[SharedPythonOptions, parser.D] = parser
18+
implicit lazy val help: Help[SharedPythonOptions] = Help.derive
19+
}

‎modules/cli-options/src/main/scala/scala/cli/commands/SharedRunOptions.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ final case class SharedRunOptions(
1717
compileCross: CompileCrossOptions = CompileCrossOptions(),
1818
@Recurse
1919
mainClass: MainClassOptions = MainClassOptions(),
20+
@Recurse
21+
sharedPython: SharedPythonOptions = SharedPythonOptions(),
2022
@Group("Run")
2123
@Hidden
2224
@HelpMessage("Run as a Spark job, using the spark-submit command")

‎modules/cli/src/main/scala/scala/cli/commands/Package.scala‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package scala.cli.commands
22

3+
import ai.kien.python.Python
34
import caseapp.*
45
import coursier.launcher.*
56
import packager.config.*
@@ -24,6 +25,7 @@ import scala.build.internal.{Runner, ScalaJsLinkerConfig}
2425
import scala.build.options.{PackageType, Platform}
2526
import scala.cli.CurrentParams
2627
import scala.cli.commands.OptionsHelper.*
28+
import scala.cli.commands.Run.orPythonDetectionError
2729
import scala.cli.commands.packaging.Spark
2830
import scala.cli.commands.util.BuildCommandHelpers
2931
import scala.cli.commands.util.CommonOps.SharedDirectoriesOptionsOps
@@ -365,7 +367,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
365367
value(buildJs(build, destPath, value(mainClass), logger))
366368

367369
case PackageType.Native =>
368-
buildNative(build, value(mainClass), destPath, logger)
370+
value(buildNative(build, value(mainClass), destPath, logger))
369371
destPath
370372

371373
case PackageType.GraalVMNativeImage =>
@@ -455,7 +457,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
455457
}
456458
destPath
457459
case PackageType.Docker =>
458-
docker(build, value(mainClass), logger)
460+
value(docker(build, value(mainClass), logger))
459461
destPath
460462
}
461463

@@ -575,7 +577,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
575577
build: Build.Successful,
576578
mainClass: String,
577579
logger: Logger
578-
): Unit = {
580+
): Either[BuildException, Unit] = either {
579581
val packageOptions = build.options.notForBloopOptions.packageOptions
580582

581583
if (build.options.platform.value == Platform.Native && (Properties.isMac || Properties.isWin)) {
@@ -616,7 +618,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
616618
build.options.platform.value match {
617619
case Platform.JVM => bootstrap(build, appPath, mainClass, () => ())
618620
case Platform.JS => buildJs(build, appPath, mainClass, logger)
619-
case Platform.Native => buildNative(build, mainClass, appPath, logger)
621+
case Platform.Native => value(buildNative(build, mainClass, appPath, logger))
620622
}
621623

622624
logger.message(
@@ -908,17 +910,32 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
908910
mainClass: String,
909911
dest: os.Path,
910912
logger: Logger
911-
): Unit = {
913+
): Either[BuildException, Unit] = either {
912914

913915
val cliOptions = build.options.scalaNativeOptions.configCliOptions()
914916

917+
val setupPython = build.options.notForBloopOptions.doSetupPython.getOrElse(false)
918+
val pythonLdFlags =
919+
if (setupPython)
920+
value {
921+
val python = Python()
922+
val flagsOrError = python.ldflags
923+
logger.debug(s"Python ldflags: $flagsOrError")
924+
flagsOrError.orPythonDetectionError
925+
}
926+
else
927+
Nil
928+
val pythonCliOptions = pythonLdFlags.flatMap(f => Seq("--linking-option", f)).toList
929+
930+
val allCliOptions = pythonCliOptions ++ cliOptions
931+
915932
val nativeWorkDir = build.inputs.nativeWorkDir
916933
os.makeDir.all(nativeWorkDir)
917934

918935
val cacheData =
919936
CachedBinary.getCacheData(
920937
build,
921-
cliOptions,
938+
allCliOptions,
922939
dest,
923940
nativeWorkDir
924941
)
@@ -928,7 +945,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
928945

929946
val classpath = build.fullClassPath.map(_.toString) :+ mainJar.toString
930947
val args =
931-
cliOptions ++
948+
allCliOptions ++
932949
logger.scalaNativeCliInternalLoggerOptions ++
933950
List[String](
934951
"--outpath",

‎modules/cli/src/main/scala/scala/cli/commands/Run.scala‎

Lines changed: 81 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
package scala.cli.commands
22

3+
import ai.kien.python.Python
34
import caseapp.*
45

6+
import java.io.File
57
import java.util.concurrent.CompletableFuture
68
import scala.build.EitherCps.{either, value}
79
import scala.build.errors.BuildException
810
import scala.build.internal.{Constants, Runner, ScalaJsLinkerConfig}
9-
import scala.build.options.{BuildOptions, JavaOpt, Platform}
11+
import scala.build.options.{BuildOptions, JavaOpt, Platform, ScalacOpt}
1012
import scala.build.*
1113
import scala.cli.CurrentParams
1214
import scala.cli.commands.run.RunMode
@@ -16,7 +18,7 @@ import scala.cli.commands.util.SharedOptionsUtil.*
1618
import scala.cli.commands.util.{BuildCommandHelpers, RunHadoop, RunSpark}
1719
import scala.cli.config.{ConfigDb, Keys}
1820
import scala.cli.internal.ProcUtil
19-
import scala.util.Properties
21+
import scala.util.{Properties, Try}
2022

2123
object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
2224
override def group = "Main"
@@ -90,7 +92,9 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
9092
}
9193
),
9294
notForBloopOptions = baseOptions.notForBloopOptions.copy(
93-
runWithManifest = options.sharedRun.useManifest
95+
runWithManifest = options.sharedRun.useManifest,
96+
python = options.sharedRun.sharedPython.python,
97+
pythonSetup = options.sharedRun.sharedPython.pythonSetup
9498
)
9599
)
96100
}
@@ -360,30 +364,86 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
360364
}
361365
value(res)
362366
case Platform.Native =>
363-
withNativeLauncher(
367+
val setupPython = build.options.notForBloopOptions.doSetupPython.getOrElse(false)
368+
val pythonLibraryPaths =
369+
if (setupPython)
370+
value {
371+
val python = Python()
372+
val pathsOrError = python.nativeLibraryPaths
373+
logger.debug(s"Python native library paths: $pathsOrError")
374+
pathsOrError.orPythonDetectionError
375+
}
376+
else
377+
Nil
378+
// seems conda doesn't add the lib directory to LD_LIBRARY_PATH (see conda/conda#308),
379+
// which prevents apps from finding libpython for example, so we update it manually here
380+
val extraEnv =
381+
if (pythonLibraryPaths.isEmpty) Map.empty
382+
else {
383+
val prependTo =
384+
if (Properties.isWin) "PATH"
385+
else if (Properties.isMac) "DYLD_LIBRARY_PATH"
386+
else "LD_LIBRARY_PATH"
387+
val currentOpt = Option(System.getenv(prependTo))
388+
val currentEntries = currentOpt
389+
.map(_.split(File.pathSeparator).toSet)
390+
.getOrElse(Set.empty)
391+
val additionalEntries = pythonLibraryPaths.filter(!currentEntries.contains(_))
392+
if (additionalEntries.isEmpty)
393+
Map.empty
394+
else {
395+
val newValue =
396+
(additionalEntries.iterator ++ currentOpt.iterator).mkString(File.pathSeparator)
397+
Map(prependTo -> newValue)
398+
}
399+
}
400+
val maybeResult = withNativeLauncher(
364401
build,
365402
mainClass,
366403
logger
367404
) { launcher =>
368405
if (showCommand)
369-
Left(launcher.toString +: args)
406+
Left(
407+
extraEnv.toVector.sorted.map { case (k, v) => s"$k=$v" } ++
408+
Seq(launcher.toString) ++
409+
args
410+
)
370411
else {
371412
val proc = Runner.runNative(
372413
launcher.toIO,
373414
args,
374415
logger,
375-
allowExecve = allowExecve
416+
allowExecve = allowExecve,
417+
extraEnv = extraEnv
376418
)
377419
Right((proc, None))
378420
}
379421
}
422+
value(maybeResult)
380423
case Platform.JVM =>
381424
runMode match {
382425
case RunMode.Default =>
426+
val baseJavaProps = build.options.javaOptions.javaOpts.toSeq.map(_.value.value)
427+
val setupPython = build.options.notForBloopOptions.doSetupPython.getOrElse(false)
428+
val pythonJavaProps =
429+
if (setupPython) {
430+
val scalapyProps = value {
431+
val python = Python()
432+
val propsOrError = python.scalapyProperties
433+
logger.debug(s"Python Java properties: $propsOrError")
434+
propsOrError.orPythonDetectionError
435+
}
436+
scalapyProps.toVector.sorted.map {
437+
case (k, v) => s"-D$k=$v"
438+
}
439+
}
440+
else
441+
Nil
442+
val allJavaOpts = pythonJavaProps ++ baseJavaProps
383443
if (showCommand) {
384444
val command = Runner.jvmCommand(
385445
build.options.javaHome().value.javaCommand,
386-
build.options.javaOptions.javaOpts.toSeq.map(_.value.value),
446+
allJavaOpts,
387447
build.fullClassPath,
388448
mainClass,
389449
args,
@@ -395,7 +455,7 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
395455
else {
396456
val proc = Runner.runJvm(
397457
build.options.javaHome().value.javaCommand,
398-
build.options.javaOptions.javaOpts.toSeq.map(_.value.value),
458+
allJavaOpts,
399459
build.fullClassPath,
400460
mainClass,
401461
args,
@@ -476,9 +536,19 @@ object Run extends ScalaCommand[RunOptions] with BuildCommandHelpers {
476536
build: Build.Successful,
477537
mainClass: String,
478538
logger: Logger
479-
)(f: os.Path => T): T = {
539+
)(f: os.Path => T): Either[BuildException, T] = {
480540
val dest = build.inputs.nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}"
481-
Package.buildNative(build, mainClass, dest, logger)
482-
f(dest)
541+
Package.buildNative(build, mainClass, dest, logger).map { _ =>
542+
f(dest)
543+
}
483544
}
545+
546+
final class PythonDetectionError(cause: Throwable) extends BuildException(
547+
s"Error detecting Python environment: ${cause.getMessage}",
548+
cause = cause
549+
)
550+
551+
extension [T](t: Try[T])
552+
def orPythonDetectionError: Either[PythonDetectionError, T] =
553+
t.toEither.left.map(new PythonDetectionError(_))
484554
}

‎modules/cli/src/main/scala/scala/cli/commands/Test.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ object Test extends ScalaCommand[TestOptions] {
215215
args,
216216
logger
217217
)
218-
}
218+
}.flatten
219219
}
220220
case Platform.JVM =>
221221
val classPath = build.fullClassPath

0 commit comments

Comments
 (0)