New cyfra DSL & compiler

.gitignore

@@ -26,3 +26,4 @@ out
 hs_err_pid*.log
 .bsp
 metals.sbt
+**/output

build.sbt

@@ -63,42 +63,37 @@ lazy val fs2Settings = Seq(libraryDependencies ++= Seq("co.fs2" %% "fs2-core" %
 lazy val utility = (project in file("cyfra-utility"))
   .settings(commonSettings)
 
-lazy val spirvTools = (project in file("cyfra-spirv-tools"))
+lazy val core = (project in file("cyfra-core"))
   .settings(commonSettings)
   .dependsOn(utility)
 
-lazy val vulkan = (project in file("cyfra-vulkan"))
+lazy val dsl = (project in file("cyfra-dsl"))
   .settings(commonSettings)
-  .dependsOn(utility)
+  .dependsOn(utility, core)
 
-lazy val dsl = (project in file("cyfra-dsl"))
+lazy val spirvTools = (project in file("cyfra-spirv-tools"))
   .settings(commonSettings)
   .dependsOn(utility)
 
 lazy val compiler = (project in file("cyfra-compiler"))
   .settings(commonSettings)
-  .dependsOn(dsl, utility)
+  .dependsOn(core, utility, spirvTools)
 
-lazy val core = (project in file("cyfra-core"))
+lazy val vulkan = (project in file("cyfra-vulkan"))
   .settings(commonSettings)
-  .dependsOn(compiler, dsl, utility, spirvTools)
+  .dependsOn(utility)
 
 lazy val runtime = (project in file("cyfra-runtime"))
   .settings(commonSettings)
-  .dependsOn(core, vulkan)
-
-lazy val foton = (project in file("cyfra-foton"))
-  .settings(commonSettings)
-  .dependsOn(compiler, dsl, runtime, utility)
+  .dependsOn(core, vulkan, spirvTools, compiler)
 
 lazy val examples = (project in file("cyfra-examples"))
   .settings(commonSettings, runnerSettings)
   .settings(libraryDependencies += "org.scala-lang.modules" % "scala-parallel-collections_3" % "1.2.0")
-  .dependsOn(foton)
+  .dependsOn(vscode, runtime, dsl)
 
 lazy val vscode = (project in file("cyfra-vscode"))
   .settings(commonSettings)
-  .dependsOn(foton)
 
 lazy val fs2interop = (project in file("cyfra-fs2"))
   .settings(commonSettings, fs2Settings)
@@ -110,7 +105,7 @@ lazy val e2eTest = (project in file("cyfra-e2e-test"))
 
 lazy val root = (project in file("."))
   .settings(name := "Cyfra")
-  .aggregate(compiler, dsl, foton, core, runtime, vulkan, examples, fs2interop)
+  .aggregate(compiler, dsl, core, runtime, vulkan, examples, fs2interop)
 
 e2eTest / Test / javaOptions ++= Seq("-Dorg.lwjgl.system.stackSize=1024", "-DuniqueLibraryNames=true")
 

cyfra-compiler/src/main/scala/io/computenode/cyfra/compiler/CompilationException.scala

@@ -0,0 +1,3 @@
+package io.computenode.cyfra.compiler
+
+class CompilationException(message: String) extends RuntimeException("Compilation Error: " + message)

cyfra-compiler/src/main/scala/io/computenode/cyfra/compiler/Compiler.scala

@@ -0,0 +1,39 @@
+package io.computenode.cyfra.compiler
+
+import io.computenode.cyfra.core.binding.GBinding
+import io.computenode.cyfra.core.expression.ExpressionBlock
+import io.computenode.cyfra.core.layout.LayoutStruct
+import io.computenode.cyfra.compiler.modules.*
+import io.computenode.cyfra.compiler.modules.CompilationModule.StandardCompilationModule
+import io.computenode.cyfra.compiler.unit.Compilation
+import io.computenode.cyfra.core.GProgram.WorkDimensions
+
+import java.nio.ByteBuffer
+
+class Compiler(verbose: "none" | "last" | "all" = "none"):
+  private val transformer = new Transformer()
+  private val modules: List[StandardCompilationModule] =
+    List(new Reordering, new StructuredControlFlow, new Variables, new Functions, new Bindings, new Constants, new Algebra, new Finalizer)
+  private val emitter = new Emitter()
+
+  def compile(bindings: Seq[GBinding[?]], body: ExpressionBlock[Unit], workgroupSize: WorkDimensions): ByteBuffer =
+    val parsedUnit =
+      val tmp = transformer.compile(body)
+      val meta = tmp.metadata.copy(bindings = bindings, workgroupSize = workgroupSize)
+      tmp.copy(metadata = meta)
+    if verbose == "all" then
+      println(s"=== ${transformer.name} ===")
+      Compilation.debugPrint(parsedUnit)
+
+    val compiledUnit = modules.foldLeft(parsedUnit): (unit, module) =>
+      val res = module.compile(unit)
+      if verbose == "all" then
+        println(s"\n=== ${module.name} ===")
+        Compilation.debugPrint(res)
+      res
+
+    if verbose == "last" then
+      println(s"\n=== Final Output ===")
+      Compilation.debugPrint(compiledUnit)
+
+    emitter.compile(compiledUnit)

cyfra-compiler/src/main/scala/io/computenode/cyfra/spirv/Opcodes.scala → cyfra-compiler/src/main/scala/io/computenode/cyfra/compiler/Spirv.scala

@@ -1,68 +1,55 @@
-package io.computenode.cyfra.spirv
+package io.computenode.cyfra.compiler
 
 import java.nio.charset.StandardCharsets
 
-private[cyfra] object Opcodes:
+private[cyfra] object Spirv:
 
-  def intToBytes(i: Int): List[Byte] =
+  private def intToBytes(i: Int): List[Byte] =
     List[Byte]((i >>> 24).asInstanceOf[Byte], (i >>> 16).asInstanceOf[Byte], (i >>> 8).asInstanceOf[Byte], (i >>> 0).asInstanceOf[Byte])
 
   private[cyfra] trait Words:
-    def toWords: List[Byte]
-
+    def toBytes: List[Byte]
     def length: Int
 
-  private[cyfra] case class Word(bytes: Array[Byte]) extends Words:
-    def toWords: List[Byte] = bytes.toList
-
-    def length = 1
-
-    override def toString = s"Word(${bytes.mkString(", ")}${if bytes.length == 4 then s" [i = ${BigInt(bytes).toInt}])" else ""}"
+  private[cyfra] case class Word private (bytes: (Byte, Byte, Byte, Byte)) extends Words:
+    def toBytes: List[Byte] = bytes.toList
+    def length: Int = 1
+    override def toString = s"Word(${bytes._4}, ${bytes._3}, ${bytes._2}, ${bytes._1})"
 
-  private[cyfra] case class WordVariable(name: String) extends Words:
-    def toWords: List[Byte] =
-      List(-1, -1, -1, -1)
-
-    def length = 1
+  object Word:
+    def apply(value: Int): Word =
+      val bytes = intToBytes(value).reverse
+      Word(bytes(0), bytes(1), bytes(2), bytes(3))
 
   private[cyfra] case class Instruction(code: Code, operands: List[Words]) extends Words:
-    override def toWords: List[Byte] =
-      code.toWords.take(2) ::: intToBytes(length).reverse.take(2) ::: operands.flatMap(_.toWords)
-
-    def length = 1 + operands.map(_.length).sum
-
-    def replaceVar(name: String, value: Int): Instruction =
-      this.copy(operands = operands.map {
-        case WordVariable(varName) if name == varName => IntWord(value)
-        case any                                      => any
-      })
-
+    def toBytes: List[Byte] =
+      code.toBytes.take(2) ::: intToBytes(length).reverse.take(2) ::: operands.flatMap(_.toBytes)
+    def length: Int = 1 + operands.map(_.length).sum
     override def toString: String = s"${code.mnemo} ${operands.mkString(", ")}"
 
   private[cyfra] case class Code(mnemo: String, opcode: Int) extends Words:
-    override def toWords: List[Byte] = intToBytes(opcode).reverse
+    def toBytes: List[Byte] = intToBytes(opcode).reverse
+    def length: Int = 1
+    override def toString: String = mnemo
 
-    override def length: Int = 1
 
   private[cyfra] case class Text(text: String) extends Words:
-    override def toWords: List[Byte] =
+    def toBytes: List[Byte] =
       val textBytes = text.getBytes(StandardCharsets.UTF_8).toList
       val complBytesLength = 4 - (textBytes.length % 4)
       val complBytes = List.fill[Byte](complBytesLength)(0)
       textBytes ::: complBytes
 
-    override def length: Int = toWords.length / 4
+    def length: Int = toBytes.length / 4
 
   private[cyfra] case class IntWord(i: Int) extends Words:
-    override def toWords: List[Byte] = intToBytes(i).reverse
-
-    override def length: Int = 1
+    def toBytes: List[Byte] = intToBytes(i).reverse
+    def length: Int = 1
+    override def toString: String = i.toString
 
   private[cyfra] case class ResultRef(result: Int) extends Words:
-    override def toWords: List[Byte] = intToBytes(result).reverse
-
-    override def length: Int = 1
-
+    def toBytes: List[Byte] = intToBytes(result).reverse
+    def length: Int = 1
     override def toString: String = s"%$result"
 
   val MagicNumber = Code("MagicNumber", 0x07230203)

cyfra-compiler/src/main/scala/io/computenode/cyfra/compiler/ir/FunctionIR.scala

@@ -0,0 +1,8 @@
+package io.computenode.cyfra.compiler.ir
+
+import io.computenode.cyfra.compiler.ir.IRs
+import io.computenode.cyfra.core.expression.Value
+import io.computenode.cyfra.core.expression.Var
+
+case class FunctionIR[A: Value](name: String, parameters: List[Var[?]]):
+  def v: Value[A] = summon[Value[A]]

cyfra-compiler/src/main/scala/io/computenode/cyfra/compiler/ir/IR.scala

@@ -0,0 +1,88 @@
+package io.computenode.cyfra.compiler.ir
+
+import io.computenode.cyfra.compiler.ir.IR
+import io.computenode.cyfra.compiler.ir.IR.RefIR
+import io.computenode.cyfra.compiler.ir.IRs
+import io.computenode.cyfra.compiler.Spirv.Code
+import io.computenode.cyfra.compiler.Spirv.Words
+import io.computenode.cyfra.core.binding.{BufferRef, GBuffer, GUniform, UniformRef}
+import io.computenode.cyfra.core.expression.*
+import io.computenode.cyfra.core.expression.given
+import io.computenode.cyfra.utility.Utility.nextId
+
+import scala.collection
+
+sealed trait IR[A: Value] extends Product:
+  val id: Int = nextId()
+  def v: Value[A] = summon[Value[A]]
+  def substitute(map: collection.Map[Int, RefIR[?]]): IR[A] =
+    val that = replace(using map)
+    if this.deepEquals(that) then this else that // not reusing IRs would break Structured Control Flow phase (as we don't want to substitute body that is being compiled)
+  def name: String = this.getClass.getSimpleName
+  protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[A] = this
+
+  def deepEquals(that: IR[?]): Boolean =
+    if this != that then return false
+
+    this.productIterator
+      .zip(that.productIterator)
+      .forall:
+        case (a: IR[?], b: IR[?])     => a.id == b.id
+        case (a: List[?], b: List[?]) =>
+          a.length == b.length &&
+          a.zip(b)
+            .forall:
+              case (x: IR[?], y: IR[?]) => x.id == y.id
+              case (x, y)               => x == y
+        case (a, b) => a == b
+
+object IR:
+  sealed trait RefIR[A: Value] extends IR[A]
+
+  case class Constant[A: Value](value: Any) extends RefIR[A]
+  case class VarDeclare[A: Value](variable: Var[A]) extends RefIR[Unit]
+  case class VarRead[A: Value](variable: Var[A]) extends RefIR[A]
+  case class VarWrite[A: Value](variable: Var[A], value: RefIR[A]) extends IR[Unit]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[Unit] = this.copy(value = value.replaced)
+  case class ReadBuffer[A: Value](buffer: BufferRef[A], index: RefIR[UInt32]) extends RefIR[A]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[A] = this.copy(index = index.replaced)
+  case class WriteBuffer[A: Value](buffer: BufferRef[A], index: RefIR[UInt32], value: RefIR[A]) extends IR[Unit]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[Unit] = this.copy(index = index.replaced, value = value.replaced)
+  case class ReadUniform[A: Value](uniform: UniformRef[A]) extends RefIR[A]
+  case class WriteUniform[A: Value](uniform: UniformRef[A], value: RefIR[A]) extends IR[Unit]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[Unit] = this.copy(value = value.replaced)
+  case class Operation[A: Value](func: BuildInFunction[A], args: List[RefIR[?]]) extends RefIR[A]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[A] = this.copy(args = args.map(_.replaced))
+  case class CallWithVar[A: Value](func: FunctionIR[A], args: List[Var[?]]) extends RefIR[A]
+  case class CallWithIR[A: Value](func: FunctionIR[A], args: List[RefIR[?]]) extends RefIR[A]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[A] = this.copy(args = args.map(_.replaced))
+  case class Branch[T: Value](cond: RefIR[Bool], ifTrue: IRs[T], ifFalse: IRs[T], break: JumpTarget[T]) extends RefIR[T]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[T] = this.copy(cond = cond.replaced)
+  case class Loop(mainBody: IRs[Unit], continueBody: IRs[Unit], break: JumpTarget[Unit], continue: JumpTarget[Unit]) extends IR[Unit]
+  case class Jump[A: Value](target: JumpTarget[A], value: RefIR[A]) extends IR[Unit]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[Unit] = this.copy(value = value.replaced)
+  case class ConditionalJump[A: Value](cond: RefIR[Bool], target: JumpTarget[A], value: RefIR[A]) extends IR[Unit]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[Unit] = this.copy(cond = cond.replaced, value = value.replaced)
+  case class Interface(ref: RefIR[?]) extends RefIR[Unit]:
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[Unit] = this.copy(ref = ref.replaced)
+  case class SvInst(op: Code, operands: List[Words | RefIR[?]]) extends IR[Unit]:
+    override def name: String = op.mnemo
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[Unit] = this.copy(operands = operands.map:
+      case r: RefIR[?] => r.replaced
+      case w           => w)
+  case class SvRef[A: Value](op: Code, tpe: Option[RefIR[Unit]], operands: List[Words | RefIR[?]]) extends RefIR[A]:
+    override def name: String = op.mnemo
+    override protected def replace(using map: collection.Map[Int, RefIR[?]]): IR[A] = this.copy(operands = operands.map:
+      case r: RefIR[?] => r.replaced
+      case w           => w)
+
+  object SvRef:
+    def apply[A: Value](op: Code, tpe: RefIR[Unit], operands: List[Words | RefIR[?]]): SvRef[A] =
+      SvRef(op, Some(tpe), operands)
+
+    def apply[A: Value](op: Code, operands: List[Words | RefIR[?]]): SvRef[A] =
+      SvRef(op, None, operands)
+
+  extension [T](ir: RefIR[T])
+    private def replaced(using map: collection.Map[Int, RefIR[?]]): RefIR[T] =
+      map.getOrElse(ir.id, ir).asInstanceOf[RefIR[T]]

cyfra-compiler/src/main/scala/io/computenode/cyfra/compiler/ir/IRs.scala

@@ -0,0 +1,68 @@
+package io.computenode.cyfra.compiler.ir
+
+import IR.*
+import io.computenode.cyfra.compiler.CompilationException
+import io.computenode.cyfra.compiler.ir.IRs.*
+import io.computenode.cyfra.core.expression.Value
+import io.computenode.cyfra.compiler.Spirv.Op
+import io.computenode.cyfra.utility.cats.{FunctionK, ~>}
+
+import scala.collection.mutable
+
+case class IRs[A: Value](result: IR[A], body: List[IR[?]]):
+
+  def prepend(ir: IR[?]): IRs[A] = IRs(result, ir :: body)
+
+  def filterOut(p: IR[?] => Boolean): (IRs[A], List[IR[?]]) =
+    val removed = mutable.Buffer.empty[IR[?]]
+    val next = flatMapReplace:
+      case x if p(x) =>
+        removed += x
+        IRs.proxy(x)(using x.v)
+      case x => IRs(x)(using x.v)
+    (next, removed.toList)
+
+  def flatMapReplace(f: IR[?] => IRs[?]): IRs[A] = flatMapReplace()(f)
+
+  def flatMapReplace(enterControlFlow: Boolean = true)(f: IR[?] => IRs[?]): IRs[A] =
+    flatMapReplaceImpl(f, mutable.Map.empty, enterControlFlow)
+
+  private def flatMapReplaceImpl(f: IR[?] => IRs[?], replacements: mutable.Map[Int, RefIR[?]], enterControlFlow: Boolean): IRs[A] =
+    val nBody = body.flatMap: (v: IR[?]) =>
+      val next = v match
+        case b: Branch[a] if enterControlFlow =>
+          given Value[a] = b.v
+          val Branch(cond, ifTrue, ifFalse, t) = b
+          val nextT = ifTrue.flatMapReplaceImpl(f, replacements, enterControlFlow)
+          val nextF = ifFalse.flatMapReplaceImpl(f, replacements, enterControlFlow)
+          Branch[a](cond, nextT, nextF, t)
+        case Loop(mainBody, continueBody, b, c) if enterControlFlow =>
+          val nextM = mainBody.flatMapReplaceImpl(f, replacements, enterControlFlow)
+          val nextC = continueBody.flatMapReplaceImpl(f, replacements, enterControlFlow)
+          Loop(nextM, nextC, b, c)
+        case other => other
+      val subst = next.substitute(replacements)
+      val IRs(result, body) = f(subst)
+      v match
+        case v: RefIR[?] => replacements(v.id) = result.asInstanceOf[RefIR[?]]
+        case _           => ()
+      body
+
+    // We neet to watch out for forward references
+    val codesWithLabels = Set(Op.OpLoopMerge, Op.OpSelectionMerge, Op.OpBranch, Op.OpBranchConditional, Op.OpSwitch)
+    val nextBody = nBody.map:
+      case x @ IR.SvInst(code, _) if codesWithLabels(code) => x.substitute(replacements) // all ops that point to labels
+      case x @ IR.SvRef(Op.OpPhi, _, args)                 =>
+        // this can contain a cyclical forward reference, let's crash if we may have to handle it
+        val safe = args.forall:
+          case ref: RefIR[?] => replacements.get(ref.id).forall(_.id == ref.id)
+          case _             => true
+        if safe then x else throw CompilationException("Forward reference detected in OpPhi")
+      case other => other
+
+    val nextResult = replacements.getOrElse(result.id, result).asInstanceOf[IR[A]]
+    IRs(nextResult, nextBody)
+
+object IRs:
+  def apply[A: Value](ir: IR[A]): IRs[A] = new IRs(ir, List(ir))
+  def proxy[A: Value](ir: IR[A]): IRs[A] = new IRs(ir, List())