Cleanup URI implementations, move parts into proper stdlibs

Author: marzipankaiser

libraries/common/char.effekt

@@ -52,6 +52,15 @@ def digitValue(char: Char, base: Int): Int / Exception[WrongFormat] = {
   digit
 }
 
+/// Encodes a number in the range 0..15 as a hex character.
+///
+/// Panics on all other numbers.
+def hexDigit(for: Int): Char = for match {
+  case i and i >= 0 and i < 10 => ('0'.toInt + i).toChar
+  case i and i >= 0 and i < 16 => ('A'.toInt + (i - 10)).toChar
+  case _ => panic(for.show ++ " is not in [0,16).")
+}
+
 /// Checks if the given character is an ASCII digit in base 10
 /// Prefer using `digitValue(c: Char)` to get the numeric value out.
 def isDigit(char: Char): Bool = result[Int, WrongFormat] { digitValue(char) }.isSuccess

libraries/common/io/uri.effekt

@@ -1,19 +1,13 @@
 import scanner
 import stream
 
-def hexDigit(for: Int): Char = for match {
-  case i and i >= 0 and i < 10 => ('0'.toInt + i).toChar
-  case i and i >= 0 and i < 16 => ('A'.toInt + (i - 10)).toChar
-  case _ => <>
-}
-
 /// %-encodes the characters for which `shouldEncode` returns true.
 /// Always %-encodes %.
 def urlencode(s: String){ shouldEncode: Char => Bool }: String = collectString {
   def encoded(c: Char): Unit = {
     do emit('%')
     val cc = c.toInt
-    if (cc >= 256){ panic("Unicode not supported") } // TODO
+    if (cc >= 256 || cc < 0){ panic("Unicode not supported") } // TODO
     do emit((cc / 16).hexDigit)
     do emit(mod(cc, 16).hexDigit)
   }
@@ -24,57 +18,61 @@ def urlencode(s: String){ shouldEncode: Char => Bool }: String = collectString {
   }
 }
 
-/// gen-delims as per RFC 3986
-def isGenDelim(c: Char): Bool = c match {
-  case ':' => true
-  case '/' => true
-  case '?' => true
-  case '#' => true
-  case '[' => true
-  case ']' => true
-  case '@' => true
-  case _ => false
-}
+namespace urichars {
+  /// gen-delims as per RFC 3986
+  def isGenDelim(c: Char): Bool = c match {
+    case ':' => true
+    case '/' => true
+    case '?' => true
+    case '#' => true
+    case '[' => true
+    case ']' => true
+    case '@' => true
+    case _ => false
+  }
 
-/// sub-delims as per RFC 3986
-def isSubDelim(c: Char): Bool = c match {
-  case '!' => true
-  case '$' => true
-  case '&' => true
-  case '\'' => true
-  case '(' => true
-  case ')' => true
-  case '*' => true
-  case '+' => true
-  case ',' => true
-  case ';' => true
-  case '=' => true
-  case _ => false
+  /// sub-delims as per RFC 3986
+  def isSubDelim(c: Char): Bool = c match {
+    case '!' => true
+    case '$' => true
+    case '&' => true
+    case '\'' => true
+    case '(' => true
+    case ')' => true
+    case '*' => true
+    case '+' => true
+    case ',' => true
+    case ';' => true
+    case '=' => true
+    case _ => false
+  }
+
+  /// Unreserved characters as per RFC 3986
+  def isUnreserved(c: Char): Bool = c match {
+    case c and c.isAlphanumeric => true
+    case '-' => true
+    case '.' => true
+    case '_' => true
+    case '~' => true
+    case _ => false
+  }
 }
 
-/// Encodes the string for urls using %-escapes
+/// Encodes the string for urls using %-escapes,
+/// escaping only url delimiters and space characters
 def urlencodePermissive(s: String): String = urlencode(s){
   case '%' => true
   case ' ' => true
-  case c and c.isGenDelim || c.isSubDelim => true
+  case c and c.urichars::isGenDelim || c.urichars::isSubDelim => true
   case _ => false
 }
 
-/// Unreserved characters as per RFC 3986
-def isUnreserved(c: Char): Bool = c match {
-  case c and c.isAlphanumeric => true
-  case '-' => true
-  case '.' => true
-  case '_' => true
-  case '~' => true
-  case _ => false
-}
 
 /// Encodes the string for urls using %-escapes,
 /// escaping everything that is not an unreserved character
 /// as per RFC 3986.
 def urlencode(s: String): String =
-  urlencode(s){ c => not(c.isUnreserved) }
+  urlencode(s){ c => not(c.urichars::isUnreserved) }
 
 
 /// Decodes %-escapes in the given string
@@ -93,112 +91,113 @@ def urldecode(s: String): String = collectString {
   }
 }
 
+/// Builder style representation of the parts of a URI.
+///
+/// Many consumers expect the operations to be called in-order.
 interface URIBuilder {
+  /// URI schema, e.g. http, https, ftp, ...
   def scheme(s: String): Unit
+  /// Userinfo part for, e.g. basic auth, e.g. user:letmein, ...
   def userinfo(a: String): Unit
+  /// Hostname (or non-host authority part), e.g. effekt-lang.org, [::1], 127.0.0.1, ...
   def host(h: String): Unit
+  /// Port, e.g. 80, 443, ...
   def port(p: Int): Unit
+  /// Path-part of the URI, commonly something like /index.html
   def path(p: String): Unit
+  /// Query part of the URI, e.g. q=12 for ...?q=12
   def query(q: String): Unit
+  /// Fragment part of the URI, e.g. a1 for ...#a1
   def fragment(f: String): Unit
 }
 
-def parseScheme(): String / { Scan[Char], stop } = {
-  with collectString
-  do emit(readIf{ c => c.isAlphabetic })
-  readWhile{ c => c.isAlphanumeric || c == '+' || c == '-' || c == '.' }
-}
-
-def unread[A, R](c: A){ body: => R / Scan[A] }: R / Scan[A] = {
-  var read = false
-  try body() with Scan[A] {
-    def peek() = if(read) { resume{do peek()} } else { resume{ () => c } }
-    def skip() = if(read) { resume{do skip[A]()} } else { resume{read = true} }
+namespace internal {
+  def parseScheme(): String / { Scan[Char], stop } = {
+    with collectString
+    do emit(readIf{ c => c.isAlphabetic })
+    readWhile{ c => c.isAlphanumeric || c == '+' || c == '-' || c == '.' }
   }
-}
-def unread[R](s: String){ body: => R / Scan[Char] }: R / Scan[Char] = {
-  var pos = 0
-  try body() with Scan[Char] {
-    def peek() = if (pos < s.length) { resume{s.unsafeCharAt(pos)} } else { resume{do peek()} }
-    def skip() = if (pos < s.length) { resume{pos = pos + 1} } else { resume{do skip[Char]()} }
-  }
-}
 
-def parseHostAndPort(): Unit / { URIBuilder, Scan[Char] } = {
-  try {
-    do peek[Char]() match {
-      case '[' => // IP-literal
-        // this is more permissive than the spec
-        do host(collectString{ readWhile{ c => c != ']' } } ++ "]")
-        readIf(']')
-      case _ =>
-        do host(collectString{ readWhile{
-          case '%' => true
-          case c and c.isUnreserved => true
-          case c and c.isSubDelim => true
-          case _ => false
-        } })
+  def parseHostAndPort(): Unit / { URIBuilder, Scan[Char] } = {
+    try {
+      do peek[Char]() match {
+        case '[' => // IP-literal
+          // this is more permissive than the spec
+          do host(collectString{ readWhile{ c => c != ']' } } ++ "]")
+          readIf(']')
+        case _ =>
+          do host(collectString{ readWhile{
+            case '%' => true
+            case c and c.urichars::isUnreserved => true
+            case c and c.urichars::isSubDelim => true
+            case _ => false
+          } })
+      }
+    } with stop { () =>
+      do host("")
+    }
+    attempt{
+      readIf(':')
+      do port(readInteger())
+    }{
+      // no port
+      ()
     }
-  } with stop { () =>
-    do host("")
-  }
-  attempt{
-    readIf(':')
-    do port(readInteger())
-  }{
-    // no port
-    ()
   }
-}
 
-def parseAuthority(): Unit / { URIBuilder, Scan[Char] } = {
-  // try parsing as userinfo@...
-  val fst = collectString{ readWhile{
-    case '%' => true
-    case ':' => true
-    case c and c.isUnreserved => true
-    case c and c.isSubDelim => true
-    case _ => false
-  } }
-  attempt{ // was userinfo
-    readIf('@')
-    do userinfo(fst)
-    parseHostAndPort()
-  }{ // was not userinfo
-    with unread(fst)
-    parseHostAndPort()
+  def parseAuthority(): Unit / { URIBuilder, Scan[Char] } = {
+    // try parsing as userinfo@...
+    val fst = collectString{ readWhile{
+      case '%' => true
+      case ':' => true
+      case c and c.urichars::isUnreserved => true
+      case c and c.urichars::isSubDelim => true
+      case _ => false
+    } }
+    attempt{ // was userinfo
+      readIf('@')
+      do userinfo(fst)
+      parseHostAndPort()
+    }{ // was not userinfo
+      with unread(fst)
+      parseHostAndPort()
+    }
   }
-}
 
-def parsePathQueryFragment(): Unit / { URIBuilder, Scan[Char], Exception[WrongFormat] } = {
-  do path(collectString{ readWhile{ 
-    case '?' => false
-    case '#' => false
-    case _ => true
-  }})
-  boundary{
-    readIf('?')
-    do query(collectString{ readWhile{ c => c != '#' }})
-  }
-  boundary{
-    readIf('#')
-    do fragment(collectString{ readWhile[Char]{ c => true } })
+  def parsePathQueryFragment(): Unit / { URIBuilder, Scan[Char], Exception[WrongFormat] } = {
+    do path(collectString{ readWhile{
+      case '?' => false
+      case '#' => false
+      case _ => true
+    }})
+    boundary{
+      readIf('?')
+      do query(collectString{ readWhile{ c => c != '#' }})
+    }
+    boundary{
+      readIf('#')
+      do fragment(collectString{ readWhile[Char]{ c => true } })
+    }
   }
 }
 
+/// Parse a (non-relative) URI into its parts, causing the respective URIBuilder events.
+/// Should at least parse all RFC3986-compliant URIs.
+///
+/// authority is returned as `host` even when it isn't one.
 def parseURI(uri: String): Unit / { URIBuilder, Exception[WrongFormat] } = {
   try {
     with feed(uri)
     with scanner[Char]
 
-    do scheme(parseScheme())
+    do scheme(internal::parseScheme())
     readIf(':')
 
     val c = read[Char]()
     if (c == '/' and do peek[Char]() == '/'){
       // starts with `//`
       readIf('/')
-      parseAuthority()
+      internal::parseAuthority()
       boundary{
         do peek[Char]() match {
           case '?' => ()
@@ -207,10 +206,10 @@ def parseURI(uri: String): Unit / { URIBuilder, Exception[WrongFormat] } = {
           case _ => do raise(WrongFormat(), "Path must be empty or start with / if there is an authority component.")
         }
       }
-      parsePathQueryFragment()
+      internal::parsePathQueryFragment()
     } else {
       with unread(c)
-      parsePathQueryFragment()
+      internal::parsePathQueryFragment()
     }
   } with stop { () =>
     do raise(WrongFormat(), "Could not parse URI")

libraries/common/scanner.effekt

@@ -118,6 +118,25 @@ def readInteger(): Int / Scan[Char] =
     readDecimal()
   }
 
+/// Handle Scan[A] in the body to first read the given character,
+/// then continue with the outside scanner.
+def unread[A, R](c: A){ body: => R / Scan[A] }: R / Scan[A] = {
+  var read = false
+  try body() with Scan[A] {
+    def peek() = if(read) { resume{do peek()} } else { resume{ () => c } }
+    def skip() = if(read) { resume{do skip[A]()} } else { resume{read = true} }
+  }
+}
+
+/// Handle Scan[Char] in the body to first read the given string,
+/// then continue with the outside scanner.
+def unread[R](s: String){ body: => R / Scan[Char] }: R / Scan[Char] = {
+  var pos = 0
+  try body() with Scan[Char] {
+    def peek() = if (pos < s.length) { resume{s.unsafeCharAt(pos)} } else { resume{do peek()} }
+    def skip() = if (pos < s.length) { resume{pos = pos + 1} } else { resume{do skip[Char]()} }
+  }
+}
 
 namespace returning {