fix(SPEC): update SPEC and fix some inconsistencies (#5206)

The change updates the SPEC to reflect the current status of Flux.
As such the SPEC is no longer a _living_ document and instead describes
current behaviors. The few IMPL issues that were not complete are left
as feature requests for the future.

Summary of the changes:

- Specific docs about universe functions have been removed
- References to the standard library have been added
- `empty` and `in` keywords have been removed
- Missing operators added
- Numeric literals removed
- Regex escape sequences have been updated in the SPEC and the
  implementation was updated to match the SPEC
- Duration literal add/sub examples have been updated
- Shorthand date time literals have been updated
- Add docs about stream type remove generator type

Author: nathanielc

docs/SPEC.md

@@ -18,10 +18,17 @@ import (
 func ParseTime(lit string) (time.Time, error) {
 	if !strings.Contains(lit, "T") {
 		// This is a date.
-		return time.Parse("2006-01-02", lit)
+		t, err := time.Parse("2006-01-02", lit)
+		if err != nil {
+			return time.Time{}, errors.New(codes.Invalid, "cannot parse date")
+		}
+		return t, nil
+	}
+	t, err := time.Parse(time.RFC3339Nano, lit)
+	if err != nil {
+		return time.Time{}, errors.New(codes.Invalid, "cannot parse date time")
 	}
-	// todo(jsternberg): need to also parse when there is no time offset.
-	return time.Parse(time.RFC3339Nano, lit)
+	return t, nil
 }
 
 // MustParseTime parses a time literal and panics in the case of an error.
@@ -138,16 +145,11 @@ func writeNextUnescapedRune(s string, builder *strings.Builder) (width int, err
 		case '$':
 			r = '$'
 		case 'x':
-			// Decode two hex chars as a single byte
-			if len(s[width:]) < 2 {
-				return 0, fmt.Errorf("invalid byte value %q", s[width:])
+			b, err := fromHexDigits(s[width:])
+			if err != nil {
+				return 0, err
 			}
-			ch1, ok1 := fromHexChar(s[width])
-			ch2, ok2 := fromHexChar(s[width+1])
-			if !ok1 || !ok2 {
-				return 0, fmt.Errorf("invalid byte value %q", s[width:])
-			}
-			builder.WriteByte((ch1 << 4) | ch2)
+			builder.WriteByte(b)
 			return width + 2, nil
 		default:
 			return 0, fmt.Errorf("invalid escape character %q", next)
@@ -160,6 +162,20 @@ func writeNextUnescapedRune(s string, builder *strings.Builder) (width int, err
 	return
 }
 
+// fromHexDigits decodes a single byte from two hex digits from the string or an error
+func fromHexDigits(s string) (byte, error) {
+	// Decode two hex chars as a single byte
+	if len(s) < 2 {
+		return 0, errors.New(codes.Invalid, "expected 2 hex characters")
+	}
+	ch1, ok1 := fromHexChar(s[0])
+	ch2, ok2 := fromHexChar(s[1])
+	if !ok1 || !ok2 {
+		return 0, fmt.Errorf("invalid byte value %q", s)
+	}
+	return ((ch1 << 4) | ch2), nil
+}
+
 // fromHexChar converts a hex character into its value and a success flag.
 func fromHexChar(c byte) (byte, bool) {
 	switch {
@@ -186,8 +202,50 @@ func ParseRegexp(lit string) (*regexp.Regexp, error) {
 	}
 
 	expr := lit[1 : len(lit)-1]
-	if index := strings.Index(expr, "\\/"); index != -1 {
-		expr = strings.Replace(expr, "\\/", "/", -1)
+	// Unescape regex literal
+	var (
+		builder    strings.Builder
+		width, pos int
+		err        error
+	)
+	builder.Grow(len(expr))
+	for pos < len(expr) {
+		width, err = writeNextUnescapedRegexRune(expr[pos:], &builder)
+		if err != nil {
+			return nil, err
+		}
+		pos += width
+	}
+	return regexp.Compile(builder.String())
+
+}
+
+// writeNextUnescapedRegexRune writes a rune to builder from s.
+// The rune is the next decoded UTF-8 rune with regex escaping rules applied.
+func writeNextUnescapedRegexRune(s string, builder *strings.Builder) (int, error) {
+	r, width := utf8.DecodeRuneInString(s)
+	if r == '\\' {
+		next, w := utf8.DecodeRuneInString(s[width:])
+		width += w
+		switch next {
+		case '/':
+			builder.WriteRune('/')
+			return width, nil
+		case 'x':
+			b, err := fromHexDigits(s[width:])
+			if err != nil {
+				return 0, err
+			}
+			builder.WriteByte(b)
+			return width + 2, nil
+		default:
+			// Standard regexp escape characters may exist,
+			// we leave them alone and let Go's regex parser validate them.
+			builder.WriteRune('\\')
+			builder.WriteRune(next)
+			return width, nil
+		}
 	}
-	return regexp.Compile(expr)
+	builder.WriteRune(r)
+	return width, nil
 }

internal/parser/strconv.go

@@ -2,6 +2,7 @@ package parser_test
 
 import (
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/influxdata/flux/ast"
@@ -165,3 +166,155 @@ func TestParseDuration(t *testing.T) {
 		})
 	}
 }
+
+func TestParseTime(t *testing.T) {
+	testCases := []struct {
+		testName string
+		time     string
+		want     time.Time
+		err      error
+	}{
+		{
+			testName: "RFC3339Nano",
+			time:     "2022-09-14T04:37:17.123456789Z",
+			want:     time.Date(2022, 9, 14, 4, 37, 17, 123456789, time.UTC),
+		},
+		{
+			testName: "millis",
+			time:     "2022-09-14T04:37:17.123Z",
+			want:     time.Date(2022, 9, 14, 4, 37, 17, 123000000, time.UTC),
+		},
+		{
+			testName: "date time offset",
+			time:     "2022-09-14T04:37:17.123456789-07:00",
+			want:     time.Date(2022, 9, 14, 4, 37, 17, 123456789, time.FixedZone("", -7*60*60)),
+		},
+		{
+			testName: "date only",
+			time:     "2022-09-14",
+			want:     time.Date(2022, 9, 14, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			testName: "date only error",
+			time:     "2022-00-14",
+			err:      errors.New(codes.Invalid, "cannot parse date"),
+		},
+		{
+			testName: "date time no offset",
+			time:     "2022-09-14T04:37:17.123456789",
+			err:      errors.New(codes.Invalid, "cannot parse date time"),
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			result, err := parser.ParseTime(tc.time)
+
+			if err != nil && tc.err == nil {
+				t.Errorf("Unexpected error: %v", err)
+			} else if tc.err != nil && tc.err == nil {
+				t.Errorf("Expected error but got nil: %v", tc.err)
+			} else if tc.err != nil && !cmp.Equal(err, tc.err) {
+				t.Errorf("Expected time error: %v", cmp.Diff(err, tc.err))
+			} else if !cmp.Equal(result, tc.want) {
+				t.Errorf("Expected time values to be eq: %v", cmp.Diff(result, tc.want))
+			}
+		})
+	}
+}
+
+func TestParseString(t *testing.T) {
+	testCases := []struct {
+		testName string
+		str      string
+		want     string
+		err      error
+	}{
+		{
+			testName: "normal",
+			str:      `"hello world"`,
+			want:     "hello world",
+		},
+		{
+			testName: "escape sequences",
+			str: `"newline\n
+carriage return\r
+horizontal tab\t
+double quote \"
+backslash \\
+dollar curly braket \${
+"`,
+
+			want: "newline\n\ncarriage return\r\nhorizontal tab\t\ndouble quote \"\nbackslash \\\ndollar curly braket ${\n",
+		},
+		{
+			testName: "hex escape sequences",
+			str:      `"\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"`,
+			want:     "日本語",
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			result, err := parser.ParseString(tc.str)
+
+			if err != nil && tc.err == nil {
+				t.Errorf("Unexpected error: %v", err)
+			} else if tc.err != nil && tc.err == nil {
+				t.Errorf("Expected error but got nil: %v", tc.err)
+			} else if tc.err != nil && !cmp.Equal(err, tc.err) {
+				t.Errorf("Expected string error: %v", cmp.Diff(err, tc.err))
+			} else if !cmp.Equal(result, tc.want) {
+				t.Errorf("Expected string values to be eq: %v", cmp.Diff(result, tc.want))
+			}
+		})
+	}
+}
+
+func TestParseRegex(t *testing.T) {
+	testCases := []struct {
+		testName string
+		str      string
+		want     string
+		err      error
+	}{
+		{
+			testName: "normal",
+			str:      `/hello world/`,
+			want:     "hello world",
+		},
+		{
+			testName: "escape sequences",
+			str:      `/forward slash \/ character classes: \w\s\d/`,
+			want:     `forward slash / character classes: \w\s\d`,
+		},
+		{
+			testName: "hex escape sequences",
+			str:      `/\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e/`,
+			want:     "日本語",
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			regex, err := parser.ParseRegexp(tc.str)
+			var result string
+			if regex != nil {
+				result = regex.String()
+			}
+
+			if err != nil && tc.err == nil {
+				t.Errorf("Unexpected error: %v", err)
+			} else if tc.err != nil && tc.err == nil {
+				t.Errorf("Expected error but got nil: %v", tc.err)
+			} else if tc.err != nil && !cmp.Equal(err, tc.err) {
+				t.Errorf("Expected regexp error: %v", cmp.Diff(err, tc.err))
+			} else if !cmp.Equal(result, tc.want) {
+				t.Errorf("Expected regexp values to be eq: %v", cmp.Diff(result, tc.want))
+			}
+		})
+	}
+}

internal/parser/strconv_test.go

@@ -1457,6 +1457,10 @@ impl<'input> Parser<'input> {
     }
 
     fn create_bad_expression(&mut self, t: Token) -> Expression {
+        let text = format!("invalid token for primary expression: {}", t.tok);
+        self.create_bad_expression_with_text(t, text)
+    }
+    fn create_bad_expression_with_text(&mut self, t: Token, text: String) -> Expression {
         Expression::Bad(Box::new(BadExpr {
             // Do not use `self.base_node_*` in order not to steal errors.
             // The BadExpr is an error per se. We want to leave errors to parents.
@@ -1467,7 +1471,7 @@ impl<'input> Parser<'input> {
                 ),
                 ..BaseNode::default()
             },
-            text: format!("invalid token for primary expression: {}", t.tok),
+            text,
             expression: None,
         }))
     }
@@ -1513,7 +1517,13 @@ impl<'input> Parser<'input> {
                 let lit = self.parse_time_literal();
                 match lit {
                     Ok(lit) => Expression::DateTime(lit),
-                    Err(terr) => self.create_bad_expression(terr.token),
+                    Err(terr) => match terr.token.tok {
+                        TokenType::Time => self.create_bad_expression_with_text(
+                            terr.token,
+                            "invalid date time literal, missing time offset".to_string(),
+                        ),
+                        _ => self.create_bad_expression(terr.token),
+                    },
                 }
             }
             TokenType::Duration => {