Add a .?>link operator to dereference link while suppressing "hidden by policy" (#9101)

msullivan · aljazerzen · web-flow · commit fc4ea5d0cb72 · 2025-10-28T11:04:50.000-07:00
Users (and us) occasionally have problems dealing with "required link hidden by access policy" errors. They can be worked around often by using backlinks and an explicit join, but that that is kind of horrible and seems to sometimes be slow. Issue #8522 proposes adding an intrinsic function to mask the error, but I really hate that. I don't think that a simple function call ought to be impacting the semantics of the expression contained within, and I think it's really unclear what its behavior would be in more complex cases. Instead, add an operator, `Obj.?>link` that does an *optional* dereference of `link`. The result is always optional, and it suppresses any "hidden by policy" error. I'm happy to argue about syntax if anyone has something they like more. I also considered `.?` but that was worse. Closes #8522, but not in the way proposed there. --------- Co-authored-by: Aljaž Mur Eržen <aljazerzen@users.noreply.github.com>
diff --git a/edb/edgeql-parser/edgeql-parser-python/src/normalize.rs b/edb/edgeql-parser/edgeql-parser-python/src/normalize.rs
@@ -215,10 +215,10 @@ fn is_operator(token: &Token) -> bool {
     use edgeql_parser::tokenizer::Kind::*;
     match token.kind {
         Assign | SubAssign | AddAssign | Arrow | Coalesce | Namespace | DoubleSplat
-        | BackwardLink | FloorDiv | Concat | GreaterEq | LessEq | NotEq | NotDistinctFrom
-        | DistinctFrom | Comma | OpenParen | CloseParen | OpenBracket | CloseBracket
-        | OpenBrace | CloseBrace | Dot | Semicolon | Colon | Add | Sub | Mul | Div | Modulo
-        | Pow | Less | Greater | Eq | Ampersand | Pipe | At => true,
+        | BackwardLink | OptionalLink | FloorDiv | Concat | GreaterEq | LessEq | NotEq
+        | NotDistinctFrom | DistinctFrom | Comma | OpenParen | CloseParen | OpenBracket
+        | CloseBracket | OpenBrace | CloseBrace | Dot | Semicolon | Colon | Add | Sub | Mul
+        | Div | Modulo | Pow | Less | Greater | Eq | Ampersand | Pipe | At => true,
         DecimalConst | FloatConst | IntConst | BigIntConst | BinStr | Parameter
         | ParameterAndType | Str | BacktickName | Keyword(_) | Ident | Substitution | EOI
         | Epsilon | StartBlock | StartExtension | StartFragment | StartMigration
diff --git a/edb/edgeql-parser/src/parser/spec.rs b/edb/edgeql-parser/src/parser/spec.rs
@@ -67,6 +67,7 @@ pub(super) fn get_token_kind(token_name: &str) -> Kind {
         "&" => Ampersand,
         "@" => At,
         ".<" => BackwardLink,
+        ".?>" => OptionalLink,
         "}" => CloseBrace,
         "]" => CloseBracket,
         ")" => CloseParen,
diff --git a/edb/edgeql-parser/src/tokenizer.rs b/edb/edgeql-parser/src/tokenizer.rs
@@ -78,6 +78,7 @@ pub enum Kind {
     Coalesce,         // ??
     Namespace,        // ::
     BackwardLink,     // .<
+    OptionalLink,     // .?>
     FloorDiv,         // //
     Concat,           // ++
     GreaterEq,        // >=
@@ -356,6 +357,16 @@ impl<'a> Tokenizer<'a> {
             },
             '.' => match iter.next() {
                 Some((_, '<')) => Ok((BackwardLink, 2)),
+                Some((_, '?')) => {
+                    if let Some((_, '>')) = iter.next() {
+                        Ok((OptionalLink, 3))
+                    } else {
+                        Err(Error::new(
+                            "`.?` is not an operator, \
+                                did you mean `.?>` ?",
+                        ))
+                    }
+                }
                 _ => Ok((Dot, 1)),
             },
             '?' => match iter.next() {
@@ -1063,6 +1074,7 @@ impl Kind {
             Ampersand => "&",
             At => "@",
             BackwardLink => ".<",
+            OptionalLink => ".?>",
             CloseBrace => "}",
             CloseBracket => "]",
             CloseParen => ")",
diff --git a/edb/edgeql/ast.py b/edb/edgeql/ast.py
@@ -406,7 +406,9 @@ class TypeIntersection(Base):
 class Ptr(Base):
     name: str
     direction: typing.Optional[str] = None
-    type: typing.Optional[str] = None
+    # @ptr has type 'property'
+    # .?>ptr has type 'optional'
+    type: typing.Optional[typing.Literal['optional', 'property']] = None
 
 
 class Splat(Base):
diff --git a/edb/edgeql/codegen.py b/edb/edgeql/codegen.py
@@ -600,6 +600,8 @@ def _visit_shape(
     def visit_Ptr(self, node: qlast.Ptr, *, quote: bool = True) -> None:
         if node.type == 'property':
             self.write('@')
+        elif node.type == 'optional':
+            self.write('?>')
         elif node.direction and node.direction != '>':
             self.write(node.direction)
 
diff --git a/edb/edgeql/compiler/inference/cardinality.py b/edb/edgeql/compiler/inference/cardinality.py
@@ -629,6 +629,10 @@ def _infer_set_inner(
 
         card = cartesian_cardinality((source_card, rptrref_card))
 
+        # "Optional derefs" (.?>) always produce an optional result.
+        if ptr.optional_deref:
+            card = cartesian_cardinality((AT_MOST_ONE, card))
+
     elif sub_expr is not None:
         card = expr_card
     else:
diff --git a/edb/edgeql/compiler/setgen.py b/edb/edgeql/compiler/setgen.py
@@ -621,12 +621,13 @@ def compile_path(expr: qlast.Path, *, ctx: context.ContextLevel) -> irast.Set:
                     direction=direction,
                     upcoming_intersections=upcoming_intersections,
                     ignore_computable=True,
+                    optional_deref=step.type == 'optional',
                     span=step.span, ctx=ctx)
 
                 assert isinstance(path_tip.expr, irast.Pointer)
                 ptrcls = typegen.ptrcls_from_ptrref(
                     path_tip.expr.ptrref, ctx=ctx)
-                if _is_computable_ptr(ptrcls, direction, ctx=ctx):
+                if _is_computable_ptr(ptrcls, path_tip.expr, ctx=ctx):
                     is_computable = True
 
         elif isinstance(step, qlast.TypeIntersection):
@@ -815,6 +816,7 @@ def ptr_step_set(
     direction: PtrDir = PtrDir.Outbound,
     span: Optional[qlast.Span],
     ignore_computable: bool = False,
+    optional_deref: bool = False,
     ctx: context.ContextLevel,
 ) -> irast.Set:
     ptrcls, path_id_ptrcls = resolve_ptr_with_intersections(
@@ -829,7 +831,9 @@ def ptr_step_set(
     return extend_path(
         path_tip, ptrcls, direction,
         path_id_ptrcls=path_id_ptrcls,
-        ignore_computable=ignore_computable, span=span,
+        ignore_computable=ignore_computable,
+        optional_deref=optional_deref,
+        span=span,
         ctx=ctx)
 
 
@@ -1116,6 +1120,7 @@ def extend_path(
     path_id_ptrcls: Optional[s_pointers.Pointer] = None,
     ignore_computable: bool = False,
     same_computable_scope: bool = False,
+    optional_deref: bool = False,
     span: Optional[qlast.Span]=None,
     ctx: context.ContextLevel,
 ) -> irast.SetE[irast.Pointer]:
@@ -1169,11 +1174,12 @@ def extend_path(
         direction=direction,
         ptrref=typegen.ptr_to_ptrref(ptrcls, ctx=ctx),
         is_definition=False,
+        optional_deref=optional_deref,
     )
     target_set = new_set(
         stype=target, path_id=path_id, span=span, expr=ptr, ctx=ctx)
 
-    is_computable = _is_computable_ptr(ptrcls, direction, ctx=ctx)
+    is_computable = _is_computable_ptr(ptrcls, ptr, ctx=ctx)
     if not ignore_computable and is_computable:
         target_set = computable_ptr_set(
             ptr,
@@ -1189,7 +1195,7 @@ def extend_path(
 
 def needs_rewrite_existence_assertion(
     ptrcls: s_pointers.PointerLike,
-    direction: PtrDir,
+    rptr: irast.Pointer,
     *,
     ctx: context.ContextLevel,
 ) -> bool:
@@ -1201,8 +1207,10 @@ def needs_rewrite_existence_assertion(
 
     return bool(
         not ctx.suppress_rewrites
+        # We *don't* need to do the rewrite when using .?>
+        and not rptr.optional_deref
         and ptrcls.get_required(ctx.env.schema)
-        and direction == PtrDir.Outbound
+        and rptr.direction == PtrDir.Outbound
         and (target := ptrcls.get_target(ctx.env.schema))
         and ctx.env.type_rewrites.get((target, False))
         and ptrcls.get_shortname(ctx.env.schema).name != '__type__'
@@ -1211,7 +1219,7 @@ def needs_rewrite_existence_assertion(
 
 def is_injected_computable_ptr(
     ptrcls: s_pointers.PointerLike,
-    direction: PtrDir,
+    rptr: irast.Pointer,
     *,
     ctx: context.ContextLevel,
 ) -> bool:
@@ -1220,14 +1228,14 @@ def is_injected_computable_ptr(
         and ptrcls not in ctx.active_computeds
         and (
             bool(ptrcls.get_schema_reflection_default(ctx.env.schema))
-            or needs_rewrite_existence_assertion(ptrcls, direction, ctx=ctx)
+            or needs_rewrite_existence_assertion(ptrcls, rptr, ctx=ctx)
         )
     )
 
 
 def _is_computable_ptr(
     ptrcls: s_pointers.PointerLike,
-    direction: PtrDir,
+    rptr: irast.Pointer,
     *,
     ctx: context.ContextLevel,
 ) -> bool:
@@ -1240,7 +1248,7 @@ def _is_computable_ptr(
 
     return (
         bool(ptrcls.get_expr(ctx.env.schema))
-        or is_injected_computable_ptr(ptrcls, direction, ctx=ctx)
+        or is_injected_computable_ptr(ptrcls, rptr, ctx=ctx)
     )
 
 
@@ -1711,8 +1719,7 @@ def computable_ptr_set(
                     op='??',
                 )
 
-            if needs_rewrite_existence_assertion(
-                    ptrcls, PtrDir.Outbound, ctx=ctx):
+            if needs_rewrite_existence_assertion(ptrcls, rptr, ctx=ctx):
                 # Wrap it in a dummy select so that we can't optimize away
                 # the assert_exists.
                 # TODO: do something less bad
diff --git a/edb/edgeql/parser/grammar/expressions.py b/edb/edgeql/parser/grammar/expressions.py
@@ -1912,6 +1912,15 @@ def reduce_DOTBW_PathStepName(self, *kids):
             direction=s_pointers.PointerDirection.Inbound
         )
 
+    def reduce_DOTQ_PathStepName(self, *kids):
+        from edb.schema import pointers as s_pointers
+
+        self.val = qlast.Ptr(
+            name=kids[1].val.name,
+            direction=s_pointers.PointerDirection.Outbound,
+            type='optional',
+        )
+
     def reduce_AT_PathNodeName(self, *kids):
         from edb.schema import pointers as s_pointers
 
diff --git a/edb/edgeql/parser/grammar/precedence.py b/edb/edgeql/parser/grammar/precedence.py
@@ -147,7 +147,7 @@ class P_PAREN(Precedence, assoc='left', tokens=('LPAREN', 'RPAREN')):
     pass
 
 
-class P_DOT(Precedence, assoc='left', tokens=('DOT', 'DOTBW')):
+class P_DOT(Precedence, assoc='left', tokens=('DOT', 'DOTBW', 'DOTQ')):
     pass
 
 
diff --git a/edb/edgeql/parser/grammar/tokens.py b/edb/edgeql/parser/grammar/tokens.py
@@ -87,6 +87,10 @@ class T_DOTBW(Token, lextoken='.<'):
     pass
 
 
+class T_DOTQ(Token, lextoken='.?>'):
+    pass
+
+
 class T_LBRACKET(Token, lextoken='['):
     pass
 
diff --git a/edb/ir/ast.py b/edb/ir/ast.py
@@ -465,6 +465,10 @@ class Pointer(Expr):
     source: Set
     ptrref: BasePointerRef
     direction: s_pointers.PointerDirection
+    # Whether to make this an optional deref (written '.?>') that
+    # suppresses any error due to looking at a required link hidden by
+    # a policy .
+    optional_deref: bool = False
     # Whether to *always* use a link table when this pointer is
     # accessed.  This is needed (for example) when a (possibly single)
     # link property is being referenced in a FOR iterator, and we
diff --git a/tests/test_edgeql_ir_card_inference.py b/tests/test_edgeql_ir_card_inference.py
@@ -1449,3 +1449,10 @@ def test_edgeql_ir_card_inference_169(self):
 % OK %
         AT_LEAST_ONE
         """
+
+    def test_edgeql_ir_card_inference_170(self):
+        """
+        select Report { u := .?>user }
+% OK %
+        u: AT_MOST_ONE
+        """
diff --git a/tests/test_edgeql_policies.py b/tests/test_edgeql_policies.py
@@ -290,6 +290,42 @@ async def test_edgeql_policies_05a(self):
                 select Ptr.tgt.b
             ''')
 
+        # A bunch using .?> that should not fail
+        await self.assert_query_result(
+            '''
+            select Ptr { t := .?>tgt }
+            ''',
+            [{'t': None}],
+        )
+
+        await self.assert_query_result(
+            '''
+            select Ptr { z := .?>tgt.b }
+            ''',
+            [{'z': None}],
+        )
+
+        await self.assert_query_result(
+            '''
+            select Ptr { z := .?>tgt.id }
+            ''',
+            [{'z': None}],
+        )
+
+        await self.assert_query_result(
+            '''
+            select Ptr.?>tgt
+            ''',
+            [],
+        )
+
+        await self.assert_query_result(
+            '''
+            select Ptr.?>tgt.b
+            ''',
+            [],
+        )
+
         await self.con.query('''
             delete Ptr
         ''')
diff --git a/tests/test_edgeql_syntax.py b/tests/test_edgeql_syntax.py
@@ -2332,6 +2332,19 @@ def test_edgeql_syntax_path_33(self):
         select Foo.bar@intersect;
         """
 
+    def test_edgeql_syntax_path_34(self):
+        """
+        SELECT Foo.?>bar;
+        SELECT Foo.?>bar@spam;
+        SELECT Foo.?>bar[IS Baz];
+
+% OK %
+
+        SELECT Foo.?>bar;
+        SELECT Foo.?>bar@spam;
+        SELECT Foo.?>bar[IS Baz];
+        """
+
     def test_edgeql_syntax_type_interpretation_01(self):
         """
         SELECT Foo[IS Bar].spam;
