avm1: More accurate semantics for instanceof and CastOp

- only coerce RHS when the LHS is object-like
- `prototype` access should ignore __proto__, properties, and version attributes

This exposes some Flash weirdness; in FP, the following code traces `false` if
placed on frame 1, but hangs if placed on any other frame.

```
a = {};
b = {};
a.__proto__ = b;
b.__proto__ = a;
trace(a instanceof b);
```

Prior to this PR, Ruffle raised an `Error::PrototypeRecursionLimit`, for slightly
wrong reasons (we tried to crawl b's `__proto__` chain to find a `prototype`
property); now it returns `false`.

This affects Gnash's `Inheritance-*` tests, as they run the above code on frame 2,
and so hang in FP.

Author: moulins

core/src/avm1/activation.rs

@@ -811,20 +811,15 @@ impl<'a, 'gc> Activation<'a, 'gc> {
 
     fn action_cast_op(&mut self) -> Result<FrameControl<'gc>, Error<'gc>> {
         let obj = self.context.avm1.pop();
-        let constr = self.context.avm1.pop().coerce_to_object_or_bare(self)?;
-
-        let is_instance_of = if let Some(obj) = obj.as_object(self) {
-            let prototype = constr
-                .get(istr!(self, "prototype"), self)?
-                .coerce_to_object_or_bare(self)?;
-            obj.is_instance_of(self, constr, prototype)?
-        } else {
-            false
-        };
+        let constr = self.context.avm1.pop();
+        // For some reason, FP does this useless extra coercion.
+        if obj.is_primitive() {
+            let _ = obj.coerce_to_object(self)?;
+        }
 
+        let is_instance_of = obj.instance_of(constr, self)?;
         let result = if is_instance_of { obj } else { Value::Null };
         self.context.avm1.push(result);
-
         Ok(FrameControl::Continue)
     }
 
@@ -1458,6 +1453,7 @@ impl<'a, 'gc> Activation<'a, 'gc> {
             }
 
             let prototype = constructor
+                // TODO(moulins): should this use `Object::prototype`?
                 .get(istr!(self, "prototype"), self)?
                 .coerce_to_object_or_bare(self)?;
             prototype.set_interfaces(self.gc(), interfaces);
@@ -1467,18 +1463,9 @@ impl<'a, 'gc> Activation<'a, 'gc> {
     }
 
     fn action_instance_of(&mut self) -> Result<FrameControl<'gc>, Error<'gc>> {
-        let constr = self.context.avm1.pop().coerce_to_object_or_bare(self)?;
+        let constr = self.context.avm1.pop();
         let obj = self.context.avm1.pop();
-
-        let result = if let Some(obj) = obj.as_object(self) {
-            let prototype = constr
-                .get(istr!(self, "prototype"), self)?
-                .coerce_to_object_or_bare(self)?;
-            obj.is_instance_of(self, constr, prototype)?
-        } else {
-            false
-        };
-
+        let result = obj.instance_of(constr, self)?;
         self.context.avm1.push(result.into());
         Ok(FrameControl::Continue)
     }

core/src/avm1/object.rs

@@ -325,6 +325,8 @@ impl<'gc> Object<'gc> {
             proto_stack.push(p);
         }
 
+        // TODO(moulins): should we guard against infinite loops here?
+        // A recursive prototype chain will hang Flash Player.
         while let Some(this_proto) = proto_stack.pop() {
             if Object::ptr_eq(this_proto, prototype) {
                 return Ok(true);
@@ -340,6 +342,7 @@ impl<'gc> Object<'gc> {
                         return Ok(true);
                     }
 
+                    // TODO(moulins): should this use `Object::prototype`?
                     if let Value::Object(o) = interface.get(istr!("prototype"), activation)? {
                         proto_stack.push(o);
                     }
@@ -361,6 +364,9 @@ impl<'gc> Object<'gc> {
 
     /// Check if this object is in the prototype chain of the specified test object.
     pub fn is_prototype_of(self, activation: &mut Activation<'_, 'gc>, other: Object<'gc>) -> bool {
+        // TODO(moulins): should we guard against infinite loops here?
+        // A recursive prototype chain will hang Flash Player.
+
         let mut proto = other.proto(activation);
 
         while let Value::Object(proto_ob) = proto {

core/src/avm1/object/script_object.rs

@@ -161,7 +161,7 @@ impl<'gc> Object<'gc> {
         ))
     }
 
-    /// Gets the value of a data property on this object.
+    /// Gets the value of a data property on this object, ignoring attributes.
     ///
     /// Doesn't look up the prototype chain and ignores virtual properties, thus cannot cause
     /// any side-effects.
@@ -177,7 +177,7 @@ impl<'gc> Object<'gc> {
             .map_or(Value::Undefined, |property| property.data())
     }
 
-    /// Sets a data property on this object.
+    /// Sets a data property on this object, ignoring attributes.
     ///
     /// Doesn't look up the prototype chain and ignores virtual properties, but still might
     /// call to watchers.
@@ -216,7 +216,8 @@ impl<'gc> Object<'gc> {
             .collect()
     }
 
-    /// Retrieve a named, non-virtual property from this object exclusively.
+    /// Retrieve a named, non-virtual property from this object exclusively, taking
+    /// attributes into account.
     ///
     /// This function should not inspect prototype chains. Instead, use
     /// `get_stored` to do ordinary property look-up and resolution.
@@ -649,6 +650,7 @@ impl<'gc> Object<'gc> {
     }
 
     /// Retrieve the `__proto__` of a given object.
+    /// (don't confuse this with `self.prototype()`!)
     ///
     /// The proto is another object used to resolve methods across a class of
     /// multiple objects. It should also be accessible as `__proto__` from
@@ -661,6 +663,13 @@ impl<'gc> Object<'gc> {
         self.get_data(istr!("__proto__"), activation)
     }
 
+    /// Retrieve the `prototype` of this object, as if it was a function.
+    /// (don't confuse this with `self.proto()`!)
+    pub fn prototype(self, activation: &mut Activation<'_, 'gc>) -> Value<'gc> {
+        // Ignore getters, __proto__, and SWF version attributes.
+        self.get_data(istr!("prototype"), activation)
+    }
+
     /// Checks if the object has a given named property.
     pub fn has_property(self, activation: &mut Activation<'_, 'gc>, name: AvmString<'gc>) -> bool {
         let dobj = match self.native_no_super() {

core/src/avm1/value.rs

@@ -552,6 +552,26 @@ impl<'gc> Value<'gc> {
         };
         Ok(result)
     }
+
+    /// ActionScript 2's `self instanceof Class`.
+    pub fn instance_of(
+        self,
+        class: Value<'gc>,
+        activation: &mut Activation<'_, 'gc>,
+    ) -> Result<bool, Error<'gc>> {
+        if !self.is_primitive() {
+            // These coercions happen even if `obj` is a dead `MovieClipReference`.
+            if let Some(class) = class.coerce_to_object(activation)? {
+                if let Some(p) = class.prototype(activation).coerce_to_object(activation)? {
+                    if let Some(obj) = self.as_object(activation) {
+                        return obj.is_instance_of(activation, class, p);
+                    }
+                }
+            }
+        }
+
+        Ok(false)
+    }
 }
 
 /// Calculate `value * 10^exp` through repeated multiplication or division.

core/src/avm1/xml/tree.rs

@@ -359,6 +359,7 @@ impl<'gc> XmlNode<'gc> {
             Some(object) => object,
             None => {
                 let xml_node = activation.prototypes().xml_node_constructor;
+                // TODO(moulins): should this use `Object::prototype`?
                 let prototype = xml_node.get(istr!("prototype"), activation).ok();
                 let object = Object::new(&activation.context.strings, prototype);
                 self.introduce_script_object(activation.gc(), object);

core/src/display_object/movie_clip.rs

@@ -1903,6 +1903,7 @@ impl<'gc> MovieClip<'gc> {
                 );
 
                 if let Ok(prototype) = constructor
+                    // TODO(moulins): should this use `Object::prototype`?
                     .get(istr!("prototype"), &mut activation)
                     .and_then(|v| v.coerce_to_object_or_bare(&mut activation))
                 {

core/src/player.rs

@@ -2175,6 +2175,7 @@ impl Player {
                         ActivationIdentifier::root("[Construct]"),
                         action.clip,
                     );
+                    // TODO(moulins): should this use `Object::prototype`?
                     if let Ok(prototype) = constructor.get(istr!("prototype"), &mut activation) {
                         if let Some(object) = action.clip.object1() {
                             object.define_value(

tests/tests/swfs/avm1/instanceof_coercions/NoisyString.as

@@ -0,0 +1,6 @@
+class NoisyString extends String {
+  function NoisyString(val) {
+    super(val);
+    trace('"' + val + '" coerced to NoisyString!');
+  }
+}

tests/tests/swfs/avm1/instanceof_coercions/Test.as

@@ -0,0 +1,93 @@
+// Compile with:
+//  mtasc -main -version 8 Test.as -swf shim.swf -keep -out test.swf
+class Test {
+
+  // shim.swf provides this function, which wraps the `CastOp` AVM1 action.
+  // arguments: (object, constructor)
+  // It is normally available through `catch(e: Class)`, but direct access
+  // makes for simpler test code, and allows inspecting the actual return value.
+  static var opcode__cast = _root.opcode__cast;
+
+  static function main(current) {
+    _global.String = NoisyString;
+
+    trace('// obj: "left", class: "right"');
+    check("left", "right");
+    trace('// obj: "left", class: Object');
+    check("left", Object);
+    trace('// obj: {}, class: "right"');
+    check({}, "right");
+    trace('');
+
+    trace('// obj: {}, class: { prototype: Object.prototype }');
+    check({}, { prototype: Object.prototype });
+
+    trace('// obj: { __proto__: "left" }, class: { prototype: "right" }');
+    check({ __proto__: "left" }, { prototype: "right" });
+    
+    trace('// obj: { __proto__: "proto" }, class: { prototype: "proto" }');
+    check({ __proto__: "proto" }, { prototype: "proto" });
+
+    trace('// obj: { __proto__: null }, class: { prototype: null }');
+    check({ __proto__: null }, { prototype: null });
+
+    // Prototype properties don't work...
+    var fakeClass = {};
+    fakeClass.addProperty("prototype", function() {
+      trace("prototype property called!");
+      return "property proto";
+    }, null);
+    trace('// obj: {}, class: { get prototype(): ... }');
+    trace('.prototype is: ' + fakeClass.prototype);
+    check({}, fakeClass);
+
+    // ...and neither do 'indirect' prototypes...
+    fakeClass = { __proto__: { prototype: {}}};
+    trace('// obj: { __proto__: X }, class: { __proto__: { prototype: X }}');
+    check({ __proto__: fakeClass.prototype }, fakeClass);
+
+    // ...nor prototypes behind 'super'.
+    var FakeClassSuper = function() { this.zuper = super; };
+    FakeClassSuper.prototype = { __proto__: { prototype: "super proto" }};
+    fakeClass = (new FakeClassSuper()).zuper;
+    trace('// obj: {}, class: super { prototype: ... }');
+    trace('.prototype is: ' + fakeClass.prototype);
+    check({}, fakeClass);
+
+    // SWF version flags are ignored.
+    fakeClass = { prototype: "SWFv9 proto" };
+    _global.ASSetPropFlags(fakeClass, "prototype", 0x2000 /* version 9 */);
+    trace('// obj: {}, class: { prototype(SWFv9): ... }');
+    trace('.prototype is: ' + fakeClass.prototype);
+    check({}, fakeClass);
+
+    trace('');
+
+    var movieclip = current.createEmptyMovieClip("mc", 1);
+    movieclip.prototype = Object.prototype;
+    testsForMovieClip(movieclip);
+    trace('// kill movieclip');
+    movieclip.removeMovieClip();
+    testsForMovieClip(movieclip);
+
+    fscommand('quit');
+  }
+
+  static function testsForMovieClip(movieclip) {
+    trace('// obj: movieclip, class: "right"');
+    check(movieclip, "right");
+    trace('// obj: movieclip, class: Object');
+    check(movieclip, Object);
+    trace('// obj: movieclip, class: { prototype: "proto" }');
+    check(movieclip, { prototype: "proto" });
+    trace('// obj: {}, class: MovieClip { prototype: Object.prototype }');
+    check({}, movieclip);
+    trace('// obj: { __proto__: movieclip }, class: { prototype: movieclip }');
+    check({ __proto__: movieclip }, { prototype: movieclip });
+  }
+
+  static function check(obj, klass) {
+    trace("instanceof: " + (obj instanceof klass));
+    trace("typeof cast: " + typeof opcode__cast(obj, klass));
+  }
+}

tests/tests/swfs/avm1/instanceof_coercions/output.txt

@@ -0,0 +1,88 @@
+// obj: "left", class: "right"
+instanceof: false
+"left" coerced to NoisyString!
+typeof cast: null
+// obj: "left", class: Object
+instanceof: false
+"left" coerced to NoisyString!
+typeof cast: null
+// obj: {}, class: "right"
+"right" coerced to NoisyString!
+instanceof: false
+"right" coerced to NoisyString!
+typeof cast: null
+
+// obj: {}, class: { prototype: Object.prototype }
+instanceof: true
+typeof cast: object
+// obj: { __proto__: "left" }, class: { prototype: "right" }
+"right" coerced to NoisyString!
+instanceof: false
+"right" coerced to NoisyString!
+typeof cast: null
+// obj: { __proto__: "proto" }, class: { prototype: "proto" }
+"proto" coerced to NoisyString!
+instanceof: false
+"proto" coerced to NoisyString!
+typeof cast: null
+// obj: { __proto__: null }, class: { prototype: null }
+instanceof: false
+typeof cast: null
+// obj: {}, class: { get prototype(): ... }
+prototype property called!
+.prototype is: property proto
+instanceof: false
+typeof cast: null
+// obj: { __proto__: X }, class: { __proto__: { prototype: X }}
+instanceof: false
+typeof cast: null
+// obj: {}, class: super { prototype: ... }
+.prototype is: super proto
+instanceof: false
+typeof cast: null
+// obj: {}, class: { prototype(SWFv9): ... }
+.prototype is: undefined
+"SWFv9 proto" coerced to NoisyString!
+instanceof: false
+"SWFv9 proto" coerced to NoisyString!
+typeof cast: null
+
+// obj: movieclip, class: "right"
+"right" coerced to NoisyString!
+instanceof: false
+"right" coerced to NoisyString!
+typeof cast: null
+// obj: movieclip, class: Object
+instanceof: true
+typeof cast: movieclip
+// obj: movieclip, class: { prototype: "proto" }
+"proto" coerced to NoisyString!
+instanceof: false
+"proto" coerced to NoisyString!
+typeof cast: null
+// obj: {}, class: MovieClip { prototype: Object.prototype }
+instanceof: true
+typeof cast: object
+// obj: { __proto__: movieclip }, class: { prototype: movieclip }
+instanceof: false
+typeof cast: null
+// kill movieclip
+// obj: movieclip, class: "right"
+"right" coerced to NoisyString!
+instanceof: false
+"right" coerced to NoisyString!
+typeof cast: null
+// obj: movieclip, class: Object
+instanceof: false
+typeof cast: null
+// obj: movieclip, class: { prototype: "proto" }
+"proto" coerced to NoisyString!
+instanceof: false
+"proto" coerced to NoisyString!
+typeof cast: null
+// obj: {}, class: MovieClip { prototype: Object.prototype }
+instanceof: false
+typeof cast: null
+// obj: { __proto__: movieclip }, class: { prototype: movieclip }
+instanceof: false
+typeof cast: null