diff --git a/tests/conformance/nonconforming.yaml b/tests/conformance/nonconforming.yaml
index 25744e50..fb2876bb 100644
--- a/tests/conformance/nonconforming.yaml
+++ b/tests/conformance/nonconforming.yaml
@@ -7,3 +7,25 @@ standard_constraints/well_known_types/duration:
 standard_constraints/well_known_types/timestamp:
  - gte_lte/invalid/above
  - lte/invalid
+
+library/is_ip:
+  - version/omitted/invalid/ipv6/g
+  #       input: [type.googleapis.com/buf.validate.conformance.cases.IsIp]:{val:":0::0"}
+  #        want: validation error (1 violation)
+  #          1. constraint_id: "library.is_ip"
+  #         got: valid
+  - version/omitted/invalid/ipv6/h
+  #       input: [type.googleapis.com/buf.validate.conformance.cases.IsIp]:{val:"0::0:"}
+  #        want: validation error (1 violation)
+  #          1. constraint_id: "library.is_ip"
+  #         got: valid
+  - version/omitted/invalid/ipv6/i
+  #       input: [type.googleapis.com/buf.validate.conformance.cases.IsIp]:{val:"0::0:"}
+  #        want: validation error (1 violation)
+  #          1. constraint_id: "library.is_ip"
+  #         got: valid
+  - version/omitted/invalid/ipv6/j
+  #       input: [type.googleapis.com/buf.validate.conformance.cases.IsIp]:{val:"::0000ffff"}
+  #        want: validation error (1 violation)
+  #          1. constraint_id: "library.is_ip"
+  #         got: unexpected error: string index out of range
diff --git a/tests/conformance/runner.py b/tests/conformance/runner.py
index 7bd6a4c1..392e97f2 100644
--- a/tests/conformance/runner.py
+++ b/tests/conformance/runner.py
@@ -72,6 +72,8 @@ def run_test_case(tc: typing.Any, result: typing.Optional[harness_pb2.TestResult
         result.runtime_error = str(e)
     except protovalidate.CompilationError as e:
         result.compilation_error = str(e)
+    except Exception as e:
+        result.unexpected_error = str(e)
     return result