|
| 1 | +import json |
| 2 | +from fastapi import HTTPException |
| 3 | + |
| 4 | +import jsonschema |
| 5 | +import xml.etree.ElementTree as ET |
| 6 | +import xmlschema |
| 7 | +import yaml |
| 8 | + |
| 9 | +from typing import List, Optional |
| 10 | + |
| 11 | +from base_detector_registry import BaseDetectorRegistry |
| 12 | +from detectors.common.scheme import ContentAnalysisResponse |
| 13 | + |
| 14 | + |
| 15 | +def is_valid_json(s: str) -> Optional[ContentAnalysisResponse]: |
| 16 | + """Detect if the text contents is not valid JSON""" |
| 17 | + try: |
| 18 | + json.loads(s) |
| 19 | + return None |
| 20 | + except (ValueError, TypeError): |
| 21 | + return ContentAnalysisResponse( |
| 22 | + start=0, |
| 23 | + end=len(s), |
| 24 | + text=s, |
| 25 | + detection="invalid_json", |
| 26 | + detection_type= "file_type", |
| 27 | + score=1.0 |
| 28 | + ) |
| 29 | + |
| 30 | + |
| 31 | +def is_valid_json_schema(s: str, schema: str) -> Optional[ContentAnalysisResponse]: |
| 32 | + """Detect if the text contents does not satisfy a provided JSON schema. To specify a schema, replace $SCHEMA with a JSON schema.""" |
| 33 | + is_valid = is_valid_json(s) |
| 34 | + if is_valid is None: |
| 35 | + msg_data = json.loads(s) |
| 36 | + else: |
| 37 | + return is_valid |
| 38 | + |
| 39 | + |
| 40 | + # validate that the schema is valid json |
| 41 | + try: |
| 42 | + schema_data = json.loads(schema) |
| 43 | + except (ValueError, TypeError): |
| 44 | + return ContentAnalysisResponse( |
| 45 | + start=0, |
| 46 | + end=len(schema), |
| 47 | + text=s, |
| 48 | + detection="invalid_schema", |
| 49 | + detection_type="file_type", |
| 50 | + score=1.0 |
| 51 | + ) |
| 52 | + |
| 53 | + # validate the schema against the message |
| 54 | + try: |
| 55 | + jsonschema.validate(instance=msg_data, schema=schema_data) |
| 56 | + return None |
| 57 | + except jsonschema.ValidationError as e: |
| 58 | + return ContentAnalysisResponse( |
| 59 | + start=0, |
| 60 | + end=len(s), |
| 61 | + text=s, |
| 62 | + detection="json_schema_mismatch", |
| 63 | + detection_type="file_type", |
| 64 | + score=1.0 |
| 65 | + ) |
| 66 | + |
| 67 | + |
| 68 | +def is_valid_yaml(s: str) -> Optional[ContentAnalysisResponse]: |
| 69 | + """Detect if the text contents is not valid YAML""" |
| 70 | + try: |
| 71 | + yaml.safe_load(s) |
| 72 | + return None |
| 73 | + except Exception: |
| 74 | + return ContentAnalysisResponse( |
| 75 | + start=0, |
| 76 | + end=len(s), |
| 77 | + text=s, |
| 78 | + detection="invalid_yaml", |
| 79 | + detection_type="file_type", |
| 80 | + score=1.0, |
| 81 | + ) |
| 82 | + |
| 83 | +def is_valid_yaml_schema(s: str, schema) -> Optional[ContentAnalysisResponse]: |
| 84 | + """Detect if the text contents does not satisfy a provided schema. To specify a schema, replace $SCHEMA with a JSON schema. That's not a typo, you validate YAML with a JSON schema!""" |
| 85 | + is_valid = is_valid_yaml(s) |
| 86 | + if is_valid is None: |
| 87 | + msg_data = yaml.safe_load(s) |
| 88 | + else: |
| 89 | + return is_valid |
| 90 | + |
| 91 | + # validate that the schema is valid json |
| 92 | + try: |
| 93 | + schema_data = json.loads(schema) |
| 94 | + except (ValueError, TypeError): |
| 95 | + return ContentAnalysisResponse( |
| 96 | + start=0, |
| 97 | + end=len(schema), |
| 98 | + text=s, |
| 99 | + detection="invalid_schema", |
| 100 | + detection_type="file_type", |
| 101 | + score=1.0 |
| 102 | + ) |
| 103 | + |
| 104 | + # validate the schema against the message |
| 105 | + try: |
| 106 | + jsonschema.validate(instance=msg_data, schema=schema_data) |
| 107 | + return None |
| 108 | + except jsonschema.ValidationError as e: |
| 109 | + return ContentAnalysisResponse( |
| 110 | + start=0, |
| 111 | + end=len(s), |
| 112 | + text=s, |
| 113 | + detection="yaml_schema_mismatch", |
| 114 | + detection_type="file_type", |
| 115 | + score=1.0 |
| 116 | + ) |
| 117 | + |
| 118 | + |
| 119 | +def is_valid_xml(s: str) -> Optional[ContentAnalysisResponse]: |
| 120 | + """Detect if the text contents is not valid XML""" |
| 121 | + try: |
| 122 | + ET.fromstring(s) |
| 123 | + return None |
| 124 | + except Exception: |
| 125 | + return ContentAnalysisResponse( |
| 126 | + start=0, |
| 127 | + end=len(s), |
| 128 | + text=s, |
| 129 | + detection="invalid_xml", |
| 130 | + detection_type="file_type", |
| 131 | + score=1.0, |
| 132 | + ) |
| 133 | + |
| 134 | + |
| 135 | +def is_valid_xml_schema(s: str, schema) -> Optional[ContentAnalysisResponse]: |
| 136 | + """Detect if the text contents does not satisfy a provided XML schema. To specify a schema, replace $SCHEMA with an XML Schema Definition (XSD)""" |
| 137 | + is_valid = is_valid_xml(s) |
| 138 | + if is_valid is not None: |
| 139 | + return is_valid |
| 140 | + try: |
| 141 | + # schema is expected to be a string containing the XSD |
| 142 | + xs = xmlschema.XMLSchema(schema) |
| 143 | + except Exception: |
| 144 | + return ContentAnalysisResponse( |
| 145 | + start=0, |
| 146 | + end=len(schema), |
| 147 | + text=s, |
| 148 | + detection="invalid_xml_schema", |
| 149 | + detection_type="file_type", |
| 150 | + score=1.0 |
| 151 | + ) |
| 152 | + |
| 153 | + try: |
| 154 | + xs.validate(s) |
| 155 | + return None |
| 156 | + except xmlschema.XMLSchemaValidationError: |
| 157 | + return ContentAnalysisResponse( |
| 158 | + start=0, |
| 159 | + end=len(s), |
| 160 | + text=s, |
| 161 | + detection="xml_schema_mismatch", |
| 162 | + detection_type="file_type", |
| 163 | + score=1.0 |
| 164 | + ) |
| 165 | + |
| 166 | + |
| 167 | + |
| 168 | +class FileTypeDetectorRegistry(BaseDetectorRegistry): |
| 169 | + def __init__(self): |
| 170 | + self.registry = { |
| 171 | + "json": is_valid_json, |
| 172 | + "xml": is_valid_xml, |
| 173 | + "yaml": is_valid_yaml, |
| 174 | + "json-with-schema:$SCHEMA": is_valid_json_schema, |
| 175 | + "xml-with-schema:$SCHEMA": is_valid_xml_schema, |
| 176 | + "yaml-with-schema:$SCHEMA": is_valid_yaml_schema, |
| 177 | + } |
| 178 | + |
| 179 | + def handle_request(self, content: str, detector_params: dict) -> List[ContentAnalysisResponse]: |
| 180 | + detections = [] |
| 181 | + if "file_type" in detector_params and isinstance(detector_params["file_type"], (list, str)): |
| 182 | + file_types = detector_params["file_type"] |
| 183 | + file_types = [file_types] if isinstance(file_types, str) else file_types |
| 184 | + for file_type in file_types: |
| 185 | + if file_type.startswith("json-with-schema"): |
| 186 | + result = is_valid_json_schema(content, file_type.split("json-with-schema:")[1]) |
| 187 | + if result is not None: |
| 188 | + detections += [result] |
| 189 | + elif file_type.startswith("yaml-with-schema"): |
| 190 | + result = is_valid_yaml_schema(content, file_type.split("yaml-with-schema:")[1]) |
| 191 | + if result is not None: |
| 192 | + detections += [result] |
| 193 | + elif file_type.startswith("xml-with-schema"): |
| 194 | + result = is_valid_xml_schema(content, file_type.split("xml-with-schema:")[1]) |
| 195 | + if result is not None: |
| 196 | + detections += [result] |
| 197 | + elif file_type in self.registry: |
| 198 | + result = self.registry[file_type](content) |
| 199 | + if result is not None: |
| 200 | + detections += [result] |
| 201 | + else: |
| 202 | + raise HTTPException(status_code=400, detail=f"Unrecognized file type: {file_type}") |
| 203 | + return detections |
0 commit comments