1+ (ns cmr.ingest.validation.generic-document-validation
2+ " Provides functions to validate the ingest generic document"
3+ (:require
4+ [cheshire.core :as json]
5+ [clojure.string :as string]
6+ [cmr.common.cache :as cache]
7+ [cmr.common.cache.in-memory-cache :as mem-cache]
8+ [cmr.common.concepts :as concepts]
9+ [cmr.common.generics :as generics]
10+ [cmr.common.log :refer (info error)]
11+ [cmr.common.services.errors :as errors]
12+ [cmr.transmit.metadata-db :as mdb]))
13+
14+ (def schema-validation-cache-key
15+ " The cache key for the schema validation functions cache."
16+ :schema-validation-functions )
17+
18+ (def SCHEMA_CACHE_TIME
19+ " The number of milliseconds schema validation functions will be cached."
20+ (* 60 60 1000 )) ; ; 1 hour
21+
22+ (defn create-schema-validation-cache
23+ " Creates a cache for schema validation functions."
24+ []
25+ (mem-cache/create-in-memory-cache :ttl {} {:ttl SCHEMA_CACHE_TIME}))
26+
27+ (defn- fetch-existing-concepts
28+ " Fetch existing concepts of the given type from metadata-db for uniqueness validation"
29+ [context concept-type]
30+ (try
31+ (mdb/find-concepts context {:latest true } concept-type)
32+ (catch Exception e
33+ (error (str " Error fetching concepts for generic validations: " .getMessage e)))
34+ [])))
35+
36+ (defn- validate-uniqueness
37+ " Validates that the combination of field values is unique in the collection.
38+ Returns a sequence of error messages if validation fails, empty sequence otherwise."
39+ [context concept fields]
40+ (let [concept-type (:concept-type concept)]
41+ (if-let [existing-concepts (fetch-existing-concepts context concept-type)]
42+ (let [; ; Helper function to extract field values from a concept
43+ get-field-values (fn [c fs]
44+ (let [metadata (json/parse-string (:metadata c) true )]
45+ (mapv #(let [field-path (rest (string/split % #"\. " ))
46+ keyword-path (keyword (first field-path))]
47+ (get-in metadata[keyword-path]))
48+ fs)))
49+
50+ ; ; Extract values for the specified fields from the current concept
51+ field-values (get-field-values concept fields)
52+
53+ ; ; Check if any other document in the collection has the same combination of values
54+ duplicate-concepts (filter (fn [existing-concept]
55+ ; ; Debug print to show values being compared
56+ (and (not= (:native-id existing-concept) (:native-id concept))
57+ (not= (:deleted existing-concept) true )
58+ (= (set (get-field-values existing-concept fields))
59+ (set field-values))))
60+ existing-concepts)]
61+ (if (seq duplicate-concepts)
62+ (let [duplicate-concept-ids (map :concept-id duplicate-concepts)
63+ field-names (mapv #(string/join " ." rest (string/split % #"\. " ))) fields)
64+ display-values (mapv str field-values)]
65+ (info " Duplicate concept IDs found: "
66+ [(format " Values %s for fields %s must be unique for concept type %s. Duplicate concept IDs: %s"
67+ (string/join " , "
68+ (string/join " , "
69+ (str concept-type)
70+ (string/join " , "
71+ []))
72+ [])))
73+
74+ (defn- validate-by-type
75+ " Validates fields based on validation type.
76+ Returns a sequence of error messages if validation fails, empty sequence otherwise."
77+ [context concept validation-type fields validation-value]
78+ (case validation-type
79+ " unique" validate-uniqueness context concept fields)
80+ ; ; Default case
81+ [(str " Unknown validation type: "
82+
83+ (defn- validate-with-schema
84+ " Validates a concept against the schema validations.
85+ Returns a sequence of error messages if validation fails, empty sequence otherwise."
86+ [context concept schema]
87+ (let [validations (:Validations schema)]
88+ (if validations
89+ (mapcat (fn [validation]
90+ (let [validation-type (:ValidationType validation)
91+ fields (:Fields validation)
92+ validation-value (:ValidationValue validation)]
93+ (validate-by-type context concept validation-type fields validation-value)))
94+ validations)
95+ [])))
96+
97+ (defn- load-schema-validation
98+ " Loads a single schema validation function for a concept type and version"
99+ [concept-type version]
100+ (try
101+ (if (generics/approved-generic? concept-type version)
102+ (let [schema-json (generics/read-schema-index concept-type version)
103+ schema (json/parse-string schema-json true )]
104+ (fn [context concept]
105+ (validate-with-schema context concept schema)))
106+ (do
107+ (error " Schema version not approved for" " version"
108+ nil ))
109+ (catch Exception e
110+ (error " Error loading schema for" " version" " :" .getMessage e))
111+ nil )))
112+
113+ (defn- extract-concept-metadata-spec
114+ " Extract metadata specification info from concept"
115+ [concept]
116+ (let [parsed-data (json/parse-string (:metadata concept))]
117+ (when parsed-data
118+ {:name (get-in parsed-data [:MetadataSpecification :Name ])
119+ :version (get-in parsed-data [:MetadataSpecification :Version ])})))
120+
121+ (defn- load-schema-validators
122+ " Loads all schema validators for all generic concept types"
123+ []
124+ (info " Loading schema validation functions for all generic concept types"
125+ (let [generic-types (concepts/get-generic-concept-types-array )
126+ validators (reduce (fn [validators concept-type]
127+ (let [current-version (generics/current-generic-version concept-type)
128+ validator (load-schema-validation concept-type current-version)]
129+ (if validator
130+ (assoc validators [concept-type current-version] validator)
131+ validators)))
132+ {}
133+ generic-types)]
134+ (info " Loaded" count validators) " schema validators"
135+ validators))
136+
137+ (defn- get-validation-functions
138+ " Gets the validation functions from cache if available, otherwise loads them"
139+ [context]
140+ (if-let [cache (cache/context->cache context schema-validation-cache-key)]
141+ ; ; Return cached validation functions if available
142+ (cache/get-value cache :validators load-schema-validators)
143+ ; ; No cache available, load directly
144+ (load-schema-validators )))
145+
146+ (defn validate-concept
147+ " Validates the given concept dynamically based on concept type.
148+ Throws a :bad-request service error if validation fails."
149+ [context concept]
150+ (let [concept-type (:concept-type concept)
151+ metadata-spec (extract-concept-metadata-spec concept)
152+ version (or (:version metadata-spec)
153+ (generics/current-generic-version concept-type))
154+
155+ ; ; Get validation functions
156+ validators (get-validation-functions context)
157+ validator-fn (get validators [concept-type version])
158+
159+ ; ; Only run validation if a validator is defined
160+ errors (when validator-fn
161+ (validator-fn context concept))]
162+ (info " Validating concept" " with schema version"
163+
164+ ; ; Throw service errors if any validation errors are found
165+ (when (seq errors)
166+ (errors/throw-service-errors :bad-request errors))))
0 commit comments