[backend] Fix strict_dynamic_mapping_exception exceptions thrown in fileIndexManager (#14665)

opencti-platform/opencti-dev/docker-compose.yml

@@ -53,7 +53,7 @@ services:
       - 1080:1080
 
   # Disabled by default, to run use:
-  # docker compose --profile opensearch up -d
+  # podman compose --profile opensearch up -d
   # in opencti configuration:
   # "elasticsearch": {
   #     "url": "http://localhost:9201",
@@ -62,7 +62,7 @@ services:
   opencti-dev-opensearch:
     profiles: [ opensearch ]
     container_name: opencti-dev-opensearch
-    image: opensearchproject/opensearch:3.5.0
+    build: ./opensearch
     volumes:
       - osdata:/usr/share/opensearch/data
       - ossnapshots:/usr/share/opensearch/snapshots

opencti-platform/opencti-dev/opensearch/Dockerfile

@@ -0,0 +1,2 @@
+FROM opensearchproject/opensearch:3.5.0
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch ingest-attachment

opencti-platform/opencti-graphql/src/database/attachment-processor-props.ts

@@ -0,0 +1,55 @@
+import type { Mutable } from '../utils/type-utils';
+
+// List of fields extracted by the attachment ingest processor.
+// The full list is available in the Elasticsearch docs:
+// (https://www.elastic.co/guide/en/elasticsearch/reference/8.19/attachment.html#attachment-fields).
+export const ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_ELASTICSEARCH = [
+  'content',
+  'title',
+  'author',
+  'keywords',
+  'date',
+  'content_type',
+  'content_length',
+  'language',
+  'modified',
+  'format',
+  // identifier,     NOT EXTRACTED
+  // contributor,    NOT EXTRACTED
+  // coverage,       NOT EXTRACTED
+  'modifier',
+  'creator_tool',
+  // publisher,      NOT EXTRACTED
+  // relation,       NOT EXTRACTED
+  // rights,         NOT EXTRACTED
+  // source,         NOT EXTRACTED
+  // type,           NOT EXTRACTED
+  'description',
+  'print_date',
+  'metadata_date',
+  // latitude,       NOT EXTRACTED
+  // longitude,      NOT EXTRACTED
+  // altitude,       NOT EXTRACTED
+  // rating,         NOT EXTRACTED
+  'comments',
+] as const;
+
+// List of fields extracted by the attachment ingest processor, for OpenSearch.
+// The full list is available in the OS docs:
+// (https://docs.opensearch.org/latest/install-and-configure/additional-plugins/ingest-attachment-plugin/#extracted-information),
+// and code shows the check rejects unknown fields with an exception:
+// https://github.com/opensearch-project/OpenSearch/blob/315481148edaa43410e2e9f1801ec903fd62ec20/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/AttachmentProcessor.java#L277
+export const ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_OPENSEARCH = [
+  'content',
+  'title',
+  'author',
+  'keywords',
+  'date',
+  'content_type',
+  'content_length',
+  'language',
+] as const;
+
+// Union type of all properties extracted by the ES or OS attachment processor
+export type AttachmentProcessorExtractedProp = Mutable<typeof ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_ELASTICSEARCH>[number]
+  | Mutable<typeof ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_OPENSEARCH>[number];

opencti-platform/opencti-graphql/src/database/engine.ts

@@ -176,6 +176,7 @@ import { DRAFT_OPERATION_CREATE, DRAFT_OPERATION_DELETE, DRAFT_OPERATION_DELETE_
 import { RELATION_SAMPLE } from '../modules/malwareAnalysis/malwareAnalysis-types';
 import { asyncMap } from '../utils/data-processing';
 import { doYield } from '../utils/eventloop-utils';
+import type { Mutable } from '../utils/type-utils';
 import { RELATION_COVERED } from '../modules/securityCoverage/securityCoverage-types';
 import type { AuthContext, AuthUser } from '../types/user';
 import type {
@@ -197,6 +198,7 @@ import { IDS_ATTRIBUTES } from '../domain/attribute-utils';
 import { schemaRelationsRefDefinition } from '../schema/schema-relationsRef';
 import type { FiltersWithNested } from './middleware-loader';
 import { pushAll, unshiftAll } from '../utils/arrayUtil';
+import { ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_ELASTICSEARCH, ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_OPENSEARCH } from './attachment-processor-props';
 
 const ELK_ENGINE = 'elk';
 const OPENSEARCH_ENGINE = 'opensearch';
@@ -309,6 +311,7 @@ export const elConfigureAttachmentProcessor = async (): Promise<boolean> => {
           attachment: {
             field: 'file_data',
             remove_binary: true,
+            properties: ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_ELASTICSEARCH as Mutable<typeof ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_ELASTICSEARCH>,
           },
         },
       ],
@@ -325,6 +328,7 @@ export const elConfigureAttachmentProcessor = async (): Promise<boolean> => {
           {
             attachment: {
               field: 'file_data',
+              properties: ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_OPENSEARCH as Mutable<typeof ATTACHMENT_PROCESSOR_EXTRACTED_PROPS_OPENSEARCH>,
             },
           },
           {
@@ -2717,11 +2721,11 @@ const buildSubQueryForFilterGroup = (
 
   const currentSubQuery = localMustFilters.length > 0
     ? {
-        bool: {
-          should: localMustFilters,
-          minimum_should_match: mode === 'or' ? 1 : localMustFilters.length,
-        },
-      }
+      bool: {
+        should: localMustFilters,
+        minimum_should_match: mode === 'or' ? 1 : localMustFilters.length,
+      },
+    }
     : null;
   return { subQuery: currentSubQuery, postFiltersTags: localPostFilterTags, resultSaltCount: localSaltCount };
 };
@@ -3094,12 +3098,12 @@ const tagFiltersForPostFiltering = (
 ) => {
   const taggedFilters: (Filter & { postFilteringTag: string })[] = filters
     ? extractFiltersFromGroup(filters, [INSTANCE_REGARDING_OF, INSTANCE_DYNAMIC_REGARDING_OF])
-        .filter((filter) => isEmptyField(filter.operator) || filter.operator === 'eq')
-        .map((filter, i) => {
-          const taggedFilter = filter as Filter & { postFilteringTag: string };
-          taggedFilter.postFilteringTag = `${i}`;
-          return taggedFilter;
-        })
+      .filter((filter) => isEmptyField(filter.operator) || filter.operator === 'eq')
+      .map((filter, i) => {
+        const taggedFilter = filter as Filter & { postFilteringTag: string };
+        taggedFilter.postFilteringTag = `${i}`;
+        return taggedFilter;
+      })
     : [];
 
   if (taggedFilters.length > 0) {

opencti-platform/opencti-graphql/src/modules/internal/document/document.ts

@@ -1,11 +1,13 @@
+import { assertType } from '../../../utils/type-utils';
+import type { AttachmentProcessorExtractedProp } from '../../../database/attachment-processor-props';
 import { ENTITY_TYPE_INTERNAL_FILE } from '../../../schema/internalObject';
 import { schemaAttributesDefinition } from '../../../schema/schema-attributes';
 import { type AttributeDefinition, createdAt, creators, entityType, id, internalId, parentTypes, refreshedAt, standardId, updatedAt } from '../../../schema/attribute-definition';
 import { ENTITY_TYPE_MARKING_DEFINITION } from '../../../schema/stixMetaObject';
 import { ABSTRACT_STIX_CORE_OBJECT } from '../../../schema/general';
 import { UPLOAD_STATUS_VALUES } from './document-domain';
 
-const attributes: Array<AttributeDefinition> = [
+const attributes = [
   id,
   internalId,
   standardId,
@@ -93,6 +95,17 @@ const attributes: Array<AttributeDefinition> = [
   { name: 'file_id', label: 'File identifier', type: 'string', format: 'short', mandatoryType: 'internal', editDefault: false, multiple: false, upsert: false, isFilterable: false },
   { name: 'entity_id', label: 'Related entity', type: 'string', format: 'id', entityTypes: [ABSTRACT_STIX_CORE_OBJECT], mandatoryType: 'internal', editDefault: false, multiple: false, upsert: false, isFilterable: false },
   { name: 'removed', label: 'Removed', type: 'boolean', mandatoryType: 'no', editDefault: false, multiple: false, upsert: false, isFilterable: false },
-];
+] as const satisfies Array<AttributeDefinition>;
+
+const attachmentAttributes = attributes[18];
+
+type AttachmentAttributeMappingNames = typeof attachmentAttributes.mappings[number]['name'][];
+
+// Make sure there's an attachment mapping for each field extracted
+// by the `attachment` ingest processor, exhaustively.
+assertType<
+  AttachmentAttributeMappingNames,
+  AttachmentProcessorExtractedProp[]
+>(attachmentAttributes.mappings.map(({ name }) => name));
 
 schemaAttributesDefinition.registerAttributes(ENTITY_TYPE_INTERNAL_FILE, attributes);

opencti-platform/opencti-graphql/src/utils/type-utils.ts

@@ -0,0 +1,27 @@
+/**
+ * Inverse operation of the built-in Readonly<T> utility type:
+ * makes all records of an object mutable.
+ */
+export type Mutable<T> = { -readonly [P in keyof T]: T[P]; };
+
+type AssertEqual<T, TExpected> = [T] extends [TExpected]
+  ? [TExpected] extends [T]
+      ? T
+      : never
+  : never;
+
+/**
+ * Compile-time type assertion utility checking that a type T
+ * is exactly of type TExpected.
+ *
+ * @example
+ * ```
+ * type Fruit = 'banana' | 'strawberry' | 'pineapple';
+ * const allFruits = ['banana' as const, 'strawberry' as const, 'pineapple' as const];
+ * // Checks for exhaustiveness
+ * assertType<typeof allFruits[number][], Fruit[]>(allFruits);
+ * ```
+ */
+export const assertType = <T, TExpected>(_x: AssertEqual<T, TExpected>) => {
+  // noop
+};

opencti-platform/opencti-graphql/tests/03-integration/01-database/index-file-test.js

@@ -99,6 +99,11 @@ describe('Indexing file test', () => {
     const result = await indexFile('test-file-to-index.html', mimeType, 'TEST_FILE_7');
     await testFileIndexing(result, mimeType);
   });
+  it('Should index file with metadata unsupported by the attachment processor config', async () => {
+    const mimeType = 'application/pdf';
+    const result = await indexFile('test-report-with-unhandled-metadata-to-index.pdf', mimeType, 'TEST_FILE_8');
+    await testFileIndexing(result, mimeType);
+  });
   it('Should find document by search query', async () => {
     const expectedFile1 = {
       _index: 'test_files-000001',

opencti-platform/opencti-graphql/tests/03-integration/04-manager/retentionManager-test.ts

@@ -286,15 +286,15 @@ describe('Retention Manager tests ', () => {
   it('should fetch the correct files to be deleted by a retention rule on files', async () => {
     // check the number of files imported in Data/import
     const files = await allFilesForPaths(testContext, ADMIN_USER, [globalPath]);
-    expect(files.length).toEqual(9); // 7 files from index-file-test + the 2 created files
+    expect(files.length).toEqual(10); // 8 files from index-file-test + the 2 created files
     // retention rule on files not modified since 2023-07-01
     const before = utcDate('2023-07-01T00:00:00.000Z');
     filesToDelete = await getElementsToDelete(context, 'file', before);
     expect(filesToDelete.edges.length).toEqual(1); // fileToTestRetentionRule is the only file that has not been modified since 'before' and with uploadStatus = complete
     expect(filesToDelete.edges[0].node.id).toEqual(fileId);
     // retention rule on all the files
     const filesToDelete2 = await getElementsToDelete(context, 'file', utcDate());
-    expect(filesToDelete2.edges.length).toEqual(8); // all the files that has not been modified since now and with uploadStatus = complete
+    expect(filesToDelete2.edges.length).toEqual(9); // all the files that has not been modified since now and with uploadStatus = complete
   });
   it('should fetch the correct files to be deleted by a retention rule on workbenches', async () => {
     // retention rule on workbenches not modified since 2023-07-01
@@ -336,7 +336,7 @@ describe('Retention Manager tests ', () => {
     // delete file
     await deleteElement(context, 'file', fileId); // should delete fileToTestRetentionRule
     const files = await allFilesForPaths(testContext, ADMIN_USER, [globalPath]);
-    expect(files.length).toEqual(8); // 7 files from index-file-test + the 2 created files - fileToTestRetentionRule that should have been deleted
+    expect(files.length).toEqual(9); // 8 files from index-file-test + the 2 created files - fileToTestRetentionRule that should have been deleted
     // delete workbench
     await deleteElement(context, 'workbench', workbench1Id); // should delete workbench1
     const workbenches = await allFilesForPaths(testContext, ADMIN_USER, [pendingPath]);