Merge pull request #2796 from sepinf-inc/#2795_FixHashSetStatus

lfcnassif · web-flow · commit 67f2dc59e7a0 · 2026-02-13T12:31:31.000-03:00
Fix hashDb set/status property name used in some parsers
diff --git a/iped-api/src/main/java/iped/properties/ExtraProperties.java b/iped-api/src/main/java/iped/properties/ExtraProperties.java
@@ -188,6 +188,12 @@ public class ExtraProperties {
 
     public static final String FACE_AGE_LABELS = "faceAge:labels";
 
+    public static final String HASHDB_PREFIX = "hashDb:";
+    public static final String STATUS_PROPERTY = "status";
+    public static final String SET_PROPERTY = "set";
+    public static final String HASHDB_STATUS = HASHDB_PREFIX + STATUS_PROPERTY;
+    public static final String HASHDB_SET = HASHDB_PREFIX + SET_PROPERTY;
+    
     /**
      * Property to be set if the evidence is a animated image (i.e. contain multiple
      * frames). Only set if the number of frames is greater than one.
diff --git a/iped-app/resources/config/conf/HashDBLookupConfig.txt b/iped-app/resources/config/conf/HashDBLookupConfig.txt
@@ -1,2 +1,2 @@
-# Excludes from the rest of processing (and from the case) known files according to hashes database (hash:status = known).
+# Excludes from the rest of processing (and from the case) known files according to hashes database (hashDb:status = known).
 excludeKnown = false
diff --git a/iped-app/resources/config/conf/metadataTypes.txt b/iped-app/resources/config/conf/metadataTypes.txt
@@ -1811,8 +1811,8 @@ fileFragment = java.lang.Boolean
 fileSystemId = java.lang.String
 format-detection = java.lang.String
 googlebot = java.lang.String
-hash:set = java.lang.String
-hash:status = java.lang.String
+hashDb:set = java.lang.String
+hashDb:status = java.lang.String
 hasSubitem = java.lang.String
 hasThumb = java.lang.Boolean
 html:"Content-Type" = java.lang.String
diff --git a/iped-app/resources/config/profiles/blind/conf/HashDBLookupConfig.txt b/iped-app/resources/config/profiles/blind/conf/HashDBLookupConfig.txt
@@ -0,0 +1 @@
+excludeKnown = true
diff --git a/iped-app/resources/config/profiles/blind/conf/HashDBLooupConfig.txt b/iped-app/resources/config/profiles/blind/conf/HashDBLooupConfig.txt
diff --git a/iped-app/src/main/java/iped/app/ui/columns/ColumnsManager.java b/iped-app/src/main/java/iped/app/ui/columns/ColumnsManager.java
@@ -305,7 +305,7 @@ void updateDinamicFields() {
                 regexFields.add(f);
             else if (f.startsWith(LanguageDetectTask.LANGUAGE_PREFIX))
                 languageFields.add(f);
-            else if (f.startsWith(HashDBLookupTask.ATTRIBUTES_PREFIX)
+            else if (f.startsWith(ExtraProperties.HASHDB_PREFIX)
                     || f.startsWith(PhotoDNALookup.PHOTO_DNA_HIT_PREFIX))
                 hashDbFields.add(f);
             else
diff --git a/iped-engine/src/main/java/iped/engine/hashdb/HashDBDataSource.java b/iped-engine/src/main/java/iped/engine/hashdb/HashDBDataSource.java
@@ -22,6 +22,7 @@
 import org.sqlite.SQLiteConfig.JournalMode;
 import org.sqlite.SQLiteConfig.SynchronousMode;
 
+import iped.properties.ExtraProperties;
 import iped.utils.HashValue;
 
 public class HashDBDataSource {
@@ -40,8 +41,8 @@ public class HashDBDataSource {
     public static final String photoDna = "photoDna";
     private static final int photoDnaBase64Len = 192;
 
-    private static final String propertySet = "set";
-    private static final String propertyStatus = "status";
+    private static final String propertySet = ExtraProperties.SET_PROPERTY;
+    private static final String propertyStatus = ExtraProperties.STATUS_PROPERTY;
     private static final String pedoStatus = "pedo";
 
     public HashDBDataSource(File dbFile) throws Exception {
diff --git a/iped-engine/src/main/java/iped/engine/task/AbstractTask.java b/iped-engine/src/main/java/iped/engine/task/AbstractTask.java
@@ -302,7 +302,7 @@ private void processMonitorTimeout(IItem evidence) throws Exception {
     }
 
     /**
-     * Indica se itens ignorados (hash:status = ignore), devem ser processados pela
+     * Indica se itens ignorados (hashDb:status = known), devem ser processados pela
      * tarefa ou não. O valor padrão é false, assim itens ignorados não são
      * processados pelas tarefas seguintes. Tarefas específicas podem sobrescrever
      * esse comportamento.
diff --git a/iped-engine/src/main/java/iped/engine/task/HashDBLookupTask.java b/iped-engine/src/main/java/iped/engine/task/HashDBLookupTask.java
@@ -27,12 +27,10 @@
 import iped.engine.hashdb.HashDBDataSource;
 import iped.parsers.util.ChildPornHashLookup;
 import iped.parsers.util.ChildPornHashLookup.LookupProvider;
+import iped.properties.ExtraProperties;
 
 public class HashDBLookupTask extends AbstractTask {
 
-    public static final String ATTRIBUTES_PREFIX = "hashDb:";
-    private static final String STATUS_PROPERTY = "status";
-    public static final String STATUS_ATTRIBUTE = ATTRIBUTES_PREFIX + STATUS_PROPERTY;
     public static final String KNOWN_VALUE = "known";
     private static final String NSRL_PRODUCT_NAME_PROPERTY = "nsrlProductName";
 
@@ -226,8 +224,8 @@ protected void process(IItem evidence) throws Exception {
                         }
                     }
                     if (!list.isEmpty()) {
-                        evidence.setExtraAttribute(ATTRIBUTES_PREFIX + key, list);
-                        if (key.equalsIgnoreCase(STATUS_PROPERTY)) {
+                        evidence.setExtraAttribute(ExtraProperties.HASHDB_PREFIX + key, list);
+                        if (key.equalsIgnoreCase(ExtraProperties.STATUS_PROPERTY)) {
                             status = list;
                         } else if (key.equalsIgnoreCase(NSRL_PRODUCT_NAME_PROPERTY)) {
                             nsrlProductName = value;
@@ -268,7 +266,7 @@ protected void process(IItem evidence) throws Exception {
                     }
                     if (modified) {
                         Collections.sort(status);
-                        evidence.setExtraAttribute(STATUS_ATTRIBUTE, status);
+                        evidence.setExtraAttribute(ExtraProperties.HASHDB_STATUS, status);
                     }
                 }
                 //Ignore only if there is a single status = "known"
diff --git a/iped-engine/src/main/java/iped/engine/task/PhotoDNATask.java b/iped-engine/src/main/java/iped/engine/task/PhotoDNATask.java
@@ -27,6 +27,7 @@
 import iped.engine.config.PhotoDNAConfig;
 import iped.engine.preview.PreviewRepositoryManager;
 import iped.parsers.util.MetadataUtil;
+import iped.properties.ExtraProperties;
 import iped.utils.IOUtil;
 import iped.utils.ImageUtil;
 
@@ -86,7 +87,7 @@ protected void process(IItem evidence) throws Exception {
         if (!evidence.isToAddToCase())
             return;
 
-        if (pdnaConfig.isSkipHashDBFiles() && evidence.getExtraAttribute(HashDBLookupTask.STATUS_ATTRIBUTE) != null)
+        if (pdnaConfig.isSkipHashDBFiles() && evidence.getExtraAttribute(ExtraProperties.HASHDB_STATUS) != null)
             return;
 
         if (evidence.getLength() != null && evidence.getLength() < pdnaConfig.getMinFileSize())
diff --git a/iped-engine/src/main/java/iped/engine/task/RemoteImageClassifierTask.java b/iped-engine/src/main/java/iped/engine/task/RemoteImageClassifierTask.java
@@ -67,6 +67,7 @@
 import iped.engine.task.die.DIETask;
 import iped.engine.task.index.IndexItem;
 import iped.parsers.util.MetadataUtil;
+import iped.properties.ExtraProperties;
 import iped.utils.ImageUtil;
 
 /**
@@ -771,7 +772,7 @@ else if (mediaType.startsWith("video")) {
         }
 
         // Skip classification of images/videos with hits on IPED hashesDB database (see 'skipHashDBFiles' config property)
-        if (skipHashDBFiles && evidence.getExtraAttribute(HashDBLookupTask.STATUS_ATTRIBUTE) != null) {
+        if (skipHashDBFiles && evidence.getExtraAttribute(ExtraProperties.HASHDB_STATUS) != null) {
             // Add skip classification info
             evidence.setExtraAttribute(AI_CLASSIFICATION_STATUS_ATTR, AI_CLASSIFICATION_SKIP_HASHDB);
             skipHashDBFilesCount.incrementAndGet();
diff --git a/iped-engine/src/main/java/iped/engine/task/regex/RegexTask.java b/iped-engine/src/main/java/iped/engine/task/regex/RegexTask.java
@@ -80,9 +80,9 @@ public class RegexTask extends AbstractTask {
         ignoredKeys.add(IndexItem.CONTAINER_TRACK_ID);
         ignoredKeys.add(IndexItem.EVIDENCE_UUID);
         ignoredKeys.add(ExtraProperties.GLOBAL_ID);
-        ignoredKeys.add(HashDBLookupTask.ATTRIBUTES_PREFIX + HashDBDataSource.ledMd5_512);
-        ignoredKeys.add(HashDBLookupTask.ATTRIBUTES_PREFIX + HashDBDataSource.ledMd5_64k);
-        ignoredKeys.add(HashDBLookupTask.ATTRIBUTES_PREFIX + HashDBDataSource.photoDna);
+        ignoredKeys.add(ExtraProperties.HASHDB_PREFIX + HashDBDataSource.ledMd5_512);
+        ignoredKeys.add(ExtraProperties.HASHDB_PREFIX + HashDBDataSource.ledMd5_64k);
+        ignoredKeys.add(ExtraProperties.HASHDB_PREFIX + HashDBDataSource.photoDna);
         ignoredKeys.add(PhotoDNALookup.PHOTO_DNA_HIT_PREFIX + HashTask.HASH.MD5.name());
         ignoredKeys.add(PhotoDNALookup.PHOTO_DNA_NEAREAST_HASH);
     }
diff --git a/iped-engine/src/main/java/iped/engine/task/transcript/AbstractTranscriptTask.java b/iped-engine/src/main/java/iped/engine/task/transcript/AbstractTranscriptTask.java
@@ -95,7 +95,7 @@ protected boolean isToProcess(IItem evidence) {
         if (evidence.getLength() == null || evidence.getLength() == 0 || !evidence.isToAddToCase() || evidence.getMetadata().get(ExtraProperties.TRANSCRIPT_ATTR) != null) {
             return false;
         }
-        if (transcriptConfig.getSkipKnownFiles() && evidence.getExtraAttribute(HashDBLookupTask.STATUS_ATTRIBUTE) != null) {
+        if (transcriptConfig.getSkipKnownFiles() && evidence.getExtraAttribute(ExtraProperties.HASHDB_STATUS) != null) {
             return false;
         }
 
diff --git a/iped-engine/src/main/java/iped/engine/task/video/VideoThumbTask.java b/iped-engine/src/main/java/iped/engine/task/video/VideoThumbTask.java
@@ -598,7 +598,7 @@ private void generateSubitems(IItem item, VideoThumbsOutputConfig config, List<F
             newItem.setChangeDate(item.getChangeDate());
 
             // replicate 'hashDb:*' item's extra attributes to subitem
-            Map<String, Object> hashDbAttrs = ((Item) item).getExtraAttributesStartWith(HashDBLookupTask.ATTRIBUTES_PREFIX);
+            Map<String, Object> hashDbAttrs = ((Item) item).getExtraAttributesStartWith(ExtraProperties.HASHDB_PREFIX);
             for (String key : hashDbAttrs.keySet()) {
                 newItem.setExtraAttribute(key, hashDbAttrs.get(key));
             }
diff --git a/iped-engine/src/main/java/iped/engine/util/ItemInfoFactory.java b/iped-engine/src/main/java/iped/engine/util/ItemInfoFactory.java
@@ -5,6 +5,7 @@
 import iped.data.IItem;
 import iped.engine.task.HashDBLookupTask;
 import iped.parsers.util.ItemInfo;
+import iped.properties.ExtraProperties;
 
 public class ItemInfoFactory {
 
@@ -17,7 +18,7 @@ public static ItemInfo getItemInfo(IItem evidence) {
 
     // Check if there is a single status, and it is "known"
     private static boolean isKnown(IItem evidence) {
-        Object hashDbStatus = evidence.getExtraAttribute(HashDBLookupTask.STATUS_ATTRIBUTE);
+        Object hashDbStatus = evidence.getExtraAttribute(ExtraProperties.HASHDB_STATUS);
         if (hashDbStatus != null) {
             if (hashDbStatus instanceof List) {
                 @SuppressWarnings("unchecked")
diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/gdrive/GDriveCloudGraphParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/gdrive/GDriveCloudGraphParser.java
@@ -163,9 +163,9 @@ private Metadata getEntryMetadata(CloudGraphEntry entry, int i, List<String> has
         metadataCloudGraphItem.add("doc_id", entry.getDoc_id());
         metadataCloudGraphItem.add("parent_doc_id", entry.getParent_doc_id());
         if (!hashSets.isEmpty()) {
-            metadataCloudGraphItem.set("hashDb:status", "pedo");
+            metadataCloudGraphItem.set(ExtraProperties.STATUS_PROPERTY, "pedo");
             for (String set : hashSets) {
-                metadataCloudGraphItem.add("hashDb:set", set);
+                metadataCloudGraphItem.add(ExtraProperties.SET_PROPERTY, set);
             }
         }
         if("yes".equalsIgnoreCase(entry.getShared()) || Boolean.valueOf(entry.getShared())) {
diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/gdrive/GDriveSnapshotParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/gdrive/GDriveSnapshotParser.java
@@ -160,9 +160,9 @@ private Metadata getEntryMetadata(SnapshotEntry entry, int i, List<String> hashS
         metadataSnapshotItem.add("child_volume", entry.getChildVolume());
         metadataSnapshotItem.add("parent_volume", entry.getParentVolume());
         if (!hashSets.isEmpty()) {
-            metadataSnapshotItem.set("hashDb:status", "pedo");
+            metadataSnapshotItem.set(ExtraProperties.STATUS_PROPERTY, "pedo");
             for (String set : hashSets) {
-                metadataSnapshotItem.add("hashDb:set", set);
+                metadataSnapshotItem.add(ExtraProperties.SET_PROPERTY, set);
             }
         }
         if("yes".equalsIgnoreCase(entry.getShared()) || Boolean.valueOf(entry.getShared())) {
diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/skype/SkypeParser.java
@@ -222,9 +222,9 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
                                 meta.set(StandardParser.INDEXER_CONTENT_TYPE, ATTACHMENT_MIME_TYPE);
                                 List<String> hashSets = ChildPornHashLookup.lookupHash(item.getHash());
                                 if (!hashSets.isEmpty()) {
-                                    meta.set("hash:status", "pedo");
+                                    meta.set(ExtraProperties.HASHDB_STATUS, "pedo");
                                     for (String set : hashSets) {
-                                        meta.add("hash:set", set);
+                                        meta.add(ExtraProperties.HASHDB_SET, set);
                                     }
                                 }
                             }
@@ -272,9 +272,9 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
                         }
                         List<String> hashSets = ChildPornHashLookup.lookupHash(t.getItem().getHash());
                         if (!hashSets.isEmpty()) {
-                            tMetadata.set("hash:status", "pedo");
+                            tMetadata.set(ExtraProperties.HASHDB_STATUS, "pedo");
                             for (String set : hashSets) {
-                                tMetadata.add("hash:set", set);
+                                tMetadata.add(ExtraProperties.HASHDB_SET, set);
                             }
                         }
                     }
diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/telegram/TelegramParser.java
@@ -295,9 +295,9 @@ private void extractMessages(String chatName, List<Message> messages, Contact ac
                 meta.set(StandardParser.INDEXER_CONTENT_TYPE, TELEGRAM_ATTACHMENT.toString());
                 meta.set(ExtraProperties.LINKED_ITEMS, BasicProps.HASH + ":" + m.getMediaHash()); //$NON-NLS-1$
                 if (!m.getChildPornSets().isEmpty()) {
-                    meta.set("hash:status", "pedo");
+                    meta.set(ExtraProperties.HASHDB_STATUS, "pedo");
                     for (String set : m.getChildPornSets()) {
-                        meta.add("hash:set", set);
+                        meta.add(ExtraProperties.HASHDB_SET, set);
                     }
                 }
                 // TODO store thumb in metadata?
diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/threema/ThreemaParser.java
@@ -382,15 +382,15 @@ private void extractMessages(String chatName, Chat c, List<Message> messages, Th
             }
             if (m.getMediaQuery() != null && m.getMediaSize() > 2) {
                 meta.set(StandardParser.INDEXER_CONTENT_TYPE, THREEMA_ATTACHMENT.toString());
-                meta.set(ExtraProperties.LINKED_ITEMS, revertEscapeQuery(m.getMediaQuery())); // $NON-NLS-1$
+                meta.set(ExtraProperties.LINKED_ITEMS, revertEscapeQuery(m.getMediaQuery()));
             }
             if (m.getMediaItem() != null && m.getMediaItem().getThumb() != null) {
                 meta.set(ExtraProperties.THUMBNAIL_BASE64, Base64.getEncoder().encodeToString(m.getMediaItem().getThumb()));
             }
             if (!m.getChildPornSets().isEmpty()) {
-                meta.set("hash:status", "pedo"); //$NON-NLS-1$ //$NON-NLS-2$
+                meta.set(ExtraProperties.HASHDB_STATUS, "pedo");
                 for (String set : m.getChildPornSets()) {
-                    meta.add("hash:set", set); //$NON-NLS-1$
+                    meta.add(ExtraProperties.HASHDB_SET, set);
                 }
             }
 
diff --git a/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java b/iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/whatsapp/WhatsAppParser.java
@@ -1082,12 +1082,12 @@ private void extractMessages(String chatName, Chat c, List<Message> messages, WA
             }
             if (m.getMediaQuery() != null && m.getMediaSize() > 2) {
                 meta.set(StandardParser.INDEXER_CONTENT_TYPE, WHATSAPP_ATTACHMENT.toString());
-                meta.set(ExtraProperties.LINKED_ITEMS, revertEscapeQuery(m.getMediaQuery())); // $NON-NLS-1$
+                meta.set(ExtraProperties.LINKED_ITEMS, revertEscapeQuery(m.getMediaQuery()));
             }
             if (!m.getChildPornSets().isEmpty()) {
-                meta.set("hash:status", "pedo"); //$NON-NLS-1$ //$NON-NLS-2$
+                meta.set(ExtraProperties.HASHDB_STATUS, "pedo");
                 for (String set : m.getChildPornSets()) {
-                    meta.add("hash:set", set); //$NON-NLS-1$
+                    meta.add(ExtraProperties.HASHDB_SET, set);
                 }
             }
 

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`		`-# Excludes from the rest of processing (and from the case) known files according to hashes database (hash:status = known).`
	`1`	`+# Excludes from the rest of processing (and from the case) known files according to hashes database (hashDb:status = known).`
`2`	`2`	`excludeKnown = false`
Original file line number	Diff line number	Diff line change
`@@ -302,7 +302,7 @@ private void processMonitorTimeout(IItem evidence) throws Exception {`
`302`	`302`	`}`
`303`	`303`
`304`	`304`	`/**`
`305`		`- * Indica se itens ignorados (hash:status = ignore), devem ser processados pela`
	`305`	`+ * Indica se itens ignorados (hashDb:status = known), devem ser processados pela`
`306`	`306`	`* tarefa ou não. O valor padrão é false, assim itens ignorados não são`
`307`	`307`	`* processados pelas tarefas seguintes. Tarefas específicas podem sobrescrever`
`308`	`308`	`* esse comportamento.`
Original file line number	Diff line number	Diff line change
`@@ -27,12 +27,10 @@`
`27`	`27`	`import iped.engine.hashdb.HashDBDataSource;`
`28`	`28`	`import iped.parsers.util.ChildPornHashLookup;`
`29`	`29`	`import iped.parsers.util.ChildPornHashLookup.LookupProvider;`
	`30`	`+import iped.properties.ExtraProperties;`
`30`	`31`
`31`	`32`	`public class HashDBLookupTask extends AbstractTask {`
`32`	`33`
`33`		`- public static final String ATTRIBUTES_PREFIX = "hashDb:";`
`34`		`- private static final String STATUS_PROPERTY = "status";`
`35`		`- public static final String STATUS_ATTRIBUTE = ATTRIBUTES_PREFIX + STATUS_PROPERTY;`
`36`	`34`	`public static final String KNOWN_VALUE = "known";`
`37`	`35`	`private static final String NSRL_PRODUCT_NAME_PROPERTY = "nsrlProductName";`
`38`	`36`
`@@ -226,8 +224,8 @@ protected void process(IItem evidence) throws Exception {`
`226`	`224`	`}`
`227`	`225`	`}`
`228`	`226`	`if (!list.isEmpty()) {`
`229`		`- evidence.setExtraAttribute(ATTRIBUTES_PREFIX + key, list);`
`230`		`- if (key.equalsIgnoreCase(STATUS_PROPERTY)) {`
	`227`	`+ evidence.setExtraAttribute(ExtraProperties.HASHDB_PREFIX + key, list);`
	`228`	`+ if (key.equalsIgnoreCase(ExtraProperties.STATUS_PROPERTY)) {`
`231`	`229`	`status = list;`
`232`	`230`	`} else if (key.equalsIgnoreCase(NSRL_PRODUCT_NAME_PROPERTY)) {`
`233`	`231`	`nsrlProductName = value;`
`@@ -268,7 +266,7 @@ protected void process(IItem evidence) throws Exception {`
`268`	`266`	`}`
`269`	`267`	`if (modified) {`
`270`	`268`	`Collections.sort(status);`
`271`		`- evidence.setExtraAttribute(STATUS_ATTRIBUTE, status);`
	`269`	`+ evidence.setExtraAttribute(ExtraProperties.HASHDB_STATUS, status);`
`272`	`270`	`}`
`273`	`271`	`}`
`274`	`272`	`//Ignore only if there is a single status = "known"`