Merge pull request #1532 from val-ms/CLAM-1859-sha256-cache

- md5 -> sha2-256 caching
- remove reliance on md5 hashes in general
- FIPS cryptographic hash limits feature to disable md5 and sha1.
  - Adds related option for ClamD, ClamScan, Freshclam, Sigtool
- ClamBC: fix crashes
- signature names that start with "Weak." won't alert anymore.
- ClamScan:
  - add `--hash-hint`, `--log-hash`, `--hash-alg`, `--file-type-hint`, `--log-file-type`
  - accurate counts for scanned bytes and bytes read
- libclamav:
  - new cl_scan*_ex() APIs
  - separate temp-directory-recursion feature from keep-temps feature
  - object id's for each layer scanned
  - scan hash hints
  - scan file type hints
  - fix double-extraction for OOXML documents
  - new scan callbacks, deprecate old scan callbacks
  - new APIs to get access to file data and metadata from scan callbacks
- metadata.json:
  - object id's
  - replace "viruses" to "alerts" and add "indicators" array
  - replace "FileMD5" with "sha2-256"
  - json store extra hashes feature
    - Related options for ClamD and ClamScan
  - object id's
- related improvements

Author: val-ms

Cargo.lock

@@ -47,12 +47,13 @@ ClamAV 1.5.0 includes the following improvements and changes:
   Added two new APIs to the public clamav.h header:
     ```c
     extern cl_error_t cl_cvdverify_ex(const char *file,
-                                      const char *certs_directory);
+                                      const char *certs_directory,
+                                      uint32_t dboptions);
 
     extern cl_error_t cl_cvdunpack_ex(const char *file,
                                       const char *dir,
-                                      bool dont_verify,
-                                      const char *certs_directory);
+                                      const char *certs_directory,
+                                      uint32_t dboptions);
     ```
     The original `cl_cvdverify` and `cl_cvdunpack` are deprecated.
 

NEWS.md

@@ -367,9 +367,6 @@
 /* yara sources are compiled in */
 #define HAVE_YARA 1
 
-/* For internal use only - DO NOT DEFINE */
-#cmakedefine HAVE__INTERNAL__SHA_COLLECT 1
-
 /* Define as const if the declaration of iconv() needs const. */
 #cmakedefine ICONV_CONST @ICONV_CONST@
 

clamav-config.h.cmake.in

@@ -398,24 +398,16 @@ int main(int argc, char *argv[])
             fprintf(stderr, "Out of memory\n");
             exit(3);
         }
-        ctx->ctx      = &cctx;
-        cctx.engine   = engine;
-        cctx.evidence = evidence_new();
+        ctx->ctx    = &cctx;
+        cctx.engine = engine;
 
         cctx.recursion_stack_size = cctx.engine->max_recursion_level;
-        cctx.recursion_stack      = calloc(sizeof(recursion_level_t), cctx.recursion_stack_size);
+        cctx.recursion_stack      = calloc(sizeof(cli_scan_layer_t), cctx.recursion_stack_size);
         if (!cctx.recursion_stack) {
             fprintf(stderr, "Out of memory\n");
             exit(3);
         }
 
-        // ctx was memset, so recursion_level starts at 0.
-        cctx.recursion_stack[cctx.recursion_level].fmap = map;
-        cctx.recursion_stack[cctx.recursion_level].type = CL_TYPE_ANY; /* ANY for the top level, because we don't yet know the type. */
-        cctx.recursion_stack[cctx.recursion_level].size = map->len;
-
-        cctx.fmap = cctx.recursion_stack[cctx.recursion_level].fmap;
-
         memset(&dbg_state, 0, sizeof(dbg_state));
         dbg_state.file     = "<libclamav>";
         dbg_state.line     = 0;
@@ -453,22 +445,34 @@ int main(int argc, char *argv[])
                 optfree(opts);
                 exit(5);
             }
-            map = fmap(fd, 0, 0, opt->strarg);
+
+            map = fmap_new(fd, 0, 0, opt->strarg, opt->strarg);
             if (!map) {
                 fprintf(stderr, "Unable to map input file %s\n", opt->strarg);
                 exit(5);
             }
+
+            // ctx was memset, so recursion_level starts at 0.
+            cctx.recursion_stack[cctx.recursion_level].fmap = map;
+            cctx.recursion_stack[cctx.recursion_level].type = CL_TYPE_ANY; /* ANY for the top level, because we don't yet know the type. */
+            cctx.recursion_stack[cctx.recursion_level].size = map->len;
+
             rc = cli_bytecode_context_setfile(ctx, map);
             if (rc != CL_SUCCESS) {
                 fprintf(stderr, "Unable to set file %s: %s\n", opt->strarg, cl_strerror(rc));
                 optfree(opts);
                 exit(5);
             }
         }
+
         /* for testing */
         ctx->hooks.match_counts  = deadbeefcounts;
         ctx->hooks.match_offsets = deadbeefcounts;
-        rc                       = cli_bytecode_run(&bcs, bc, ctx);
+
+        /*
+         * Run the bytecode.
+         */
+        rc = cli_bytecode_run(&bcs, bc, ctx);
         if (rc != CL_SUCCESS) {
             fprintf(stderr, "Unable to run bytecode: %s\n", cl_strerror(rc));
         } else {
@@ -479,12 +483,15 @@ int main(int argc, char *argv[])
             if (debug_flag)
                 printf("[clambc] Bytecode returned: 0x%llx\n", (long long)v);
         }
+
         cli_bytecode_context_destroy(ctx);
         if (map)
-            funmap(map);
-        cl_engine_free(engine);
+            fmap_free(map);
+        if (cctx.recursion_stack[cctx.recursion_level].evidence) {
+            evidence_free(cctx.recursion_stack[cctx.recursion_level].evidence);
+        }
         free(cctx.recursion_stack);
-        evidence_free(cctx.evidence);
+        cl_engine_free(engine);
     }
     cli_bytecode_destroy(bc);
     cli_bytecode_done(&bcs);

clambc/bcrun.c

@@ -616,8 +616,12 @@ int main(int argc, char **argv)
 
         cl_engine_set_clcb_virus_found(engine, clamd_virus_found_cb);
 
-        if (optget(opts, "LeaveTemporaryFiles")->enabled)
+        if (optget(opts, "LeaveTemporaryFiles")->enabled) {
+            /* Set the engine to keep temporary files */
             cl_engine_set_num(engine, CL_ENGINE_KEEPTMP, 1);
+            /* Also set the engine to create temporary directory structure */
+            cl_engine_set_num(engine, CL_ENGINE_TMPDIR_RECURSION, 1);
+        }
 
         if (optget(opts, "ForceToDisk")->enabled)
             cl_engine_set_num(engine, CL_ENGINE_FORCETODISK, 1);
@@ -705,6 +709,12 @@ int main(int argc, char **argv)
             }
         }
 
+        if (optget(opts, "FIPSCryptoHashLimits")->enabled) {
+            dboptions |= CL_DB_FIPS_LIMITS;
+            cl_engine_set_num(engine, CL_ENGINE_FIPS_LIMITS, 1);
+            logg(LOGG_INFO_NF, "FIPS crypto hash limits enabled.\n");
+        }
+
         if ((ret = cl_load(dbdir, engine, &sigs, dboptions))) {
             logg(LOGG_ERROR, "%s\n", cl_strerror(ret));
             ret = 1;

clamd/clamd.c

@@ -94,7 +94,7 @@ void msg_callback(enum cl_msg severity, const char *fullmsg, const char *msg, vo
     }
 }
 
-void hash_callback(int fd, unsigned long long size, const unsigned char *md5, const char *virname, void *ctx)
+void hash_callback(int fd, unsigned long long size, const char *md5, const char *virname, void *ctx)
 {
     struct cb_context *c = ctx;
     UNUSEDPARAM(fd);
@@ -103,8 +103,8 @@ void hash_callback(int fd, unsigned long long size, const unsigned char *md5, co
     if (!c)
         return;
     c->virsize = size;
-    strncpy(c->virhash, (const char *)md5, 32);
-    c->virhash[32] = '\0';
+    strncpy(c->virhash, md5, MD5_HASH_SIZE * 2);
+    c->virhash[MD5_HASH_SIZE * 2] = '\0';
 }
 
 void clamd_virus_found_cb(int fd, const char *virname, void *ctx)

clamd/scanner.c

@@ -69,7 +69,7 @@ cl_error_t scanfd(const client_conn_t *conn, unsigned long int *scanned, const s
 int scanstream(int odesc, unsigned long int *scanned, const struct cl_engine *engine, struct cl_scan_options *options, const struct optstruct *opts, char term);
 cl_error_t scan_callback(STATBUF *sb, char *filename, const char *msg, enum cli_ftw_reason reason, struct cli_ftw_cbdata *data);
 int scan_pathchk(const char *path, struct cli_ftw_cbdata *data);
-void hash_callback(int fd, unsigned long long size, const unsigned char *md5, const char *virname, void *ctx);
+void hash_callback(int fd, unsigned long long size, const char *md5, const char *virname, void *ctx);
 void msg_callback(enum cl_msg severity, const char *fullmsg, const char *msg, void *ctx);
 void clamd_virus_found_cb(int fd, const char *virname, void *context);
 

clamd/scanner.h

@@ -1379,11 +1379,6 @@ int recvloop(int *socketds, unsigned nsockets, struct cl_engine *engine, unsigne
             options.heuristic |= CL_SCAN_HEURISTIC_STRUCTURED_SSN_STRIPPED;
     }
 
-#ifdef HAVE__INTERNAL__SHA_COLLECT
-    if (optget(opts, "DevCollectHashes")->enabled)
-        options.dev |= CL_SCAN_DEV_COLLECT_SHA;
-#endif
-
     if (optget(opts, "GenerateMetadataJson")->enabled) {
         options.general |= CL_SCAN_GENERAL_COLLECT_METADATA;
     }
@@ -1396,6 +1391,10 @@ int recvloop(int *socketds, unsigned nsockets, struct cl_engine *engine, unsigne
         options.general |= CL_SCAN_GENERAL_STORE_PDF_URIS;
     }
 
+    if (optget(opts, "JsonStoreExtraHashes")->enabled) {
+        options.general |= CL_SCAN_GENERAL_STORE_EXTRA_HASHES;
+    }
+
     selfchk = optget(opts, "SelfCheck")->numarg;
     if (!selfchk) {
         logg(LOGG_INFO, "Self checking disabled.\n");