Merge pull request #1856 from hashtopolis/686-rebuild-chunk-cache-helper

added helpers 'rebuildChunkCache' and 'rescanGlobalFiles'

Author: s3inlc

ci/apiv2/test_chunk.py

@@ -39,3 +39,13 @@ def test_helper_purge_task(self):
 
         chunks = Chunk.objects.filter(taskId=retval['task'].id)
         self.assertEqual(len(chunks), 0)
+
+    def test_helper_rebuild_chunk_cache(self):
+        # Note: it is currently only tested that the call to the helper works, but not that it would fix anything correctly.
+        # The problem is, that we cannot set the values of chunks and hashlists to "wrong" values via the API.
+
+        self.create_test_object()
+
+        helper = Helper()
+        response = helper.rebuild_chunk_cache()
+        self.assertEqual({"Rebuild": "Success", "correctedChunks": 0, "correctedHashlists": 0}, response)

ci/apiv2/test_file.py

@@ -57,3 +57,14 @@ def test_range_request_get_file(self):
     def test_bulk_delete(self):
         files = [self.create_test_object(delete=False) for i in range(5)]
         File.objects.delete_many(files)
+
+    def test_helper_rescan_global_files(self):
+        model_obj1 = self.create_test_object()
+        model_obj2 = self.create_test_object()
+
+        helper = Helper()
+        data = helper.rescan_global_files()
+        self.assertEqual(data, {"Rescan": "Success"})
+
+        check_obj1 = File.objects.get(fileId=model_obj1.id)
+        self.assertEqual(3, check_obj1.lineCount)

src/api/v2/index.php

@@ -357,7 +357,9 @@ public static function addCORSheaders(Request $request, $response) {
 require __DIR__ . "/../../inc/apiv2/helper/importCrackedHashes.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/importFile.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/purgeTask.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/rebuildChunkCache.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/recountFileLines.routes.php";
+require __DIR__ . "/../../inc/apiv2/helper/rescanGlobalFiles.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/resetChunk.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/resetUserPassword.routes.php";
 require __DIR__ . "/../../inc/apiv2/helper/searchHashes.routes.php";

src/inc/apiv2/helper/rebuildChunkCache.routes.php

@@ -0,0 +1,47 @@
+<?php
+
+use DBA\File;
+use DBA\Config;
+use Psr\Container\ContainerExceptionInterface;
+use Psr\Container\NotFoundExceptionInterface;
+
+require_once(dirname(__FILE__) . "/../common/AbstractHelperAPI.class.php");
+
+class RebuildChunkCacheHelperAPI extends AbstractHelperAPI {
+  public static function getBaseUri(): string {
+    return "/api/v2/helper/rebuildChunkCache";
+  }
+  
+  public static function getAvailableMethods(): array {
+    return ['POST'];
+  }
+  
+  public function getRequiredPermissions(string $method): array {
+    return [Config::PERM_UPDATE];
+  }
+  
+  public function getFormFields(): array {
+    return [];
+  }
+  
+  public static function getResponse(): array {
+    return ["Rebuild" => "Success"];
+  }
+  
+  /**
+   * Endpoint to recount files for when there is size mismatch
+   * @param $data
+   * @return object|array|null
+   * @throws ContainerExceptionInterface
+   * @throws NotFoundExceptionInterface
+   */
+  public function actionPost($data): object|array|null {
+    $result = ConfigUtils::rebuildCache();
+    $response = $this->getResponse();
+    $response["correctedChunks"] = $result[0];
+    $response["correctedHashlists"] = $result[1];
+    return $response;
+  }
+}
+
+RebuildChunkCacheHelperAPI::register($app);

src/inc/apiv2/helper/rescanGlobalFiles.routes.php

@@ -0,0 +1,45 @@
+<?php
+
+use DBA\File;
+use DBA\Config;
+use Psr\Container\ContainerExceptionInterface;
+use Psr\Container\NotFoundExceptionInterface;
+
+require_once(dirname(__FILE__) . "/../common/AbstractHelperAPI.class.php");
+
+class RescanGlobalFilesHelperAPI extends AbstractHelperAPI {
+  public static function getBaseUri(): string {
+    return "/api/v2/helper/rescanGlobalFiles";
+  }
+  
+  public static function getAvailableMethods(): array {
+    return ['POST'];
+  }
+  
+  public function getRequiredPermissions(string $method): array {
+    return [Config::PERM_UPDATE];
+  }
+  
+  public function getFormFields(): array {
+    return [];
+  }
+  
+  public static function getResponse(): array {
+    return ["Rescan" => "Success"];
+  }
+  
+  /**
+   * Endpoint to recount files for when there is size mismatch
+   * @param $data
+   * @return object|array|null
+   * @throws ContainerExceptionInterface
+   * @throws NotFoundExceptionInterface
+   * @throws HTMessages
+   */
+  public function actionPost($data): object|array|null {
+    ConfigUtils::scanFiles();
+    return $this->getResponse();
+  }
+}
+
+RescanGlobalFilesHelperAPI::register($app);