Remove IndexMap document list (distributed-system-analysis#3606)

* Remove IndexMap document list

PBENCH-1315

The production server, with "only" 108,728 indexed datasets (many more still
haven't been migrated from the passthrough server), currently claims 84.1Gb of
PostgreSQL storage just for the `IndexMap` table. Most of this consists of a
list of each Opensearch document ID in order to allow using bulk update and
delete operations to manage the index. This is straining the capacity of our
RDU2 PostgreSQL server.

As an alternative, this PR removes the document list and instead of the bulk
update and delete operations uses `_delete_by_query` and `_update_by_query`
searching for documents in the appropriate indices (which we still store in
the `IndexMap`) by parent dataset resource ID.

Along the way, I noticed that (oops) we were missing the `"authorization"`
subdocument in some of our Elasticsearch documents, which would impact the
authenticated search API behaviors. And I acted on a deprecation warning for
a camelCase template keyword by replacing it with a snake_case alternative.

Author: dbutenhof

lib/pbench/cli/server/report.py

@@ -1,6 +1,5 @@
 from collections import defaultdict
 import datetime
-import json
 import re
 from threading import Thread
 import time
@@ -323,13 +322,9 @@ def report_sql():
     indices = set()
     root_size = 0
     index_size = 0
-    document_count = 0
-    document_size = 0
-    json_size = 0
-    document_int_size = 0
 
-    query = select(IndexMap.root, IndexMap.index, IndexMap.documents)
-    for root, index, documents in Database.db_session.execute(
+    query = select(IndexMap.root, IndexMap.index)
+    for root, index in Database.db_session.execute(
         query, execution_options={"stream_results": True}
     ).yield_per(500):
         record_count += 1
@@ -338,12 +333,6 @@ def report_sql():
         indices.add(index)
         root_size += len(root)
         index_size += len(index)
-        json_size += len(json.dumps(documents))
-        for d in documents:
-            document_count += 1
-            document_size += len(d)
-            i = int(d, base=16)
-            document_int_size += (i.bit_length() + 7) / 8
         unique_root_size = sum(len(r) for r in roots)
         unique_index_size = sum(len(i) for i in indices)
 
@@ -359,11 +348,6 @@ def report_sql():
         f" deduped: {humanize.naturalsize(unique_index_size)} for index "
         f"names, {humanize.naturalsize(unique_root_size)} for root names"
     )
-    detailer.message(
-        f" {humanize.naturalsize(document_size)} for {document_count:,d} document IDs, "
-        f"{humanize.naturalsize(json_size)} as JSON, "
-        f"{humanize.naturalsize(document_int_size)} as (Python) ints"
-    )
 
 
 def report_states():

lib/pbench/server/api/__init__.py

@@ -23,7 +23,7 @@
 from pbench.server.api.resources.datasets_metadata import DatasetsMetadata
 from pbench.server.api.resources.datasets_visualize import DatasetsVisualize
 from pbench.server.api.resources.endpoint_configure import EndpointConfig
-from pbench.server.api.resources.query_apis.dataset import Datasets
+from pbench.server.api.resources.query_apis.datasets.datasets import Datasets
 from pbench.server.api.resources.query_apis.datasets.datasets_detail import (
     DatasetsDetail,
 )

lib/pbench/server/api/resources/__init__.py

@@ -1030,7 +1030,7 @@ class ApiAttributes:
     """
 
     action: str
-    operation_name: OperationName
+    operation_name: Optional[OperationName]
     require_stable: bool
     require_map: bool