Add hack folder with various dev scripts (#145)

Author: inelpandzic

hack/change_stream.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# poetry run python3 .dev/change_stream.py mongodb://adm:pass@rs00:30000
+
+import sys
+from signal import SIG_DFL, SIGINT, signal
+
+import bson.json_util as json
+import pymongo
+
+MONGODB_URI = "mongodb://adm:pass@rs00:30000"
+
+ignoreCheckpoints = True  # set to False to see all changes, including checkpoints
+
+if __name__ == "__main__":
+    signal(SIGINT, SIG_DFL)
+
+    if len(sys.argv) > 1:
+        url = sys.argv[1]
+        print(f"Using MongoDB URI: {url}")
+    else:
+        url = MONGODB_URI
+
+    m = pymongo.MongoClient(url)
+    for change in m.watch(show_expanded_events=True):
+        del change["_id"]
+        del change["wallTime"]
+
+        ns = change["ns"]["db"]
+        
+        if ignoreCheckpoints and ns == "percona_link_mongodb":
+            continue
+
+        if coll := change["ns"].get("coll"):
+            ns += "." + coll
+        ts = change["clusterTime"]
+        doc = {
+            "ts": f"{ts.time}.{ts.inc:03}",
+            "ns": ns,
+            "op": change["operationType"][0],
+        }
+        if val := change.get("updateDescription"):
+            doc["updateDesc"] = val
+        if val := change.get("operationDescription"):
+            doc["desc"] = val
+        if val := change.get("fullDocument"):
+            doc["fullDoc"] = val
+        if val := change.get("txnNumber"):
+            doc["txn"] = val
+
+        print(json.dumps(doc))

hack/compare-all.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+import hashlib
+
+import bson
+
+import pymongo
+from pymongo import ASCENDING, MongoClient
+from pymongo.collection import Collection
+
+SRC_URI = "mongodb://adm:pass@rs00:30000"
+TGT_URI = "mongodb://adm:pass@rs10:30100"
+
+SRC_SH_URI = "mongodb://adm:pass@src-mongos:27017"
+TGT_SH_URI = "mongodb://adm:pass@tgt-mongos:29017"
+
+src = pymongo.MongoClient(SRC_SH_URI)
+tgt = pymongo.MongoClient(TGT_SH_URI)
+
+
+def list_databases(client: MongoClient):
+    """List all databases in the given MongoClient."""
+    for name in client.list_database_names():
+        if name not in ("admin", "config", "local", "percona_mongolink"):
+            yield name
+
+
+def list_collections(client: MongoClient, db: str):
+    """List all namespaces in the given database."""
+    for name in client[db].list_collection_names():
+        if not name.startswith("system."):
+            yield name
+
+
+def list_all_namespaces(client: MongoClient):
+    """Return all namespaces in the target MongoDB."""
+    for db in list_databases(client):
+        for coll in list_collections(client, db):
+            yield f"{db}.{coll}"
+
+
+def _coll_content(coll: Collection, sort=None):
+    """Get the content and hash of the given collection."""
+    if not sort:
+        sort = [("_id", ASCENDING)]
+
+    count, md5 = 0, hashlib.md5()
+    for data in coll.find_raw_batches(sort=sort):
+        md5.update(data)
+        count += len(bson.decode_all(data))
+    return count, md5.hexdigest()
+
+
+def compare_namespace(source: MongoClient, target: MongoClient, db: str, coll: str, sort=None):
+    """Compare the given namespace between source and target MongoDB."""
+    ns = f"{db}.{coll}"
+
+    source_options = source[db][coll].options()
+    target_options = target[db][coll].options()
+    assert source_options == target_options, f"{ns}: {source_options=} != {target_options=}"
+
+    if "viewOn" not in source_options:
+        source_indexes = source[db][coll].index_information()
+        target_indexes = target[db][coll].index_information()
+        assert source_indexes == target_indexes, f"{ns}: {source_indexes=} != {target_indexes=}"
+
+    source_count, source_hash = _coll_content(source[db][coll], sort)
+    target_count, target_hash = _coll_content(target[db][coll], sort)
+    assert source_count == target_count, f"{ns}: {source_count=} != {target_count=}"
+    assert source_hash == target_hash, f"{ns}: {source_hash=} != {target_hash=}"
+
+
+# Main ===========================================================================
+
+sort = None
+
+source_dbs = set(list_databases(src))
+target_dbs = set(list_databases(tgt))
+assert source_dbs == target_dbs, f"{source_dbs} != {target_dbs}"
+
+for db in source_dbs:
+    print(f"Comparing {db}...")
+    source_colls = set(list_collections(src, db))
+    target_colls = set(list_collections(tgt, db))
+    print(f"  {db}: {source_colls} vs {target_colls}")
+    assert source_colls == target_colls, f"{db} :: {source_colls} != {target_colls}"
+
+    for coll in source_colls:
+        print(f"  Comparing {db}.{coll}...")
+        compare_namespace(src, tgt, db, coll, sort)

hack/drop-all.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import pymongo
+
+SRC_URI = "mongodb://adm:pass@rs00:30000"
+TGT_URI = "mongodb://adm:pass@rs10:30100"
+
+SRC_SH_URI = "mongodb://adm:pass@src-mongos:27017"
+TGT_SH_URI = "mongodb://adm:pass@tgt-mongos:29017"
+
+src = pymongo.MongoClient(SRC_SH_URI)
+tgt = pymongo.MongoClient(TGT_SH_URI)
+
+
+def drop_all_from_src_and_tgt():
+    """Drop all databases and collections from source and target MongoDB."""
+    for db in src.list_database_names():
+        if db not in ("admin", "config", "local", "percona_mongolink"):
+            src.drop_database(db)
+
+    for db in tgt.list_database_names():
+        if db not in ("admin", "config", "local", "percona_mongolink"):
+            tgt.drop_database(db)
+
+
+# Main ===========================================================================
+
+drop_all_from_src_and_tgt()
+print("Dropped all databases and collections from source and target MongoDB.")

hack/oplog_stream.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+import bson.json_util as json
+import pymongo
+
+if __name__ == "__main__":
+    MONGODB_URI = "mongodb://adm:pass@rs10:30100,rs11:30101,rs12:30102"
+    m = pymongo.MongoClient(MONGODB_URI, readPreference="primary")
+    for change in m.local["oplog.rs"].find():
+        del change["_id"]
+        del change["wallTime"]
+        print(json.dumps(change))