feat(integration-tests): Test package compression. (#1801)

Co-authored-by: Jack Luo <jack.luo@yscope.com>
Co-authored-by: Junhao Liao <junhao.liao@yscope.com>

Author: 3 people

integration-tests/.pytest.ini

@@ -11,6 +11,7 @@ env =
     D:CLP_BUILD_DIR=../build
     D:CLP_CORE_BINS_DIR=../build/core
     D:CLP_PACKAGE_DIR=../build/clp-package
+    D:INTEGRATION_TESTS_PROJECT_ROOT=./
 log_cli = True
 log_cli_date_format = %Y-%m-%d %H:%M:%S,%f
 log_cli_format = %(name)s %(asctime)s [%(levelname)s] %(message)s

integration-tests/tests/fixtures/path_configs.py

@@ -31,5 +31,8 @@ def fixt_package_path_config(
     """Provides paths for the clp-package directory and its contents."""
     return PackagePathConfig(
         clp_package_dir=resolve_path_env_var("CLP_PACKAGE_DIR"),
+        package_test_scripts_dir=(
+            resolve_path_env_var("INTEGRATION_TESTS_PROJECT_ROOT") / "tests" / "package_tests"
+        ),
         test_root_dir=integration_test_path_config.test_root_dir,
     )

integration-tests/tests/package_tests/clp_json/data/json-multifile/README.md

@@ -0,0 +1,16 @@
+# Description of logs
+
+**Type**
+JSON
+
+**Number of files**
+5
+
+**Number of events per file**
+8
+
+**Beginning timestamp (milliseconds)**
+1310138944000
+
+**End timestamp (milliseconds)**
+1311208074120

integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-08.jsonl

@@ -0,0 +1,8 @@
+{"timestamp":1310138944000,"mission":"STS-135","mission_day_index":0,"event":"SRB_IGNITION_CONFIRMED","subsystem":"PROP","level":"INFO","detail":"Solid rocket boosters report stable ignition and rising thrust","line_index":0}
+{"timestamp":1310139148373,"mission":"STS-135","mission_day_index":0,"event":"CLEAR_OF_TOWER","subsystem":"GUIDANCE","level":"INFO","detail":"Vehicle cleared tower, roll program initiated, tracking nominal trajectory","line_index":1}
+{"timestamp":1310139352746,"mission":"STS-135","mission_day_index":0,"event":"ROLL_PROGRAM_COMPLETE","subsystem":"GUIDANCE","level":"INFO","detail":"Roll program complete, heads down attitude achieved for ascent","line_index":2}
+{"timestamp":1310139557119,"mission":"STS-135","mission_day_index":0,"event":"MAX_Q_THROTTLE","subsystem":"PROP","level":"INFO","detail":"Main engines throttled to manage maximum dynamic pressure on stack","line_index":3}
+{"timestamp":1310139761493,"mission":"STS-135","mission_day_index":0,"event":"SRB_SEPARATION","subsystem":"PROP","level":"INFO","detail":"Solid rocket boosters separation confirmed, separation motors firing nominally","line_index":4}
+{"timestamp":1310139965866,"mission":"STS-135","mission_day_index":0,"event":"PRESS_TO_MECO","subsystem":"GUIDANCE","level":"INFO","detail":"Performance marks indicate press to main engine cutoff achieved","line_index":5}
+{"timestamp":1310140170239,"mission":"STS-135","mission_day_index":0,"event":"MAIN_ENGINE_CUTOFF","subsystem":"PROP","level":"INFO","detail":"All three SSME engines shutdown, engine parameters within expected bands","line_index":6}
+{"timestamp":1310140374613,"mission":"STS-135","mission_day_index":0,"event":"EXTERNAL_TANK_SEPARATION","subsystem":"STRUCTURES","level":"INFO","detail":"External tank separation command executed, umbilical disconnect nominal","line_index":7}

integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-09.jsonl

@@ -0,0 +1,8 @@
+{"timestamp":1310169600000,"mission":"STS-135","mission_day_index":1,"event":"RNDZ_BURN","subsystem":"GUIDANCE","level":"INFO","detail":"Rendezvous correction burn complete, relative motion profile matches ISS plan","line_index":0}
+{"timestamp":1310170175999,"mission":"STS-135","mission_day_index":1,"event":"RPM_MANEUVER","subsystem":"GUIDANCE","level":"INFO","detail":"R bar pitch maneuver complete for station imaging and TPS assessment","line_index":1}
+{"timestamp":1310170751999,"mission":"STS-135","mission_day_index":1,"event":"FINAL_APPROACH","subsystem":"GUIDANCE","level":"INFO","detail":"Final approach corridor maintained, closing rate within docking constraints","line_index":2}
+{"timestamp":1310171327999,"mission":"STS-135","mission_day_index":1,"event":"SOFT_DOCK","subsystem":"DOCKING","level":"INFO","detail":"Soft capture confirmed at forward port, relative motion damped","line_index":3}
+{"timestamp":1310171903999,"mission":"STS-135","mission_day_index":1,"event":"HARD_DOCK","subsystem":"DOCKING","level":"INFO","detail":"Structural latches engaged, ISS and shuttle in hard dock configuration","line_index":4}
+{"timestamp":1310172479999,"mission":"STS-135","mission_day_index":1,"event":"HATCH_OPEN","subsystem":"STRUCTURES","level":"INFO","detail":"Hatch open for ingress, pressure equalization verified within safe band","line_index":5}
+{"timestamp":1310173055999,"mission":"STS-135","mission_day_index":1,"event":"MPLM_BERTHING","subsystem":"ROBOTICS","level":"INFO","detail":"Multipurpose logistics module berthed to Node, structural attach points verified","line_index":6}
+{"timestamp":1310173631999,"mission":"STS-135","mission_day_index":1,"event":"CARGO_TRANSFER","subsystem":"PAYLOAD","level":"INFO","detail":"Cargo transfer underway according to integrated ISS and shuttle transfer list","line_index":7}

integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-11.jsonl

@@ -0,0 +1,8 @@
+{"timestamp":1310342400000,"mission":"STS-135","mission_day_index":3,"event":"JOINT_OPERATIONS_PLANNING","subsystem":"TIMELINE","level":"INFO","detail":"Daily planning conference complete, joint tasks synchronized","line_index":0}
+{"timestamp":1310342975999,"mission":"STS-135","mission_day_index":3,"event":"EVA_PREP","subsystem":"EVA","level":"INFO","detail":"Extravehicular mobility units serviced, prebreathe protocols initiated","line_index":1}
+{"timestamp":1310343551999,"mission":"STS-135","mission_day_index":3,"event":"EVA_TASK_STATUS","subsystem":"EVA","level":"INFO","detail":"Truss and payload bay tasks progressing, consumables within expected usage","line_index":2}
+{"timestamp":1310344127999,"mission":"STS-135","mission_day_index":3,"event":"MPLM_UNBERTH","subsystem":"ROBOTICS","level":"INFO","detail":"Logistics module unberthed and reinstalled in payload bay","line_index":3}
+{"timestamp":1310344703999,"mission":"STS-135","mission_day_index":3,"event":"ISS_SYSTEMS_HANDOFF","subsystem":"COMMS","level":"INFO","detail":"Command and telemetry handoff between shuttle and station completed","line_index":4}
+{"timestamp":1310345279999,"mission":"STS-135","mission_day_index":3,"event":"MIDDECK_PAYLOAD_CHECK","subsystem":"PAYLOAD","level":"INFO","detail":"Middeck experiment racks powered and data recording verified","line_index":5}
+{"timestamp":1310345855999,"mission":"STS-135","mission_day_index":3,"event":"ENVIRONMENTAL_MONITORING","subsystem":"ECLSS","level":"INFO","detail":"Atmospheric sampling shows stable CO2 and trace contaminant levels","line_index":6}
+{"timestamp":1310346431999,"mission":"STS-135","mission_day_index":3,"event":"JOINT_CREW_MEETING","subsystem":"TIMELINE","level":"INFO","detail":"Joint crew status review completed, no blocking issues reported","line_index":7}

integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-19.jsonl

@@ -0,0 +1,8 @@
+{"timestamp":1311033600000,"mission":"STS-135","mission_day_index":11,"event":"UNDOCK_COMMAND","subsystem":"DOCKING","level":"INFO","detail":"Undock command executed, structural latches release verified","line_index":0}
+{"timestamp":1311034175999,"mission":"STS-135","mission_day_index":11,"event":"SEPARATION_BURNS","subsystem":"GUIDANCE","level":"INFO","detail":"Separation maneuvers complete, increasing range rate from station","line_index":1}
+{"timestamp":1311034751999,"mission":"STS-135","mission_day_index":11,"event":"TPS_INSPECTION","subsystem":"THERMAL","level":"INFO","detail":"Thermal protection system sensor data and imagery show no critical damage","line_index":2}
+{"timestamp":1311035327999,"mission":"STS-135","mission_day_index":11,"event":"DEORBIT_BURN","subsystem":"PROP","level":"INFO","detail":"Deorbit burn complete, entry interface conditions within design envelope","line_index":3}
+{"timestamp":1311035903999,"mission":"STS-135","mission_day_index":11,"event":"ENTRY_INTERFACE","subsystem":"GUIDANCE","level":"INFO","detail":"Vehicle at entry interface, guidance following nominal drag corridor","line_index":4}
+{"timestamp":1311036479999,"mission":"STS-135","mission_day_index":11,"event":"COMM_BLACKOUT","subsystem":"COMMS","level":"INFO","detail":"Expected communications blackout region entered, tracking via ground radar only","line_index":5}
+{"timestamp":1311037055999,"mission":"STS-135","mission_day_index":11,"event":"SUBSONIC_TRANSITION","subsystem":"GUIDANCE","level":"INFO","detail":"Shuttle transitions to subsonic flight, control surfaces active","line_index":6}
+{"timestamp":1311037631999,"mission":"STS-135","mission_day_index":11,"event":"WHEELS_STOP","subsystem":"TIMELINE","level":"INFO","detail":"Vehicle stopped on runway, rollout distance within predicted range","line_index":7}

integration-tests/tests/package_tests/clp_json/data/json-multifile/logs/sts-135-2011-07-21.jsonl

@@ -0,0 +1,8 @@
+{"timestamp":1311206400000,"mission":"STS-135","mission_day_index":13,"event":"PAYLOAD_UNLOAD","subsystem":"PAYLOAD","level":"INFO","detail":"Post flight payload processing under way, canisters removed from bay","line_index":0}
+{"timestamp":1311206639160,"mission":"STS-135","mission_day_index":13,"event":"VEHICLE_SAFE","subsystem":"GROUND","level":"INFO","detail":"Hazardous systems safed, access teams cleared for orbiter entry","line_index":1}
+{"timestamp":1311206878320,"mission":"STS-135","mission_day_index":13,"event":"DATA_ARCHIVE","subsystem":"GROUND","level":"INFO","detail":"Flight data downlinked and archived for post mission analysis","line_index":2}
+{"timestamp":1311207117480,"mission":"STS-135","mission_day_index":13,"event":"CREW_DEBRIEF","subsystem":"TIMELINE","level":"INFO","detail":"Crew debrief with mission control completed without anomalies","line_index":3}
+{"timestamp":1311207356640,"mission":"STS-135","mission_day_index":13,"event":"VEHICLE_TOW","subsystem":"GROUND","level":"INFO","detail":"Atlantis towed from runway to Orbiter Processing Facility","line_index":4}
+{"timestamp":1311207595800,"mission":"STS-135","mission_day_index":13,"event":"RETIREMENT_CONFIG","subsystem":"GROUND","level":"INFO","detail":"Systems configured for extended safe power down and display conversion","line_index":5}
+{"timestamp":1311207834960,"mission":"STS-135","mission_day_index":13,"event":"PUBLIC_EVENT","subsystem":"OUTREACH","level":"INFO","detail":"Crew participates in public event describing program closeout activities","line_index":6}
+{"timestamp":1311208074120,"mission":"STS-135","mission_day_index":13,"event":"HARDWARE_REUSE_ASSESSMENT","subsystem":"ENGINEERING","level":"INFO","detail":"Component assessments identify candidates for museum display","line_index":7}

integration-tests/tests/package_tests/clp_json/test_clp_json.py

@@ -12,9 +12,11 @@
 
 from tests.utils.asserting_utils import (
     validate_package_instance,
+    verify_package_compression,
 )
 from tests.utils.clp_mode_utils import CLP_API_SERVER_COMPONENT, CLP_BASE_COMPONENTS
-from tests.utils.config import PackageInstance, PackageModeConfig
+from tests.utils.config import PackageCompressionJob, PackageInstance, PackageModeConfig
+from tests.utils.package_utils import run_package_compression_script
 
 logger = logging.getLogger(__name__)
 
@@ -54,20 +56,43 @@ def test_clp_json_startup(fixt_package_instance: PackageInstance) -> None:
 
 
 @pytest.mark.compression
-def test_clp_json_compression(fixt_package_instance: PackageInstance) -> None:
+def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInstance) -> None:
     """
-    Validate that the `clp-json` package successfully compresses some dataset.
+    Validate that the `clp-json` package successfully compresses the `json-multifile` dataset.
 
     :param fixt_package_instance:
     """
     validate_package_instance(fixt_package_instance)
 
-    # TODO: compress some dataset and check the correctness of compression.
-    assert True
+    # Clear archives before compressing.
+    package_test_config = fixt_package_instance.package_test_config
+    package_path_config = package_test_config.path_config
+    package_path_config.clear_package_archives()
 
-    log_msg = "test_clp_json_compression was successful."
+    # Compress a dataset.
+    compression_job = PackageCompressionJob(
+        path_to_original_dataset=(
+            package_path_config.clp_json_test_data_path / "json-multifile" / "logs"
+        ),
+        options=[
+            "--timestamp-key",
+            "timestamp",
+            "--dataset",
+            "json_multifile",
+        ],
+        positional_args=None,
+    )
+    run_package_compression_script(compression_job, package_test_config)
+
+    # Check the correctness of compression.
+    verify_package_compression(compression_job.path_to_original_dataset, package_test_config)
+
+    log_msg = "test_clp_json_compression_json_multifile was successful."
     logger.info(log_msg)
 
+    # Clear archives.
+    package_path_config.clear_package_archives()
+
 
 @pytest.mark.search
 def test_clp_json_search(fixt_package_instance: PackageInstance) -> None:
@@ -78,11 +103,15 @@ def test_clp_json_search(fixt_package_instance: PackageInstance) -> None:
     """
     validate_package_instance(fixt_package_instance)
 
-    # TODO: compress some dataset and check the correctness of compression.
+    # TODO: compress a dataset
+
+    # TODO: check the correctness of the compression
 
     # TODO: search through that dataset and check the correctness of the search results.
 
     assert True
 
     log_msg = "test_clp_json_search was successful."
     logger.info(log_msg)
+
+    # TODO: clean up clp-package/var/data, clp-package/var/log, and clp-package/var/tmp

integration-tests/tests/package_tests/clp_text/data/text-multifile/README.md

@@ -0,0 +1,16 @@
+# Description of logs
+
+**Type**
+Unstructured text
+
+**Number of files**
+5
+
+**Number of events per file**
+10
+
+**Beginning timestamp (milliseconds)**
+92572380000
+
+**End timestamp (milliseconds)**
+93614008000