adds incremental override and resource folder to state.json (#358)

* adds incremental override and resource folder to state.json

* handles cases when state doesn't have incremental override

* uses .get(i) in comparison

* tidies

Author: CarlosCoelhoSL

digital_land/cli.py

@@ -642,15 +642,36 @@ def config_load_cmd(ctx, config_path):
     default="pipeline",
     help="directory containing the pipeline",
 )
+@click.option(
+    "--resource-dir",
+    type=click.Path(),
+    default="collection/resource",
+    help="directory containing resources",
+)
+@click.option("--incremental-override", type=click.BOOL, default=False)
 @click.option(
     "--output-path",
     "-o",
     type=click.Path(),
     default="state.json",
     help="path of the output state file",
 )
-def save_state_cmd(specification_dir, collection_dir, pipeline_dir, output_path):
-    save_state(specification_dir, collection_dir, pipeline_dir, output_path)
+def save_state_cmd(
+    specification_dir,
+    collection_dir,
+    pipeline_dir,
+    resource_dir,
+    incremental_override,
+    output_path,
+):
+    save_state(
+        specification_dir,
+        collection_dir,
+        pipeline_dir,
+        resource_dir,
+        incremental_override,
+        output_path,
+    )
 
 
 @cli.command(
@@ -675,17 +696,43 @@ def save_state_cmd(specification_dir, collection_dir, pipeline_dir, output_path)
     default="pipeline",
     help="directory containing the pipeline",
 )
+@click.option(
+    "--resource-dir",
+    type=click.Path(),
+    default="collection/resource",
+    help="directory containing resources",
+)
+@click.option("--incremental-override", type=click.BOOL, default=False)
 @click.option(
     "--state-path",
     type=click.Path(),
     default="state.json",
     help="path of the output state file",
 )
-def check_state_cmd(specification_dir, collection_dir, pipeline_dir, state_path):
+def check_state_cmd(
+    specification_dir,
+    collection_dir,
+    pipeline_dir,
+    resource_dir,
+    incremental_override,
+    state_path,
+):
     # If the state isn't the same, use a non-zero return code so scripts can
     # detect this, and print a message. If it is the same, exit silenty wirh a
     # 0 retun code.
-    diffs = compare_state(specification_dir, collection_dir, pipeline_dir, state_path)
+
+    if incremental_override:
+        print("State comparison skipped as incremental override enabled")
+        sys.exit(1)
+
+    diffs = compare_state(
+        specification_dir,
+        collection_dir,
+        pipeline_dir,
+        resource_dir,
+        incremental_override,
+        state_path,
+    )
     if diffs:
         print(f"State differs from {state_path} - {', '.join(diffs)}")
         sys.exit(1)

digital_land/commands.py

@@ -1405,29 +1405,51 @@ def organisation_check(**kwargs):
     package.check(lpa_path, output_path)
 
 
-def save_state(specification_dir, collection_dir, pipeline_dir, output_path):
+def save_state(
+    specification_dir,
+    collection_dir,
+    pipeline_dir,
+    resource_dir,
+    incremental_override,
+    output_path,
+):
     state = State.build(
         specification_dir=specification_dir,
         collection_dir=collection_dir,
         pipeline_dir=pipeline_dir,
+        resource_dir=resource_dir,
+        incremental_override=incremental_override,
     )
     state.save(
         output_path=output_path,
     )
 
 
-def compare_state(specification_dir, collection_dir, pipeline_dir, state_path):
+def compare_state(
+    specification_dir,
+    collection_dir,
+    pipeline_dir,
+    resource_dir,
+    incremental_override,
+    state_path,
+):
     """Compares the current state against the one in state_path.
     Returns a list of different elements, or None if they are the same."""
     current = State.build(
         specification_dir=specification_dir,
         collection_dir=collection_dir,
         pipeline_dir=pipeline_dir,
+        resource_dir=resource_dir,
+        incremental_override=incremental_override,
     )
+    # in here current incremental override must be false
 
     compare = State.load(state_path)
+    # we don't want to include whether the previous state was an incremental override in comparison
+    current.pop("incremental_override", None)
+    compare.pop("incremental_override", None)
 
     if current == compare:
         return None
 
-    return [i for i in current.keys() if current[i] != compare[i]]
+    return [i for i in current.keys() if current[i] != compare.get(i, "")]

digital_land/state.py

@@ -12,7 +12,13 @@ def __init__(self, data):
         for k, v in data.items():
             self.__setitem__(k, v)
 
-    def build(specification_dir, collection_dir, pipeline_dir):
+    def build(
+        specification_dir,
+        collection_dir,
+        pipeline_dir,
+        resource_dir,
+        incremental_override,
+    ):
         """Build a state object from the current configuration and code"""
         return State(
             {
@@ -21,7 +27,9 @@ def build(specification_dir, collection_dir, pipeline_dir):
                 "collection": State.get_dir_hash(
                     collection_dir, ["log/", "log.csv", "pipeline.mk", "resource/"]
                 ),
+                "resource": State.get_dir_hash(resource_dir),
                 "pipeline": State.get_dir_hash(pipeline_dir),
+                "incremental_override": incremental_override,
             }
         )
 

tests/data/state/resource/example_resource

@@ -0,0 +1 @@
+resource here

tests/data/state/resource_diff/different_resource

@@ -0,0 +1 @@
+this resource is different

tests/e2e/test_state.py

@@ -7,6 +7,7 @@
 specification_hash = "ebe620f5228d01170b1857bad3e738aa432f5fd6"
 collection_hash = "ed4c5979268ad880f7edbdc2047cfcfa6b9ee3b4"
 pipeline_hash = "4a5a778d678db812e4f3d498a5aaa6f39af38d10"
+resource_hash = "063e908c6695671063dee27c534bf3471aa3f5d5"
 
 
 def get_code_hash():
@@ -26,6 +27,8 @@ def test_state(tmp_path):
         specification_dir=os.path.join(test_data_dir, "specification"),
         collection_dir=os.path.join(test_data_dir, "collection"),
         pipeline_dir=os.path.join(test_data_dir, "pipeline"),
+        resource_dir=os.path.join(test_data_dir, "resource"),
+        incremental_override=True,
         output_path=state_path,
     )
 
@@ -36,18 +39,24 @@ def test_state(tmp_path):
             "code",
             "specification",
             "collection",
+            "resource",
             "pipeline",
+            "incremental_override",
         ]
         assert state_data["code"] == get_code_hash()
         assert state_data["specification"] == specification_hash
         assert state_data["collection"] == collection_hash
         assert state_data["pipeline"] == pipeline_hash
+        assert state_data["resource"] == resource_hash
+        assert state_data["incremental_override"]
 
     assert (
         compare_state(
             specification_dir=os.path.join(test_data_dir, "specification"),
             collection_dir=os.path.join(test_data_dir, "collection"),
             pipeline_dir=os.path.join(test_data_dir, "pipeline"),
+            resource_dir=os.path.join(test_data_dir, "resource"),
+            incremental_override=True,
             state_path=state_path,
         )
         is None
@@ -58,6 +67,8 @@ def test_state(tmp_path):
             specification_dir=os.path.join(test_data_dir, "specification"),
             collection_dir=os.path.join(test_data_dir, "collection_exclude"),
             pipeline_dir=os.path.join(test_data_dir, "pipeline"),
+            resource_dir=os.path.join(test_data_dir, "resource"),
+            incremental_override=True,
             state_path=state_path,
         )
         is None
@@ -67,5 +78,30 @@ def test_state(tmp_path):
         specification_dir=os.path.join(test_data_dir, "specification"),
         collection_dir=os.path.join(test_data_dir, "collection_blank"),
         pipeline_dir=os.path.join(test_data_dir, "pipeline"),
+        resource_dir=os.path.join(test_data_dir, "resource"),
+        incremental_override=True,
         state_path=state_path,
     ) == ["collection"]
+
+    assert compare_state(
+        specification_dir=os.path.join(test_data_dir, "specification"),
+        collection_dir=os.path.join(test_data_dir, "collection"),
+        pipeline_dir=os.path.join(test_data_dir, "pipeline"),
+        resource_dir=os.path.join(test_data_dir, "resource_diff"),
+        incremental_override=True,
+        state_path=state_path,
+    ) == ["resource"]
+
+    # we shouldn't include the incremental override value in state comparison
+    # so test it isn't flagged if different
+    assert (
+        compare_state(
+            specification_dir=os.path.join(test_data_dir, "specification"),
+            collection_dir=os.path.join(test_data_dir, "collection"),
+            pipeline_dir=os.path.join(test_data_dir, "pipeline"),
+            resource_dir=os.path.join(test_data_dir, "resource"),
+            incremental_override=False,
+            state_path=state_path,
+        )
+        is None
+    )

tests/integration/test_state.py

@@ -65,6 +65,8 @@ def test_state_build_persist(tmp_path):
         os.path.join(test_dir, "specification"),
         os.path.join(test_dir, "collection"),
         os.path.join(test_dir, "pipeline"),
+        os.path.join(test_dir, "resource"),
+        True,
     )
     state_1.save(tmp_json)
 
@@ -77,6 +79,8 @@ def test_state_build_persist(tmp_path):
         os.path.join(test_dir, "specification"),
         os.path.join(test_dir, "collection_blank"),
         os.path.join(test_dir, "pipeline"),
+        os.path.join(test_dir, "resource"),
+        True,
     )
 
     # Check that's different from the first one