diff --git a/digital_land/cli.py b/digital_land/cli.py index ca92ef3f4..6aa15884c 100644 --- a/digital_land/cli.py +++ b/digital_land/cli.py @@ -642,6 +642,13 @@ def config_load_cmd(ctx, config_path): default="pipeline", help="directory containing the pipeline", ) +@click.option( + "--resource-dir", + type=click.Path(), + default="collection/resource", + help="directory containing resources", +) +@click.option("--incremental-override", type=click.BOOL, default=False) @click.option( "--output-path", "-o", @@ -649,8 +656,22 @@ def config_load_cmd(ctx, config_path): default="state.json", help="path of the output state file", ) -def save_state_cmd(specification_dir, collection_dir, pipeline_dir, output_path): - save_state(specification_dir, collection_dir, pipeline_dir, output_path) +def save_state_cmd( + specification_dir, + collection_dir, + pipeline_dir, + resource_dir, + incremental_override, + output_path, +): + save_state( + specification_dir, + collection_dir, + pipeline_dir, + resource_dir, + incremental_override, + output_path, + ) @cli.command( @@ -675,17 +696,43 @@ def save_state_cmd(specification_dir, collection_dir, pipeline_dir, output_path) default="pipeline", help="directory containing the pipeline", ) +@click.option( + "--resource-dir", + type=click.Path(), + default="collection/resource", + help="directory containing resources", +) +@click.option("--incremental-override", type=click.BOOL, default=False) @click.option( "--state-path", type=click.Path(), default="state.json", help="path of the output state file", ) -def check_state_cmd(specification_dir, collection_dir, pipeline_dir, state_path): +def check_state_cmd( + specification_dir, + collection_dir, + pipeline_dir, + resource_dir, + incremental_override, + state_path, +): # If the state isn't the same, use a non-zero return code so scripts can # detect this, and print a message. If it is the same, exit silenty wirh a # 0 retun code. - diffs = compare_state(specification_dir, collection_dir, pipeline_dir, state_path) + + if incremental_override: + print("State comparison skipped as incremental override enabled") + sys.exit(1) + + diffs = compare_state( + specification_dir, + collection_dir, + pipeline_dir, + resource_dir, + incremental_override, + state_path, + ) if diffs: print(f"State differs from {state_path} - {', '.join(diffs)}") sys.exit(1) diff --git a/digital_land/commands.py b/digital_land/commands.py index 870eeab9c..eb26a4035 100644 --- a/digital_land/commands.py +++ b/digital_land/commands.py @@ -1405,29 +1405,51 @@ def organisation_check(**kwargs): package.check(lpa_path, output_path) -def save_state(specification_dir, collection_dir, pipeline_dir, output_path): +def save_state( + specification_dir, + collection_dir, + pipeline_dir, + resource_dir, + incremental_override, + output_path, +): state = State.build( specification_dir=specification_dir, collection_dir=collection_dir, pipeline_dir=pipeline_dir, + resource_dir=resource_dir, + incremental_override=incremental_override, ) state.save( output_path=output_path, ) -def compare_state(specification_dir, collection_dir, pipeline_dir, state_path): +def compare_state( + specification_dir, + collection_dir, + pipeline_dir, + resource_dir, + incremental_override, + state_path, +): """Compares the current state against the one in state_path. Returns a list of different elements, or None if they are the same.""" current = State.build( specification_dir=specification_dir, collection_dir=collection_dir, pipeline_dir=pipeline_dir, + resource_dir=resource_dir, + incremental_override=incremental_override, ) + # in here current incremental override must be false compare = State.load(state_path) + # we don't want to include whether the previous state was an incremental override in comparison + current.pop("incremental_override", None) + compare.pop("incremental_override", None) if current == compare: return None - return [i for i in current.keys() if current[i] != compare[i]] + return [i for i in current.keys() if current[i] != compare.get(i, "")] diff --git a/digital_land/state.py b/digital_land/state.py index 81f4fcc32..380eb99e3 100644 --- a/digital_land/state.py +++ b/digital_land/state.py @@ -12,7 +12,13 @@ def __init__(self, data): for k, v in data.items(): self.__setitem__(k, v) - def build(specification_dir, collection_dir, pipeline_dir): + def build( + specification_dir, + collection_dir, + pipeline_dir, + resource_dir, + incremental_override, + ): """Build a state object from the current configuration and code""" return State( { @@ -21,7 +27,9 @@ def build(specification_dir, collection_dir, pipeline_dir): "collection": State.get_dir_hash( collection_dir, ["log/", "log.csv", "pipeline.mk", "resource/"] ), + "resource": State.get_dir_hash(resource_dir), "pipeline": State.get_dir_hash(pipeline_dir), + "incremental_override": incremental_override, } ) diff --git a/tests/data/state/resource/example_resource b/tests/data/state/resource/example_resource new file mode 100644 index 000000000..4d400557e --- /dev/null +++ b/tests/data/state/resource/example_resource @@ -0,0 +1 @@ +resource here \ No newline at end of file diff --git a/tests/data/state/resource_diff/different_resource b/tests/data/state/resource_diff/different_resource new file mode 100644 index 000000000..34fa90ce9 --- /dev/null +++ b/tests/data/state/resource_diff/different_resource @@ -0,0 +1 @@ +this resource is different \ No newline at end of file diff --git a/tests/e2e/test_state.py b/tests/e2e/test_state.py index 2e38d37f9..51ee5bd0a 100644 --- a/tests/e2e/test_state.py +++ b/tests/e2e/test_state.py @@ -7,6 +7,7 @@ specification_hash = "ebe620f5228d01170b1857bad3e738aa432f5fd6" collection_hash = "ed4c5979268ad880f7edbdc2047cfcfa6b9ee3b4" pipeline_hash = "4a5a778d678db812e4f3d498a5aaa6f39af38d10" +resource_hash = "063e908c6695671063dee27c534bf3471aa3f5d5" def get_code_hash(): @@ -26,6 +27,8 @@ def test_state(tmp_path): specification_dir=os.path.join(test_data_dir, "specification"), collection_dir=os.path.join(test_data_dir, "collection"), pipeline_dir=os.path.join(test_data_dir, "pipeline"), + resource_dir=os.path.join(test_data_dir, "resource"), + incremental_override=True, output_path=state_path, ) @@ -36,18 +39,24 @@ def test_state(tmp_path): "code", "specification", "collection", + "resource", "pipeline", + "incremental_override", ] assert state_data["code"] == get_code_hash() assert state_data["specification"] == specification_hash assert state_data["collection"] == collection_hash assert state_data["pipeline"] == pipeline_hash + assert state_data["resource"] == resource_hash + assert state_data["incremental_override"] assert ( compare_state( specification_dir=os.path.join(test_data_dir, "specification"), collection_dir=os.path.join(test_data_dir, "collection"), pipeline_dir=os.path.join(test_data_dir, "pipeline"), + resource_dir=os.path.join(test_data_dir, "resource"), + incremental_override=True, state_path=state_path, ) is None @@ -58,6 +67,8 @@ def test_state(tmp_path): specification_dir=os.path.join(test_data_dir, "specification"), collection_dir=os.path.join(test_data_dir, "collection_exclude"), pipeline_dir=os.path.join(test_data_dir, "pipeline"), + resource_dir=os.path.join(test_data_dir, "resource"), + incremental_override=True, state_path=state_path, ) is None @@ -67,5 +78,30 @@ def test_state(tmp_path): specification_dir=os.path.join(test_data_dir, "specification"), collection_dir=os.path.join(test_data_dir, "collection_blank"), pipeline_dir=os.path.join(test_data_dir, "pipeline"), + resource_dir=os.path.join(test_data_dir, "resource"), + incremental_override=True, state_path=state_path, ) == ["collection"] + + assert compare_state( + specification_dir=os.path.join(test_data_dir, "specification"), + collection_dir=os.path.join(test_data_dir, "collection"), + pipeline_dir=os.path.join(test_data_dir, "pipeline"), + resource_dir=os.path.join(test_data_dir, "resource_diff"), + incremental_override=True, + state_path=state_path, + ) == ["resource"] + + # we shouldn't include the incremental override value in state comparison + # so test it isn't flagged if different + assert ( + compare_state( + specification_dir=os.path.join(test_data_dir, "specification"), + collection_dir=os.path.join(test_data_dir, "collection"), + pipeline_dir=os.path.join(test_data_dir, "pipeline"), + resource_dir=os.path.join(test_data_dir, "resource"), + incremental_override=False, + state_path=state_path, + ) + is None + ) diff --git a/tests/integration/test_state.py b/tests/integration/test_state.py index 5e7be1e85..500175708 100644 --- a/tests/integration/test_state.py +++ b/tests/integration/test_state.py @@ -65,6 +65,8 @@ def test_state_build_persist(tmp_path): os.path.join(test_dir, "specification"), os.path.join(test_dir, "collection"), os.path.join(test_dir, "pipeline"), + os.path.join(test_dir, "resource"), + True, ) state_1.save(tmp_json) @@ -77,6 +79,8 @@ def test_state_build_persist(tmp_path): os.path.join(test_dir, "specification"), os.path.join(test_dir, "collection_blank"), os.path.join(test_dir, "pipeline"), + os.path.join(test_dir, "resource"), + True, ) # Check that's different from the first one