Skip to content

Commit bcf2b2f

Browse files
authored
v1 API tests for add_to_snapshot and remove_from_snapshot + other (#2651)
#### Reference Issues/PRs <!--Example: Fixes #1234. See also #3456.--> #### What does this implement or fix? Documentation is changed to reflect actual behavior Added new test for add_to_snapshot and remove_from_snapshot methods to checks behavior when we supply non existing symbol or version. Additional simple test for listing and removing incompletes also added as all coverage was for v2 #### Any other comments? #### Checklist <details> <summary> Checklist for code changes... </summary> - [ ] Have you updated the relevant docstrings, documentation and copyright notice? - [ ] Is this contribution tested against [all ArcticDB's features](../docs/mkdocs/docs/technical/contributing.md)? - [ ] Do all exceptions introduced raise appropriate [error messages](https://docs.arcticdb.io/error_messages/)? - [ ] Are API changes highlighted in the PR description? - [ ] Is the PR labelled as enhancement or bug so it appears in autogenerated release notes? </details> <!-- Thanks for contributing a Pull Request to ArcticDB! Please ensure you have taken a look at: - ArcticDB's Code of Conduct: https://github.com/man-group/ArcticDB/blob/master/CODE_OF_CONDUCT.md - ArcticDB's Contribution Licensing: https://github.com/man-group/ArcticDB/blob/master/docs/mkdocs/docs/technical/contributing.md#contribution-licensing --> --------- Co-authored-by: Georgi Rusev <Georgi Rusev>
1 parent d78b668 commit bcf2b2f

File tree

2 files changed

+109
-1
lines changed

2 files changed

+109
-1
lines changed

python/arcticdb/version_store/_store.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2679,6 +2679,9 @@ def add_to_snapshot(
26792679
"""
26802680
Add items to a snapshot. Will replace if the snapshot already contains an entry for a particular symbol.
26812681
2682+
Note: attempt to add non-existing symbol or version to a snapshot will not fail, but will have no effect
2683+
on the snapshot.
2684+
26822685
Parameters
26832686
----------
26842687
snap_name : `str`
@@ -2696,6 +2699,9 @@ def remove_from_snapshot(self, snap_name: str, symbols: List[str], versions: Lis
26962699
"""
26972700
Remove items from a snapshot
26982701
2702+
Note: attempt to remove non-existing symbol or version from a snapshot will not fail, but will have no effect
2703+
on the snapshot.
2704+
26992705
Parameters
27002706
----------
27012707
snap_name : `str`

python/tests/integration/arcticdb/version_store/test_basic_operations_scenarios.py

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from datetime import timedelta, timezone
2424

2525
from arcticdb.exceptions import ArcticNativeException, SortingException
26-
from arcticdb_ext.version_store import StreamDescriptorMismatch
26+
from arcticdb_ext.version_store import StreamDescriptorMismatch, NoSuchVersionException
2727

2828
from arcticdb_ext.exceptions import (
2929
UnsortedDataException,
@@ -582,3 +582,105 @@ def check_incomplete_staged(sym: str, remove_staged: bool = True) -> None:
582582
# Complex structures can be staged by default
583583
lib.stage(symbol, get_metadata())
584584
check_incomplete_staged(symbol)
585+
586+
587+
def test_add_to_snapshot_and_remove_from_snapshots_scenarios(basic_store):
588+
lib: NativeVersionStore = basic_store
589+
lib.write("s1", 100)
590+
lib.write("s2", 200)
591+
592+
lib.snapshot("snap")
593+
lib.write("s3", 300)
594+
lib.write("s1", 101)
595+
lib.write("s1", 102)
596+
lib.write("s1", 103)
597+
lib.write("s2", 201)
598+
lib.write("s4", 400)
599+
600+
# We can add empty list of symbols without error
601+
lib.add_to_snapshot("snap", [])
602+
# We can remove nothing without error
603+
lib.remove_from_snapshot("snap", [], [])
604+
605+
# add to snapshot operation succeeds even symbol does not exist
606+
lib.add_to_snapshot("snap", ["ss"])
607+
# remove from snapshot operation succeeds even symbol does not exist
608+
lib.remove_from_snapshot("snap", ["FDFGEREG"], [213])
609+
610+
# remove from snapshot operation succeeds even symbol exists but version does not exist
611+
lib.remove_from_snapshot("snap", ["s2"], [2])
612+
lib.add_to_snapshot("snap", ["s2", "s1"], [4343, 45949345])
613+
614+
# Verify the snapshot state is not changed
615+
assert 100 == lib.read("s1", as_of="snap").data
616+
assert 200 == lib.read("s2", as_of="snap").data
617+
with pytest.raises(NoSuchVersionException):
618+
lib.read("s3", as_of="snap")
619+
620+
# Verify mixing of existing and non-existing symbols result
621+
# in proper versions of existing symbols added to the snapshot
622+
lib.add_to_snapshot("snap", [" ", 5443, "ss", "s1", "s4"])
623+
assert 103 == lib.read("s1", as_of="snap").data
624+
assert 400 == lib.read("s4", as_of="snap").data
625+
assert 200 == lib.read("s2", as_of="snap").data
626+
with pytest.raises(NoSuchVersionException):
627+
lib.read("s3", as_of="snap")
628+
629+
# Verify mixing of existing and non-existing symbols and versions result
630+
# in proper versions of existing symbols added to the snapshot
631+
lib.add_to_snapshot("snap", ["Go home ...", "WELCOME!", "s1", "s2", "s2"], [1, 1, 1, 1, 4])
632+
assert 101 == lib.read("s1", as_of="snap").data
633+
assert 400 == lib.read("s4", as_of="snap").data
634+
assert 201 == lib.read("s2", as_of="snap").data
635+
with pytest.raises(NoSuchVersionException):
636+
lib.read("s3", as_of="snap")
637+
638+
# Mix of valid and invalid symbols and versions does not affect removal from snapshot
639+
lib.remove_from_snapshot("snap", ["s11", "s1", "s2", "s1", "s2"], [33, 222, 123, 1, 1])
640+
assert 400 == lib.read("s4", as_of="snap").data
641+
for symbol in ["s1", "s2", "s3"]:
642+
with pytest.raises(NoSuchVersionException):
643+
lib.read(symbol, as_of="snap")
644+
645+
646+
@pytest.mark.xfail(True, reason="Negative version numbers does not work, issue 10060901137")
647+
def test_add_to_snapshot_with_negative_numbers(basic_store):
648+
lib: NativeVersionStore = basic_store
649+
lib.write("s1", 100)
650+
lib.snapshot("snap")
651+
lib.write("s1", 101)
652+
lib.write("s1", 102)
653+
lib.write("s1", 103)
654+
655+
# Lets check negative number version handling
656+
lib.add_to_snapshot("snap", ["s1"], [-1])
657+
assert 102 == lib.read("s1", as_of="snap").data
658+
lib.add_to_snapshot("snap", ["s1"], [-2])
659+
assert 101 == lib.read("s1", as_of="snap").data
660+
661+
662+
@pytest.mark.parametrize("dynamic_schema", [True, False])
663+
def test_remove_incomplete_for_v1_API(version_store_and_real_s3_basic_store_factory, dynamic_schema):
664+
"""Testing staging and removing incomplete series for v1 API"""
665+
666+
lib: NativeVersionStore = version_store_and_real_s3_basic_store_factory(
667+
dynamic_schema=dynamic_schema, segment_row_size=10
668+
)
669+
sym = "any symbol will do until don't"
670+
name = "series_name"
671+
length_of_series = np.random.randint(5, 26, size=10)
672+
673+
for iter, length in enumerate(length_of_series):
674+
timestamp = pd.Timestamp(f"{1990 + iter}-1-1")
675+
series = generate_random_series(np.float64, length, name, start_time=timestamp, seed=None)
676+
if iter == 0:
677+
lib.write(sym, series)
678+
else:
679+
lib.stage(sym, series, validate_index=False, sort_on_index=False)
680+
681+
assert lib.list_symbols_with_incomplete_data() == [sym]
682+
lib.remove_incomplete("") # non-existing symbol
683+
lib.remove_incomplete("any name will do") # non-existing symbol
684+
assert lib.list_symbols_with_incomplete_data() == [sym]
685+
lib.remove_incomplete(sym)
686+
assert lib.list_symbols_with_incomplete_data() == []

0 commit comments

Comments
 (0)