|
23 | 23 | from datetime import timedelta, timezone
|
24 | 24 |
|
25 | 25 | from arcticdb.exceptions import ArcticNativeException, SortingException
|
26 |
| -from arcticdb_ext.version_store import StreamDescriptorMismatch |
| 26 | +from arcticdb_ext.version_store import StreamDescriptorMismatch, NoSuchVersionException |
27 | 27 |
|
28 | 28 | from arcticdb_ext.exceptions import (
|
29 | 29 | UnsortedDataException,
|
@@ -582,3 +582,105 @@ def check_incomplete_staged(sym: str, remove_staged: bool = True) -> None:
|
582 | 582 | # Complex structures can be staged by default
|
583 | 583 | lib.stage(symbol, get_metadata())
|
584 | 584 | check_incomplete_staged(symbol)
|
| 585 | + |
| 586 | + |
| 587 | +def test_add_to_snapshot_and_remove_from_snapshots_scenarios(basic_store): |
| 588 | + lib: NativeVersionStore = basic_store |
| 589 | + lib.write("s1", 100) |
| 590 | + lib.write("s2", 200) |
| 591 | + |
| 592 | + lib.snapshot("snap") |
| 593 | + lib.write("s3", 300) |
| 594 | + lib.write("s1", 101) |
| 595 | + lib.write("s1", 102) |
| 596 | + lib.write("s1", 103) |
| 597 | + lib.write("s2", 201) |
| 598 | + lib.write("s4", 400) |
| 599 | + |
| 600 | + # We can add empty list of symbols without error |
| 601 | + lib.add_to_snapshot("snap", []) |
| 602 | + # We can remove nothing without error |
| 603 | + lib.remove_from_snapshot("snap", [], []) |
| 604 | + |
| 605 | + # add to snapshot operation succeeds even symbol does not exist |
| 606 | + lib.add_to_snapshot("snap", ["ss"]) |
| 607 | + # remove from snapshot operation succeeds even symbol does not exist |
| 608 | + lib.remove_from_snapshot("snap", ["FDFGEREG"], [213]) |
| 609 | + |
| 610 | + # remove from snapshot operation succeeds even symbol exists but version does not exist |
| 611 | + lib.remove_from_snapshot("snap", ["s2"], [2]) |
| 612 | + lib.add_to_snapshot("snap", ["s2", "s1"], [4343, 45949345]) |
| 613 | + |
| 614 | + # Verify the snapshot state is not changed |
| 615 | + assert 100 == lib.read("s1", as_of="snap").data |
| 616 | + assert 200 == lib.read("s2", as_of="snap").data |
| 617 | + with pytest.raises(NoSuchVersionException): |
| 618 | + lib.read("s3", as_of="snap") |
| 619 | + |
| 620 | + # Verify mixing of existing and non-existing symbols result |
| 621 | + # in proper versions of existing symbols added to the snapshot |
| 622 | + lib.add_to_snapshot("snap", [" ", 5443, "ss", "s1", "s4"]) |
| 623 | + assert 103 == lib.read("s1", as_of="snap").data |
| 624 | + assert 400 == lib.read("s4", as_of="snap").data |
| 625 | + assert 200 == lib.read("s2", as_of="snap").data |
| 626 | + with pytest.raises(NoSuchVersionException): |
| 627 | + lib.read("s3", as_of="snap") |
| 628 | + |
| 629 | + # Verify mixing of existing and non-existing symbols and versions result |
| 630 | + # in proper versions of existing symbols added to the snapshot |
| 631 | + lib.add_to_snapshot("snap", ["Go home ...", "WELCOME!", "s1", "s2", "s2"], [1, 1, 1, 1, 4]) |
| 632 | + assert 101 == lib.read("s1", as_of="snap").data |
| 633 | + assert 400 == lib.read("s4", as_of="snap").data |
| 634 | + assert 201 == lib.read("s2", as_of="snap").data |
| 635 | + with pytest.raises(NoSuchVersionException): |
| 636 | + lib.read("s3", as_of="snap") |
| 637 | + |
| 638 | + # Mix of valid and invalid symbols and versions does not affect removal from snapshot |
| 639 | + lib.remove_from_snapshot("snap", ["s11", "s1", "s2", "s1", "s2"], [33, 222, 123, 1, 1]) |
| 640 | + assert 400 == lib.read("s4", as_of="snap").data |
| 641 | + for symbol in ["s1", "s2", "s3"]: |
| 642 | + with pytest.raises(NoSuchVersionException): |
| 643 | + lib.read(symbol, as_of="snap") |
| 644 | + |
| 645 | + |
| 646 | +@pytest.mark.xfail(True, reason="Negative version numbers does not work, issue 10060901137") |
| 647 | +def test_add_to_snapshot_with_negative_numbers(basic_store): |
| 648 | + lib: NativeVersionStore = basic_store |
| 649 | + lib.write("s1", 100) |
| 650 | + lib.snapshot("snap") |
| 651 | + lib.write("s1", 101) |
| 652 | + lib.write("s1", 102) |
| 653 | + lib.write("s1", 103) |
| 654 | + |
| 655 | + # Lets check negative number version handling |
| 656 | + lib.add_to_snapshot("snap", ["s1"], [-1]) |
| 657 | + assert 102 == lib.read("s1", as_of="snap").data |
| 658 | + lib.add_to_snapshot("snap", ["s1"], [-2]) |
| 659 | + assert 101 == lib.read("s1", as_of="snap").data |
| 660 | + |
| 661 | + |
| 662 | +@pytest.mark.parametrize("dynamic_schema", [True, False]) |
| 663 | +def test_remove_incomplete_for_v1_API(version_store_and_real_s3_basic_store_factory, dynamic_schema): |
| 664 | + """Testing staging and removing incomplete series for v1 API""" |
| 665 | + |
| 666 | + lib: NativeVersionStore = version_store_and_real_s3_basic_store_factory( |
| 667 | + dynamic_schema=dynamic_schema, segment_row_size=10 |
| 668 | + ) |
| 669 | + sym = "any symbol will do until don't" |
| 670 | + name = "series_name" |
| 671 | + length_of_series = np.random.randint(5, 26, size=10) |
| 672 | + |
| 673 | + for iter, length in enumerate(length_of_series): |
| 674 | + timestamp = pd.Timestamp(f"{1990 + iter}-1-1") |
| 675 | + series = generate_random_series(np.float64, length, name, start_time=timestamp, seed=None) |
| 676 | + if iter == 0: |
| 677 | + lib.write(sym, series) |
| 678 | + else: |
| 679 | + lib.stage(sym, series, validate_index=False, sort_on_index=False) |
| 680 | + |
| 681 | + assert lib.list_symbols_with_incomplete_data() == [sym] |
| 682 | + lib.remove_incomplete("") # non-existing symbol |
| 683 | + lib.remove_incomplete("any name will do") # non-existing symbol |
| 684 | + assert lib.list_symbols_with_incomplete_data() == [sym] |
| 685 | + lib.remove_incomplete(sym) |
| 686 | + assert lib.list_symbols_with_incomplete_data() == [] |
0 commit comments