Skip to content

Commit 2fafc3d

Browse files
normen662normen662
committed
save point -- in the middle of just mess
1 parent 72ccc88 commit 2fafc3d

34 files changed

+4432
-11
lines changed

‎fdb-extensions/fdb-extensions.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ dependencies {
2626
}
2727
api(libs.fdbJava)
2828
implementation(libs.guava)
29+
implementation(libs.half4j)
2930
implementation(libs.slf4j.api)
3031
compileOnly(libs.jsr305)
3132

‎fdb-extensions/src/main/java/com/apple/foundationdb/async/hnsw/AbstractChangeSet.java

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* AbstractChangeSet.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2023 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.foundationdb.async.hnsw;
22+
23+
import com.apple.foundationdb.Transaction;
24+
import com.apple.foundationdb.async.hnsw.Node.ChangeSet;
25+
26+
import javax.annotation.Nonnull;
27+
import javax.annotation.Nullable;
28+
29+
/**
30+
* Abstract base implementations for all {@link ChangeSet}s.
31+
* @param <S> slot type class
32+
* @param <N> node type class (self type)
33+
*/
34+
public abstract class AbstractChangeSet<S extends NodeSlot, N extends AbstractNode<S, N>> implements ChangeSet {
35+
@Nullable
36+
private final ChangeSet previousChangeSet;
37+
38+
@Nonnull
39+
private final N node;
40+
41+
private final int level;
42+
43+
AbstractChangeSet(@Nullable final ChangeSet previousChangeSet, @Nonnull final N node, final int level) {
44+
this.previousChangeSet = previousChangeSet;
45+
this.node = node;
46+
this.level = level;
47+
}
48+
49+
@Override
50+
public void apply(@Nonnull final Transaction transaction) {
51+
if (previousChangeSet != null) {
52+
previousChangeSet.apply(transaction);
53+
}
54+
}
55+
56+
/**
57+
* Previous change set in the chain of change sets. Can be {@code null} if there is no previous change set.
58+
* @return the previous change set in the chain of change sets
59+
*/
60+
@Nullable
61+
public ChangeSet getPreviousChangeSet() {
62+
return previousChangeSet;
63+
}
64+
65+
/**
66+
* The node this change set applies to.
67+
* @return the node this change set applies to
68+
*/
69+
@Nonnull
70+
public N getNode() {
71+
return node;
72+
}
73+
74+
/**
75+
* The level we should use when maintaining the node slot index. If {@code level < 0}, do not maintain the node slot
76+
* index.
77+
* @return the level used when maintaing the node slot index
78+
*/
79+
public int getLevel() {
80+
return level;
81+
}
82+
83+
/**
84+
* Returns whether this change set needs to also update the node slot index. There are scenarios where we
85+
* do not need to update such an index in general. For instance, the user may not want to use such an index.
86+
* In addition to that, there are change set implementations that should not update the index even if such and index
87+
* is maintained in general. For instance, the moved-in slots were already persisted in the database before the
88+
* move-in operation. We should not update the node slot index in such a case.
89+
* @return {@code true} if we need to update the node slot index, {@code false} otherwise
90+
*/
91+
public boolean isUpdateNodeSlotIndex() {
92+
return level >= 0;
93+
}
94+
}

‎fdb-extensions/src/main/java/com/apple/foundationdb/async/hnsw/AbstractNode.java

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* AbstractNode.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2023 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.foundationdb.async.hnsw;
22+
23+
import com.apple.foundationdb.tuple.Tuple;
24+
import com.christianheina.langx.half4j.Half;
25+
import com.google.common.collect.ImmutableList;
26+
27+
import javax.annotation.Nonnull;
28+
import java.util.List;
29+
30+
/**
31+
* TODO.
32+
* @param <N> node type class.
33+
*/
34+
abstract class AbstractNode<N extends Neighbor> implements Node<N> {
35+
@Nonnull
36+
private final Tuple primaryKey;
37+
38+
@Nonnull
39+
private final Vector<Half> vector;
40+
41+
@Nonnull
42+
private final List<N> neighbors;
43+
44+
protected AbstractNode(@Nonnull final Tuple primaryKey, @Nonnull final Vector<Half> vector,
45+
@Nonnull final List<N> neighbors) {
46+
this.primaryKey = primaryKey;
47+
this.vector = vector;
48+
this.neighbors = ImmutableList.copyOf(neighbors);
49+
}
50+
51+
@Nonnull
52+
@Override
53+
public Tuple getPrimaryKey() {
54+
return primaryKey;
55+
}
56+
57+
@Nonnull
58+
public Vector<Half> getVector() {
59+
return vector;
60+
}
61+
62+
@Nonnull
63+
@Override
64+
public List<N> getNeighbors() {
65+
return neighbors;
66+
}
67+
68+
@Nonnull
69+
@Override
70+
public N getNeighbor(final int index) {
71+
return neighbors.get(index);
72+
}
73+
}

‎fdb-extensions/src/main/java/com/apple/foundationdb/async/hnsw/AbstractStorageAdapter.java

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
/*
2+
* AbstractStorageAdapter.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2023 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.foundationdb.async.hnsw;
22+
23+
import com.apple.foundationdb.ReadTransaction;
24+
import com.apple.foundationdb.Transaction;
25+
import com.apple.foundationdb.subspace.Subspace;
26+
import com.apple.foundationdb.tuple.Tuple;
27+
28+
import javax.annotation.Nonnull;
29+
import javax.annotation.Nullable;
30+
import java.math.BigInteger;
31+
import java.util.List;
32+
import java.util.Objects;
33+
import java.util.concurrent.CompletableFuture;
34+
import java.util.function.Function;
35+
36+
/**
37+
* Implementations and attributes common to all concrete implementations of {@link StorageAdapter}.
38+
*/
39+
abstract class AbstractStorageAdapter implements StorageAdapter {
40+
public static final byte SUBSPACE_PREFIX_ENTRY_NODE = 0x01;
41+
public static final byte SUBSPACE_PREFIX_DATA = 0x02;
42+
43+
@Nonnull
44+
private final HNSW.Config config;
45+
@Nonnull
46+
private final Subspace subspace;
47+
@Nonnull
48+
private final OnWriteListener onWriteListener;
49+
@Nonnull
50+
private final OnReadListener onReadListener;
51+
52+
private final Subspace entryNodeSubspace;
53+
private final Subspace dataSubspace;
54+
55+
protected AbstractStorageAdapter(@Nonnull final HNSW.Config config, @Nonnull final Subspace subspace,
56+
@Nonnull final Subspace nodeSlotIndexSubspace,
57+
@Nonnull final Function<HNSW.Point, BigInteger> hilbertValueFunction,
58+
@Nonnull final OnWriteListener onWriteListener,
59+
@Nonnull final OnReadListener onReadListener) {
60+
this.config = config;
61+
this.subspace = subspace;
62+
this.onWriteListener = onWriteListener;
63+
this.onReadListener = onReadListener;
64+
65+
this.entryNodeSubspace = subspace.subspace(Tuple.from(SUBSPACE_PREFIX_ENTRY_NODE));
66+
this.dataSubspace = subspace.subspace(Tuple.from(SUBSPACE_PREFIX_DATA));
67+
}
68+
69+
@Override
70+
@Nonnull
71+
public HNSW.Config getConfig() {
72+
return config;
73+
}
74+
75+
@Override
76+
@Nonnull
77+
public Subspace getSubspace() {
78+
return subspace;
79+
}
80+
81+
@Nullable
82+
@Override
83+
public Subspace getSecondarySubspace() {
84+
return null;
85+
}
86+
87+
@Override
88+
@Nonnull
89+
public Subspace getEntryNodeSubspace() {
90+
return entryNodeSubspace;
91+
}
92+
93+
@Override
94+
@Nonnull
95+
public Subspace getDataSubspace() {
96+
return dataSubspace;
97+
}
98+
99+
@Override
100+
@Nonnull
101+
public OnWriteListener getOnWriteListener() {
102+
return onWriteListener;
103+
}
104+
105+
@Override
106+
@Nonnull
107+
public OnReadListener getOnReadListener() {
108+
return onReadListener;
109+
}
110+
111+
@Override
112+
public void writeNodes(@Nonnull final Transaction transaction, @Nonnull final List<? extends Node> nodes) {
113+
for (final Node node : nodes) {
114+
writeNode(transaction, node);
115+
}
116+
}
117+
118+
protected void writeNode(@Nonnull final Transaction transaction, @Nonnull final Node node) {
119+
final Node.ChangeSet changeSet = node.getChangeSet();
120+
if (changeSet == null) {
121+
return;
122+
}
123+
124+
changeSet.apply(transaction);
125+
getOnWriteListener().onNodeWritten(node);
126+
}
127+
128+
@Nonnull
129+
public byte[] packWithSubspace(final byte[] key) {
130+
return getSubspace().pack(key);
131+
}
132+
133+
@Nonnull
134+
public byte[] packWithSubspace(final Tuple tuple) {
135+
return getSubspace().pack(tuple);
136+
}
137+
138+
@Nonnull
139+
@Override
140+
public CompletableFuture<Node> scanNodeIndexAndFetchNode(@Nonnull final ReadTransaction transaction,
141+
final int level,
142+
@Nonnull final BigInteger hilbertValue,
143+
@Nonnull final Tuple key,
144+
final boolean isInsertUpdate) {
145+
Objects.requireNonNull(nodeSlotIndexAdapter);
146+
return nodeSlotIndexAdapter.scanIndexForNodeId(transaction, level, hilbertValue, key, isInsertUpdate)
147+
.thenCompose(nodeId -> nodeId == null
148+
? CompletableFuture.completedFuture(null)
149+
: fetchNode(transaction, nodeId));
150+
}
151+
152+
@Override
153+
public void insertIntoNodeIndexIfNecessary(@Nonnull final Transaction transaction, final int level,
154+
@Nonnull final NodeSlot nodeSlot) {
155+
if (!getConfig().isUseNodeSlotIndex() || !(nodeSlot instanceof ChildSlot)) {
156+
return;
157+
}
158+
159+
Objects.requireNonNull(nodeSlotIndexAdapter);
160+
nodeSlotIndexAdapter.writeChildSlot(transaction, level, (ChildSlot)nodeSlot);
161+
}
162+
163+
@Override
164+
public void deleteFromNodeIndexIfNecessary(@Nonnull final Transaction transaction, final int level,
165+
@Nonnull final NodeSlot nodeSlot) {
166+
if (!getConfig().isUseNodeSlotIndex() || !(nodeSlot instanceof ChildSlot)) {
167+
return;
168+
}
169+
170+
Objects.requireNonNull(nodeSlotIndexAdapter);
171+
nodeSlotIndexAdapter.clearChildSlot(transaction, level, (ChildSlot)nodeSlot);
172+
}
173+
174+
@Nonnull
175+
@Override
176+
public CompletableFuture<Node> fetchNode(@Nonnull final ReadTransaction transaction, @Nonnull final byte[] nodeId) {
177+
return getOnWriteListener().onAsyncReadForWrite(fetchNodeInternal(transaction, nodeId).thenApply(this::checkNode));
178+
}
179+
180+
@Nonnull
181+
protected abstract CompletableFuture<Node> fetchNodeInternal(@Nonnull ReadTransaction transaction, @Nonnull byte[] nodeId);
182+
183+
/**
184+
* Method to perform basic invariant check(s) on a newly-fetched node.
185+
*
186+
* @param node the node to check
187+
* @param <N> the type param for the node in order for this method to not be lossy on the type of the node that
188+
* was passed in
189+
*
190+
* @return the node that was passed in
191+
*/
192+
@Nullable
193+
private <N extends Node> N checkNode(@Nullable final N node) {
194+
if (node != null && (node.size() < getConfig().getMinM() || node.size() > getConfig().getMaxM())) {
195+
if (!node.isRoot()) {
196+
throw new IllegalStateException("packing of non-root is out of valid range");
197+
}
198+
}
199+
return node;
200+
}
201+
202+
@Nonnull
203+
abstract <S extends NodeSlot, N extends AbstractNode<S, N>> AbstractChangeSet<S, N>
204+
newInsertChangeSet(@Nonnull N node, int level, @Nonnull List<S> insertedSlots);
205+
206+
@Nonnull
207+
abstract <S extends NodeSlot, N extends AbstractNode<S, N>> AbstractChangeSet<S, N>
208+
newUpdateChangeSet(@Nonnull N node, int level, @Nonnull S originalSlot, @Nonnull S updatedSlot);
209+
210+
@Nonnull
211+
abstract <S extends NodeSlot, N extends AbstractNode<S, N>> AbstractChangeSet<S, N>
212+
newDeleteChangeSet(@Nonnull N node, int level, @Nonnull List<S> deletedSlots);
213+
}

0 commit comments

Comments
 (0)