Skip to content

Commit 556e802

Browse files
niknal357niknal357
committed
Merge branch 'eval-fuzzing'
2 parents 9f51f5a + a2f8be5 commit 556e802

File tree

5 files changed

+333
-2
lines changed

5 files changed

+333
-2
lines changed

‎buildandruntests.cmd‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
cmake --build --preset tests
22
if %ERRORLEVEL% == 0 (
3-
rm -r coverage
4-
call .\opencpp_coverage.cmd
3+
call "build-tests/Debug/Connection_Machine_tests.exe" --gtest_filter=*Fuzz*
54
)

‎src/backend/evaluator/layers/layer4_replacer.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,10 @@ void Replacer::mergeBusLane(SimPauseGuard& pauseGuard, int layer, int junctionOv
119119
Replacement& replacement = makeReplacement(layer);
120120
middle_id_t newJunctionId = replacement.getNewId();
121121
replacement.addGate(pauseGuard, BlockType::JUNCTION, newJunctionId);
122+
existingJunctionIds.insert(newJunctionId);
123+
replacement.addRevertAction([this, newJunctionId]() {
124+
existingJunctionIds.erase(newJunctionId);
125+
});
122126
std::queue<BlockLane> mergeQueue;
123127
std::unordered_set<BlockLane, BlockLane::Hash> visited;
124128
mergeQueue.push({ id, laneId });

‎src/backend/evaluator/layers/layer4_replacer.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,28 @@ class Replacer {
153153
// connections that are connected to the junctions through the outputs of other gates
154154
// A -> JUNCTION, A -> B, A -> B is a connection to reroute because B should actually pull from the junction
155155
logic_state_t defaultState { logic_state_t::FLOATING };
156+
157+
nlohmann::json dumpState() const {
158+
nlohmann::json stateJson;
159+
stateJson["outputsGoingIntoJunctions"] = nlohmann::json::array();
160+
for (const EvalConnectionPoint& point : outputsGoingIntoJunctions) {
161+
stateJson["outputsGoingIntoJunctions"].push_back(point.dumpState());
162+
}
163+
stateJson["inputsPullingFromJunctions"] = nlohmann::json::array();
164+
for (const EvalConnection& conn : inputsPullingFromJunctions) {
165+
stateJson["inputsPullingFromJunctions"].push_back(conn.dumpState());
166+
}
167+
stateJson["junctionIds"] = nlohmann::json::array();
168+
for (const middle_id_t& id : junctionIds) {
169+
stateJson["junctionIds"].push_back(id.get());
170+
}
171+
stateJson["connectionsToReroute"] = nlohmann::json::array();
172+
for (const EvalConnection& conn : connectionsToReroute) {
173+
stateJson["connectionsToReroute"].push_back(conn.dumpState());
174+
}
175+
stateJson["defaultState"] = static_cast<uint8_t>(defaultState);
176+
return stateJson;
177+
}
156178
};
157179

158180
struct BusFloodFillResult {

‎src/backend/evaluator/simulator/logicSimulator.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,13 @@ void LogicSimulator::processPendingStateChanges() {
154154
const StateChange& change = localQueue.front();
155155

156156
extendDataVectors(change.id);
157+
if (!gateLocations.contains(change.id)) {
158+
continue;
159+
}
160+
GateLocation& gateLocation = gateLocations.at(change.id);
161+
if (gateLocation.gateType == SimGateType::CONSTANT) {
162+
continue;
163+
}
157164

158165
statesA[change.id] = change.state;
159166
statesB[change.id] = change.state;
@@ -172,11 +179,19 @@ void LogicSimulator::setState(simulator_id_t id, logic_state_t st) {
172179
return;
173180
}
174181
// we don't want to freeze up if the mutexes are locked, so we'll only set the state if we can successfully lock. otherwise, we'll wait until the next tick to set the states.
182+
175183
std::unique_lock lkMain(mainDataMutex, std::try_to_lock);
176184
std::unique_lock lkA(statesAMutex, std::try_to_lock);
177185

178186
if (lkMain.owns_lock() && lkA.owns_lock()) {
179187
extendDataVectors(id);
188+
if (!gateLocations.contains(id)) {
189+
return;
190+
}
191+
GateLocation& gateLocation = gateLocations.at(id);
192+
if (gateLocation.gateType == SimGateType::CONSTANT) {
193+
return;
194+
}
180195
statesA[id] = st;
181196
statesB[id] = st;
182197
setStateUsed[replayHead] = true;

‎tests/evaluator/fuzzing/basic.cpp‎

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,291 @@
1+
#include <gtest/gtest.h>
2+
#include "environment/environment.h"
3+
#include "backend/evaluator/evaluator.h"
4+
#include "backend/container/block/blockDefs.h"
5+
#include "backend/blockData/blockDataManager.h"
6+
#include "computerAPI/directoryManager.h"
7+
8+
class BasicFuzzingEvaluatorTest : public ::testing::TestWithParam<uint64_t> {
9+
protected:
10+
void SetUp() override;
11+
void TearDown() override;
12+
std::mt19937_64 gen;
13+
Environment environment {false};
14+
SharedCircuit circuit = nullptr;
15+
SharedEvaluator tEval = nullptr; // testing evaluator
16+
SharedEvaluator rEval = nullptr; // reference evaluator
17+
};
18+
19+
void BasicFuzzingEvaluatorTest::SetUp() {
20+
gen.seed(GetParam());
21+
circuit_id_t circuitId = environment.getBackend().getCircuitManager().createNewCircuit(false);
22+
circuit = environment.getBackend().getCircuit(circuitId);
23+
evaluator_id_t evalId = environment.getBackend().createEvaluator(circuitId).value();
24+
tEval = environment.getBackend().getEvaluator(evalId);
25+
ASSERT_TRUE(tEval->isPause());
26+
}
27+
28+
void BasicFuzzingEvaluatorTest::TearDown() {
29+
circuit.reset();
30+
tEval.reset();
31+
rEval.reset();
32+
}
33+
34+
std::optional<connection_end_id_t> getRandomConnectionEnd(const BlockData* blockData, std::mt19937_64& gen, bool wantInput) {
35+
if (blockData->isDefaultData()) {
36+
if (wantInput) {
37+
return connection_end_id_t(0);
38+
} else {
39+
return connection_end_id_t(1);
40+
}
41+
}
42+
int numConnections = blockData->getBidirectionalConnectionCount().get();
43+
if (wantInput) numConnections += blockData->getInputConnectionCount().get();
44+
else numConnections += blockData->getOutputConnectionCount().get();
45+
if (numConnections == 0) return std::nullopt;
46+
std::uniform_int_distribution<int> dist(0, numConnections - 1);
47+
int index = dist(gen);
48+
const std::unordered_map<connection_end_id_t, BlockData::ConnectionData>& connections = blockData->getConnections();
49+
int i = 0;
50+
for (auto& pair : connections) {
51+
if (wantInput) {
52+
if (pair.second.portType == BlockData::ConnectionData::PortType::INPUT || pair.second.portType == BlockData::ConnectionData::PortType::BIDIRECTIONAL) {
53+
if (i == index) return pair.first;
54+
i++;
55+
}
56+
} else {
57+
if (pair.second.portType == BlockData::ConnectionData::PortType::OUTPUT || pair.second.portType == BlockData::ConnectionData::PortType::BIDIRECTIONAL) {
58+
if (i == index) return pair.first;
59+
i++;
60+
}
61+
}
62+
}
63+
return std::nullopt;
64+
}
65+
66+
TEST_P(BasicFuzzingEvaluatorTest, FuzzInteractions) {
67+
bool runRealistic = gen() % 2 == 0;
68+
tEval->setRealistic(runRealistic);
69+
BlockDataManager& blockDataManager = environment.getBackend().getBlockDataManager();
70+
std::uniform_int_distribution<int> distPos(-20, 20);
71+
std::vector<block_id_t> blockIds;
72+
int numEditOperations = 5000;
73+
int numTestOperations = 100;
74+
int numTicksBetweenTests = 3;
75+
int numStatesSetPerTest = 50;
76+
for (int i = 0; i < numEditOperations; ++i) {
77+
int operation = gen() % 8; // 0,1: place, 2: remove, 3,4,5: connect, 6,7: disconnect
78+
if (operation <= 1) { // place block
79+
Orientation orientation(Rotation(gen() % 4), (gen() % 2) == 0);
80+
BlockType blockType = BlockType((gen() % blockDataManager.maxBlockId()) + 1);
81+
if (!blockDataManager.blockExists(blockType)) {
82+
continue;
83+
}
84+
Position pos(distPos(gen), distPos(gen));
85+
bool success = circuit->tryInsertBlock(pos, orientation, blockType);
86+
if (!success) continue;
87+
const Block* block = circuit->getBlockContainer().getBlock(pos);
88+
ASSERT_NE(block, nullptr);
89+
blockIds.push_back(block->id());
90+
} else if (operation <= 2) {
91+
if (blockIds.empty()) continue;
92+
std::uniform_int_distribution<size_t> distIndex(0, blockIds.size() - 1);
93+
size_t index = distIndex(gen);
94+
block_id_t blockId = blockIds[index];
95+
const Block* block = circuit->getBlockContainer().getBlock(blockId);
96+
ASSERT_NE(block, nullptr);
97+
Position pos = block->getPosition();
98+
ASSERT_TRUE(circuit->tryRemoveBlock(pos));
99+
std::swap(blockIds[index], blockIds.back());
100+
blockIds.pop_back();
101+
} else if (operation <= 5) { // connect
102+
if (blockIds.empty()) continue;
103+
block_id_t blockIdA = blockIds[gen() % blockIds.size()];
104+
block_id_t blockIdB = blockIds[gen() % blockIds.size()];
105+
const Block* blockA = circuit->getBlockContainer().getBlock(blockIdA);
106+
const Block* blockB = circuit->getBlockContainer().getBlock(blockIdB);
107+
ASSERT_NE(blockA, nullptr);
108+
ASSERT_NE(blockB, nullptr);
109+
const BlockData* blockDataA = blockDataManager.getBlockData(blockA->type());
110+
const BlockData* blockDataB = blockDataManager.getBlockData(blockB->type());
111+
ASSERT_NE(blockDataA, nullptr);
112+
ASSERT_NE(blockDataB, nullptr);
113+
std::optional<connection_end_id_t> connA = getRandomConnectionEnd(blockDataA, gen, false);
114+
std::optional<connection_end_id_t> connB = getRandomConnectionEnd(blockDataB, gen, true);
115+
if (!connA || !connB) continue;
116+
circuit->tryCreateConnection({blockIdA, *connA}, {blockIdB, *connB});
117+
} else if (operation <= 7) { // disconnect
118+
if (blockIds.empty()) continue;
119+
block_id_t blockIdA = blockIds[gen() % blockIds.size()];
120+
block_id_t blockIdB = blockIds[gen() % blockIds.size()];
121+
const Block* blockA = circuit->getBlockContainer().getBlock(blockIdA);
122+
const Block* blockB = circuit->getBlockContainer().getBlock(blockIdB);
123+
ASSERT_NE(blockA, nullptr);
124+
ASSERT_NE(blockB, nullptr);
125+
const BlockData* blockDataA = blockDataManager.getBlockData(blockA->type());
126+
const BlockData* blockDataB = blockDataManager.getBlockData(blockB->type());
127+
ASSERT_NE(blockDataA, nullptr);
128+
ASSERT_NE(blockDataB, nullptr);
129+
std::optional<connection_end_id_t> connA = getRandomConnectionEnd(blockDataA, gen, false);
130+
std::optional<connection_end_id_t> connB = getRandomConnectionEnd(blockDataB, gen, true);
131+
if (!connA || !connB) continue;
132+
circuit->tryRemoveConnection({blockIdA, *connA}, {blockIdB, *connB});
133+
}
134+
}
135+
136+
// SAVE CIRCUIT TO FILE
137+
{
138+
CircuitFileManager& circuitFileManager = environment.getCircuitFileManager();
139+
const std::filesystem::path savePath =
140+
DirectoryManager::getConfigDirectory() / "tmp" / ("BasicFuzzingCircuit_" + std::to_string(GetParam()) + ".cir");
141+
std::filesystem::create_directories(savePath.parent_path());
142+
ASSERT_TRUE(circuitFileManager.saveToFile(savePath.string(), circuit->getUUID()));
143+
logInfo("Saved fuzzing circuit to " + savePath.string(), "BasicFuzzingEvaluatorTest");
144+
}
145+
146+
logInfo("Creating reference evaluator", "BasicFuzzingEvaluatorTest");
147+
evaluator_id_t evalId = environment.getBackend().createEvaluator(circuit->getCircuitId()).value();
148+
rEval = environment.getBackend().getEvaluator(evalId);
149+
rEval->setRealistic(runRealistic);
150+
ASSERT_TRUE(tEval->isPause());
151+
ASSERT_TRUE(rEval->isPause());
152+
tEval->resetStates();
153+
rEval->resetStates();
154+
std::vector<simulator_id_t> simulatorIdsTest;
155+
std::vector<simulator_id_t> simulatorIdsRef;
156+
std::unordered_map<block_id_t, Position> blockIdToPosition;
157+
std::vector<std::string> ps;
158+
for (block_id_t blockId : blockIds) {
159+
const Block* block = circuit->getBlockContainer().getBlock(blockId);
160+
ASSERT_NE(block, nullptr);
161+
Position pos = block->getPosition();
162+
blockIdToPosition[blockId] = pos;
163+
simulatorIdsTest.push_back(tEval->getBlockSimulatorId(pos));
164+
simulatorIdsRef.push_back(rEval->getBlockSimulatorId(pos));
165+
ps.push_back("B " + pos.toString());
166+
const BlockData* blockData = blockDataManager.getBlockData(block->type());
167+
ASSERT_NE(blockData, nullptr);
168+
if (blockData->isDefaultData()) {
169+
std::variant<simulator_id_t, std::vector<simulator_id_t>> simIdTest = tEval->getPinSimulatorId(pos);
170+
std::variant<simulator_id_t, std::vector<simulator_id_t>> simIdRef = rEval->getPinSimulatorId(pos);
171+
if (std::holds_alternative<simulator_id_t>(simIdTest) && std::holds_alternative<simulator_id_t>(simIdRef)) {
172+
simulatorIdsTest.push_back(std::get<simulator_id_t>(simIdTest));
173+
simulatorIdsRef.push_back(std::get<simulator_id_t>(simIdRef));
174+
ps.push_back("P " + pos.toString());
175+
} else if (std::holds_alternative<std::vector<simulator_id_t>>(simIdTest) && std::holds_alternative<std::vector<simulator_id_t>>(simIdRef)) {
176+
std::vector<simulator_id_t>& vecTest = std::get<std::vector<simulator_id_t>>(simIdTest);
177+
std::vector<simulator_id_t>& vecRef = std::get<std::vector<simulator_id_t>>(simIdRef);
178+
simulatorIdsTest.insert(simulatorIdsTest.end(), vecTest.begin(), vecTest.end());
179+
simulatorIdsRef.insert(simulatorIdsRef.end(), vecRef.begin(), vecRef.end());
180+
for (size_t i = 0; i < vecTest.size(); ++i) {
181+
ps.push_back("P " + pos.toString() + "[" + std::to_string(i) + "]");
182+
}
183+
} else {
184+
FAIL() << "Mismatched simulator ID types for pin at position " << pos.toString();
185+
}
186+
} else {
187+
const std::unordered_map<connection_end_id_t, BlockData::ConnectionData>& connections = blockData->getConnections();
188+
for (const auto& [connectionId, connectionData] : connections) {
189+
if (connectionData.portType == BlockData::ConnectionData::PortType::INPUT) {
190+
continue;
191+
}
192+
std::optional<Position> portPositionOpt = block->getConnectionPosition(connectionId);
193+
ASSERT_TRUE(portPositionOpt.has_value());
194+
Position portPosition = portPositionOpt.value();
195+
std::variant<simulator_id_t, std::vector<simulator_id_t>> simIdTest = tEval->getPinSimulatorId(portPosition);
196+
std::variant<simulator_id_t, std::vector<simulator_id_t>> simIdRef = rEval->getPinSimulatorId(portPosition);
197+
if (std::holds_alternative<simulator_id_t>(simIdTest) && std::holds_alternative<simulator_id_t>(simIdRef)) {
198+
simulatorIdsTest.push_back(std::get<simulator_id_t>(simIdTest));
199+
simulatorIdsRef.push_back(std::get<simulator_id_t>(simIdRef));
200+
ps.push_back("P " + portPosition.toString());
201+
} else if (std::holds_alternative<std::vector<simulator_id_t>>(simIdTest) && std::holds_alternative<std::vector<simulator_id_t>>(simIdRef)) {
202+
std::vector<simulator_id_t>& vecTest = std::get<std::vector<simulator_id_t>>(simIdTest);
203+
std::vector<simulator_id_t>& vecRef = std::get<std::vector<simulator_id_t>>(simIdRef);
204+
ASSERT_EQ(vecTest.size(), vecRef.size()) << "Mismatched simulator ID vector sizes for pin at position " << portPosition.toString();
205+
simulatorIdsTest.insert(simulatorIdsTest.end(), vecTest.begin(), vecTest.end());
206+
simulatorIdsRef.insert(simulatorIdsRef.end(), vecRef.begin(), vecRef.end());
207+
for (size_t i = 0; i < vecTest.size(); ++i) {
208+
ps.push_back("P " + portPosition.toString() + "[" + std::to_string(i) + "]");
209+
}
210+
} else {
211+
FAIL() << "Mismatched simulator ID types for pin at position " << portPosition.toString();
212+
}
213+
}
214+
}
215+
}
216+
217+
ASSERT_EQ(simulatorIdsTest.size(), simulatorIdsRef.size());
218+
for (int i = 0; i < numTestOperations; ++i) {
219+
// set random states
220+
for (int j = 0; j < numStatesSetPerTest; ++j) {
221+
block_id_t blockId = blockIds[gen() % blockIds.size()];
222+
Position pos = blockIdToPosition.at(blockId);
223+
logic_state_t state = logic_state_t(gen() % 4);
224+
rEval->setState(pos, state);
225+
tEval->setState(pos, state);
226+
}
227+
228+
// compare states
229+
std::vector<logic_state_t> statesTest = tEval->getStatesFromSimulatorIds(simulatorIdsTest);
230+
std::vector<logic_state_t> statesRef = rEval->getStatesFromSimulatorIds(simulatorIdsRef);
231+
ASSERT_EQ(statesTest.size(), statesRef.size());
232+
for (size_t k = 0; k < statesTest.size(); ++k) {
233+
// if (statesTest[k] != statesRef[k]) {
234+
// simulator_id_t testSimId = simulatorIdsTest[k];
235+
// simulator_id_t refSimId = simulatorIdsRef[k];
236+
// logInfo("Mismatch at simulator ID index {} (simulator ID test: {}, ref: {}) at p {}", "BasicFuzzingEvaluatorTest", k, testSimId.get(), refSimId.get(), ps.at(k));
237+
// int stepBackAmount = 6;
238+
// for (int m = 0; m < stepBackAmount; ++m) {
239+
// tEval->stepBack();
240+
// rEval->stepBack();
241+
// }
242+
// for (size_t m = 0; m < stepBackAmount; ++m) {
243+
// logInfo("After stepping back {} ticks:", "BasicFuzzingEvaluatorTest", stepBackAmount - m);
244+
// logic_state_t stateTest = tEval->getStateFromSimulatorId(testSimId);
245+
// logic_state_t stateRef = rEval->getStateFromSimulatorId(refSimId);
246+
// logInfo(" Test evaluator state: {}, Reference evaluator state: {}", "BasicFuzzingEvaluatorTest", logicstate_to_string(stateTest), logicstate_to_string(stateRef));
247+
// rEval->stepForward();
248+
// tEval->stepForward();
249+
// }
250+
// }
251+
ASSERT_EQ(statesTest[k], statesRef[k]) << "Mismatch at simulator ID index " << k << " at p " << ps.at(k);
252+
}
253+
254+
// tick
255+
tEval->tickStep(numTicksBetweenTests);
256+
rEval->tickStep(numTicksBetweenTests);
257+
258+
// compare states
259+
statesTest = tEval->getStatesFromSimulatorIds(simulatorIdsTest);
260+
statesRef = rEval->getStatesFromSimulatorIds(simulatorIdsRef);
261+
ASSERT_EQ(statesTest.size(), statesRef.size());
262+
for (size_t k = 0; k < statesTest.size(); ++k) {
263+
// if (statesTest[k] != statesRef[k]) {
264+
// simulator_id_t testSimId = simulatorIdsTest[k];
265+
// simulator_id_t refSimId = simulatorIdsRef[k];
266+
// logInfo("Mismatch at simulator ID index {} (simulator ID test: {}, ref: {}) at p {}", "BasicFuzzingEvaluatorTest", k, testSimId.get(), refSimId.get(), ps.at(k));
267+
// int stepBackAmount = 6;
268+
// for (int m = 0; m < stepBackAmount; ++m) {
269+
// tEval->stepBack();
270+
// rEval->stepBack();
271+
// }
272+
// for (size_t m = 0; m < stepBackAmount; ++m) {
273+
// logInfo("After stepping back {} ticks:", "BasicFuzzingEvaluatorTest", stepBackAmount - m);
274+
// logic_state_t stateTest = tEval->getStateFromSimulatorId(testSimId);
275+
// logic_state_t stateRef = rEval->getStateFromSimulatorId(refSimId);
276+
// logInfo(" Test evaluator state: {}, Reference evaluator state: {}", "BasicFuzzingEvaluatorTest", logicstate_to_string(stateTest), logicstate_to_string(stateRef));
277+
// rEval->stepForward();
278+
// tEval->stepForward();
279+
// }
280+
// }
281+
ASSERT_EQ(statesTest[k], statesRef[k]) << "Mismatch at simulator ID index " << k << " at p " << ps.at(k);
282+
}
283+
}
284+
}
285+
286+
INSTANTIATE_TEST_SUITE_P(
287+
RandomSeeds,
288+
BasicFuzzingEvaluatorTest,
289+
::testing::Range(uint64_t(0), uint64_t(10))
290+
// ::testing::Values(uint64_t(3))
291+
);

0 commit comments

Comments
 (0)