Skip to content

Commit 5bb9ac5

Browse files
edwbakeredwbaker
committed
list-rejects
1 parent a18a589 commit 5bb9ac5

File tree

6 files changed

+153
-13
lines changed

6 files changed

+153
-13
lines changed

‎README.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,24 @@ sensor-data transform input.out output.out
4747

4848
# Filter by date range
4949
sensor-data transform --min-date 2026-01-01 --max-date 2026-01-31 input.out
50+
```
51+
52+
### list-rejects
53+
54+
List rejected readings (inverse of transform filters). Useful for inspecting which readings would be filtered out.
55+
56+
```bash
57+
# Show error readings that would be removed
58+
sensor-data list-rejects --remove-errors input.out
59+
60+
# Show readings that would be filtered by --clean
61+
sensor-data list-rejects --clean input.out
62+
63+
# Show rows with empty values
64+
cat data.out | sensor-data list-rejects --not-empty value
65+
66+
# Show readings outside date range
67+
sensor-data list-rejects --min-date 2026-01-01 input.out
5068

5169
# Remove error readings
5270
sensor-data transform --remove-errors input.out output.out

‎debian/sensor-data.1‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@ If no input files are specified, data is read from stdin.
2020
Transform JSON or CSV sensor data files to JSON or CSV format.
2121
Each sensor reading becomes a row in the output.
2222
.TP
23+
.B list\-rejects
24+
List rejected readings (inverse of transform filters).
25+
Outputs readings that would be filtered OUT by the specified filters.
26+
Accepts the same options as transform.
27+
.TP
2328
.B count
2429
Count sensor data readings that match the specified filters.
2530
Outputs a single number representing the count of matching readings.

‎include/sensor_data_transformer.h‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ class SensorDataTransformer : public CommandBase {
2929
std::string outputFile;
3030
std::string outputFormat; // "json" or "csv"
3131
bool removeWhitespace;
32+
bool rejectMode; // If true, output rejected readings instead of accepted
3233

3334
// Column discovery
3435
std::set<std::string> allKeys;
@@ -41,6 +42,11 @@ class SensorDataTransformer : public CommandBase {
4142
*/
4243
bool hasActiveFilters() const;
4344

45+
/**
46+
* Check if a reading should be output (respects rejectMode)
47+
*/
48+
bool shouldOutputReading(const std::map<std::string, std::string>& reading);
49+
4450
/**
4551
* Execute sc-prototype command and parse columns from JSON output
4652
*/
@@ -86,8 +92,9 @@ class SensorDataTransformer : public CommandBase {
8692
public:
8793
/**
8894
* Construct transformer from command line arguments
95+
* @param rejectMode If true, output rejected readings instead of accepted
8996
*/
90-
SensorDataTransformer(int argc, char* argv[]);
97+
SensorDataTransformer(int argc, char* argv[], bool rejectMode = false);
9198

9299
/**
93100
* Execute the transformation
@@ -98,6 +105,11 @@ class SensorDataTransformer : public CommandBase {
98105
* Print usage information
99106
*/
100107
static void printTransformUsage(const char* progName);
108+
109+
/**
110+
* Print usage information for list-rejects command
111+
*/
112+
static void printListRejectsUsage(const char* progName);
101113
};
102114

103115
#endif // SENSOR_DATA_TRANSFORMER_H

‎src/sensor-data.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ void printUsage(const char* progName) {
2323
std::cerr << std::endl;
2424
std::cerr << "Commands:" << std::endl;
2525
std::cerr << " transform Transform JSON or CSV sensor data files" << std::endl;
26+
std::cerr << " list-rejects List rejected readings (inverse of transform filters)" << std::endl;
2627
std::cerr << " count Count sensor data readings (with optional filters)" << std::endl;
2728
std::cerr << " list-errors List error readings in sensor data files" << std::endl;
2829
std::cerr << " summarise-errors Summarise error readings with counts" << std::endl;
@@ -76,6 +77,16 @@ int main(int argc, char* argv[]) {
7677
std::cerr << "Error: " << e.what() << std::endl;
7778
return 1;
7879
}
80+
} else if (command == "list-rejects") {
81+
try {
82+
std::vector<char*> newArgv = buildSubcommandArgv(argc, argv);
83+
SensorDataTransformer transformer(static_cast<int>(newArgv.size()), newArgv.data(), true);
84+
transformer.transform();
85+
return 0;
86+
} catch (const std::exception& e) {
87+
std::cerr << "Error: " << e.what() << std::endl;
88+
return 1;
89+
}
7990
} else if (command == "count") {
8091
try {
8192
std::vector<char*> newArgv = buildSubcommandArgv(argc, argv);

‎src/sensor_data_transformer.cpp‎

Lines changed: 59 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,19 @@
66

77
// ===== Private helper methods =====
88

9+
/**
10+
* Check if a reading should be output based on filters and rejectMode.
11+
* In normal mode: output if passes filters
12+
* In reject mode: output if fails filters (but not for empty/whitespace-only readings)
13+
*/
14+
bool SensorDataTransformer::shouldOutputReading(const std::map<std::string, std::string>& reading) {
15+
bool passesFilter = shouldIncludeReading(reading);
16+
return rejectMode ? !passesFilter : passesFilter;
17+
}
18+
919
bool SensorDataTransformer::hasActiveFilters() const {
10-
return !notEmptyColumns.empty() || !onlyValueFilters.empty() ||
11-
!excludeValueFilters.empty() || removeErrors || removeWhitespace ||
20+
return rejectMode || !notEmptyColumns.empty() || !onlyValueFilters.empty() ||
21+
!excludeValueFilters.empty() || !allowedValues.empty() || removeErrors || removeWhitespace ||
1222
removeEmptyJson || minDate > 0 || maxDate > 0;
1323
}
1424

@@ -138,7 +148,7 @@ void SensorDataTransformer::writeRowsFromFile(const std::string& filename, std::
138148
reading[csvHeaders[i]] = fields[i];
139149
}
140150

141-
if (!shouldIncludeReading(reading)) continue;
151+
if (!shouldOutputReading(reading)) continue;
142152
writeRow(reading, headers, outfile);
143153
}
144154
} else {
@@ -149,7 +159,7 @@ void SensorDataTransformer::writeRowsFromFile(const std::string& filename, std::
149159
auto readings = JsonParser::parseJsonLine(line);
150160
for (const auto& reading : readings) {
151161
if (reading.empty()) continue;
152-
if (!shouldIncludeReading(reading)) continue;
162+
if (!shouldOutputReading(reading)) continue;
153163
writeRow(reading, headers, outfile);
154164
}
155165
}
@@ -194,7 +204,7 @@ void SensorDataTransformer::writeRowsFromFileJson(const std::string& filename, s
194204
reading[csvHeaders[i]] = fields[i];
195205
}
196206

197-
if (!shouldIncludeReading(reading)) continue;
207+
if (!shouldOutputReading(reading)) continue;
198208

199209
if (!firstOutput) outfile << "\n";
200210
firstOutput = false;
@@ -217,7 +227,7 @@ void SensorDataTransformer::writeRowsFromFileJson(const std::string& filename, s
217227

218228
for (const auto& reading : readings) {
219229
if (reading.empty()) continue;
220-
if (shouldIncludeReading(reading)) {
230+
if (shouldOutputReading(reading)) {
221231
filtered.push_back(reading);
222232
}
223233
}
@@ -263,7 +273,7 @@ void SensorDataTransformer::processStdinData(const std::vector<std::string>& lin
263273
reading[csvHeaders[i]] = fields[i];
264274
}
265275

266-
if (!shouldIncludeReading(reading)) continue;
276+
if (!shouldOutputReading(reading)) continue;
267277
writeRow(reading, headers, outfile);
268278
}
269279
} else {
@@ -273,7 +283,7 @@ void SensorDataTransformer::processStdinData(const std::vector<std::string>& lin
273283
auto readings = JsonParser::parseJsonLine(line);
274284
for (const auto& reading : readings) {
275285
if (reading.empty()) continue;
276-
if (!shouldIncludeReading(reading)) continue;
286+
if (!shouldOutputReading(reading)) continue;
277287
writeRow(reading, headers, outfile);
278288
}
279289
}
@@ -306,7 +316,7 @@ void SensorDataTransformer::processStdinDataJson(const std::vector<std::string>&
306316
reading[csvHeaders[i]] = fields[i];
307317
}
308318

309-
if (!shouldIncludeReading(reading)) continue;
319+
if (!shouldOutputReading(reading)) continue;
310320

311321
if (!firstOutput) outfile << "\n";
312322
firstOutput = false;
@@ -330,17 +340,22 @@ void SensorDataTransformer::writeRow(const std::map<std::string, std::string>& r
330340

331341
// ===== Constructor =====
332342

333-
SensorDataTransformer::SensorDataTransformer(int argc, char* argv[])
343+
SensorDataTransformer::SensorDataTransformer(int argc, char* argv[], bool rejectModeParam)
334344
: outputFormat("")
335345
, removeWhitespace(false)
346+
, rejectMode(rejectModeParam)
336347
, numThreads(4)
337348
, usePrototype(false) {
338349

339350
// Check for help flag first
340351
for (int i = 1; i < argc; ++i) {
341352
std::string arg = argv[i];
342353
if (arg == "--help" || arg == "-h") {
343-
printTransformUsage(argv[0]);
354+
if (rejectMode) {
355+
printListRejectsUsage(argv[0]);
356+
} else {
357+
printTransformUsage(argv[0]);
358+
}
344359
exit(0);
345360
}
346361
}
@@ -450,7 +465,7 @@ void SensorDataTransformer::transform() {
450465

451466
for (const auto& reading : readings) {
452467
if (reading.empty()) continue;
453-
if (shouldIncludeReading(reading)) {
468+
if (shouldOutputReading(reading)) {
454469
filtered.push_back(reading);
455470
}
456471
}
@@ -714,3 +729,35 @@ void SensorDataTransformer::printTransformUsage(const char* progName) {
714729
std::cerr << " " << progName << " transform --only-value type:temperature -r -e .out -o output.csv /logs" << std::endl;
715730
std::cerr << " " << progName << " transform --only-value type:temperature --only-value unit:C -o output.csv /logs" << std::endl;
716731
}
732+
733+
void SensorDataTransformer::printListRejectsUsage(const char* progName) {
734+
std::cerr << "Usage: " << progName << " list-rejects [options] [<input_file(s)_or_directory(ies)>]" << std::endl;
735+
std::cerr << std::endl;
736+
std::cerr << "List rejected sensor readings (inverse of transform)." << std::endl;
737+
std::cerr << "Outputs readings that would be filtered OUT by the specified filters." << std::endl;
738+
std::cerr << "Accepts the same options as 'transform'." << std::endl;
739+
std::cerr << std::endl;
740+
std::cerr << "Options:" << std::endl;
741+
std::cerr << " -o, --output <file> Output file (default: stdout)" << std::endl;
742+
std::cerr << " -if, --input-format <fmt> Input format for stdin: json or csv (default: json)" << std::endl;
743+
std::cerr << " -of, --output-format <fmt> Output format: json or csv (default: json)" << std::endl;
744+
std::cerr << " -r, --recursive Recursively process subdirectories" << std::endl;
745+
std::cerr << " -v Verbose output (show progress)" << std::endl;
746+
std::cerr << " -V Very verbose output (show detailed progress)" << std::endl;
747+
std::cerr << " -e, --extension <ext> Filter files by extension (e.g., .out or out)" << std::endl;
748+
std::cerr << " -d, --depth <n> Maximum recursion depth (0 = current dir only)" << std::endl;
749+
std::cerr << " --not-empty <column> List rows where column IS empty" << std::endl;
750+
std::cerr << " --only-value <col:val> List rows where column does NOT have this value" << std::endl;
751+
std::cerr << " --exclude-value <col:val> List rows where column HAS this value" << std::endl;
752+
std::cerr << " --allowed-values <col> <values|file> List rows where column is NOT in allowed list" << std::endl;
753+
std::cerr << " --remove-errors List error readings (DS18B20 value=85 or -127)" << std::endl;
754+
std::cerr << " --remove-empty-json List empty JSON input lines" << std::endl;
755+
std::cerr << " --clean Shorthand for --remove-empty-json --not-empty value --remove-errors" << std::endl;
756+
std::cerr << " --min-date <date> List readings before this date" << std::endl;
757+
std::cerr << " --max-date <date> List readings after this date" << std::endl;
758+
std::cerr << std::endl;
759+
std::cerr << "Examples:" << std::endl;
760+
std::cerr << " " << progName << " list-rejects --remove-errors sensor1.out # Show error readings" << std::endl;
761+
std::cerr << " " << progName << " list-rejects --clean sensor1.out # Show filtered readings" << std::endl;
762+
std::cerr << " cat data.out | " << progName << " list-rejects --not-empty value # Show rows with empty value" << std::endl;
763+
}

‎tests/test_transform.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,6 +1014,53 @@ else
10141014
FAILED=$((FAILED + 1))
10151015
fi
10161016

1017+
# Test 27: list-rejects command outputs rejected readings
1018+
echo ""
1019+
echo "Test 27: list-rejects outputs rejected readings"
1020+
input='[{"sensor": "ds18b20", "value": "22.5"}]
1021+
[{"sensor": "ds18b20", "value": "85"}]
1022+
[{"sensor": "ds18b20", "value": "23.0"}]'
1023+
result=$(echo "$input" | ./sensor-data list-rejects --remove-errors)
1024+
# Should output only the error reading (value=85)
1025+
if echo "$result" | grep -q '"value": "85"' && ! echo "$result" | grep -q '"value": "22.5"'; then
1026+
echo " ✓ PASS"
1027+
PASSED=$((PASSED + 1))
1028+
else
1029+
echo " ✗ FAIL - Expected only error reading (value=85)"
1030+
echo " Got: $result"
1031+
FAILED=$((FAILED + 1))
1032+
fi
1033+
1034+
# Test 28: list-rejects with --not-empty shows empty value rows
1035+
echo ""
1036+
echo "Test 28: list-rejects --not-empty shows rows with empty values"
1037+
input='[{"sensor": "ds18b20", "value": "22.5"}]
1038+
[{"sensor": "ds18b20", "value": ""}]
1039+
[{"sensor": "ds18b20", "value": "23.0"}]'
1040+
result=$(echo "$input" | ./sensor-data list-rejects --not-empty value)
1041+
# Should output only the row with empty value
1042+
if echo "$result" | grep -q '"value": ""' && ! echo "$result" | grep -q '"value": "22.5"'; then
1043+
echo " ✓ PASS"
1044+
PASSED=$((PASSED + 1))
1045+
else
1046+
echo " ✗ FAIL - Expected only row with empty value"
1047+
echo " Got: $result"
1048+
FAILED=$((FAILED + 1))
1049+
fi
1050+
1051+
# Test 29: list-rejects --help shows list-rejects usage
1052+
echo ""
1053+
echo "Test 29: list-rejects --help shows usage"
1054+
result=$(./sensor-data list-rejects --help 2>&1) || true
1055+
if echo "$result" | grep -q "list-rejects"; then
1056+
echo " ✓ PASS"
1057+
PASSED=$((PASSED + 1))
1058+
else
1059+
echo " ✗ FAIL - Expected list-rejects usage text"
1060+
echo " Got: $result"
1061+
FAILED=$((FAILED + 1))
1062+
fi
1063+
10171064
# Summary
10181065
echo ""
10191066
echo "================================"

0 commit comments

Comments
 (0)