Skip to content

Commit 4b5cbd9

Browse files
Bill-hbrhbrBill-hbrhbr
committed
Add binding for file API
1 parent 26e91a6 commit 4b5cbd9

File tree

11 files changed

+134
-9
lines changed

11 files changed

+134
-9
lines changed

‎src/clp_ffi_js/binding_types.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,8 @@ namespace clp_ffi_js {
66
EMSCRIPTEN_BINDINGS(ClpFfiJsBindingTypes) {
77
// JS types used as inputs
88
emscripten::register_type<DataArrayTsType>("Uint8Array");
9+
10+
// JS types used as outputs
11+
emscripten::register_type<StringArrayTsType>("string[]");
912
}
1013
} // namespace clp_ffi_js

‎src/clp_ffi_js/binding_types.hpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
namespace clp_ffi_js {
77
// JS types used as inputs
88
EMSCRIPTEN_DECLARE_VAL_TYPE(DataArrayTsType);
9+
10+
// JS types used as outputs
11+
EMSCRIPTEN_DECLARE_VAL_TYPE(StringArrayTsType);
912
} // namespace clp_ffi_js
1013

1114
#endif // CLP_FFI_JS_BINDING_TYPES_HPP

‎src/clp_ffi_js/sfa/ClpArchiveReader.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import {getModule} from "./module.js";
2+
import type {FileInfoArray} from "./types.js";
23

34
import type {ClpSfaReader as WasmClpArchiveReader} from "#clp-ffi-js/node";
45

@@ -44,6 +45,26 @@ class ClpArchiveReader {
4445
return this.#getWasmReader().getEventCount();
4546
}
4647

48+
/**
49+
* Gets source file names in range-index order.
50+
*
51+
* @return Source file names in range-index order.
52+
* @throws {Error} If the reader has been closed.
53+
*/
54+
getFileNames (): string[] {
55+
return this.#getWasmReader().getFileNames();
56+
}
57+
58+
/**
59+
* Gets source file metadata in range-index order.
60+
*
61+
* @return Source file metadata in range-index order.
62+
* @throws {Error} If the reader has been closed.
63+
*/
64+
getFileInfos (): FileInfoArray {
65+
return this.#getWasmReader().getFileInfos() as FileInfoArray;
66+
}
67+
4768
/**
4869
* Releases the underlying WASM resources. After calling this method, the reader is no longer
4970
* usable and any subsequent method calls will throw.

‎src/clp_ffi_js/sfa/SfaReader.cpp‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
namespace clp_ffi_js::sfa {
1919
using clp_ffi_js::DataArrayTsType;
20+
using clp_ffi_js::StringArrayTsType;
2021

2122
auto SfaReader::create(DataArrayTsType const& data_array) -> std::unique_ptr<SfaReader> {
2223
auto const length{data_array["length"].as<size_t>()};
@@ -44,13 +45,41 @@ auto SfaReader::create(DataArrayTsType const& data_array) -> std::unique_ptr<Sfa
4445

4546
return std::unique_ptr<SfaReader>{new SfaReader{std::move(reader_result.value())}};
4647
}
48+
49+
auto SfaReader::get_file_names() const -> StringArrayTsType {
50+
auto file_names{emscripten::val::array()};
51+
for (auto const& file_name : m_reader.get_file_names()) {
52+
file_names.call<void>("push", emscripten::val(file_name));
53+
}
54+
return StringArrayTsType{file_names};
55+
}
56+
57+
auto SfaReader::get_file_infos() const -> FileInfoArrayTsType {
58+
auto file_infos{emscripten::val::array()};
59+
for (auto const& file_info : m_reader.get_file_infos()) {
60+
auto entry{emscripten::val::object()};
61+
entry.set("fileName", emscripten::val(file_info.get_file_name()));
62+
entry.set("logEventIdxStart", emscripten::val(file_info.get_start_index()));
63+
entry.set("logEventIdxEnd", emscripten::val(file_info.get_end_index()));
64+
entry.set("logEventCount", emscripten::val(file_info.get_event_count()));
65+
file_infos.call<void>("push", entry);
66+
}
67+
return FileInfoArrayTsType{file_infos};
68+
}
4769
} // namespace clp_ffi_js::sfa
4870

4971
EMSCRIPTEN_BINDINGS(SfaReader) {
72+
emscripten::register_type<clp_ffi_js::sfa::FileInfoArrayTsType>(
73+
"Array<{fileName: string, logEventIdxStart: bigint, logEventIdxEnd: bigint, "
74+
"logEventCount: bigint}>"
75+
);
76+
5077
emscripten::class_<clp_ffi_js::sfa::SfaReader>("ClpSfaReader")
5178
.constructor(
5279
&clp_ffi_js::sfa::SfaReader::create,
5380
emscripten::return_value_policy::take_ownership()
5481
)
55-
.function("getEventCount", &clp_ffi_js::sfa::SfaReader::get_event_count);
82+
.function("getEventCount", &clp_ffi_js::sfa::SfaReader::get_event_count)
83+
.function("getFileNames", &clp_ffi_js::sfa::SfaReader::get_file_names)
84+
.function("getFileInfos", &clp_ffi_js::sfa::SfaReader::get_file_infos);
5685
}

‎src/clp_ffi_js/sfa/SfaReader.hpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,14 @@
66
#include <utility>
77

88
#include <clp_s/ffi/sfa/ClpArchiveReader.hpp>
9+
#include <emscripten/val.h>
910

1011
#include <clp_ffi_js/binding_types.hpp>
1112

1213
namespace clp_ffi_js::sfa {
14+
EMSCRIPTEN_DECLARE_VAL_TYPE(FileInfoArrayTsType);
15+
EMSCRIPTEN_DECLARE_VAL_TYPE(LogEventArrayTsType);
16+
1317
class SfaReader {
1418
public:
1519
/**
@@ -24,6 +28,10 @@ class SfaReader {
2428

2529
[[nodiscard]] auto get_event_count() const -> uint64_t { return m_reader.get_event_count(); }
2630

31+
[[nodiscard]] auto get_file_names() const -> clp_ffi_js::StringArrayTsType;
32+
33+
[[nodiscard]] auto get_file_infos() const -> FileInfoArrayTsType;
34+
2735
private:
2836
explicit SfaReader(clp_s::ffi::sfa::ClpArchiveReader&& reader) : m_reader(std::move(reader)) {}
2937

‎src/clp_ffi_js/sfa/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
11
export {ClpArchiveReader} from "./ClpArchiveReader.js";
2+
export type {
3+
FieldValue, FileInfo, FileInfoArray,
4+
} from "./types.js";

‎src/clp_ffi_js/sfa/types.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/**
2+
* Source file metadata from the archive's range index.
3+
*/
4+
interface FileInfo {
5+
fileName: string;
6+
logEventIdxStart: bigint;
7+
logEventIdxEnd: bigint;
8+
logEventCount: bigint;
9+
}
10+
11+
type FileInfoArray = FileInfo[];
12+
13+
type FieldValue =
14+
number |
15+
string |
16+
boolean |
17+
null |
18+
{[key: string]: FieldValue} |
19+
FieldValue[];
20+
21+
export type {
22+
FieldValue,
23+
FileInfo,
24+
FileInfoArray,
25+
};

‎taskfiles/deps.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ tasks:
126126
>> "{{.G_DEPS_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake"
127127
128128
clp:
129-
#internal: true
129+
internal: true
130130
vars:
131131
LIB_NAME: "clp"
132132
OUTPUT_DIR: "{{.G_DEPS_DIR}}/{{.LIB_NAME}}"

‎test/ClpArchiveReader.test.ts‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,37 @@
1-
import {ClpArchiveReader} from "clp-ffi-js/sfa";
21
import {
32
afterEach,
43
describe,
54
expect,
65
it,
76
} from "vitest";
87

9-
import {loadTestData} from "./utils.js";
8+
import {
9+
ClpArchiveReader,
10+
type FieldValue,
11+
} from "clp-ffi-js/sfa";
12+
import {
13+
loadTestData,
14+
} from "./utils.js";
1015

1116

17+
const CLP_JSON_TEST_LOG_FILES_EXPECTED_FILE_COUNT = 9;
18+
const CLP_JSON_TEST_LOG_FILES_EXPECTED_EVENT_COUNT = 132n;
1219
const COCKROACHDB_EXPECTED_EVENT_COUNT = 200000n;
1320
const POSTGRESQL_EXPECTED_EVENT_COUNT = 1000000n;
1421

1522
describe("ClpArchiveReader", () => {
1623
let reader: ClpArchiveReader | null = null;
24+
let readerWoTs: ClpArchiveReader | null = null;
1725

1826
afterEach(() => {
1927
if (null !== reader) {
2028
reader.close();
2129
reader = null;
2230
}
31+
if (null !== readerWoTs) {
32+
readerWoTs.close();
33+
readerWoTs = null;
34+
}
2335
});
2436

2537
it("should read postgresql sfa archive from buffer", async () => {
@@ -33,7 +45,24 @@ describe("ClpArchiveReader", () => {
3345
const data = await loadTestData("cockroachdb.clp");
3446
reader = ClpArchiveReader.create(data);
3547

48+
const data_wo_ts = await loadTestData("cockroachdb_wo_ts.clp");
49+
readerWoTs = ClpArchiveReader.create(data_wo_ts);
50+
3651
expect(reader.getEventCount()).toBe(COCKROACHDB_EXPECTED_EVENT_COUNT);
52+
expect(readerWoTs.getEventCount()).toBe(COCKROACHDB_EXPECTED_EVENT_COUNT);
53+
});
54+
55+
it("should read clp_json_test_log_files sfa archive from buffer", async () => {
56+
const data = await loadTestData("clp_json_test_log_files.clp");
57+
reader = await ClpArchiveReader.create(data);
58+
59+
const fileNames = reader.getFileNames();
60+
expect(fileNames.length).toBe(CLP_JSON_TEST_LOG_FILES_EXPECTED_FILE_COUNT);
61+
62+
const fileInfos = reader.getFileInfos();
63+
expect(fileInfos.length).toBe(CLP_JSON_TEST_LOG_FILES_EXPECTED_FILE_COUNT);
64+
65+
expect(reader.getEventCount()).toBe(CLP_JSON_TEST_LOG_FILES_EXPECTED_EVENT_COUNT);
3766
});
3867

3968
it("should throw when calling getEventCount after close", async () => {

‎test/constants.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,19 @@ const TEST_DATA_WEB_BASE_PATH = "/test/data/";
2424
* Remote sources used to download fixture files during global setup.
2525
*/
2626
const TEST_DATA_URLS: Readonly<Record<string, string>> = {
27+
"clp_json_test_log_files.clp":
28+
"https://yscope.s3.us-east-2.amazonaws.com/sample-sfa/clp_json_test_log_files.clp",
29+
"cockroachdb.clp":
30+
"https://yscope.s3.us-east-2.amazonaws.com/sample-sfa/cockroachdb.clp",
31+
"cockroachdb_wo_ts.clp":
32+
"https://yscope.s3.us-east-2.amazonaws.com/sample-sfa/cockroachdb_wo_ts.clp",
33+
"postgresql.clp":
34+
"https://yscope.s3.us-east-2.amazonaws.com/sample-sfa/postgresql.clp",
2735
"structured-cockroachdb.clp.zst":
2836
"https://yscope.s3.us-east-2.amazonaws.com/sample-logs/cockroachdb.clp.zst",
2937
"unstructured-yarn.clp.zst":
3038
"https://yscope.s3.us-east-2.amazonaws.com/sample-logs/" +
3139
"yarn-ubuntu-resourcemanager-ip-172-31-17-135.log.1.clp.zst",
32-
"postgresql.clp":
33-
"https://yscope.s3.us-east-2.amazonaws.com/sample-sfa/postgresql.clp",
34-
"cockroachdb.clp":
35-
"https://yscope.s3.us-east-2.amazonaws.com/sample-sfa/cockroachdb.clp",
3640
};
3741

3842
/**

0 commit comments

Comments
 (0)