Skip to content

Commit 0a67cff

Browse files
risenWrisenW
committed
Add support for passing frame config to io functions
1 parent eb2ed39 commit 0a67cff

File tree

13 files changed

+207
-28
lines changed

13 files changed

+207
-28
lines changed

‎src/danfojs-base/io/browser/io.csv.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
* limitations under the License.
1313
* ==========================================================================
1414
*/
15-
import { CsvInputOptionsBrowser, CsvOutputOptionsBrowser, ArrayType2D} from "../../shared/types"
15+
import { CsvInputOptionsBrowser, CsvOutputOptionsBrowser, ArrayType2D } from "../../shared/types"
1616
import { DataFrame, NDframe, Series } from '../../'
1717
import Papa from 'papaparse'
1818

@@ -46,13 +46,15 @@ import Papa from 'papaparse'
4646
* ```
4747
*/
4848
const $readCSV = async (file: any, options?: CsvInputOptionsBrowser): Promise<DataFrame> => {
49+
const frameConfig = options?.frameConfig || {}
50+
4951
return new Promise(resolve => {
5052
Papa.parse(file, {
5153
header: true,
5254
...options,
5355
download: true,
5456
complete: results => {
55-
const df = new DataFrame(results.data);
57+
const df = new DataFrame(results.data, frameConfig);
5658
resolve(df);
5759
}
5860
});
@@ -75,14 +77,16 @@ const $readCSV = async (file: any, options?: CsvInputOptionsBrowser): Promise<Da
7577
* ```
7678
*/
7779
const $streamCSV = async (file: string, callback: (df: DataFrame) => void, options: CsvInputOptionsBrowser,): Promise<null> => {
80+
const frameConfig = options?.frameConfig || {}
81+
7882
return new Promise(resolve => {
7983
let count = -1
8084
Papa.parse(file, {
8185
...options,
8286
header: true,
8387
download: true,
8488
step: results => {
85-
const df = new DataFrame([results.data], { index: [count++] });
89+
const df = new DataFrame([results.data], { ...frameConfig, index: [count++] });
8690
callback(df);
8791
},
8892
complete: () => resolve(null)

‎src/danfojs-base/io/browser/io.excel.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ import XLSX from 'xlsx';
4545
* ```
4646
*/
4747
const $readExcel = async (file: any, options?: ExcelInputOptionsBrowser) => {
48-
const { sheet, method, headers } = { sheet: 0, method: "GET", headers: {}, ...options }
48+
const { sheet, method, headers, frameConfig } = { sheet: 0, method: "GET", headers: {}, frameConfig: {}, ...options }
4949

5050
if (typeof file === "string" && file.startsWith("http")) {
5151

@@ -59,7 +59,7 @@ const $readExcel = async (file: any, options?: ExcelInputOptionsBrowser) => {
5959
const workbook = XLSX.read(arrBufInt8, { type: "array" })
6060
const worksheet = workbook.Sheets[workbook.SheetNames[sheet]];
6161
const data = XLSX.utils.sheet_to_json(worksheet);
62-
const df = new DataFrame(data);
62+
const df = new DataFrame(data, frameConfig);
6363
resolve(df);
6464
});
6565
}).catch((err) => {
@@ -73,7 +73,7 @@ const $readExcel = async (file: any, options?: ExcelInputOptionsBrowser) => {
7373
const workbook = XLSX.read(arrBufInt8, { type: "array" })
7474
const worksheet = workbook.Sheets[workbook.SheetNames[sheet]];
7575
const data = XLSX.utils.sheet_to_json(worksheet);
76-
const df = new DataFrame(data);
76+
const df = new DataFrame(data, frameConfig);
7777
return df;
7878
} else {
7979
throw new Error("ParamError: File not supported. file must be a url or an input File object")

‎src/danfojs-base/io/browser/io.json.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ import { DataFrame, NDframe, Series } from '../../'
4848
* ```
4949
*/
5050
const $readJSON = async (file: any, options?: JsonInputOptionsBrowser) => {
51-
const { method, headers } = { method: "GET", headers: {}, ...options }
51+
const { method, headers, frameConfig } = { method: "GET", headers: {}, frameConfig: {}, ...options }
5252

5353
if (typeof file === "string" && file.startsWith("http")) {
5454

@@ -58,7 +58,7 @@ const $readJSON = async (file: any, options?: JsonInputOptionsBrowser) => {
5858
throw new Error(`Failed to load ${file}`)
5959
}
6060
response.json().then(json => {
61-
resolve(new DataFrame(json));
61+
resolve(new DataFrame(json, frameConfig));
6262
});
6363
}).catch((err) => {
6464
throw new Error(err)
@@ -71,7 +71,7 @@ const $readJSON = async (file: any, options?: JsonInputOptionsBrowser) => {
7171
reader.readAsText(file);
7272
reader.onload = (event) => {
7373
const jsonObj = JSON.parse(event?.target?.result as string);
74-
resolve(new DataFrame(jsonObj));
74+
resolve(new DataFrame(jsonObj, frameConfig));
7575
}
7676
})
7777
} else {

‎src/danfojs-base/io/node/io.csv.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ import fs from 'fs'
4848
* ```
4949
*/
5050
const $readCSV = async (filePath: string, options?: CsvInputOptionsNode): Promise<DataFrame> => {
51+
const frameConfig = options?.frameConfig || {}
52+
5153
if (filePath.startsWith("http") || filePath.startsWith("https")) {
5254
return new Promise(resolve => {
5355
const optionsWithDefaults = {
@@ -65,7 +67,7 @@ const $readCSV = async (filePath: string, options?: CsvInputOptionsNode): Promis
6567
});
6668

6769
parseStream.on("finish", () => {
68-
resolve(new DataFrame(data));
70+
resolve(new DataFrame(data, frameConfig));
6971
});
7072
});
7173

@@ -76,7 +78,7 @@ const $readCSV = async (filePath: string, options?: CsvInputOptionsNode): Promis
7678
header: true,
7779
...options,
7880
complete: results => {
79-
const df = new DataFrame(results.data);
81+
const df = new DataFrame(results.data, frameConfig);
8082
resolve(df);
8183
}
8284
});
@@ -100,6 +102,7 @@ const $readCSV = async (filePath: string, options?: CsvInputOptionsNode): Promis
100102
* ```
101103
*/
102104
const $streamCSV = async (filePath: string, callback: (df: DataFrame) => void, options?: CsvInputOptionsNode): Promise<null> => {
105+
const frameConfig = options?.frameConfig || {}
103106

104107
if (filePath.startsWith("http") || filePath.startsWith("https")) {
105108
const optionsWithDefaults = {
@@ -113,7 +116,7 @@ const $streamCSV = async (filePath: string, callback: (df: DataFrame) => void, o
113116
dataStream.pipe(parseStream);
114117

115118
parseStream.on("data", (chunk: any) => {
116-
const df = new DataFrame([chunk], { index: [count++] });
119+
const df = new DataFrame([chunk], { ...frameConfig, index: [count++], });
117120
callback(df);
118121
});
119122

@@ -131,7 +134,7 @@ const $streamCSV = async (filePath: string, callback: (df: DataFrame) => void, o
131134
header: true,
132135
...options,
133136
step: results => {
134-
const df = new DataFrame([results.data], { index: [count++] });
137+
const df = new DataFrame([results.data], { ...frameConfig, index: [count++] });
135138
callback(df);
136139
},
137140
complete: () => resolve(null)

‎src/danfojs-base/io/node/io.excel.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ import XLSX from 'xlsx';
4646
* ```
4747
*/
4848
const $readExcel = async (filePath: string, options: ExcelInputOptionsNode = {}) => {
49-
const { sheet, method, headers } = { sheet: 0, method: "GET", headers: {}, ...options }
49+
const { sheet, method, headers, frameConfig } = { sheet: 0, method: "GET", headers: {}, frameConfig: {}, ...options }
5050

5151
if (filePath.startsWith("http") || filePath.startsWith("https")) {
5252

@@ -60,7 +60,7 @@ const $readExcel = async (filePath: string, options: ExcelInputOptionsNode = {})
6060
const workbook = XLSX.read(arrBufInt8, { type: "array" })
6161
const worksheet = workbook.Sheets[workbook.SheetNames[sheet]];
6262
const data = XLSX.utils.sheet_to_json(worksheet);
63-
const df = new DataFrame(data);
63+
const df = new DataFrame(data, frameConfig);
6464
resolve(df);
6565
});
6666
}).catch((err) => {
@@ -73,7 +73,7 @@ const $readExcel = async (filePath: string, options: ExcelInputOptionsNode = {})
7373
const workbook = XLSX.readFile(filePath);
7474
const worksheet = workbook.Sheets[workbook.SheetNames[sheet]];
7575
const data = XLSX.utils.sheet_to_json(worksheet);
76-
const df = new DataFrame(data);
76+
const df = new DataFrame(data, frameConfig);
7777
resolve(df);
7878
});
7979
}

‎src/danfojs-base/io/node/io.json.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import fs from 'fs'
3434
* ```
3535
*/
3636
const $readJSON = async (filePath: string, options: JsonInputOptionsNode = {}) => {
37-
const { method, headers } = { method: "GET", headers: {}, ...options }
37+
const { method, headers, frameConfig } = { method: "GET", headers: {}, frameConfig: {}, ...options }
3838

3939
if (filePath.startsWith("http") || filePath.startsWith("https")) {
4040

@@ -44,7 +44,7 @@ const $readJSON = async (filePath: string, options: JsonInputOptionsNode = {}) =
4444
throw new Error(`Failed to load ${filePath}`)
4545
}
4646
response.json().then(json => {
47-
resolve(new DataFrame(json));
47+
resolve(new DataFrame(json, frameConfig));
4848
});
4949
}).catch((err) => {
5050
throw new Error(err)
@@ -54,7 +54,7 @@ const $readJSON = async (filePath: string, options: JsonInputOptionsNode = {}) =
5454
} else {
5555
return new Promise(resolve => {
5656
const file = fs.readFileSync(filePath, "utf8")
57-
const df = new DataFrame(JSON.parse(file));
57+
const df = new DataFrame(JSON.parse(file), frameConfig);
5858
resolve(df);
5959
});
6060
}
@@ -80,14 +80,14 @@ const $streamJSON = async (
8080
callback: (df: DataFrame) => void,
8181
options?: request.RequiredUriUrl & request.CoreOptions,
8282
) => {
83-
const { method, headers } = { method: "GET", headers: {}, ...options }
83+
const { method, headers, frameConfig } = { method: "GET", headers: {}, frameConfig: {}, ...options }
8484
if (filePath.startsWith("http") || filePath.startsWith("https")) {
8585
return new Promise(resolve => {
8686
let count = -1
8787
const dataStream = request({ url: filePath, method, headers })
8888
const pipeline = dataStream.pipe(parser()).pipe(streamArray());
8989
pipeline.on('data', ({ value }) => {
90-
const df = new DataFrame([value], { index: [count++] });
90+
const df = new DataFrame([value], { ...frameConfig, index: [count++] });
9191
callback(df);
9292
});
9393
pipeline.on('end', () => resolve(null));
@@ -99,7 +99,7 @@ const $streamJSON = async (
9999
const fileStream = fs.createReadStream(filePath)
100100
const pipeline = fileStream.pipe(parser()).pipe(streamArray());
101101
pipeline.on('data', ({ value }) => {
102-
const df = new DataFrame([value], { index: [count++] });
102+
const df = new DataFrame([value], { ...frameConfig, index: [count++] });
103103
callback(df);
104104
});
105105
pipeline.on('end', () => resolve(null));

‎src/danfojs-base/shared/types.ts‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -374,13 +374,36 @@ export interface IPlotlyLib {
374374
table(plotConfig?: PlotConfigObject): void
375375
}
376376

377-
export interface CsvInputOptionsBrowser extends ParseConfig { }
378-
export type ExcelInputOptionsBrowser = { sheet?: number, method?: string, headers?: any }
379-
export type JsonInputOptionsBrowser = { method?: string, headers?: any }
377+
export interface CsvInputOptionsBrowser extends ParseConfig {
378+
frameConfig?: BaseDataOptionType
379+
}
380+
export type ExcelInputOptionsBrowser = {
381+
sheet?: number,
382+
method?: string,
383+
headers?: any,
384+
frameConfig?: BaseDataOptionType
385+
}
386+
export type JsonInputOptionsBrowser = {
387+
method?: string,
388+
headers?: any,
389+
frameConfig?: BaseDataOptionType
390+
}
380391

381-
export interface CsvInputOptionsNode extends ParseConfig { }
382-
export type ExcelInputOptionsNode = { sheet?: number, method?: string, headers?: HeadersInit }
383-
export type JsonInputOptionsNode = { method?: string, headers?: HeadersInit }
392+
export interface CsvInputOptionsNode extends ParseConfig {
393+
frameConfig?: BaseDataOptionType
394+
}
395+
396+
export type ExcelInputOptionsNode = {
397+
sheet?: number,
398+
method?: string,
399+
headers?: HeadersInit
400+
frameConfig?: BaseDataOptionType
401+
}
402+
export type JsonInputOptionsNode = {
403+
method?: string,
404+
headers?: HeadersInit
405+
frameConfig?: BaseDataOptionType
406+
}
384407

385408
export type CsvOutputOptionsBrowser = { fileName?: string, sep?: string, header?: boolean, download?: boolean };
386409
export type ExcelOutputOptionsBrowser = { fileName?: string, sheetName?: string };

‎src/danfojs-browser/tests/io/csv.reader.test.js‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,40 @@ describe("readCSV", function () {
2424
]);
2525
});
2626

27+
it("Read remote csv file with config works", async function () {
28+
const remoteFile = "https://raw.githubusercontent.com/opensource9ja/danfojs/dev/danfojs-node/tests/samples/titanic.csv";
29+
const frameConfig = {
30+
columns: [
31+
'A',
32+
'B',
33+
'C',
34+
'D',
35+
'E',
36+
'F',
37+
'G',
38+
'H'
39+
]
40+
};
41+
let df = await dfd.readCSV(remoteFile, { header: true, preview: 5, frameConfig });
42+
assert.deepEqual(df.shape, [ 5, 8 ]);
43+
assert.deepEqual(df.columns, [
44+
'A',
45+
'B',
46+
'C',
47+
'D',
48+
'E',
49+
'F',
50+
'G',
51+
'H'
52+
]);
53+
assert.deepEqual(df.dtypes, [
54+
'int32', 'int32',
55+
'string', 'string',
56+
'int32', 'int32',
57+
'int32', 'float32'
58+
]);
59+
});
60+
2761
});
2862

2963
// describe("streamCSV", function () {

‎src/danfojs-browser/tests/io/excel.reader.test.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,29 @@ describe("readExcel", function () {
1717
]);
1818
assert.deepEqual(df.shape, [ 82, 4 ]);
1919
});
20+
21+
it("Read remote excel file with config works", async function () {
22+
const remoteFile = "https://raw.githubusercontent.com/opensource9ja/danfojs/dev/danfojs-node/tests/samples/SampleData.xlsx";
23+
const frameConfig = {
24+
columns: [
25+
'A',
26+
'B',
27+
'C',
28+
'D'
29+
]
30+
};
31+
let df = await dfd.readExcel(remoteFile, { frameConfig });
32+
assert.deepEqual(df.columns, [
33+
'A',
34+
'B',
35+
'C',
36+
'D'
37+
]);
38+
assert.deepEqual(df.dtypes, [
39+
'int32', 'float32',
40+
'float32', 'float32'
41+
]);
42+
assert.deepEqual(df.shape, [ 82, 4 ]);
43+
});
2044
});
2145

‎src/danfojs-browser/tests/io/json.reader.test.js‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,29 @@ describe("readJSON", function () {
1717
'string', 'string'
1818
]);
1919
});
20+
21+
it("Read remote csv file with config works", async function () {
22+
const remoteFile = "https://raw.githubusercontent.com/opensource9ja/danfojs/dev/danfojs-node/tests/samples/book.json";
23+
const frameConfig = {
24+
columns: [
25+
'A',
26+
'B',
27+
'C',
28+
'D'
29+
]
30+
};
31+
const df = await dfd.readJSON(remoteFile, { frameConfig });
32+
assert.deepEqual(df.columns, [
33+
'A',
34+
'B',
35+
'C',
36+
'D'
37+
]);
38+
assert.deepEqual(df.dtypes, [
39+
'int32', 'string',
40+
'string', 'string'
41+
]);
42+
});
2043
});
2144

2245
describe("toJSON", function () {

0 commit comments

Comments
 (0)