Skip to content

Commit 54ec311

Browse files
rbiseck3ryannikolaidis
rbiseck3
and
ryannikolaidis
authored
feat/migrate onedrive src (#3295)
### Description Migrate the onedrive source connector to v2, adding in more rich content pulled from the response of the SDK to add further metadata to the FIleData produced by the indexer. --------- Co-authored-by: ryannikolaidis <[email protected]> Co-authored-by: rbiseck3 <[email protected]>
1 parent 6939bff commit 54ec311

File tree

6 files changed

+742
-380
lines changed

6 files changed

+742
-380
lines changed

‎test_unstructured_ingest/expected-structured-output/onedrive/utic-test-ingest-fixtures/fake-text.txt.json‎

Lines changed: 90 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,128 +1,134 @@
11
[
22
{
3+
"type": "NarrativeText",
34
"element_id": "fc01503614e0f12f585427cccf81cf86",
5+
"text": "This is a test document to use for unit tests.",
46
"metadata": {
5-
"data_source": {
6-
"date_created": "2023-08-24T03:00:09+00:00",
7-
"date_modified": "2023-08-24T03:00:09+00:00",
8-
"record_locator": {
9-
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
10-
"user_pname": "[email protected]"
11-
},
12-
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
13-
},
147
"filename": "fake-text.txt",
15-
"filetype": "text/plain",
168
"languages": [
179
"eng"
18-
]
19-
},
20-
"text": "This is a test document to use for unit tests.",
21-
"type": "NarrativeText"
10+
],
11+
"filetype": "text/plain",
12+
"data_source": {
13+
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
14+
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
15+
"record_locator": {
16+
"user_pname": "[email protected]",
17+
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
18+
},
19+
"date_created": "1692846009.0",
20+
"date_modified": "1692846009.0"
21+
}
22+
}
2223
},
2324
{
25+
"type": "Address",
2426
"element_id": "d06d10c0722ac08a2488076a48e858d5",
27+
"text": "Doylestown, PA 18901",
2528
"metadata": {
26-
"data_source": {
27-
"date_created": "2023-08-24T03:00:09+00:00",
28-
"date_modified": "2023-08-24T03:00:09+00:00",
29-
"record_locator": {
30-
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
31-
"user_pname": "[email protected]"
32-
},
33-
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
34-
},
3529
"filename": "fake-text.txt",
36-
"filetype": "text/plain",
3730
"languages": [
3831
"eng"
39-
]
40-
},
41-
"text": "Doylestown, PA 18901",
42-
"type": "Address"
32+
],
33+
"filetype": "text/plain",
34+
"data_source": {
35+
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
36+
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
37+
"record_locator": {
38+
"user_pname": "[email protected]",
39+
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
40+
},
41+
"date_created": "1692846009.0",
42+
"date_modified": "1692846009.0"
43+
}
44+
}
4345
},
4446
{
47+
"type": "Title",
4548
"element_id": "a190164de573571375ecf759a5027a3a",
49+
"text": "Important points:",
4650
"metadata": {
47-
"data_source": {
48-
"date_created": "2023-08-24T03:00:09+00:00",
49-
"date_modified": "2023-08-24T03:00:09+00:00",
50-
"record_locator": {
51-
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
52-
"user_pname": "[email protected]"
53-
},
54-
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
55-
},
5651
"filename": "fake-text.txt",
57-
"filetype": "text/plain",
5852
"languages": [
5953
"eng"
60-
]
61-
},
62-
"text": "Important points:",
63-
"type": "Title"
54+
],
55+
"filetype": "text/plain",
56+
"data_source": {
57+
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
58+
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
59+
"record_locator": {
60+
"user_pname": "[email protected]",
61+
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
62+
},
63+
"date_created": "1692846009.0",
64+
"date_modified": "1692846009.0"
65+
}
66+
}
6467
},
6568
{
69+
"type": "ListItem",
6670
"element_id": "64c58eb106608ad05424e47bbcdef7dc",
71+
"text": "Hamburgers are delicious",
6772
"metadata": {
68-
"data_source": {
69-
"date_created": "2023-08-24T03:00:09+00:00",
70-
"date_modified": "2023-08-24T03:00:09+00:00",
71-
"record_locator": {
72-
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
73-
"user_pname": "[email protected]"
74-
},
75-
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
76-
},
7773
"filename": "fake-text.txt",
78-
"filetype": "text/plain",
7974
"languages": [
8075
"eng"
81-
]
82-
},
83-
"text": "Hamburgers are delicious",
84-
"type": "ListItem"
76+
],
77+
"filetype": "text/plain",
78+
"data_source": {
79+
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
80+
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
81+
"record_locator": {
82+
"user_pname": "[email protected]",
83+
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
84+
},
85+
"date_created": "1692846009.0",
86+
"date_modified": "1692846009.0"
87+
}
88+
}
8589
},
8690
{
91+
"type": "ListItem",
8792
"element_id": "9210d7882755a60fc82272a0e93df94f",
93+
"text": "Dogs are the best",
8894
"metadata": {
89-
"data_source": {
90-
"date_created": "2023-08-24T03:00:09+00:00",
91-
"date_modified": "2023-08-24T03:00:09+00:00",
92-
"record_locator": {
93-
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
94-
"user_pname": "[email protected]"
95-
},
96-
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
97-
},
9895
"filename": "fake-text.txt",
99-
"filetype": "text/plain",
10096
"languages": [
10197
"eng"
102-
]
103-
},
104-
"text": "Dogs are the best",
105-
"type": "ListItem"
98+
],
99+
"filetype": "text/plain",
100+
"data_source": {
101+
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
102+
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
103+
"record_locator": {
104+
"user_pname": "[email protected]",
105+
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
106+
},
107+
"date_created": "1692846009.0",
108+
"date_modified": "1692846009.0"
109+
}
110+
}
106111
},
107112
{
113+
"type": "ListItem",
108114
"element_id": "970dd4ac687529e2d2126e0a51cb7c27",
115+
"text": "I love fuzzy blankets",
109116
"metadata": {
110-
"data_source": {
111-
"date_created": "2023-08-24T03:00:09+00:00",
112-
"date_modified": "2023-08-24T03:00:09+00:00",
113-
"record_locator": {
114-
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
115-
"user_pname": "[email protected]"
116-
},
117-
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
118-
},
119117
"filename": "fake-text.txt",
120-
"filetype": "text/plain",
121118
"languages": [
122119
"eng"
123-
]
124-
},
125-
"text": "I love fuzzy blankets",
126-
"type": "ListItem"
120+
],
121+
"filetype": "text/plain",
122+
"data_source": {
123+
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
124+
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
125+
"record_locator": {
126+
"user_pname": "[email protected]",
127+
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
128+
},
129+
"date_created": "1692846009.0",
130+
"date_modified": "1692846009.0"
131+
}
132+
}
127133
}
128134
]

0 commit comments

Comments
 (0)