Skip to content

Commit 9589843

Browse files
mateuszkuprowskimateuszkuprowski
authored
feat: Improved google drive precheck (#376)
* Improved google drive precheck Added test for new precheack Added new fixtures
1 parent 56e69a4 commit 9589843

File tree

9 files changed

+745
-4
lines changed

9 files changed

+745
-4
lines changed

‎CHANGELOG.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
## 0.5.2-dev1
2+
3+
### Enchancements
4+
5+
* **Improved google drive precheck mechanism**
6+
* **Added integration tests for google drive precheck and connector**
7+
18
## 0.5.2-dev0
29

310
### Enhancements

‎test/integration/connectors/expected_results/google_drive/directory_structure.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"directory_structure": [
3+
"fake.docx",
4+
"nested/fake.docx",
5+
"recalibrating-risk-report.pdf",
6+
"test-drive-doc.docx"
7+
]
8+
}

‎test/integration/connectors/expected_results/google_drive/file_data/117qrVqiCoR5EjYMsDHGdy3UMkEtKr9Q8.json‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
{
2+
"identifier": "117qrVqiCoR5EjYMsDHGdy3UMkEtKr9Q8",
3+
"connector_type": "google_drive",
4+
"source_identifiers": {
5+
"filename": "test-drive-doc.docx",
6+
"fullpath": "utic-test-ingest-fixtures/test-drive-doc.docx",
7+
"rel_path": "test-drive-doc.docx"
8+
},
9+
"metadata": {
10+
"url": "https://drive.google.com/uc?id=117qrVqiCoR5EjYMsDHGdy3UMkEtKr9Q8&export=download",
11+
"version": "32",
12+
"record_locator": {
13+
"file_id": "117qrVqiCoR5EjYMsDHGdy3UMkEtKr9Q8"
14+
},
15+
"date_created": "1686809758.931",
16+
"date_modified": "1686809744.0",
17+
"date_processed": null,
18+
"permissions_data": [
19+
{
20+
"id": "anyoneWithLink",
21+
"type": "anyone",
22+
"kind": "drive#permission",
23+
"role": "reader",
24+
"allowFileDiscovery": false
25+
},
26+
{
27+
"id": "18298851591250030956",
28+
"displayName": "[email protected]",
29+
"type": "user",
30+
"kind": "drive#permission",
31+
"photoLink": "https://lh3.googleusercontent.com/a/ACg8ocJok2KRwwYvrEDkeZVCYosHOMoa52GZa2qIIC1jScCRoFLHaQ=s64",
32+
"emailAddress": "[email protected]",
33+
"role": "writer",
34+
"deleted": false,
35+
"pendingOwner": false
36+
},
37+
{
38+
"id": "04774006893477068632",
39+
"displayName": "ryan",
40+
"type": "user",
41+
"kind": "drive#permission",
42+
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjXeWpu7QcZuYqIl3p1mwqzS8XGFJ4RqA3Xjljfkm1DcFZ9M7A=s64",
43+
"emailAddress": "[email protected]",
44+
"role": "owner",
45+
"deleted": false,
46+
"pendingOwner": false
47+
},
48+
{
49+
"id": "09147371668407854156",
50+
"displayName": "roman",
51+
"type": "user",
52+
"kind": "drive#permission",
53+
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjWoGrFCgXcF6CtiBIBLnAfM68qUnQaJOcgvg3qzfQ3W8Ch6dA=s64",
54+
"emailAddress": "[email protected]",
55+
"role": "writer",
56+
"deleted": false,
57+
"pendingOwner": false
58+
}
59+
],
60+
"filesize_bytes": null
61+
},
62+
"additional_metadata": {
63+
"fileExtension": "docx",
64+
"md5Checksum": "b6bd26fa317493cf447882754dac5e9d",
65+
"mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
66+
"headRevisionId": "0B5wM1A2PIvPWOVFKa0U4MHQ1T2E0b0wzSWtGSkRZR1hwRUpzPQ",
67+
"webViewLink": "https://docs.google.com/document/d/117qrVqiCoR5EjYMsDHGdy3UMkEtKr9Q8/edit?usp=drivesdk&ouid=116756094827511368604&rtpof=true&sd=true",
68+
"id": "117qrVqiCoR5EjYMsDHGdy3UMkEtKr9Q8",
69+
"originalFilename": "test-drive-doc.docx",
70+
"capabilities": {
71+
"canChangeViewersCanCopyContent": false,
72+
"canEdit": true,
73+
"canCopy": true,
74+
"canComment": true,
75+
"canAddChildren": false,
76+
"canDelete": false,
77+
"canDownload": true,
78+
"canListChildren": false,
79+
"canRemoveChildren": false,
80+
"canRename": true,
81+
"canTrash": false,
82+
"canReadRevisions": true,
83+
"canChangeCopyRequiresWriterPermission": false,
84+
"canMoveItemIntoTeamDrive": false,
85+
"canUntrash": false,
86+
"canModifyContent": true,
87+
"canMoveItemOutOfDrive": false,
88+
"canAddMyDriveParent": false,
89+
"canRemoveMyDriveParent": true,
90+
"canMoveItemWithinDrive": true,
91+
"canShare": true,
92+
"canMoveChildrenWithinDrive": false,
93+
"canModifyContentRestriction": true,
94+
"canChangeSecurityUpdateEnabled": false,
95+
"canAcceptOwnership": false,
96+
"canReadLabels": false,
97+
"canModifyLabels": false,
98+
"canModifyEditorContentRestriction": true,
99+
"canModifyOwnerContentRestriction": false,
100+
"canRemoveContentRestriction": false
101+
},
102+
"permissionIds": [
103+
"anyoneWithLink",
104+
"18298851591250030956",
105+
"04774006893477068632",
106+
"09147371668407854156"
107+
],
108+
"sha1Checksum": "70daaa25dd03bc56192ccc18fc25f53aa06b7b36",
109+
"sha256Checksum": "9dd205eea9d2fa9f4e7a993697c8d5b6c4ec7a92ccd12fff772b1dff084deca8"
110+
},
111+
"reprocess": false,
112+
"local_download_path": "/tmp/tmp9k8p4569/test-drive-doc.docx",
113+
"display_name": null
114+
}

‎test/integration/connectors/expected_results/google_drive/file_data/1SpQuE7jHz9nMt5hfQXsiok1SgIdRYX5o.json‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
{
2+
"identifier": "1SpQuE7jHz9nMt5hfQXsiok1SgIdRYX5o",
3+
"connector_type": "google_drive",
4+
"source_identifiers": {
5+
"filename": "fake.docx",
6+
"fullpath": "utic-test-ingest-fixtures/fake.docx",
7+
"rel_path": "fake.docx"
8+
},
9+
"metadata": {
10+
"url": "https://drive.google.com/uc?id=1SpQuE7jHz9nMt5hfQXsiok1SgIdRYX5o&export=download",
11+
"version": "36",
12+
"record_locator": {
13+
"file_id": "1SpQuE7jHz9nMt5hfQXsiok1SgIdRYX5o"
14+
},
15+
"date_created": "1686809759.687",
16+
"date_modified": "1686809743.0",
17+
"date_processed": null,
18+
"permissions_data": [
19+
{
20+
"id": "anyoneWithLink",
21+
"type": "anyone",
22+
"kind": "drive#permission",
23+
"role": "reader",
24+
"allowFileDiscovery": false
25+
},
26+
{
27+
"id": "18298851591250030956",
28+
"displayName": "[email protected]",
29+
"type": "user",
30+
"kind": "drive#permission",
31+
"photoLink": "https://lh3.googleusercontent.com/a/ACg8ocJok2KRwwYvrEDkeZVCYosHOMoa52GZa2qIIC1jScCRoFLHaQ=s64",
32+
"emailAddress": "[email protected]",
33+
"role": "writer",
34+
"deleted": false,
35+
"pendingOwner": false
36+
},
37+
{
38+
"id": "04774006893477068632",
39+
"displayName": "ryan",
40+
"type": "user",
41+
"kind": "drive#permission",
42+
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjXeWpu7QcZuYqIl3p1mwqzS8XGFJ4RqA3Xjljfkm1DcFZ9M7A=s64",
43+
"emailAddress": "[email protected]",
44+
"role": "owner",
45+
"deleted": false,
46+
"pendingOwner": false
47+
},
48+
{
49+
"id": "09147371668407854156",
50+
"displayName": "roman",
51+
"type": "user",
52+
"kind": "drive#permission",
53+
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjWoGrFCgXcF6CtiBIBLnAfM68qUnQaJOcgvg3qzfQ3W8Ch6dA=s64",
54+
"emailAddress": "[email protected]",
55+
"role": "writer",
56+
"deleted": false,
57+
"pendingOwner": false
58+
}
59+
],
60+
"filesize_bytes": null
61+
},
62+
"additional_metadata": {
63+
"fileExtension": "docx",
64+
"md5Checksum": "5ddd25c0e5d5f6996a93c154830dc7aa",
65+
"mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
66+
"headRevisionId": "0B5wM1A2PIvPWOEoxZ2FjcDRsRUJsb2sveEQzTmJZZ1luY3JVPQ",
67+
"webViewLink": "https://docs.google.com/document/d/1SpQuE7jHz9nMt5hfQXsiok1SgIdRYX5o/edit?usp=drivesdk&ouid=116756094827511368604&rtpof=true&sd=true",
68+
"id": "1SpQuE7jHz9nMt5hfQXsiok1SgIdRYX5o",
69+
"originalFilename": "fake.docx",
70+
"capabilities": {
71+
"canChangeViewersCanCopyContent": false,
72+
"canEdit": true,
73+
"canCopy": true,
74+
"canComment": true,
75+
"canAddChildren": false,
76+
"canDelete": false,
77+
"canDownload": true,
78+
"canListChildren": false,
79+
"canRemoveChildren": false,
80+
"canRename": true,
81+
"canTrash": false,
82+
"canReadRevisions": true,
83+
"canChangeCopyRequiresWriterPermission": false,
84+
"canMoveItemIntoTeamDrive": false,
85+
"canUntrash": false,
86+
"canModifyContent": true,
87+
"canMoveItemOutOfDrive": false,
88+
"canAddMyDriveParent": false,
89+
"canRemoveMyDriveParent": true,
90+
"canMoveItemWithinDrive": true,
91+
"canShare": true,
92+
"canMoveChildrenWithinDrive": false,
93+
"canModifyContentRestriction": true,
94+
"canChangeSecurityUpdateEnabled": false,
95+
"canAcceptOwnership": false,
96+
"canReadLabels": false,
97+
"canModifyLabels": false,
98+
"canModifyEditorContentRestriction": true,
99+
"canModifyOwnerContentRestriction": false,
100+
"canRemoveContentRestriction": false
101+
},
102+
"permissionIds": [
103+
"anyoneWithLink",
104+
"18298851591250030956",
105+
"04774006893477068632",
106+
"09147371668407854156"
107+
],
108+
"sha1Checksum": "07efd2a6bd828801d8d1d4bd77fcb15120bbad94",
109+
"sha256Checksum": "690b185fb657a76dfe5d304911e720f8998b328edb6677cbf2c846152284dfb3"
110+
},
111+
"reprocess": false,
112+
"local_download_path": "/tmp/tmp9k8p4569/fake.docx",
113+
"display_name": null
114+
}

‎test/integration/connectors/expected_results/google_drive/file_data/1m1TUgyLv0hHdlsuL7DOWBAKQtvrhWNiV.json‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
{
2+
"identifier": "1m1TUgyLv0hHdlsuL7DOWBAKQtvrhWNiV",
3+
"connector_type": "google_drive",
4+
"source_identifiers": {
5+
"filename": "recalibrating-risk-report.pdf",
6+
"fullpath": "utic-test-ingest-fixtures/recalibrating-risk-report.pdf",
7+
"rel_path": "recalibrating-risk-report.pdf"
8+
},
9+
"metadata": {
10+
"url": "https://drive.google.com/uc?id=1m1TUgyLv0hHdlsuL7DOWBAKQtvrhWNiV&export=download",
11+
"version": "7",
12+
"record_locator": {
13+
"file_id": "1m1TUgyLv0hHdlsuL7DOWBAKQtvrhWNiV"
14+
},
15+
"date_created": "1718723636.34",
16+
"date_modified": "1676196572.0",
17+
"date_processed": null,
18+
"permissions_data": [
19+
{
20+
"id": "anyoneWithLink",
21+
"type": "anyone",
22+
"kind": "drive#permission",
23+
"role": "reader",
24+
"allowFileDiscovery": false
25+
},
26+
{
27+
"id": "18298851591250030956",
28+
"displayName": "[email protected]",
29+
"type": "user",
30+
"kind": "drive#permission",
31+
"photoLink": "https://lh3.googleusercontent.com/a/ACg8ocJok2KRwwYvrEDkeZVCYosHOMoa52GZa2qIIC1jScCRoFLHaQ=s64",
32+
"emailAddress": "[email protected]",
33+
"role": "writer",
34+
"deleted": false,
35+
"pendingOwner": false
36+
},
37+
{
38+
"id": "04774006893477068632",
39+
"displayName": "ryan",
40+
"type": "user",
41+
"kind": "drive#permission",
42+
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjXeWpu7QcZuYqIl3p1mwqzS8XGFJ4RqA3Xjljfkm1DcFZ9M7A=s64",
43+
"emailAddress": "[email protected]",
44+
"role": "writer",
45+
"deleted": false,
46+
"pendingOwner": false
47+
},
48+
{
49+
"id": "09147371668407854156",
50+
"displayName": "roman",
51+
"type": "user",
52+
"kind": "drive#permission",
53+
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjWoGrFCgXcF6CtiBIBLnAfM68qUnQaJOcgvg3qzfQ3W8Ch6dA=s64",
54+
"emailAddress": "[email protected]",
55+
"role": "owner",
56+
"deleted": false,
57+
"pendingOwner": false
58+
}
59+
],
60+
"filesize_bytes": null
61+
},
62+
"additional_metadata": {
63+
"fileExtension": "pdf",
64+
"md5Checksum": "e690f37ef36368a509d150f373a0bbe0",
65+
"mimeType": "application/pdf",
66+
"headRevisionId": "0B8fLPtk3k4KOdlRMRDkvM0JCQkdtTU1NdjdpK0ZlSTJadXNjPQ",
67+
"webViewLink": "https://drive.google.com/file/d/1m1TUgyLv0hHdlsuL7DOWBAKQtvrhWNiV/view?usp=drivesdk",
68+
"id": "1m1TUgyLv0hHdlsuL7DOWBAKQtvrhWNiV",
69+
"originalFilename": "recalibrating-risk-report.pdf",
70+
"capabilities": {
71+
"canChangeViewersCanCopyContent": false,
72+
"canEdit": true,
73+
"canCopy": true,
74+
"canComment": true,
75+
"canAddChildren": false,
76+
"canDelete": false,
77+
"canDownload": true,
78+
"canListChildren": false,
79+
"canRemoveChildren": false,
80+
"canRename": true,
81+
"canTrash": false,
82+
"canReadRevisions": true,
83+
"canChangeCopyRequiresWriterPermission": false,
84+
"canMoveItemIntoTeamDrive": false,
85+
"canUntrash": false,
86+
"canModifyContent": true,
87+
"canMoveItemOutOfDrive": false,
88+
"canAddMyDriveParent": false,
89+
"canRemoveMyDriveParent": true,
90+
"canMoveItemWithinDrive": true,
91+
"canShare": true,
92+
"canMoveChildrenWithinDrive": false,
93+
"canModifyContentRestriction": true,
94+
"canChangeSecurityUpdateEnabled": false,
95+
"canAcceptOwnership": false,
96+
"canReadLabels": false,
97+
"canModifyLabels": false,
98+
"canModifyEditorContentRestriction": true,
99+
"canModifyOwnerContentRestriction": false,
100+
"canRemoveContentRestriction": false
101+
},
102+
"permissionIds": [
103+
"anyoneWithLink",
104+
"18298851591250030956",
105+
"04774006893477068632",
106+
"09147371668407854156"
107+
],
108+
"sha1Checksum": "57e64b7be1e099ecf9dcd9832bc09c81ffc5fc44",
109+
"sha256Checksum": "076d8d243312e0b03a0fe2191e924c37fe159dd01d6b6ce3a3b2b7c642c36245"
110+
},
111+
"reprocess": false,
112+
"local_download_path": "/tmp/tmp9k8p4569/recalibrating-risk-report.pdf",
113+
"display_name": null
114+
}

0 commit comments

Comments
 (0)