Skip to content

Commit c16862e

Browse files
natygyoonnatygyoon
authored
feat: add --metadata-include and --metadata-exclude parameters to unstructured-ingest (#368)
* added metadata in/exclude params * updated process_file * existing tests * remove default behavior * changelog and ci * line length * import * import * import sorted * import * type * line length * main * ci * json * dict * type ignore * lint * unit tests for process_file * lint * type changed to Optional(str) * ci * line length * added mutex check * nit
1 parent d5a0fce commit c16862e

17 files changed

+175
-8
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
### Features
88

9+
* Add `--metadata-include` and `--metadata-exclude` parameters to `unstructured-ingest`
910
* Add `clean_non_ascii_chars` to remove non-ascii characters from unicode string
1011

1112
### Fixes

‎test_unstructured_ingest/test-ingest-azure.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
44
cd "$SCRIPT_DIR"/.. || exit 1
55

66
PYTHONPATH=. ./unstructured/ingest/main.py \
7+
--metadata-exclude filename \
78
--remote-url abfs://container1/ \
89
--azure-account-name azureunstructured1 \
910
--structured-output-dir azure-ingest-output \

‎test_unstructured_ingest/test-ingest-biomed-api.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ if [[ "$(find test_unstructured_ingest/expected-structured-output/biomed-ingest-
1010
fi
1111

1212
PYTHONPATH=. ./unstructured/ingest/main.py \
13+
--metadata-exclude filename \
1314
--biomed-api-from "2019-01-02" \
1415
--biomed-api-until "2019-01-02+00:03:10" \
1516
--structured-output-dir biomed-ingest-output-api \

‎test_unstructured_ingest/test-ingest-biomed-path.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ if [[ "$(find test_unstructured_ingest/expected-structured-output/biomed-ingest-
1010
fi
1111

1212
PYTHONPATH=. ./unstructured/ingest/main.py \
13+
--metadata-exclude filename \
1314
--biomed-path "oa_pdf/07/07/sbaa031.073.PMC7234218.pdf" \
1415
--structured-output-dir biomed-ingest-output-path \
1516
--num-processes 2 \

‎test_unstructured_ingest/test-ingest-github.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@ if [[ "$CI" == "true" ]]; then
1212
fi
1313

1414

15-
PYTHONPATH=. ./unstructured/ingest/main.py --github-url dcneiner/Downloadify --git-file-glob '*.html,*.txt' --structured-output-dir github-downloadify-output --verbose
15+
PYTHONPATH=. ./unstructured/ingest/main.py \
16+
--metadata-exclude filename \
17+
--github-url dcneiner/Downloadify \
18+
--git-file-glob '*.html,*.txt' \
19+
--structured-output-dir github-downloadify-output \
20+
--verbose
1621

1722
if ! diff -ru test_unstructured_ingest/expected-structured-output/github-downloadify github-downloadify-output ; then
1823
echo

‎test_unstructured_ingest/test-ingest-gitlab.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
44
cd "$SCRIPT_DIR"/.. || exit 1
55

66
PYTHONPATH=. ./unstructured/ingest/main.py \
7+
--metadata-exclude filename \
78
--gitlab-url https://gitlab.com/gitlab-com/content-sites/docsy-gitlab \
89
--git-file-glob '*.md,*.txt' \
910
--structured-output-dir gitlab-ingest-output \

‎test_unstructured_ingest/test-ingest-s3.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ if [[ "$(find test_unstructured_ingest/expected-structured-output/s3-small-batch
99
exit 1
1010
fi
1111

12-
PYTHONPATH=. ./unstructured/ingest/main.py --s3-url s3://utic-dev-tech-fixtures/small-pdf-set/ --s3-anonymous --structured-output-dir s3-small-batch-output
12+
PYTHONPATH=. ./unstructured/ingest/main.py \
13+
--metadata-exclude filename \
14+
--s3-url s3://utic-dev-tech-fixtures/small-pdf-set/ \
15+
--s3-anonymous \
16+
--structured-output-dir s3-small-batch-output \
1317

1418
if ! diff -ru test_unstructured_ingest/expected-structured-output/s3-small-batch s3-small-batch-output ; then
1519
echo

‎test_unstructured_ingest/test-ingest-wikipedia.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
44
cd "$SCRIPT_DIR"/.. || exit 1
55

66
PYTHONPATH=. ./unstructured/ingest/main.py \
7+
--metadata-exclude filename \
78
--wikipedia-page-title "Open Source Software" \
89
--structured-output-dir wikipedia-ingest-output \
910
--num-processes 2 \

‎test_unstructured_ingest/test_interfaces.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import os
2+
import pathlib
3+
4+
import pytest
5+
6+
from unstructured.ingest.connector.git import GitIngestDoc, SimpleGitConfig
7+
8+
DIRECTORY = pathlib.Path(__file__).parent.resolve()
9+
EXAMPLE_DOCS_DIRECTORY = os.path.join(DIRECTORY, "..", "example-docs")
10+
11+
test_files = [
12+
"layout-parser-paper-fast.jpg",
13+
"layout-parser-paper-fast.pdf",
14+
]
15+
16+
17+
@pytest.mark.parametrize("filename", test_files)
18+
def test_process_file_include_filename(filename: str):
19+
ingest_doc = GitIngestDoc(
20+
path=filename,
21+
config=SimpleGitConfig(
22+
download_dir=EXAMPLE_DOCS_DIRECTORY,
23+
metadata_include="filename",
24+
),
25+
)
26+
isd_elems = ingest_doc.process_file()
27+
28+
for elem in isd_elems:
29+
for k in elem["metadata"]:
30+
assert k == "filename"
31+
32+
33+
@pytest.mark.parametrize("filename", test_files)
34+
def test_process_file_include_filename_pagenum(filename: str):
35+
ingest_doc = GitIngestDoc(
36+
path=filename,
37+
config=SimpleGitConfig(
38+
download_dir=EXAMPLE_DOCS_DIRECTORY,
39+
metadata_include="filename,page_number",
40+
),
41+
)
42+
isd_elems = ingest_doc.process_file()
43+
44+
for elem in isd_elems:
45+
for k in elem["metadata"]:
46+
assert k in ["filename", "page_number"]
47+
48+
49+
@pytest.mark.parametrize("filename", test_files)
50+
def test_process_file_exclude_filename(filename: str):
51+
ingest_doc = GitIngestDoc(
52+
path=filename,
53+
config=SimpleGitConfig(
54+
download_dir=EXAMPLE_DOCS_DIRECTORY,
55+
metadata_exclude="filename",
56+
),
57+
)
58+
isd_elems = ingest_doc.process_file()
59+
60+
for elem in isd_elems:
61+
for k in elem["metadata"]:
62+
assert k != "filename"
63+
64+
65+
@pytest.mark.parametrize("filename", test_files)
66+
def test_process_file_exclude_filename_pagenum(filename: str):
67+
ingest_doc = GitIngestDoc(
68+
path=filename,
69+
config=SimpleGitConfig(
70+
download_dir=EXAMPLE_DOCS_DIRECTORY,
71+
metadata_exclude="filename,page_number",
72+
),
73+
)
74+
isd_elems = ingest_doc.process_file()
75+
76+
for elem in isd_elems:
77+
for k in elem["metadata"]:
78+
assert k not in ["filename", "page_number"]

‎unstructured/ingest/connector/biomed.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from datetime import datetime
66
from ftplib import FTP, error_perm
77
from pathlib import Path
8-
from typing import List, Union
8+
from typing import List, Optional, Union
99

1010
import requests
1111
from bs4 import BeautifulSoup
@@ -48,6 +48,8 @@ class SimpleBiomedConfig(BaseConnectorConfig):
4848
output_dir: str
4949
re_download: bool = False
5050
preserve_downloads: bool = False
51+
metadata_include: Optional[str] = None
52+
metadata_exclude: Optional[str] = None
5153

5254
def _validate_date_args(self, date):
5355
date_formats = ["%Y-%m-%d", "%Y-%m-%d+%H:%M:%S"]

0 commit comments

Comments
 (0)