webextract/pyproject.toml at main · HimashaHerath/webextract · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "llm-webextract"
version = "1.2.4"
description = "AI-powered web content extraction with Large Language Models"
readme = "README.md"
license = {text = "MIT"}
authors = [
    {name = "Himasha Herath", email = "himasha626@gmail.com"}
]
keywords = ["web scraping", "llm", "ai", "content extraction", "playwright", "ollama", "openai", "anthropic"]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.8",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Internet :: WWW/HTTP :: Browsers",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Topic :: Text Processing :: Markup :: HTML",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
]
requires-python = ">=3.8"
dependencies = [
    "playwright>=1.40.0",
    "beautifulsoup4>=4.12.0",
    "pydantic>=2.0.0",
    "typer>=0.9.0",
    "rich>=13.0.0",
    "ollama>=0.1.7",
    "lxml>=4.9.0",
]

[project.optional-dependencies]
dev = [
    "pytest>=6.0",
    "pytest-asyncio",
    "black",
    "flake8",
    "mypy",
    "isort",
    "pylint",
    "safety",
    "bandit",
    "pre-commit",
]
openai = ["openai>=1.0.0"]
anthropic = ["anthropic>=0.8.0"]
all = [
    "openai>=1.0.0",
    "anthropic>=0.8.0",
]

[project.scripts]
llm-webextract = "webextract.cli:main"

[project.urls]
Homepage = "https://github.com/HimashaHerath/webextract"
Repository = "https://github.com/HimashaHerath/webextract"
Issues = "https://github.com/HimashaHerath/webextract/issues"

[tool.black]
line-length = 100
target-version = ['py38']
include = '\.pyi?$'
extend-exclude = '''
/(
  \.eggs
  | \.git
  | \.hg
  | \.mypy_cache
  | \.tox
  | \.venv
  | _build
  | buck-out
  | build
  | dist
)/
'''

[tool.isort]
profile = "black"
line_length = 100
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
use_parentheses = true
ensure_newline_before_comments = true

[tool.flake8]
max-line-length = 100
extend-ignore = ["E203", "W503"]
exclude = [
    ".git",
    "__pycache__",
    "dist",
    "build",
    "*.egg-info",
]

[tool.mypy]
python_version = "3.8"
warn_return_any = true
warn_unused_configs = true
ignore_missing_imports = true

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
python_functions = ["test_*"]
addopts = "-v --tb=short"