clean up deptry config and dependencies

Author: aaronsteers

pyproject.toml

@@ -43,13 +43,13 @@ jsonref = "~0.2"
 jsonschema = "~4.17.3"  # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0
 packaging = "*"  # Transitive dependency used directly in code
 pandas = "2.2.2"
-psutil = "6.1.0"
+psutil = "6.1.0" # TODO: Remove if unused
 pydantic = "^2.7"
 pyrate-limiter = "~3.1.0"
 python-dateutil = "^2.9.0"
 python-ulid = "^3.0.0"
 PyYAML = "^6.0.1"
-rapidfuzz = "^3.10.1"
+rapidfuzz = "^3.10.1" # TODO: Remove if unused
 requests = "*"
 requests_cache = "*"
 typing-extensions = "*"  # Transitive dependency used directly in code
@@ -60,14 +60,14 @@ cohere = { version = "4.21", optional = true }
 fastavro = { version = "~1.8.0", optional = true }
 langchain = { version = "0.1.16", optional = true }
 langchain_core = { version = "0.1.42", optional = true }
-markdown = { version = "*", optional = true }
-openai = { version = "0.27.9", extras = ["embeddings"], optional = true }
+markdown = { version = "*", optional = true }  # TODO: Remove if unused
+openai = { version = "0.27.9", extras = ["embeddings"], optional = true }  # Used indirectly by langchain library
 pdf2image = { version = "1.16.3", optional = true }
-"pdfminer.six" = { version = "20221105", optional = true }
+"pdfminer.six" = { version = "20221105", optional = true }  # Used indirectly by unstructured library
 pyarrow = { version = "^19.0.0", optional = true }
-pytesseract = { version = "0.3.10", optional = true }
-python-calamine = { version = "0.2.3", optional = true }
-python-snappy = { version = "0.7.3", optional = true }
+pytesseract = { version = "0.3.10", optional = true }  # Used indirectly by unstructured library
+python-calamine = { version = "0.2.3", optional = true }  # Used indirectly by unstructured library
+python-snappy = { version = "0.7.3", optional = true }  # Used indirectly by unstructured library
 tiktoken = { version = "0.8.0", optional = true }
 nltk = { version = "3.9.1", optional = true }
 # This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen.
@@ -76,7 +76,7 @@ numpy = "<2"
 unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true }
 "unstructured.pytesseract" = { version = ">=0.3.12", optional = true }
 pyjwt = "^2.8.0"
-cryptography = ">=44.0.0,<45.0.0"
+cryptography = ">=44.0.0,<45.0.0"  # Constrained as transitive dependency due to a bug in newer versions
 pytz = "2024.2"
 orjson = "^3.10.7"
 serpyco-rs = "^1.10.2"
@@ -201,24 +201,30 @@ poetry_extras = ["file-based", "vector-db-based"]
 poe_tasks = ["check-ci"]
 mount_docker_socket = true
 
+[tool.deptry]
+exclude = [
+  "bin",
+  "docs",
+  "unit_tests",
+]
+
 [tool.deptry.per_rule_ignores]
 # This is a mapping of rules and package names to be ignored for that rule.
 
-# DEP001: Unused direct dependencies - packages that are declared as dependencies but not imported
-# https://deptry.com/rules-violations/#unused-dependencies-dep001
+# DEP001: Project should not contain missing dependencies.
+# https://deptry.com/rules-violations/#missing-dependencies-dep001
 DEP001 = [
-  # Optional dependencies that might be imported conditionally
-  "dagger",  # Used in bin scripts but not a core dependency
-  "source_declarative_manifest"  # Used in tests but not a core dependency
+  # These are imported but not declared:
+  "dagger",  # Used in `bin` scripts but not in main code
+  "source_declarative_manifest"  # Imported only in dynamic import tests, not in main code
 ]
 
-# DEP002: Missing dependencies - packages that are imported but not declared as dependencies
+# DEP002:
 # https://deptry.com/rules-violations/#missing-dependencies-dep002
 DEP002 = [
-  # Dependencies that are imported indirectly through other packages
-  "sqlalchemy",  # Used through extras
-  # Optional dependencies in extras
-  "psutil",
+  "cryptography",  # Constrained as transitive dependency due to a bug in newer versions
+
+  # TODO: Remove these dependencies if not needed:
   "rapidfuzz",
   "cohere",
   "markdown",
@@ -230,23 +236,20 @@ DEP002 = [
   "python-snappy",
   "tiktoken",
   "unstructured.pytesseract",
-  "cryptography"
 ]
 
 # DEP003: Transitive dependencies - packages that are imported directly
 # https://deptry.com/rules-violations/#transitive-dependencies-dep003
 DEP003 = [
   # Transitive dependencies that are imported directly
-  "pydantic_core"  # Pydantic internals
+  "pydantic_core"  # Pydantic internals, no need to define as separate dependency
 ]
 
 # DEP004: Dev dependencies imported in main code - packages that are imported in non-test code but declared as dev dependencies
 # https://deptry.com/rules-violations/#misplaced-development-dependencies-dep004
 DEP004 = [
-  "pdoc",  # Only used for generating documentation. Not a runtime dependency.
-  "pytest",  # Test dependencies
-  "freezegun",  # Test dependencies
-  "requests_mock",  # Test dependencies
-  "_pytest",  # Test dependencies
-  "pympler"  # Test dependencies
+  # The `airbyte_cdk.test.utils` module is main code.
+  # These should probably be declared in a `tests` extra (TODO):
+  "pytest",
+  "requests_mock",
 ]