feat/env var user agent (#452)

* use env var for user agent with default

* expose github connector

* update changelog

* fix github

* fix version

* fix github

* drop user agent as field

Author: rbiseck3

CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.6.4
+
+### Features
+
+* **Support env var for user agent settings**
+
+### Fixes
+
+* **Expose Github connector**
+
 ## 0.6.3
 
 ### Features

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.6.3"  # pragma: no cover
+__version__ = "0.6.4"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -31,6 +31,8 @@
 from .delta_table import delta_table_destination_entry
 from .discord import CONNECTOR_TYPE as DISCORD_CONNECTOR_TYPE
 from .discord import discord_source_entry
+from .github import CONNECTOR_TYPE as GITHUB_CONNECTOR_TYPE
+from .github import github_source_entry
 from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
 from .gitlab import gitlab_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
@@ -124,3 +126,4 @@
 add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
 
 add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)
+add_source_entry(source_type=GITHUB_CONNECTOR_TYPE, entry=github_source_entry)

unstructured_ingest/v2/processes/connectors/databricks/volumes.py

@@ -68,7 +68,6 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
         "Databricks workspace endpoint or the "
         "Databricks accounts endpoint.",
     )
-    user_agent: str = "unstructuredio_oss"
 
     def wrap_error(self, e: Exception) -> Exception:
         from databricks.sdk.errors.base import DatabricksError
@@ -101,7 +100,9 @@ def get_client(self) -> "WorkspaceClient":
         config = Config(
             host=self.host,
             **self.access_config.get_secret_value().model_dump(),
-        ).with_user_agent_extra("PyDatabricksSdk", self.user_agent)
+        ).with_user_agent_extra(
+            "PyDatabricksSdk", os.getenv("UNSTRUCTURED_USER_AGENT", "unstructuredio_oss")
+        )
 
         return WorkspaceClient(config=config)
 

unstructured_ingest/v2/processes/connectors/github.py

@@ -1,12 +1,10 @@
-import fnmatch
 from dataclasses import dataclass, field
 from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional
 from urllib.parse import urlparse
 from uuid import NAMESPACE_DNS, uuid5
 
-import requests
 from pydantic import Field, Secret, field_validator
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
@@ -34,6 +32,7 @@
     from github import ContentFile, GitTreeElement, Repository
     from github import Github as GithubClient
     from github.GithubException import GithubException
+    from requests import HTTPError
 
 CONNECTOR_TYPE = "github"
 
@@ -77,7 +76,7 @@ def wrap_github_exception(self, e: "GithubException") -> Exception:
         logger.debug(f"unhandled github error: {e}")
         return e
 
-    def wrap_http_error(self, e: requests.HTTPError) -> Exception:
+    def wrap_http_error(self, e: "HTTPError") -> Exception:
         status_code = e.response.status_code
         if status_code == 401:
             return UserAuthError(f"Unauthorized access to Github: {e.response.text}")
@@ -88,12 +87,14 @@ def wrap_http_error(self, e: requests.HTTPError) -> Exception:
         logger.debug(f"unhandled http error: {e}")
         return e
 
+    @requires_dependencies(["requests"], extras="github")
     def wrap_error(self, e: Exception) -> Exception:
         from github.GithubException import GithubException
+        from requests import HTTPError
 
         if isinstance(e, GithubException):
             return self.wrap_github_exception(e=e)
-        if isinstance(e, requests.HTTPError):
+        if isinstance(e, HTTPError):
             return self.wrap_http_error(e=e)
         logger.debug(f"unhandled error: {e}")
         return e
@@ -106,11 +107,6 @@ class GithubIndexerConfig(IndexerConfig):
     recursive: bool = Field(
         description="Recursively index all files in the repository", default=True
     )
-    file_glob: Optional[list[str]] = Field(
-        default=None,
-        description="file globs to limit which types of " "files are accepted",
-        examples=["*.pdf", "*.html"],
-    )
 
 
 @dataclass
@@ -137,19 +133,8 @@ def list_files(self) -> list["GitTreeElement"]:
         file_elements = [
             element for element in git_tree.tree if element.size is not None and element.size > 0
         ]
-        if self.index_config.file_glob:
-            file_elements = self.filter_files(files=file_elements)
         return file_elements
 
-    def filter_files(self, files: list["GitTreeElement"]) -> list["GitTreeElement"]:
-        filtered_files = []
-        for file in files:
-            path = file.path
-            for pattern in self.index_config.file_glob:
-                if fnmatch.filter([path], pattern):
-                    filtered_files.append(file)
-        return filtered_files
-
     def convert_element(self, element: "GitTreeElement") -> FileData:
         full_path = (
             f"{self.connection_config.get_full_url()}/blob/{self.get_branch()}/{element.path}"
@@ -204,7 +189,10 @@ def get_file(self, file_data: FileData) -> "ContentFile":
             raise UserError(f"File not found: {path}")
         return content_file
 
+    @requires_dependencies(["requests"], extras="github")
     def get_contents(self, content_file: "ContentFile") -> bytes:
+        import requests
+
         if content_file.decoded_content:
             return content_file.decoded_content
         download_url = content_file.download_url

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -1,4 +1,5 @@
 import json
+import os
 from contextlib import contextmanager
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Generator, Optional
@@ -42,7 +43,6 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
     access_config: Secret[DatabricksDeltaTablesAccessConfig]
     server_hostname: str = Field(description="server hostname connection config value")
     http_path: str = Field(description="http path connection config value")
-    user_agent: str = "unstructuredio_oss"
 
     @requires_dependencies(["databricks"], extras="databricks-delta-tables")
     def get_credentials_provider(self) -> "oauth_service_principal":
@@ -86,7 +86,9 @@ def get_connection(self, **connect_kwargs) -> Generator["DeltaTableConnection",
         from databricks.sql import connect
 
         connect_kwargs = connect_kwargs or {}
-        connect_kwargs["_user_agent_entry"] = self.user_agent
+        connect_kwargs["_user_agent_entry"] = os.getenv(
+            "UNSTRUCTURED_USER_AGENT", "unstructuredio_oss"
+        )
         connect_kwargs["server_hostname"] = connect_kwargs.get(
             "server_hostname", self.server_hostname
         )