Merge pull request #15 from LaunchPlatform/inbox

Inbox

Author: fangpenlin

.circleci/config.yml

@@ -11,30 +11,26 @@ jobs:
       - image: cimg/python:3.10.14
     steps:
       - checkout
-      - python/install-packages:
-          pkg-manager: poetry
-          args: --all-extras
+      - run:
+          name: Install uv
+          command: pip install uv
       - run:
           name: Run test
-          command: poetry run python -m pytest ./tests -svvvv
+          command: uv run python -m pytest ./tests -svvvv
   build-and-publish:
     docker:
       - image: cimg/python:3.10.14
     steps:
       - checkout
-      - python/install-packages:
-          pkg-manager: poetry
-          args: --all-extras
       - run:
-          name: config
-          command: |
-            poetry config http-basic.pypi "__token__" "${POETRY_PYPI_TOKEN_PYPI}"
+          name: Install uv
+          command: pip install uv
       - run:
           name: Build
-          command: poetry build
+          command: uv build
       - run:
           name: Publish
-          command: poetry publish
+          command: uv publish
 
 workflows:
   build-and-publish:

beanhub_cli/api_helpers.py

@@ -31,11 +31,3 @@ def callee(*args, **kwargs):
         return callee
 
     return decorator
-
-
-def make_auth_client(base_url: str, token: str) -> "AuthenticatedClient":
-    from .internal_api.client import AuthenticatedClient
-
-    return AuthenticatedClient(
-        base_url=base_url, prefix="", auth_header_name="access-token", token=token
-    )

beanhub_cli/connect/config.py beanhub_cli/auth.pybeanhub_cli/connect/config.py renamed to beanhub_cli/auth.py

@@ -3,14 +3,14 @@
 import sys
 import typing
 
-from ..api_helpers import make_auth_client
-from ..config import load_config
+from .config import load_config
+from .internal_api.client import AuthenticatedClient
 
 logger = logging.getLogger(__name__)
 
 
 @dataclasses.dataclass
-class ConnectConfig:
+class AuthConfig:
     token: str
     username: str
     repo: str
@@ -23,7 +23,7 @@ def parse_repo(repo: str | None) -> typing.Tuple[str | None, str | None]:
 
 
 # TODO: maybe extract this part to a shared env for connect command?
-def ensure_config(api_base_url: str, repo: str | None) -> ConnectConfig:
+def ensure_auth_config(api_base_url: str, repo: str | None) -> AuthConfig:
     config = load_config()
     if config is None or config.access_token is None:
         logger.error(
@@ -35,7 +35,7 @@ def ensure_config(api_base_url: str, repo: str | None) -> ConnectConfig:
             "No repo provided, try to determine which repo to use automatically ..."
         )
 
-        from ..internal_api.api.repo import list_repo
+        from .internal_api.api.repo import list_repo
 
         with make_auth_client(
             base_url=api_base_url, token=config.access_token.token
@@ -63,8 +63,14 @@ def ensure_config(api_base_url: str, repo: str | None) -> ConnectConfig:
         username, repo_name = parse_repo(
             repo if repo is not None else config.repo.default
         )
-    return ConnectConfig(
+    return AuthConfig(
         token=config.access_token.token,
         username=username,
         repo=repo_name,
     )
+
+
+def make_auth_client(base_url: str, token: str) -> AuthenticatedClient:
+    return AuthenticatedClient(
+        base_url=base_url, prefix="", auth_header_name="access-token", token=token
+    )

beanhub_cli/connect/file_io.py

@@ -7,21 +7,35 @@
 import time
 
 import click
+import httpx
 import rich
+from nacl.encoding import URLSafeBase64Encoder
+from nacl.public import PrivateKey
+from nacl.public import SealedBox
 from rich import box
 from rich.markup import escape
 from rich.padding import Padding
 from rich.table import Table
 
 from ..api_helpers import handle_api_exception
-from ..api_helpers import make_auth_client
+from ..auth import AuthConfig
+from ..auth import ensure_auth_config
+from ..auth import make_auth_client
 from ..environment import Environment
 from ..environment import pass_env
-from ..utils import check_imports
-from ..utils import ExtraDepsSet
+from ..internal_api.api.connect import create_dump_request
+from ..internal_api.api.connect import create_sync_batch
+from ..internal_api.api.connect import get_dump_request
+from ..internal_api.api.connect import get_sync_batch
+from ..internal_api.models import CreateDumpRequestRequest
+from ..internal_api.models import CreateDumpRequestResponse
+from ..internal_api.models import CreateSyncBatchResponse
+from ..internal_api.models import DumpRequestState
+from ..internal_api.models import GetDumpRequestResponse
+from ..internal_api.models import GetSyncBatchResponse
+from ..internal_api.models import HTTPValidationError
+from ..internal_api.models import PlaidItemSyncState
 from .cli import cli
-from .config import ConnectConfig
-from .config import ensure_config
 
 logger = logging.getLogger(__name__)
 
@@ -30,14 +44,7 @@
 SPOOLED_FILE_MAX_SIZE = 1024 * 1024 * 5
 
 
-def run_sync(env: Environment, config: ConnectConfig):
-    from ..internal_api.api.connect import create_sync_batch
-    from ..internal_api.api.connect import get_sync_batch
-    from ..internal_api.models import CreateSyncBatchResponse
-    from ..internal_api.models import GetSyncBatchResponse
-    from ..internal_api.models import HTTPValidationError
-    from ..internal_api.models import PlaidItemSyncState
-
+def run_sync(env: Environment, config: AuthConfig):
     GOOD_TERMINAL_SYNC_STATES = frozenset(
         [
             PlaidItemSyncState.IMPORT_COMPLETE,
@@ -147,10 +154,9 @@ def run_sync(env: Environment, config: ConnectConfig):
     help='Which repository to run sync on, in "<username>/<repo_name>" format',
 )
 @pass_env
-@check_imports(ExtraDepsSet.LOGIN, logger)
 @handle_api_exception(logger)
 def sync(env: Environment, repo: str | None):
-    config = ensure_config(api_base_url=env.api_base_url, repo=repo)
+    config = ensure_auth_config(api_base_url=env.api_base_url, repo=repo)
     run_sync(env, config)
     env.logger.info("done")
 
@@ -162,7 +168,7 @@ def sync(env: Environment, repo: str | None):
     "-r",
     "--repo",
     type=str,
-    help='Which repository to run sync on, in "<username>/<repo_name>" format',
+    help='Which repository to run dump on, in "<username>/<repo_name>" format',
 )
 @click.option(
     "-s",
@@ -183,7 +189,6 @@ def sync(env: Environment, repo: str | None):
     help="Allow unsafe tar extraction, mostly for Python < 3.11",
 )
 @pass_env
-@check_imports(ExtraDepsSet.CONNECT, logger)
 @handle_api_exception(logger)
 def dump(
     env: Environment,
@@ -192,24 +197,13 @@ def dump(
     output_accounts: str | None,
     unsafe_tar_extract: bool,
 ):
-    import httpx
-    from ..internal_api.api.connect import create_dump_request
-    from ..internal_api.api.connect import get_dump_request
-    from ..internal_api.models import CreateDumpRequestRequest
-    from ..internal_api.models import CreateDumpRequestResponse
-    from ..internal_api.models import DumpRequestState
-    from ..internal_api.models import GetDumpRequestResponse
-    from nacl.encoding import URLSafeBase64Encoder
-    from nacl.public import PrivateKey
-    from nacl.public import SealedBox
-
     if not hasattr(tarfile, "data_filter") and not unsafe_tar_extract:
         logger.error(
             "You need to use Python >= 3.11 in order to safely unpack the downloaded tar file, or you need to pass "
             "in --unsafe-tar-extract argument to allow unsafe tar file extracting"
         )
         sys.exit(-1)
-    config = ensure_config(api_base_url=env.api_base_url, repo=repo)
+    config = ensure_auth_config(api_base_url=env.api_base_url, repo=repo)
     if sync:
         run_sync(env, config)
 
@@ -270,8 +264,8 @@ def dump(
         logger.info("Decrypting downloaded file ...")
 
         # delay import for testing purpose
-        from .encryption import decrypt_file
-        from .file_io import extract_tar
+        from ..encryption import decrypt_file
+        from ..file_io import extract_tar
 
         decrypt_file(
             input_file=encrypted_file, output_file=decrypted_file, key=key, iv=iv
@@ -298,8 +292,8 @@ def dump(
             )
 
             # delay import for testing purpose
-            from .encryption import decrypt_file
-            from .file_io import extract_tar
+            from ..encryption import decrypt_file
+            from ..file_io import extract_tar
 
             decrypt_file(
                 input_file=encrypted_file, output_file=decrypted_file, key=key, iv=iv

beanhub_cli/connect/main.py

@@ -1,14 +1,14 @@
 import io
 
+from cryptography.hazmat.primitives import padding
+from cryptography.hazmat.primitives.ciphers import algorithms
+from cryptography.hazmat.primitives.ciphers import Cipher
+from cryptography.hazmat.primitives.ciphers import modes
+
 
 def decrypt_file(
     input_file: io.BytesIO, output_file: io.BytesIO, iv: bytes, key: bytes
 ):
-    from cryptography.hazmat.primitives import padding
-    from cryptography.hazmat.primitives.ciphers import algorithms
-    from cryptography.hazmat.primitives.ciphers import Cipher
-    from cryptography.hazmat.primitives.ciphers import modes
-
     cipher = Cipher(algorithms.AES256(key), modes.CBC(iv))
     decryptor = cipher.decryptor()
     padder = padding.PKCS7(128).unpadder()

beanhub_cli/connect/encryption.py beanhub_cli/encryption.pybeanhub_cli/connect/encryption.py renamed to beanhub_cli/encryption.py

@@ -0,0 +1,55 @@
+import io
+import logging
+import pathlib
+import sys
+import tarfile
+
+
+def extract_tar(input_file: io.BytesIO, logger: logging.Logger):
+    with tarfile.open(fileobj=input_file, mode="r:gz") as tar_file:
+        if hasattr(tarfile, "data_filter"):
+            tar_file.extractall(filter="data")
+        else:
+            logger.warning("Performing unsafe tar file extracting")
+            tar_file.extractall()
+
+
+def extract_inbox_tar(
+    input_file: io.BytesIO,
+    email_output_paths: dict[str, pathlib.Path],
+    workdir_path: pathlib.Path,
+    unsafe_tar_extract: bool,
+    logger: logging.Logger,
+):
+    with tarfile.open(fileobj=input_file, mode="r:gz") as tar_file:
+        for member in tar_file:
+            if not member.isreg():
+                continue
+            member_path = pathlib.PurePosixPath(member.name)
+            email_id = member_path.stem
+            output_path = email_output_paths.get(email_id)
+            if output_path is None:
+                logger.error("Cannot find output path for email %s", email_id)
+                sys.exit(-1)
+            output_path.parent.mkdir(exist_ok=True, parents=True)
+            logger.info(
+                "Writing email [green]%s[/] to [green]%s[/]",
+                email_id,
+                output_path,
+                extra={"markup": True, "highlighter": None},
+            )
+            full_output_path = workdir_path / output_path
+            has_data_filter = hasattr(tarfile, "data_filter")
+            if not has_data_filter and not unsafe_tar_extract:
+                logger.error(
+                    "You need to use Python >= 3.11 in order to safely unpack the downloaded tar file, or you need to pass "
+                    "in --unsafe-tar-extract argument to allow unsafe tar file extracting"
+                )
+                sys.exit(-1)
+            member.name = output_path.name
+            tar_file.extract(
+                member,
+                full_output_path.parent,
+                set_attrs=False,
+                filter="data" if has_data_filter else None,
+            )

beanhub_cli/file_io.py

@@ -46,7 +46,7 @@
     "--workdir",
     type=click.Path(exists=True, dir_okay=True, file_okay=False),
     default=str(pathlib.Path.cwd()),
-    help="The beanhub project path to work on",
+    help="The BeanHub project path to work on",
 )
 @click.option(
     "-b",
@@ -111,7 +111,7 @@ def main(
     env.logger.info("Skipped %s transactions", len(unprocessed_txns))
 
     beanfile_path = (workdir_path / pathlib.Path(beanfile)).resolve()
-    if workdir_path.resolve().absolute() not in beanfile_path.absolute().parents:
+    if not beanfile_path.is_relative_to(workdir_path.resolve()):
         env.logger.error(
             "The provided beanfile path %s is not a sub-path of workdir %s",
             beanfile_path,

beanhub_cli/import_cli.py

@@ -0,0 +1,11 @@
+from ..aliase import AliasedGroup
+from ..cli import cli as root_cli
+
+
+@root_cli.group(
+    name="inbox",
+    help="BeanHub inbox features, such as dump (login required) or extract.",
+    cls=AliasedGroup,
+)
+def cli():
+    pass