Spruce up project a bit

AGENTS.md

@@ -1,29 +1,25 @@
-# Absolute edict
-
 **DO NOT BE OBSEQUIOUS**
 
-# For Agent mode
-
 **NEVER use TEST_MODEL_NAME or "test" embedding model outside of test files**
 
-Never run git commands that many any changes. (`git status` and `git diff` are fine)
+Never run git commands that make any changes. (`git status` and `git diff` are fine)
 
-**NEVER COMMIT CODE. Do not run `git commit` or any other git commands that make changes to the repository. Not even `git add`**
+**NEVER COMMIT CODE. Do not run `git commit` or any other git commands
+that make changes to the repository. Not even `git add`**
 
 When moving, copying or deleting files, use the git commands: `git mv`, `git cp`, `git rm`
 
-When the working directory is ~/typeagent-py:
-
 - Don't use '!' on the command line, it's some bash magic (even inside single quotes)
 - Activate `.venv`: make venv; source .venv/bin/activate
-- To get API keys in ad-hoc code, run `typeagent.aitools.utils.load_dotenv()`
-- Use pytest to run tests in test/
-- Use pyright to check type annotations in tools/, test/, typeagent/
+- To get API keys in ad-hoc code, call `typeagent.aitools.utils.load_dotenv()`
+- Use `pytest test` to run tests in test/
+- Use `pyright` to check type annotations in tools/, test/, typeagent/, gmail/
 - Ignore build/, dist/
 - You can also use the pylance extension for type checking in VS Code
 - Use `make check` to type-check all files
 - Use `make test` to run all tests
 - Use `make check test` to run `make check` and if it passes also run `make test`
+- Use `make format` to format all files using `black`. Do this before reporting success.
 
 ## Package Management with uv
 
@@ -56,8 +52,7 @@ please follow these guidelines:
 
 * Use `Literal` for unions of string literals
 * Keep union notation (`X | Y`) for other unions
-* Use `Protocol` for interfaces whose name starts with `I`
-  followed by a capital letter
+* Use `Protocol` for interfaces whose name starts with `I` followed by a capital letter
 * Use `dataclass` for other classes and structured types
 * Use `type` for type aliases (`PascalCase` again)
 * Use `list`, `tuple`, `dict`, `set` etc., not `List` etc.

Makefile

@@ -8,11 +8,11 @@ all: venv format check test build
 
 .PHONY: format
 format: venv
-	.venv/bin/black typeagent test tools $(FLAGS)
+	.venv/bin/black -tpy312 -tpy313 -tpy314 typeagent test tools gmail $(FLAGS)
 
 .PHONY: check
 check: venv
-	.venv/bin/pyright --pythonpath .venv/bin/python typeagent test tools
+	.venv/bin/pyright --pythonpath .venv/bin/python typeagent test tools gmail
 
 .PHONY: test
 test: venv

README.md

@@ -12,7 +12,7 @@ This is an in-progress project aiming at a Pythonic translation of
 
 ### Documentation
 
-(To be written. Sorry.)
+Found in the [docs directory](docs/README.md)
 
 ## Trademarks
 

TADA.md

@@ -4,13 +4,14 @@ Talk at PyBay is on Sat, Oct 18 in SF
 
 ## Software
 
-- We should change the commit state machine into a context manager
-  ('with storage_provider: ...') rather than requiring the user to do
-  the commit/rollback logic
+Minor:
 
 - Distinguish between release deps and build/dev deps?
+- Improve load_dotenv() (don't look for `<repo>/ts/.env`, use one loop)
 
-### Specifically for VTT import (minor):
+### Specifically for VTT import:
+
+Minor:
 
 - Reduce duplication between ingest_vtt.py and typeagent/transcripts/
 - `get_transcript_speakers` and `get_transcript_duration` should not

docs/README.md

@@ -0,0 +1,14 @@
+# Typeagent docs
+
+## TBD
+
+For now we have:
+
+### High-level API:
+
+- create_conversation
+- [conversation.query](query-method.md)
+
+### Other
+
+- [architecture design](typeagent-architecture.md)

gmail/gmail_dump.py

@@ -1,29 +1,85 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-from pathlib import Path
+import argparse
+import time
 from base64 import urlsafe_b64decode as b64d
+from pathlib import Path
+
+from google.oauth2.credentials import Credentials
 from google_auth_oauthlib.flow import InstalledAppFlow
 from googleapiclient.discovery import build
-from google.oauth2.credentials import Credentials
 
 SCOPES = ["https://www.googleapis.com/auth/gmail.readonly"]
-CREDS = "client_secret.json"
-TOKEN = Path("token.json")
-OUT = Path("mail_dump"); OUT.mkdir(exist_ok=True)
-
-def get_creds():
-    if TOKEN.exists():
-        return Credentials.from_authorized_user_file(TOKEN, SCOPES)
-    flow = InstalledAppFlow.from_client_secrets_file(CREDS, SCOPES)
+OUT = Path("mail_dump")
+
+
+def get_creds(creds_dir: Path):
+    token_file = creds_dir / "token.json"
+    client_secret_file = creds_dir / "client_secret.json"
+
+    if token_file.exists():
+        return Credentials.from_authorized_user_file(token_file, SCOPES)
+    flow = InstalledAppFlow.from_client_secrets_file(client_secret_file, SCOPES)
     creds = flow.run_local_server(port=0)
-    TOKEN.write_text(creds.to_json())
+    token_file.write_text(creds.to_json())
     return creds
 
-svc = build("gmail", "v1", credentials=get_creds())
 
-resp = svc.users().messages().list(userId="me", maxResults=50, q="").execute()
-for m in resp.get("messages", []):
-    raw = svc.users().messages().get(userId="me", id=m["id"], format="raw").execute()["raw"]
-    Path(OUT / f"{m['id']}.eml").write_bytes(b64d(raw.encode()))
-print("Done.")
+def main():
+    parser = argparse.ArgumentParser(
+        description="Download Gmail messages as .eml files"
+    )
+    parser.add_argument(
+        "--max-results",
+        type=int,
+        default=50,
+        help="Maximum number of messages to download (default: 50)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=OUT,
+        help="Output directory for .eml files (default: mail_dump)",
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        default="",
+        help="Gmail search query (default: empty, returns all messages)",
+    )
+    parser.add_argument(
+        "--creds-dir",
+        type=Path,
+        default=Path("."),
+        help="Directory containing client_secret.json and token.json (default: current directory)",
+    )
+    args = parser.parse_args()
+
+    args.output_dir.mkdir(exist_ok=True)
+
+    svc = build("gmail", "v1", credentials=get_creds(args.creds_dir))
+
+    start_time = time.time()
+    resp = (
+        svc.users()
+        .messages()
+        .list(userId="me", maxResults=args.max_results, q=args.query)
+        .execute()
+    )
+    count = 0
+    for m in resp.get("messages", []):
+        raw = (
+            svc.users()
+            .messages()
+            .get(userId="me", id=m["id"], format="raw")
+            .execute()["raw"]
+        )
+        Path(args.output_dir / f"{m['id']}.eml").write_bytes(b64d(raw.encode()))
+        count += 1
+    elapsed = time.time() - start_time
+    print(f"Downloaded {count} messages to {args.output_dir} in {elapsed:.1f}s")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "typeagent"
-version = "0.2.6"
+version = "0.2.7"
 description = "TypeAgent implements an agentic memory framework."
 readme = { file = "README.md", content-type = "text/markdown" }
 authors = [
@@ -54,8 +54,8 @@ dependencies = [
 ]
 
 [project.urls]
-Homepage = "https://github.com/microsoft/TypeAgent/tree/main"
-
+GitHub = "https://github.com/microsoft/typeagent-py"
+Documentation = "https://github.com/microsoft/typeagent-py/tree/main/docs/README.md"
 [tool.setuptools]
 # Needed so setuptools doesn't complain about testdata.
 packages = [