Skip to content

Commit 52259f4

Browse files
authored
Add opt-out telemetry (#655)
* Add opt-out telemetry * Fix test name, add docker dirs * Add future import * Mypy: install missing types * Mypy noninteractive * Handle wrong token error
1 parent ca9cae7 commit 52259f4

File tree

12 files changed

+234
-9
lines changed

12 files changed

+234
-9
lines changed

.github/workflows/test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@ jobs:
2626
pip install .[ai]
2727
pip list
2828
- name: Run type checker
29-
run: mypy --ignore-missing-imports --exclude build .
29+
run: mypy --install-types --non-interactive --ignore-missing-imports --exclude build .
3030
- name: Test with pytest
3131
run: pytest

.vscode/extensions.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"recommendations": [
33
"ms-python.python",
44
"ms-python.black-formatter",
5-
"editorconfig.editorconfig"
5+
"ms-python.isort"
66
]
77
}
88

Dockerfile

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ ENV OMP_NUM_THREADS=1
3939
RUN mkdir /app/src && mkdir /app/config && touch /app/config/config.cfg
4040
RUN mkdir /app/static && touch /app/static/index.html
4141
RUN mkdir /app/db && mkdir /app/media && mkdir /app/indexdir && mkdir /app/users
42-
RUN mkdir /app/thumbnail_cache
42+
RUN mkdir /app/thumbnail_cache && mkdir /app/request_cache && mkdir /app/persistent_cache
4343
RUN mkdir /app/cache && mkdir /app/cache/reports && mkdir /app/cache/export
4444
RUN mkdir /app/tmp && mkdir /app/persist
4545
RUN mkdir -p /root/gramps/gramps$GRAMPS_VERSION/plugins
@@ -49,6 +49,8 @@ ENV GRAMPSWEB_MEDIA_BASE_DIR=/app/media
4949
ENV GRAMPSWEB_SEARCH_INDEX_DB_URI=sqlite:////app/indexdir/search_index.db
5050
ENV GRAMPSWEB_STATIC_PATH=/app/static
5151
ENV GRAMPSWEB_THUMBNAIL_CACHE_CONFIG__CACHE_DIR=/app/thumbnail_cache
52+
ENV GRAMPSWEB_REQUEST_CACHE_CONFIG__CACHE_DIR=/app/request_cache
53+
ENV GRAMPSWEB_PERSISTENT_CACHE_CONFIG__CACHE_DIR=/app/persistent_cache
5254
ENV GRAMPSWEB_REPORT_DIR=/app/cache/reports
5355
ENV GRAMPSWEB_EXPORT_DIR=/app/cache/export
5456
ENV GRAMPSHOME=/root

gramps_webapi/api/cache.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
thumbnail_cache = Cache()
2323
request_cache = Cache()
24+
persistent_cache = Cache()
2425

2526

2627
def get_db_last_change_timestamp(tree_id: str) -> int | float | None:

gramps_webapi/api/tasks.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
transaction_to_json,
5454
)
5555
from .search import get_search_indexer, get_semantic_search_indexer
56+
from .telemetry import get_telemetry_payload, send_telemetry, update_telemetry_timestamp
5657
from .util import (
5758
abort_with_message,
5859
check_quota_people,
@@ -547,7 +548,7 @@ def process_transactions(
547548
finally:
548549
close_db(db_handle)
549550
return trans_dict
550-
551+
551552

552553
def handle_delete(trans: DbTxn, class_name: str, handle: str) -> None:
553554
"""Handle a delete action."""
@@ -606,3 +607,12 @@ def update_search_indices_from_transaction(
606607
indexer_semantic.add_or_update_object(handle, db_handle, class_name)
607608
finally:
608609
close_db(db_handle)
610+
611+
612+
@shared_task()
613+
def send_telemetry_task(tree: str):
614+
"""Send telemetry"""
615+
data = get_telemetry_payload(tree_id=tree)
616+
# if the request fails, an exception will be raised
617+
send_telemetry(data=data)
618+
update_telemetry_timestamp()

gramps_webapi/api/telemetry.py

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
"""Optional telemetry for Gramps Web API."""
2+
3+
from __future__ import annotations
4+
5+
import os
6+
import time
7+
import uuid
8+
9+
import requests
10+
from flask import current_app
11+
12+
from gramps_webapi.api.cache import persistent_cache
13+
from gramps_webapi.auth.passwords import hash_password_salt
14+
from gramps_webapi.const import (
15+
TELEMETRY_ENDPOINT,
16+
TELEMETRY_SERVER_ID_KEY,
17+
TELEMETRY_TIMESTAMP_KEY,
18+
)
19+
20+
21+
def send_telemetry(data: dict[str, str | int | float]) -> None:
22+
"""Send telemetry"""
23+
response = requests.post(TELEMETRY_ENDPOINT, json=data, timeout=30)
24+
response.raise_for_status() # Raise exception for HTTP errors
25+
26+
27+
def should_send_telemetry() -> bool:
28+
"""Whether telemetry should be sent."""
29+
if current_app.config.get("DISABLE_TELEMETRY"):
30+
return False
31+
if os.getenv("FLASK_RUN_FROM_CLI"):
32+
# Flask development server, not a production environment (hopefully!)
33+
return False
34+
if (os.environ.get("PYTEST_CURRENT_TEST") or current_app.testing) and not os.getenv(
35+
"MOCK_TELEMETRY"
36+
):
37+
# do not send telemetry during tests unless MOCK_TELEMETRY is set
38+
return False
39+
# only send telemetry if it has not been sent in the last 24 hours
40+
if time.time() - telemetry_last_sent() < 24 * 60 * 60:
41+
return False
42+
return True
43+
44+
45+
def telemetry_last_sent() -> float:
46+
"""Timestamp when telemetry was last sent successfully."""
47+
return persistent_cache.get(TELEMETRY_TIMESTAMP_KEY) or 0.0
48+
49+
50+
def update_telemetry_timestamp() -> None:
51+
"""Update the telemetry timestamp."""
52+
persistent_cache.set(TELEMETRY_TIMESTAMP_KEY, time.time())
53+
54+
55+
def generate_server_uuid() -> str:
56+
"""Generate a random, unique server UUID."""
57+
return uuid.uuid4().hex
58+
59+
60+
def generate_tree_uuid(tree_id: str, server_uuid: str) -> str:
61+
"""Generate a unique tree UUID for the given tree ID and server UUID.
62+
63+
The tree UUID is uniquely determined for a given tree ID and server
64+
UUID but does not allow reconstructing the tree ID.
65+
"""
66+
return hash_password_salt(password=tree_id, salt=server_uuid.encode()).hex()
67+
68+
69+
def get_telemetry_payload(tree_id: str) -> dict[str, str | int | float]:
70+
"""Get the telemetry payload for the given tree ID."""
71+
if not tree_id:
72+
raise ValueError("Tree ID must not be empty")
73+
server_uuid = persistent_cache.get(TELEMETRY_SERVER_ID_KEY)
74+
if not server_uuid:
75+
server_uuid = generate_server_uuid()
76+
persistent_cache.set(TELEMETRY_SERVER_ID_KEY, server_uuid)
77+
tree_uuid = generate_tree_uuid(tree_id=tree_id, server_uuid=server_uuid)
78+
return {
79+
"server_uuid": server_uuid,
80+
"tree_uuid": tree_uuid,
81+
"timestamp": time.time(),
82+
}

gramps_webapi/api/util.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121

2222
from __future__ import annotations
2323

24-
import hashlib
2524
import io
2625
import json
2726
import logging
@@ -46,6 +45,7 @@
4645
request,
4746
)
4847
from flask_jwt_extended import get_jwt, get_jwt_identity
48+
from flask_jwt_extended.exceptions import WrongTokenError
4949
from gramps.cli.clidbman import NAME_FILE, CLIDbManager
5050
from gramps.gen.config import config
5151
from gramps.gen.const import GRAMPS_LOCALE
@@ -373,7 +373,11 @@ def get_tree_from_jwt() -> str | None:
373373
Needs request context. Can return None if no tree ID is present,
374374
e.g. for e-mail confirmation or password reset tokens.
375375
"""
376-
claims = get_jwt()
376+
try:
377+
claims = get_jwt()
378+
except WrongTokenError:
379+
# wrong token type, so no tree ID
380+
return None
377381
return claims.get("tree")
378382

379383

gramps_webapi/app.py

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,18 @@
2727
from flask import Flask, abort, g, send_from_directory
2828
from flask_compress import Compress
2929
from flask_cors import CORS
30-
from flask_jwt_extended import JWTManager
30+
from flask_jwt_extended import JWTManager, verify_jwt_in_request
31+
from flask_jwt_extended.exceptions import WrongTokenError
3132
from gramps.gen.config import config as gramps_config
3233
from gramps.gen.config import set as setconfig
3334

3435
from .api import api_blueprint
35-
from .api.cache import request_cache, thumbnail_cache
36+
from .api.cache import persistent_cache, request_cache, thumbnail_cache
3637
from .api.ratelimiter import limiter
3738
from .api.search.embeddings import load_model
38-
from .api.util import close_db
39+
from .api.tasks import run_task, send_telemetry_task
40+
from .api.telemetry import should_send_telemetry
41+
from .api.util import close_db, get_tree_from_jwt
3942
from .auth import user_db
4043
from .config import DefaultConfig, DefaultConfigJWT
4144
from .const import API_PREFIX, ENV_CONFIG_FILE, TREE_MULTI
@@ -145,6 +148,7 @@ def create_app(config: Optional[Dict[str, Any]] = None, config_from_env: bool =
145148

146149
request_cache.init_app(app, config=app.config["REQUEST_CACHE_CONFIG"])
147150
thumbnail_cache.init_app(app, config=app.config["THUMBNAIL_CACHE_CONFIG"])
151+
persistent_cache.init_app(app, config=app.config["PERSISTENT_CACHE_CONFIG"])
148152

149153
# enable CORS for /api/... resources
150154
if app.config.get("CORS_ORIGINS"):
@@ -180,6 +184,23 @@ def send_static(path):
180184
# instantiate celery
181185
create_celery(app)
182186

187+
@app.before_request
188+
def maybe_send_telemetry() -> None:
189+
"""Send telementry if needed."""
190+
try:
191+
if verify_jwt_in_request(optional=True) is None:
192+
# for requests without JWT, do nothing
193+
return None
194+
except WrongTokenError:
195+
# for the refresh token endpoint, this will fail
196+
return None
197+
tree_id = get_tree_from_jwt()
198+
if not tree_id:
199+
# for endpoints that don't require a JWT, we do nothing
200+
return None
201+
if should_send_telemetry():
202+
run_task(send_telemetry_task, tree=tree_id)
203+
183204
@app.teardown_appcontext
184205
def close_db_connection(exception) -> None:
185206
"""Close the Gramps database after every request."""

gramps_webapi/config.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@ class DefaultConfig(object):
5151
"CACHE_THRESHOLD": 1000,
5252
"CACHE_DEFAULT_TIMEOUT": 0,
5353
}
54+
PERSISTENT_CACHE_CONFIG = {
55+
"CACHE_TYPE": "FileSystemCache",
56+
"CACHE_DIR": str(Path.cwd() / "persistent_cache"),
57+
"CACHE_THRESHOLD": 0,
58+
"CACHE_DEFAULT_TIMEOUT": 0,
59+
}
5460
POSTGRES_USER = None
5561
POSTGRES_PASSWORD = None
5662
POSTGRES_HOST = "localhost"
@@ -69,6 +75,7 @@ class DefaultConfig(object):
6975
LLM_MODEL = ""
7076
LLM_MAX_CONTEXT_LENGTH = 50000
7177
VECTOR_EMBEDDING_MODEL = ""
78+
DISABLE_TELEMETRY = False
7279

7380

7481
class DefaultConfigJWT(object):

gramps_webapi/const.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,3 +195,8 @@
195195

196196
# list of exporters (by file extension) that are not allowed
197197
DISABLED_EXPORTERS = ["gpkg"]
198+
199+
# Settings for the opt-out telemetry
200+
TELEMETRY_ENDPOINT = "https://telemetry-cloud-run-442080026669.europe-west1.run.app"
201+
TELEMETRY_TIMESTAMP_KEY = "telemetry_last_sent"
202+
TELEMETRY_SERVER_ID_KEY = "telemetry_server_uuid"

0 commit comments

Comments
 (0)