Skip to content

Commit afe38f9

Browse files
kelkawi-akelkawi-a
authored
[CSS-20889] - Fix cache backend for global async queries (#90)
* fix cache backend for global async queries * fix scaling integration test * debug scaling itest * update redis db partitions * enable feature flag in scaling itest * remove extra logs * set cache timeout from charm config * update trivyignore * switch to self-hosted runners * update microk8s to 1.35 * Update integration_test.yaml
1 parent ee98cc3 commit afe38f9

File tree

7 files changed

+56
-36
lines changed

7 files changed

+56
-36
lines changed

‎.github/workflows/integration_test.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ jobs:
1313
uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
1414
secrets: inherit
1515
with:
16-
channel: 1.28-strict/stable
16+
test-timeout: 90
17+
channel: 1.35-strict/stable
1718
modules: '["test_charm.py", "test_scaling.py", "test_upgrades.py", "test_major_upgrades.py"]'
1819
juju-channel: 3/stable
1920
self-hosted-runner: false

‎.trivyignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
gcp-service-account
2+
CVE-2025-68121

‎templates/superset_config.py‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -43,32 +43,40 @@
4343

4444
# Redis caching
4545
CACHE_CONFIG = {
46-
"CACHE_TYPE": "redis",
47-
"CACHE_REDIS_URL": f"redis://{os.getenv('REDIS_HOST')}:{os.getenv('REDIS_PORT')}/0",
46+
"CACHE_TYPE": "RedisCache",
47+
"CACHE_DEFAULT_TIMEOUT": int(os.getenv("REDIS_TIMEOUT", 300)),
48+
"CACHE_REDIS_HOST": os.getenv("REDIS_HOST"),
49+
"CACHE_REDIS_PORT": int(os.getenv("REDIS_PORT")),
50+
"CACHE_REDIS_DB": 0,
4851
}
4952
# TALISMAN_ENABLED=True
5053
FILTER_STATE_CACHE_CONFIG = {
5154
"CACHE_TYPE": "RedisCache",
52-
"CACHE_DEFAULT_TIMEOUT": 86400, # 24 hours
55+
"CACHE_DEFAULT_TIMEOUT": int(os.getenv("REDIS_TIMEOUT", 300)),
5356
"CACHE_KEY_PREFIX": "superset_filter_cache",
54-
"CACHE_REDIS_URL": f"redis://{os.getenv('REDIS_HOST')}:{os.getenv('REDIS_PORT')}/1",
57+
"CACHE_REDIS_HOST": os.getenv("REDIS_HOST"),
58+
"CACHE_REDIS_PORT": int(os.getenv("REDIS_PORT")),
59+
"CACHE_REDIS_DB": 1,
5560
}
5661
EXPLORE_FORM_DATA_CACHE_CONFIG = {
5762
"CACHE_TYPE": "RedisCache",
58-
"CACHE_DEFAULT_TIMEOUT": 86400, # 24 hours
63+
"CACHE_DEFAULT_TIMEOUT": int(os.getenv("REDIS_TIMEOUT", 300)),
5964
"CACHE_KEY_PREFIX": "superset_explore_cache",
60-
"CACHE_REDIS_URL": f"redis://{os.getenv('REDIS_HOST')}:{os.getenv('REDIS_PORT')}/2",
65+
"CACHE_REDIS_HOST": os.getenv("REDIS_HOST"),
66+
"CACHE_REDIS_PORT": int(os.getenv("REDIS_PORT")),
67+
"CACHE_REDIS_DB": 2,
6168
}
6269
DATA_CACHE_CONFIG = {
63-
"CACHE_TYPE": "SupersetMetastoreCache",
64-
"CACHE_KEY_PREFIX": "superset_results",
65-
"CACHE_DEFAULT_TIMEOUT": 86400, # 24 hours
66-
"CACHE_REDIS_URL": f"redis://{os.getenv('REDIS_HOST')}:{os.getenv('REDIS_PORT')}/3",
70+
"CACHE_TYPE": "RedisCache",
71+
"CACHE_DEFAULT_TIMEOUT": int(os.getenv("REDIS_TIMEOUT", 300)),
72+
"CACHE_REDIS_HOST": os.getenv("REDIS_HOST"),
73+
"CACHE_REDIS_PORT": int(os.getenv("REDIS_PORT")),
74+
"CACHE_REDIS_DB": 3,
6775
}
6876

6977
RESULTS_BACKEND = RedisCache(
7078
host=os.getenv("REDIS_HOST"),
71-
port=os.getenv("REDIS_PORT"),
79+
port=int(os.getenv("REDIS_PORT")),
7280
key_prefix="superset_results",
7381
)
7482

@@ -155,12 +163,13 @@ class CeleryConfig(object):
155163
imports = (
156164
"superset.sql_lab",
157165
"superset.tasks",
166+
"superset.tasks.async_queries",
158167
)
159168
result_backend = (
160169
f"redis://{os.getenv('REDIS_HOST')}:{os.getenv('REDIS_PORT')}/5"
161170
)
162171
worker_log_level = "DEBUG"
163-
worker_prefetch_multiplier = 10
172+
worker_prefetch_multiplier = 1
164173
task_acks_late = True
165174
task_annotations = {
166175
"sql_lab.get_sql_results": {
@@ -238,14 +247,16 @@ class CeleryConfig(object):
238247

239248
# Asynchronous queries
240249
GLOBAL_ASYNC_QUERIES_REDIS_STREAM_PREFIX = "async-events-"
241-
GLOBAL_ASYNC_QUERIES_JWT_SECRET = os.getenv("GLOBAL_ASYNC_QUERIES_JWT")
242-
GLOBAL_ASYNC_QUERIES_CACHE_BACKEND = RedisCache(
243-
host=os.getenv("REDIS_HOST"),
244-
port=int(os.getenv("REDIS_PORT")),
245-
key_prefix="superset_results",
246-
default_timeout=int(os.getenv("REDIS_TIMEOUT", 300)),
247-
)
248-
GLOBAL_ASYNC_QUERIES_POLLING_DELAY = os.getenv("GLOBAL_ASYNC_QUERIES_POLLING_DELAY")
250+
GLOBAL_ASYNC_QUERIES_JWT_SECRET = os.environ["GLOBAL_ASYNC_QUERIES_JWT"]
251+
GLOBAL_ASYNC_QUERIES_CACHE_BACKEND = {
252+
"CACHE_TYPE": "RedisCache",
253+
"CACHE_KEY_PREFIX": "superset_gaq_",
254+
"CACHE_DEFAULT_TIMEOUT": int(os.getenv("REDIS_TIMEOUT", 300)),
255+
"CACHE_REDIS_HOST": os.getenv("REDIS_HOST"),
256+
"CACHE_REDIS_PORT": int(os.getenv("REDIS_PORT")),
257+
"CACHE_REDIS_DB": 6,
258+
}
259+
GLOBAL_ASYNC_QUERIES_POLLING_DELAY = int(os.getenv("GLOBAL_ASYNC_QUERIES_POLLING_DELAY", "500"))
249260
SECRET_KEY = os.getenv("SUPERSET_SECRET_KEY")
250261

251262
# Log rotation

‎tests/integration/conftest.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ async def deploy(ops_test: OpsTest, charm: str, charm_image: str):
8080
"charm-function": function,
8181
"superset-secret-key": SUPERSET_SECRET_KEY,
8282
"server-alias": UI_NAME,
83+
"feature-flags": "GLOBAL_ASYNC_QUERIES",
8384
}
8485

8586
# Load examples for the UI charm

‎tests/integration/helpers.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
CHARM_FUNCTIONS = {"app-gunicorn": "ui", "beat": "beat", "worker": "worker"}
2020
SCALABLE_SERVICES = {"app-gunicorn": "ui", "worker": "worker"}
2121
API_AUTH_PAYLOAD = {
22-
"username": "admin",
23-
"password": "admin",
22+
"username": "admin", # nosec
23+
"password": "admin", # nosec
2424
"provider": "db",
2525
}
2626
APP_NAME = "superset-k8s"
@@ -254,6 +254,11 @@ async def get_active_workers(ops_test: OpsTest):
254254
redis_ip = status["applications"][REDIS_NAME]["units"][f"{REDIS_NAME}/0"][
255255
"address"
256256
]
257-
app = Celery("superset", broker=f"redis://{redis_ip}:6379/4")
258-
active_workers = app.control.inspect().active()
257+
app = Celery(
258+
"superset",
259+
broker=f"redis://{redis_ip}:6379/4",
260+
)
261+
262+
logger.info("Checking active workers...")
263+
active_workers = app.control.inspect(timeout=10).active()
259264
return active_workers

‎tests/integration/test_scaling.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ async def deploy(ops_test: OpsTest, charm: str, charm_image: str):
5252
"charm-function": function,
5353
"superset-secret-key": SUPERSET_SECRET_KEY,
5454
"server-alias": UI_NAME,
55+
"feature-flags": "GLOBAL_ASYNC_QUERIES",
5556
}
5657

5758
await deploy_and_relate_superset_charm(

‎tests/unit/test_charm.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,9 @@ def test_ready(self):
8282
"startup": "enabled",
8383
"environment": {
8484
"ALLOW_IMAGE_DOMAINS": None,
85-
"SUPERSET_SECRET_KEY": "example-pass",
85+
"SUPERSET_SECRET_KEY": "example-pass", # nosec
8686
"ADMIN_USER": "unique-user",
87-
"ADMIN_PASSWORD": "admin",
87+
"ADMIN_PASSWORD": "admin", # nosec
8888
"CHARM_FUNCTION": "app-gunicorn",
8989
"SQL_ALCHEMY_URI": "postgresql://postgres_user:admin@myhost:5432/superset",
9090
"REDIS_HOST": "redis-host",
@@ -93,8 +93,8 @@ def test_ready(self):
9393
"SQLALCHEMY_POOL_SIZE": 5,
9494
"SQLALCHEMY_POOL_TIMEOUT": 300,
9595
"SQLALCHEMY_MAX_OVERFLOW": 5,
96-
"GOOGLE_KEY": None,
97-
"GOOGLE_SECRET": None,
96+
"GOOGLE_KEY": None, # nosec
97+
"GOOGLE_SECRET": None, # nosec
9898
"OAUTH_DOMAIN": None,
9999
"OAUTH_ADMIN_EMAIL": "admin@superset.com",
100100
"SELF_REGISTRATION_ROLE": "Public",
@@ -188,8 +188,8 @@ def test_config_changed(self):
188188
"SQLALCHEMY_POOL_SIZE": 5,
189189
"SQLALCHEMY_POOL_TIMEOUT": 300,
190190
"SQLALCHEMY_MAX_OVERFLOW": 5,
191-
"GOOGLE_KEY": None,
192-
"GOOGLE_SECRET": None,
191+
"GOOGLE_KEY": None, # nosec
192+
"GOOGLE_SECRET": None, # nosec
193193
"OAUTH_DOMAIN": None,
194194
"OAUTH_ADMIN_EMAIL": "admin@superset.com",
195195
"SELF_REGISTRATION_ROLE": "Public",
@@ -435,7 +435,7 @@ def test_smtp_handling_with_secret(
435435
secret_contents = {
436436
"host": "localhost",
437437
"port": "1025",
438-
"username": "admin",
438+
"username": "admin", # nosec
439439
"password": "testpassword", # nosec
440440
"email": "admin@example.com",
441441
"ssl": "false",
@@ -492,7 +492,7 @@ def test_smtp_handling_with_inaccessible_secret(
492492
secret_contents = {
493493
"host": "localhost",
494494
"port": "1025",
495-
"username": "admin",
495+
"username": "admin", # nosec
496496
"password": "testpassword", # nosec
497497
"email": "admin@example.com",
498498
"ssl": "false",
@@ -525,7 +525,7 @@ def test_smtp_handling_with_improper_secret(
525525

526526
secret_contents = {
527527
"port": "1025",
528-
"username": "admin",
528+
"username": "admin", # nosec
529529
"password": "testpassword", # nosec
530530
"email": "admin@example.com",
531531
"ssl": "false",
@@ -597,6 +597,6 @@ def database_provider_databag():
597597
"""
598598
return {
599599
"endpoints": "myhost:5432,anotherhost:2345",
600-
"username": "postgres_user",
601-
"password": "admin",
600+
"username": "postgres_user", # nosec
601+
"password": "admin", # nosec
602602
}

0 commit comments

Comments
 (0)