Skip to content

Commit 1158b99

Browse files
ptiurinptiurin
committed
fix bug when expired cache from file would be loaded into memory
1 parent 8d5a8a2 commit 1158b99

File tree

2 files changed

+143
-22
lines changed

2 files changed

+143
-22
lines changed

‎examples/dbapi.ipynb‎

Lines changed: 125 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
},
1313
{
1414
"cell_type": "code",
15-
"execution_count": null,
15+
"execution_count": 1,
1616
"id": "bdd3e404",
1717
"metadata": {},
1818
"outputs": [],
@@ -33,17 +33,44 @@
3333
},
3434
{
3535
"cell_type": "code",
36-
"execution_count": null,
36+
"execution_count": 2,
3737
"id": "0ce8b2d3",
3838
"metadata": {},
3939
"outputs": [],
4040
"source": [
41-
"client_id = \"\"\n",
42-
"client_secret = \"\"\n",
43-
"account_name = \"\"\n",
44-
"engine_name = \"\" # Optional\n",
45-
"database_name = \"\" # Optional\n",
46-
"api_endpoint = DEFAULT_API_URL # Optional"
41+
"client_id = \"fN7Gr5c6lTWrzDol6RwyDOlDQVZFuoUd\"\n",
42+
"client_secret = \"Lh2fznC_iiiV4CEYcOedCAc4Q1FxqwyaXbj5s0MiUoQhVNapvylLuRMvuEnKkySu\"\n",
43+
"account_name = \"developer\"\n",
44+
"engine_name = \"petro_test\" # Optional\n",
45+
"database_name = \"petro_test\" # Optional\n",
46+
"api_endpoint = \"api.staging.firebolt.io\" # Optional"
47+
]
48+
},
49+
{
50+
"cell_type": "code",
51+
"execution_count": 3,
52+
"id": "60eec74a",
53+
"metadata": {},
54+
"outputs": [],
55+
"source": [
56+
"import logging\n",
57+
"\n",
58+
"# Configure logging to display messages in the notebook\n",
59+
"logging.basicConfig(level=logging.WARNING) # Set global level to WARNING to avoid noise\n",
60+
"\n",
61+
"# Set up cache-specific logger with DEBUG level\n",
62+
"cache_logger = logging.getLogger(\"firebolt.utils.cache\")\n",
63+
"cache_logger.setLevel(logging.DEBUG)\n",
64+
"\n",
65+
"# Create a console handler for the cache logger if it doesn't have one\n",
66+
"if not cache_logger.handlers:\n",
67+
" console_handler = logging.StreamHandler()\n",
68+
" console_handler.setLevel(logging.DEBUG)\n",
69+
" formatter = logging.Formatter(\"%(name)s - %(levelname)s - %(message)s\")\n",
70+
" console_handler.setFormatter(formatter)\n",
71+
" cache_logger.addHandler(console_handler)\n",
72+
" # Prevent messages from propagating to the root logger to avoid duplication\n",
73+
" cache_logger.propagate = False"
4774
]
4875
},
4976
{
@@ -56,10 +83,53 @@
5683
},
5784
{
5885
"cell_type": "code",
59-
"execution_count": null,
86+
"execution_count": 4,
87+
"id": "176bb75d",
88+
"metadata": {},
89+
"outputs": [
90+
{
91+
"name": "stderr",
92+
"output_type": "stream",
93+
"text": [
94+
"firebolt.utils.cache - DEBUG - Clearing memory cache\n"
95+
]
96+
}
97+
],
98+
"source": [
99+
"from firebolt.utils.cache import _firebolt_cache\n",
100+
"\n",
101+
"_firebolt_cache.clear()"
102+
]
103+
},
104+
{
105+
"cell_type": "code",
106+
"execution_count": 5,
60107
"id": "646869f7",
61108
"metadata": {},
62-
"outputs": [],
109+
"outputs": [
110+
{
111+
"name": "stderr",
112+
"output_type": "stream",
113+
"text": [
114+
"firebolt.utils.cache - DEBUG - Decryption failed for /Users/petrotiurin/Library/Application Support/firebolt/JG__FqhfFJ-fIbZg_O1xcDiwRJ-Vk1jPnGpPpMMKy8h0jnS179yIl6vfG1hIM8dSkOs8ZiZy6VSjcrD62gWt-imOzPAvxmhSMOFbrSquG5iGCms6SXjQ0UPlUpa4zI0qgf6ncH0Y76TfY7mipq81PZGK69GGfcOFYenkBsoyPgM.txt\n",
115+
"firebolt.utils.cache - DEBUG - Setting value in cache\n",
116+
"firebolt.utils.cache - DEBUG - Writing data to /Users/petrotiurin/Library/Application Support/firebolt/JG__FqhfFJ-fIbZg_O1xcDiwRJ-Vk1jPnGpPpMMKy8h0jnS179yIl6vfG1hIM8dSkOs8ZiZy6VSjcrD62gWt-imOzPAvxmhSMOFbrSquG5iGCms6SXjQ0UPlUpa4zI0qgf6ncH0Y76TfY7mipq81PZGK69GGfcOFYenkBsoyPgM.txt\n",
117+
"firebolt.utils.cache - DEBUG - Cache hit in memory\n",
118+
"firebolt.utils.cache - DEBUG - Cache hit in memory\n",
119+
"firebolt.utils.cache - DEBUG - Setting value in cache\n",
120+
"firebolt.utils.cache - DEBUG - Writing data to /Users/petrotiurin/Library/Application Support/firebolt/JG__FqhfFJ-fIbZg_O1xcDiwRJ-Vk1jPnGpPpMMKy8h0jnS179yIl6vfG1hIM8dSkOs8ZiZy6VSjcrD62gWt-imOzPAvxmhSMOFbrSquG5iGCms6SXjQ0UPlUpa4zI0qgf6ncH0Y76TfY7mipq81PZGK69GGfcOFYenkBsoyPgM.txt\n",
121+
"firebolt.utils.cache - DEBUG - Cache hit in memory\n",
122+
"firebolt.utils.cache - DEBUG - Setting value in cache\n",
123+
"firebolt.utils.cache - DEBUG - Writing data to /Users/petrotiurin/Library/Application Support/firebolt/JG__FqhfFJ-fIbZg_O1xcDiwRJ-Vk1jPnGpPpMMKy8h0jnS179yIl6vfG1hIM8dSkOs8ZiZy6VSjcrD62gWt-imOzPAvxmhSMOFbrSquG5iGCms6SXjQ0UPlUpa4zI0qgf6ncH0Y76TfY7mipq81PZGK69GGfcOFYenkBsoyPgM.txt\n",
124+
"firebolt.utils.cache - DEBUG - Cache hit in memory\n",
125+
"firebolt.utils.cache - DEBUG - Setting value in cache\n",
126+
"firebolt.utils.cache - DEBUG - Writing data to /Users/petrotiurin/Library/Application Support/firebolt/JG__FqhfFJ-fIbZg_O1xcDiwRJ-Vk1jPnGpPpMMKy8h0jnS179yIl6vfG1hIM8dSkOs8ZiZy6VSjcrD62gWt-imOzPAvxmhSMOFbrSquG5iGCms6SXjQ0UPlUpa4zI0qgf6ncH0Y76TfY7mipq81PZGK69GGfcOFYenkBsoyPgM.txt\n",
127+
"firebolt.utils.cache - DEBUG - Cache hit in memory\n",
128+
"firebolt.utils.cache - DEBUG - Setting value in cache\n",
129+
"firebolt.utils.cache - DEBUG - Writing data to /Users/petrotiurin/Library/Application Support/firebolt/JG__FqhfFJ-fIbZg_O1xcDiwRJ-Vk1jPnGpPpMMKy8h0jnS179yIl6vfG1hIM8dSkOs8ZiZy6VSjcrD62gWt-imOzPAvxmhSMOFbrSquG5iGCms6SXjQ0UPlUpa4zI0qgf6ncH0Y76TfY7mipq81PZGK69GGfcOFYenkBsoyPgM.txt\n"
130+
]
131+
}
132+
],
63133
"source": [
64134
"# create a connection based on provided credentials\n",
65135
"connection = connect(\n",
@@ -84,10 +154,39 @@
84154
},
85155
{
86156
"cell_type": "code",
87-
"execution_count": null,
157+
"execution_count": 11,
88158
"id": "4bce6586",
89159
"metadata": {},
90-
"outputs": [],
160+
"outputs": [
161+
{
162+
"name": "stderr",
163+
"output_type": "stream",
164+
"text": [
165+
"/Users/petrotiurin/Development/firebolt-sdk/src/firebolt/db/connection.py:365: UserWarning: Unclosed <firebolt.db.connection.Connection object at 0x105ad8740>\n",
166+
" warn(f\"Unclosed {self!r}\", UserWarning)\n",
167+
"firebolt.utils.cache - DEBUG - Cache hit in memory for key\n",
168+
"firebolt.utils.cache - DEBUG - Setting value in cache for key\n",
169+
"firebolt.utils.cache - DEBUG - Writing data to /Users/petrotiurin/Library/Application Support/firebolt/JG__FqhfFJ-fIbZg_O1xcDiwRJ-Vk1jPnGpPpMMKy8h0jnS179yIl6vfG1hIM8dSkOs8ZiZy6VSjcrD62gWt-imOzPAvxmhSMOFbrSquG5iGCms6SXjQ0UPlUpa4zI0qgf6ncH0Y76TfY7mipq81PZGK69GGfcOFYenkBsoyPgM.txt: {\"id\": \"43a03510f62846cbb398449eacb51853\", \"expiry_time\": 1755856645, \"system_engine\": {\"url\": \"https://01hnj9r1xrx3a4t3kb1ec7qs2b.api.us-east-1.staging.firebolt.io\", \"params\": {}}, \"databases\": {\"petro_test\": {\"name\": \"petro_test\"}}, \"engines\": {\"petro_test\": {\"url\": \"https://developer-firebolt.api.us-east-1.staging.firebolt.io\", \"params\": {\"database\": \"petro_test\", \"engine\": \"petro_test\"}}}, \"token\": \"eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Ik1kMmVfbjlsTlBiSGFPTWJFR3JQZyJ9.eyJodHRwczovL2ZpcmVib2x0LmlvL2NsYWltcyI6eyJjb25uZWN0aW9uX3ByZWZlcmVuY2UiOiJQUkVGRVJfUFVCTElDIiwiZ2xvYmFsX3JvbGVzIjpbIk9SR0FOSVpBVElPTl9BRE1JTiJdLCJpZCI6ImMxNzJkNjJlLWFiM2EtNDVjZS1hYTdkLTA3M2ZlMTRlYzk1NiIsImlzX3N1cHBvcnQiOmZhbHNlLCJvcmdhbml6YXRpb25faWQiOiI5N2FjNzdiZC1iMWY4LTRkNmItOWNkZS02Zjc3ZWVmMWRmODkiLCJvcmdhbml6YXRpb25fbmFtZSI6ImZpcmVib2x0Iiwic2VydmljZV9hY2NvdW50X2lkIjoiYmEzNDcyN2ItMzA1YS00NzM1LTllMTItNDEwMTYwMGM2OTRhIn0sImlzcyI6Imh0dHBzOi8vaWQuc3RhZ2luZy5maXJlYm9sdC5pby8iLCJzdWIiOiJmTjdHcjVjNmxUV3J6RG9sNlJ3eURPbERRVlpGdW9VZEBjbGllbnRzIiwiYXVkIjoiaHR0cHM6Ly9hcGkuZmlyZWJvbHQuaW8iLCJpYXQiOjE3NTU4NTMwNDUsImV4cCI6MTc1NTg2MDI0NSwic2NvcGUiOiJzZXJ2aWNlLWFjY291bnQiLCJndHkiOiJjbGllbnQtY3JlZGVudGlhbHMiLCJhenAiOiJmTjdHcjVjNmxUV3J6RG9sNlJ3eURPbERRVlpGdW9VZCJ9.FHdXEie-c1W6rlsEZaXhD5DoQj8wRFHAhLQBBd7wDQR3uBzHRJOMPnMt6pJuaS3p2FwFeghMu7u3FAYPKz-6qm1QiFOnAapXuAHYZT-_Ft_Pbux0O5dmfP7Vy8DAKW9gpL8djkwg5Hf8LOjzTXPT_iawSZt4reDoG8uokwBpDyqbzqFfsn8bRLEObgbBaPFGLIzR_gLlWvYUKvxcPccykwBjAqmcN2FDRB-9aYetIs6y0C00MYSjzSfOVMH7Y7cVx_SAPooVPmm3gkFJa2FBjYBVNjJz8mm5NJw5_jE7bWQjkrc4bq6O7eNOhmLbIkI7p7QyzNECsnklUUU8hSaoAg\"}\n"
170+
]
171+
},
172+
{
173+
"name": "stdout",
174+
"output_type": "stream",
175+
"text": [
176+
"Writing to file: /Users/petrotiurin/Library/Application Support/firebolt/JG__FqhfFJ-fIbZg_O1xcDiwRJ-Vk1jPnGpPpMMKy8h0jnS179yIl6vfG1hIM8dSkOs8ZiZy6VSjcrD62gWt-imOzPAvxmhSMOFbrSquG5iGCms6SXjQ0UPlUpa4zI0qgf6ncH0Y76TfY7mipq81PZGK69GGfcOFYenkBsoyPgM.txt\n"
177+
]
178+
},
179+
{
180+
"data": {
181+
"text/plain": [
182+
"0"
183+
]
184+
},
185+
"execution_count": 11,
186+
"metadata": {},
187+
"output_type": "execute_result"
188+
}
189+
],
91190
"source": [
92191
"cursor.execute(\n",
93192
" \"create fact table if not exists test_table (id int, name text, dt datetime) primary index id\"\n",
@@ -134,10 +233,19 @@
134233
},
135234
{
136235
"cell_type": "code",
137-
"execution_count": null,
236+
"execution_count": 12,
138237
"id": "71744dc5",
139238
"metadata": {},
140-
"outputs": [],
239+
"outputs": [
240+
{
241+
"name": "stdout",
242+
"output_type": "stream",
243+
"text": [
244+
"Description: [Column(name='id', type_code=<class 'int'>, display_size=None, internal_size=None, precision=None, scale=None, null_ok=None), Column(name='name', type_code=<class 'str'>, display_size=None, internal_size=None, precision=None, scale=None, null_ok=None), Column(name='dt', type_code=<class 'datetime.datetime'>, display_size=None, internal_size=None, precision=None, scale=None, null_ok=None)]\n",
245+
"Rowcount: 3\n"
246+
]
247+
}
248+
],
141249
"source": [
142250
"cursor.execute(\"select * from test_table\")\n",
143251
"print(\"Description: \", cursor.description)\n",
@@ -439,9 +547,9 @@
439547
],
440548
"metadata": {
441549
"kernelspec": {
442-
"display_name": "Python 3 Trio",
550+
"display_name": "venv",
443551
"language": "python",
444-
"name": "python3-trio"
552+
"name": "python3"
445553
},
446554
"language_info": {
447555
"codemirror_mode": {
@@ -453,7 +561,7 @@
453561
"name": "python",
454562
"nbconvert_exporter": "python",
455563
"pygments_lexer": "ipython3",
456-
"version": "3.9.20"
564+
"version": "3.9.19"
457565
}
458566
},
459567
"nbformat": 4,

‎src/firebolt/utils/cache.py‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -143,12 +143,13 @@ def _is_expired(self, value: T) -> bool:
143143
return False
144144

145145
@noop_if_disabled
146-
def set(self, key: ReprCacheable, value: T) -> None:
146+
def set(self, key: ReprCacheable, value: T, preserve_expiry: bool = False) -> None:
147147
if not self.disabled:
148148
# Set expiry_time for ConnectionInfo objects
149149
if hasattr(value, "expiry_time"):
150-
current_time = int(time.time())
151-
value.expiry_time = current_time + CACHE_EXPIRY_SECONDS
150+
if not preserve_expiry or value.expiry_time is None:
151+
current_time = int(time.time())
152+
value.expiry_time = current_time + CACHE_EXPIRY_SECONDS
152153

153154
s_key = self.create_key(key)
154155
self._cache[s_key] = value
@@ -275,11 +276,23 @@ def get(self, key: SecureCacheKey) -> Optional[ConnectionInfo]:
275276
raw_data = self._read_data_json(file_path, encrypter)
276277
if not raw_data:
277278
return None
279+
278280
logger.debug("Cache hit on disk")
279281
data = ConnectionInfo(**raw_data)
280282

281-
# Add to memory cache and return
282-
self.memory_cache.set(key, data)
283+
# Check if the loaded data is expired
284+
if self.memory_cache._is_expired(data):
285+
# Data is expired, delete the file and return None
286+
try:
287+
if path.exists(file_path):
288+
os.remove(file_path)
289+
logger.debug("Deleted expired file %s", file_path)
290+
except OSError:
291+
logger.debug("Failed to delete expired file %s", file_path)
292+
return None
293+
294+
# Data is not expired, add to memory cache preserving original expiry time
295+
self.memory_cache.set(key, data, preserve_expiry=True)
283296
return data
284297

285298
def set(self, key: SecureCacheKey, value: ConnectionInfo) -> None:

0 commit comments

Comments
 (0)