Skip to content

Commit c90db97

Browse files
ceviancevian
committed
Add ability to have predicates on uuid timestamp
1 parent 2ae8fa6 commit c90db97

File tree

2 files changed

+133
-47
lines changed

2 files changed

+133
-47
lines changed

‎nbs/00_vector.ipynb

Lines changed: 120 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
"metadata": {},
4747
"outputs": [],
4848
"source": [
49-
"_ = load_dotenv(find_dotenv())\n",
49+
"_ = load_dotenv(find_dotenv(), override=True)\n",
5050
"service_url = os.environ['TIMESCALE_SERVICE_URL']"
5151
]
5252
},
@@ -455,15 +455,25 @@
455455
" operator = self.operators_mapping[operator]\n",
456456
" else:\n",
457457
" raise ValueError(\"Invalid clause format\")\n",
458-
" \n",
458+
"\n",
459+
" index = len(params)+1\n",
460+
" param_name = f\"${index}\"\n",
461+
"\n",
462+
" if field == '__uuid_timestamp':\n",
463+
" #convert str to timestamp in the database, it's better at it than python\n",
464+
" if isinstance(value, str):\n",
465+
" where_conditions.append(f\"uuid_timestamp(id) {operator} ({param_name}::text)::timestamptz\")\n",
466+
" else:\n",
467+
" where_conditions.append(f\"uuid_timestamp(id) {operator} {param_name}\")\n",
468+
" params.append(value)\n",
469+
" continue\n",
470+
"\n",
459471
" field_cast = ''\n",
460472
" if isinstance(value, int):\n",
461473
" field_cast = '::int'\n",
462474
" elif isinstance(value, float):\n",
463475
" field_cast = '::numeric' \n",
464476
"\n",
465-
" index = len(params)+1\n",
466-
" param_name = f\"${index}\"\n",
467477
" where_conditions.append(f\"(metadata->>'{field}'){field_cast} {operator} {param_name}\")\n",
468478
" params.append(value) \n",
469479
"\n",
@@ -830,7 +840,7 @@
830840
"text/markdown": [
831841
"---\n",
832842
"\n",
833-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L321){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
843+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L475){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
834844
"\n",
835845
"### QueryBuilder.get_create_query\n",
836846
"\n",
@@ -841,7 +851,7 @@
841851
"text/plain": [
842852
"---\n",
843853
"\n",
844-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L321){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
854+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L475){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
845855
"\n",
846856
"### QueryBuilder.get_create_query\n",
847857
"\n",
@@ -1137,7 +1147,7 @@
11371147
"text/markdown": [
11381148
"---\n",
11391149
"\n",
1140-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L653){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1150+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L843){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
11411151
"\n",
11421152
"### Async.create_tables\n",
11431153
"\n",
@@ -1148,7 +1158,7 @@
11481158
"text/plain": [
11491159
"---\n",
11501160
"\n",
1151-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L653){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1161+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L843){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
11521162
"\n",
11531163
"### Async.create_tables\n",
11541164
"\n",
@@ -1176,7 +1186,7 @@
11761186
"text/markdown": [
11771187
"---\n",
11781188
"\n",
1179-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L653){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1189+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L843){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
11801190
"\n",
11811191
"### Async.create_tables\n",
11821192
"\n",
@@ -1187,7 +1197,7 @@
11871197
"text/plain": [
11881198
"---\n",
11891199
"\n",
1190-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L653){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1200+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L843){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
11911201
"\n",
11921202
"### Async.create_tables\n",
11931203
"\n",
@@ -1227,7 +1237,7 @@
12271237
"text/markdown": [
12281238
"---\n",
12291239
"\n",
1230-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L753){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1240+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L944){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
12311241
"\n",
12321242
"### Async.search\n",
12331243
"\n",
@@ -1250,7 +1260,7 @@
12501260
"text/plain": [
12511261
"---\n",
12521262
"\n",
1253-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L753){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1263+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L944){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
12541264
"\n",
12551265
"### Async.search\n",
12561266
"\n",
@@ -1490,24 +1500,52 @@
14901500
" (uuid_from_time(specific_datetime), {\"key\": \"val\"}, \"the brown fox\", [1.0, 1.2])\n",
14911501
"])\n",
14921502
"assert not await vec.table_is_empty()\n",
1493-
"rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7), specific_datetime+timedelta(days=7)))\n",
1494-
"assert len(rec) == 1\n",
1495-
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": specific_datetime-timedelta(days=7), \"__end_date\": specific_datetime+timedelta(days=7)})\n",
1496-
"assert len(rec) == 1\n",
1497-
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": str(specific_datetime-timedelta(days=7)), \"__end_date\": str(specific_datetime+timedelta(days=7))})\n",
1498-
"assert len(rec) == 1\n",
1499-
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": str(specific_datetime-timedelta(days=7))})\n",
1500-
"assert len(rec) == 2\n",
1501-
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__end_date\": str(specific_datetime+timedelta(days=7))})\n",
1502-
"assert len(rec) == 1\n",
1503-
"rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7), specific_datetime-timedelta(days=2)))\n",
1504-
"assert len(rec) == 0\n",
1505-
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": specific_datetime-timedelta(days=7), \"__end_date\": specific_datetime-timedelta(days=2)})\n",
1506-
"assert len(rec) == 0\n",
1507-
"rec = await vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": str(specific_datetime-timedelta(days=7)), \"__end_date\": str(specific_datetime-timedelta(days=2))})\n",
1508-
"assert len(rec) == 0\n",
1509-
"rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7)))\n",
1510-
"assert len(rec) == 2\n",
1503+
"\n",
1504+
"#check all the possible ways to specify a date range\n",
1505+
"async def search_date(start_date, end_date, expected):\n",
1506+
" #using uuid_time_filter\n",
1507+
" rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(start_date, end_date))\n",
1508+
" assert len(rec) == expected\n",
1509+
" \n",
1510+
" #using filters\n",
1511+
" filter = {}\n",
1512+
" if start_date is not None:\n",
1513+
" filter[\"__start_date\"] = start_date\n",
1514+
" if end_date is not None:\n",
1515+
" filter[\"__end_date\"] = end_date\n",
1516+
" rec = await vec.search([1.0, 2.0], limit=4, filter=filter)\n",
1517+
" assert len(rec) == expected\n",
1518+
" #using filters with string dates\n",
1519+
" filter = {}\n",
1520+
" if start_date is not None:\n",
1521+
" filter[\"__start_date\"] = str(start_date)\n",
1522+
" if end_date is not None:\n",
1523+
" filter[\"__end_date\"] = str(end_date)\n",
1524+
" rec = await vec.search([1.0, 2.0], limit=4, filter=filter)\n",
1525+
" assert len(rec) == expected\n",
1526+
" #using predicates\n",
1527+
" predicates = []\n",
1528+
" if start_date is not None:\n",
1529+
" predicates.append((\"__uuid_timestamp\", \">=\", start_date))\n",
1530+
" if end_date is not None:\n",
1531+
" predicates.append((\"__uuid_timestamp\", \"<\", end_date))\n",
1532+
" rec = await vec.search([1.0, 2.0], limit=4, predicates=Predicates(*predicates))\n",
1533+
" assert len(rec) == expected\n",
1534+
" #using predicates with string dates\n",
1535+
" predicates = []\n",
1536+
" if start_date is not None:\n",
1537+
" predicates.append((\"__uuid_timestamp\", \">=\", str(start_date)))\n",
1538+
" if end_date is not None:\n",
1539+
" predicates.append((\"__uuid_timestamp\", \"<\", str(end_date)))\n",
1540+
" rec = await vec.search([1.0, 2.0], limit=4, predicates=Predicates(*predicates))\n",
1541+
" assert len(rec) == expected\n",
1542+
"\n",
1543+
"await search_date(specific_datetime-timedelta(days=7), specific_datetime+timedelta(days=7), 1)\n",
1544+
"await search_date(specific_datetime-timedelta(days=7), None, 2)\n",
1545+
"await search_date(None, specific_datetime+timedelta(days=7), 1)\n",
1546+
"await search_date(specific_datetime-timedelta(days=7), specific_datetime-timedelta(days=2), 0)\n",
1547+
"\n",
1548+
"#check timedelta handling\n",
15111549
"rec = await vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(start_date=specific_datetime, time_delta=timedelta(days=7)))\n",
15121550
"assert len(rec) == 1\n",
15131551
"#end is exclusive\n",
@@ -1877,7 +1915,7 @@
18771915
"text/markdown": [
18781916
"---\n",
18791917
"\n",
1880-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L955){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1918+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1147){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
18811919
"\n",
18821920
"### Sync.create_tables\n",
18831921
"\n",
@@ -1888,7 +1926,7 @@
18881926
"text/plain": [
18891927
"---\n",
18901928
"\n",
1891-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L955){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1929+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1147){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
18921930
"\n",
18931931
"### Sync.create_tables\n",
18941932
"\n",
@@ -1916,7 +1954,7 @@
19161954
"text/markdown": [
19171955
"---\n",
19181956
"\n",
1919-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L935){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1957+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1127){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
19201958
"\n",
19211959
"### Sync.upsert\n",
19221960
"\n",
@@ -1932,7 +1970,7 @@
19321970
"text/plain": [
19331971
"---\n",
19341972
"\n",
1935-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L935){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1973+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1127){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
19361974
"\n",
19371975
"### Sync.upsert\n",
19381976
"\n",
@@ -1965,7 +2003,7 @@
19652003
"text/markdown": [
19662004
"---\n",
19672005
"\n",
1968-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1072){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
2006+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1262){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
19692007
"\n",
19702008
"### Sync.search\n",
19712009
"\n",
@@ -1988,7 +2026,7 @@
19882026
"text/plain": [
19892027
"---\n",
19902028
"\n",
1991-
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1072){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
2029+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L1262){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
19922030
"\n",
19932031
"### Sync.search\n",
19942032
"\n",
@@ -2205,15 +2243,53 @@
22052243
" #time in 2018\n",
22062244
" (uuid_from_time(specific_datetime), {\"key\": \"val\"}, \"the brown fox\", [1.0, 1.2])\n",
22072245
"])\n",
2246+
"\n",
2247+
"def search_date(start_date, end_date, expected):\n",
2248+
" #using uuid_time_filter\n",
2249+
" rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(start_date, end_date))\n",
2250+
" assert len(rec) == expected\n",
2251+
" \n",
2252+
" #using filters\n",
2253+
" filter = {}\n",
2254+
" if start_date is not None:\n",
2255+
" filter[\"__start_date\"] = start_date\n",
2256+
" if end_date is not None:\n",
2257+
" filter[\"__end_date\"] = end_date\n",
2258+
" rec = vec.search([1.0, 2.0], limit=4, filter=filter)\n",
2259+
" assert len(rec) == expected\n",
2260+
" #using filters with string dates\n",
2261+
" filter = {}\n",
2262+
" if start_date is not None:\n",
2263+
" filter[\"__start_date\"] = str(start_date)\n",
2264+
" if end_date is not None:\n",
2265+
" filter[\"__end_date\"] = str(end_date)\n",
2266+
" rec = vec.search([1.0, 2.0], limit=4, filter=filter)\n",
2267+
" assert len(rec) == expected\n",
2268+
" #using predicates\n",
2269+
" predicates = []\n",
2270+
" if start_date is not None:\n",
2271+
" predicates.append((\"__uuid_timestamp\", \">=\", start_date))\n",
2272+
" if end_date is not None:\n",
2273+
" predicates.append((\"__uuid_timestamp\", \"<\", end_date))\n",
2274+
" rec = vec.search([1.0, 2.0], limit=4, predicates=Predicates(*predicates))\n",
2275+
" assert len(rec) == expected\n",
2276+
" #using predicates with string dates\n",
2277+
" predicates = []\n",
2278+
" if start_date is not None:\n",
2279+
" predicates.append((\"__uuid_timestamp\", \">=\", str(start_date)))\n",
2280+
" if end_date is not None:\n",
2281+
" predicates.append((\"__uuid_timestamp\", \"<\", str(end_date)))\n",
2282+
" rec = vec.search([1.0, 2.0], limit=4, predicates=Predicates(*predicates))\n",
2283+
" assert len(rec) == expected\n",
2284+
"\n",
22082285
"assert not vec.table_is_empty()\n",
2209-
"rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7), specific_datetime+timedelta(days=7)))\n",
2210-
"assert len(rec) == 1\n",
2211-
"rec = vec.search([1.0, 2.0], limit=4, filter={\"__start_date\": specific_datetime-timedelta(days=7), \"__end_date\": specific_datetime+timedelta(days=7)})\n",
2212-
"assert len(rec) == 1\n",
2213-
"rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7), specific_datetime-timedelta(days=2)))\n",
2214-
"assert len(rec) == 0\n",
2215-
"rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(specific_datetime-timedelta(days=7)))\n",
2216-
"assert len(rec) == 2\n",
2286+
"\n",
2287+
"search_date(specific_datetime-timedelta(days=7), specific_datetime+timedelta(days=7), 1)\n",
2288+
"search_date(specific_datetime-timedelta(days=7), None, 2)\n",
2289+
"search_date(None, specific_datetime+timedelta(days=7), 1)\n",
2290+
"search_date(specific_datetime-timedelta(days=7), specific_datetime-timedelta(days=2), 0)\n",
2291+
"\n",
2292+
"#check timedelta handling\n",
22172293
"rec = vec.search([1.0, 2.0], limit=4, uuid_time_filter=UUIDTimeRange(start_date=specific_datetime, time_delta=timedelta(days=7)))\n",
22182294
"assert len(rec) == 1\n",
22192295
"#end is exclusive\n",

‎timescale_vector/client.py

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -351,15 +351,25 @@ def build_query(self, params: List) -> Tuple[str, List]:
351351
operator = self.operators_mapping[operator]
352352
else:
353353
raise ValueError("Invalid clause format")
354-
354+
355+
index = len(params)+1
356+
param_name = f"${index}"
357+
358+
if field == '__uuid_timestamp':
359+
#convert str to timestamp in the database, it's better at it than python
360+
if isinstance(value, str):
361+
where_conditions.append(f"uuid_timestamp(id) {operator} ({param_name}::text)::timestamptz")
362+
else:
363+
where_conditions.append(f"uuid_timestamp(id) {operator} {param_name}")
364+
params.append(value)
365+
continue
366+
355367
field_cast = ''
356368
if isinstance(value, int):
357369
field_cast = '::int'
358370
elif isinstance(value, float):
359371
field_cast = '::numeric'
360372

361-
index = len(params)+1
362-
param_name = f"${index}"
363373
where_conditions.append(f"(metadata->>'{field}'){field_cast} {operator} {param_name}")
364374
params.append(value)
365375

0 commit comments

Comments
 (0)