Skip to content

Commit 19ac338

Browse files
awln-temporalawln-temporal
authored
Add Archetype search attributes for CHASM Visibility (#8390)
## What changed? Adds Archetype search attributes schema for CHASM Visibility. Functions similarly to custom search attributes, where pre-allocated column names or explicit properties, called field names, will map to user aliased Archetype search attributes. Adds Archetype search attributes to ClusterMetadata to save all pre-allocated field name to type mappings in the Cassandra backed ClusterMetadata store. ## Why? Required for CHASM Visibility support. Component authors will register a component with tagged attribute values for querying (List, Count APIs for now), and executions of the registered component will be tagged with these attributes. Archetype name used for querying is left out of this schema change, may require further discussion. ## How did you test it? - [X] built - [X] run locally and tested manually - [ ] added new unit test(s) - [ ] added new functional test(s)
1 parent 3c51318 commit 19ac338

File tree

13 files changed

+495
-12
lines changed

13 files changed

+495
-12
lines changed

‎schema/elasticsearch/visibility/index_template_v7.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
./versioned/v9/index_template_v7.json
1+
./versioned/v11/index_template_v7.json

‎schema/elasticsearch/visibility/versioned/v11/index_template_v7.json‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
{
2+
"order": 0,
3+
"index_patterns": ["temporal_visibility_v1*"],
4+
"settings": {
5+
"index": {
6+
"number_of_shards": "1",
7+
"number_of_replicas": "0",
8+
"auto_expand_replicas": "0-2",
9+
"search.idle.after": "365d",
10+
"sort.field": ["CloseTime", "StartTime", "RunId"],
11+
"sort.order": ["desc", "desc", "desc"],
12+
"sort.missing": ["_first", "_first", "_first"]
13+
}
14+
},
15+
"mappings": {
16+
"dynamic": "false",
17+
"properties": {
18+
"NamespaceId": {
19+
"type": "keyword"
20+
},
21+
"TemporalNamespaceDivision": {
22+
"type": "keyword"
23+
},
24+
"WorkflowId": {
25+
"type": "keyword"
26+
},
27+
"RunId": {
28+
"type": "keyword"
29+
},
30+
"WorkflowType": {
31+
"type": "keyword"
32+
},
33+
"StartTime": {
34+
"type": "date_nanos"
35+
},
36+
"ExecutionTime": {
37+
"type": "date_nanos"
38+
},
39+
"CloseTime": {
40+
"type": "date_nanos"
41+
},
42+
"ExecutionDuration": {
43+
"type": "long"
44+
},
45+
"ExecutionStatus": {
46+
"type": "keyword"
47+
},
48+
"TaskQueue": {
49+
"type": "keyword"
50+
},
51+
"TemporalChangeVersion": {
52+
"type": "keyword"
53+
},
54+
"BatcherNamespace": {
55+
"type": "keyword"
56+
},
57+
"BatcherUser": {
58+
"type": "keyword"
59+
},
60+
"BinaryChecksums": {
61+
"type": "keyword"
62+
},
63+
"HistoryLength": {
64+
"type": "long"
65+
},
66+
"StateTransitionCount": {
67+
"type": "long"
68+
},
69+
"TemporalScheduledStartTime": {
70+
"type": "date_nanos"
71+
},
72+
"TemporalScheduledById": {
73+
"type": "keyword"
74+
},
75+
"TemporalSchedulePaused": {
76+
"type": "boolean"
77+
},
78+
"HistorySizeBytes": {
79+
"type": "long"
80+
},
81+
"BuildIds": {
82+
"type": "keyword"
83+
},
84+
"ParentWorkflowId": {
85+
"type": "keyword"
86+
},
87+
"ParentRunId": {
88+
"type": "keyword"
89+
},
90+
"RootWorkflowId": {
91+
"type": "keyword"
92+
},
93+
"RootRunId": {
94+
"type": "keyword"
95+
},
96+
"TemporalPauseInfo": {
97+
"type": "keyword"
98+
},
99+
"TemporalWorkerDeploymentVersion": {
100+
"type": "keyword"
101+
},
102+
"TemporalWorkflowVersioningBehavior": {
103+
"type": "keyword"
104+
},
105+
"TemporalWorkerDeployment": {
106+
"type": "keyword"
107+
},
108+
"TemporalReportedProblems": {
109+
"type": "keyword"
110+
},
111+
"TemporalBool01": {
112+
"type": "boolean"
113+
},
114+
"TemporalBool02": {
115+
"type": "boolean"
116+
},
117+
"TemporalDatetime01": {
118+
"type": "date_nanos"
119+
},
120+
"TemporalDatetime02": {
121+
"type": "date_nanos"
122+
},
123+
"TemporalDouble01": {
124+
"type": "scaled_float",
125+
"scaling_factor": 10000
126+
},
127+
"TemporalDouble02": {
128+
"type": "scaled_float",
129+
"scaling_factor": 10000
130+
},
131+
"TemporalInt01": {
132+
"type": "long"
133+
},
134+
"TemporalInt02": {
135+
"type": "long"
136+
},
137+
"TemporalKeyword01": {
138+
"type": "keyword"
139+
},
140+
"TemporalKeyword02": {
141+
"type": "keyword"
142+
},
143+
"TemporalKeyword03": {
144+
"type": "keyword"
145+
},
146+
"TemporalKeyword04": {
147+
"type": "keyword"
148+
},
149+
"TemporalKeywordList01": {
150+
"type": "keyword"
151+
},
152+
"TemporalKeywordList02": {
153+
"type": "keyword"
154+
}
155+
}
156+
},
157+
"aliases": {}
158+
}

‎schema/elasticsearch/visibility/versioned/v11/upgrade.sh‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu -o pipefail
4+
5+
# Prerequisites:
6+
# - jq
7+
# - curl
8+
9+
# Input parameters.
10+
: "${ES_SCHEME:=http}"
11+
: "${ES_SERVER:=127.0.0.1}"
12+
: "${ES_PORT:=9200}"
13+
: "${ES_USER:=}"
14+
: "${ES_PWD:=}"
15+
: "${ES_VERSION:=v7}"
16+
: "${ES_VIS_INDEX_V1:=temporal_visibility_v1_dev}"
17+
: "${AUTO_CONFIRM:=}"
18+
: "${SLICES_COUNT:=auto}"
19+
20+
es_endpoint="${ES_SCHEME}://${ES_SERVER}:${ES_PORT}"
21+
22+
echo "=== Step 0. Sanity check if Elasticsearch index is accessible ==="
23+
24+
if ! curl --silent --fail --user "${ES_USER}":"${ES_PWD}" "${es_endpoint}/${ES_VIS_INDEX_V1}/_stats/docs" --write-out "\n"; then
25+
echo "Elasticsearch index ${ES_VIS_INDEX_V1} is not accessible at ${es_endpoint}."
26+
exit 1
27+
fi
28+
29+
echo "=== Step 1. Add new builtin search attributes ==="
30+
31+
new_mapping='
32+
{
33+
"properties": {
34+
"TemporalBool01": {
35+
"type": "boolean"
36+
},
37+
"TemporalBool02": {
38+
"type": "boolean"
39+
},
40+
"TemporalDatetime01": {
41+
"type": "date_nanos"
42+
},
43+
"TemporalDatetime02": {
44+
"type": "date_nanos"
45+
},
46+
"TemporalDouble01": {
47+
"type": "scaled_float",
48+
"scaling_factor": 10000
49+
},
50+
"TemporalDouble02": {
51+
"type": "scaled_float",
52+
"scaling_factor": 10000
53+
},
54+
"TemporalInt01": {
55+
"type": "long"
56+
},
57+
"TemporalInt02": {
58+
"type": "long"
59+
},
60+
"TemporalKeyword01": {
61+
"type": "keyword"
62+
},
63+
"TemporalKeyword02": {
64+
"type": "keyword"
65+
},
66+
"TemporalKeyword03": {
67+
"type": "keyword"
68+
},
69+
"TemporalKeyword04": {
70+
"type": "keyword"
71+
},
72+
"TemporalKeywordList01": {
73+
"type": "keyword"
74+
},
75+
"TemporalKeywordList02": {
76+
"type": "keyword"
77+
}
78+
}
79+
}
80+
'
81+
82+
if [ -z "${AUTO_CONFIRM}" ]; then
83+
read -p "Add new builtin search attributes to the index ${ES_VIS_INDEX_V1}? (N/y)" -n 1 -r
84+
echo
85+
else
86+
REPLY="y"
87+
fi
88+
if [ "${REPLY}" = "y" ]; then
89+
curl --silent --fail --user "${ES_USER}":"${ES_PWD}" -X PUT "${es_endpoint}/${ES_VIS_INDEX_V1}/_mapping" -H "Content-Type: application/json" --data-binary "$new_mapping" | jq
90+
# Wait for mapping changes to go through.
91+
until curl --silent --user "${ES_USER}":"${ES_PWD}" "${es_endpoint}/_cluster/health/${ES_VIS_INDEX_V1}" | jq --exit-status '.status=="green" | .'; do
92+
echo "Waiting for Elasticsearch index ${ES_VIS_INDEX_V1} become green."
93+
sleep 1
94+
done
95+
fi

‎schema/embed.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func ElasticsearchClusterSettings() (string, error) {
5858

5959
// ElasticsearchIndexTemplate returns the embedded index template for Elasticsearch v7 (latest version)
6060
func ElasticsearchIndexTemplate() (string, error) {
61-
data, err := assets.ReadFile("elasticsearch/visibility/versioned/v9/index_template_v7.json")
61+
data, err := assets.ReadFile("elasticsearch/visibility/versioned/v11/index_template_v7.json")
6262
if err != nil {
6363
return "", err
6464
}

‎schema/mysql/v8/version.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ package v8
66
const Version = "1.18"
77

88
// VisibilityVersion is the MySQL visibility database release version
9-
const VisibilityVersion = "1.10"
9+
const VisibilityVersion = "1.11"

‎schema/mysql/v8/visibility/schema.sql‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,3 +168,55 @@ CREATE FULLTEXT INDEX by_text_03 ON custom_search_attributes (Text03);
168168
CREATE INDEX by_keyword_list_01 ON custom_search_attributes (namespace_id, (CAST(KeywordList01 AS CHAR(255) ARRAY)));
169169
CREATE INDEX by_keyword_list_02 ON custom_search_attributes (namespace_id, (CAST(KeywordList02 AS CHAR(255) ARRAY)));
170170
CREATE INDEX by_keyword_list_03 ON custom_search_attributes (namespace_id, (CAST(KeywordList03 AS CHAR(255) ARRAY)));
171+
172+
CREATE TABLE chasm_search_attributes (
173+
namespace_id CHAR(64) NOT NULL,
174+
run_id CHAR(64) NOT NULL,
175+
_version BIGINT NOT NULL DEFAULT 0,
176+
search_attributes JSON NULL,
177+
178+
-- Pre-allocated CHASM search attributes
179+
TemporalBool01 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.TemporalBool01"),
180+
TemporalBool02 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.TemporalBool02"),
181+
TemporalDatetime01 DATETIME(6) GENERATED ALWAYS AS (
182+
CONVERT_TZ(
183+
REGEXP_REPLACE(search_attributes->>"$.TemporalDatetime01", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''),
184+
SUBSTR(REPLACE(search_attributes->>"$.TemporalDatetime01", 'Z', '+00:00'), -6, 6),
185+
'+00:00'
186+
)
187+
),
188+
TemporalDatetime02 DATETIME(6) GENERATED ALWAYS AS (
189+
CONVERT_TZ(
190+
REGEXP_REPLACE(search_attributes->>"$.TemporalDatetime02", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''),
191+
SUBSTR(REPLACE(search_attributes->>"$.TemporalDatetime02", 'Z', '+00:00'), -6, 6),
192+
'+00:00'
193+
)
194+
),
195+
TemporalDouble01 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.TemporalDouble01"),
196+
TemporalDouble02 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.TemporalDouble02"),
197+
TemporalInt01 BIGINT GENERATED ALWAYS AS (search_attributes->"$.TemporalInt01"),
198+
TemporalInt02 BIGINT GENERATED ALWAYS AS (search_attributes->"$.TemporalInt02"),
199+
TemporalKeyword01 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalKeyword01"),
200+
TemporalKeyword02 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalKeyword02"),
201+
TemporalKeyword03 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalKeyword03"),
202+
TemporalKeyword04 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalKeyword04"),
203+
TemporalKeywordList01 JSON GENERATED ALWAYS AS (search_attributes->"$.TemporalKeywordList01"),
204+
TemporalKeywordList02 JSON GENERATED ALWAYS AS (search_attributes->"$.TemporalKeywordList02"),
205+
206+
PRIMARY KEY (namespace_id, run_id)
207+
);
208+
209+
CREATE INDEX by_temporal_bool_01 ON chasm_search_attributes (namespace_id, TemporalBool01);
210+
CREATE INDEX by_temporal_bool_02 ON chasm_search_attributes (namespace_id, TemporalBool02);
211+
CREATE INDEX by_temporal_datetime_01 ON chasm_search_attributes (namespace_id, TemporalDatetime01);
212+
CREATE INDEX by_temporal_datetime_02 ON chasm_search_attributes (namespace_id, TemporalDatetime02);
213+
CREATE INDEX by_temporal_double_01 ON chasm_search_attributes (namespace_id, TemporalDouble01);
214+
CREATE INDEX by_temporal_double_02 ON chasm_search_attributes (namespace_id, TemporalDouble02);
215+
CREATE INDEX by_temporal_int_01 ON chasm_search_attributes (namespace_id, TemporalInt01);
216+
CREATE INDEX by_temporal_int_02 ON chasm_search_attributes (namespace_id, TemporalInt02);
217+
CREATE INDEX by_temporal_keyword_01 ON chasm_search_attributes (namespace_id, TemporalKeyword01);
218+
CREATE INDEX by_temporal_keyword_02 ON chasm_search_attributes (namespace_id, TemporalKeyword02);
219+
CREATE INDEX by_temporal_keyword_03 ON chasm_search_attributes (namespace_id, TemporalKeyword03);
220+
CREATE INDEX by_temporal_keyword_04 ON chasm_search_attributes (namespace_id, TemporalKeyword04);
221+
CREATE INDEX by_temporal_keyword_list_01 ON chasm_search_attributes (namespace_id, (CAST(TemporalKeywordList01 AS CHAR(255) ARRAY)));
222+
CREATE INDEX by_temporal_keyword_list_02 ON chasm_search_attributes (namespace_id, (CAST(TemporalKeywordList02 AS CHAR(255) ARRAY)));

‎schema/mysql/v8/visibility/versioned/v1.11/add_chasm_search_attributes.sql‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
CREATE TABLE chasm_search_attributes (
2+
namespace_id CHAR(64) NOT NULL,
3+
run_id CHAR(64) NOT NULL,
4+
_version BIGINT NOT NULL DEFAULT 0,
5+
search_attributes JSON NULL,
6+
7+
-- Pre-allocated CHASM search attributes
8+
TemporalBool01 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.TemporalBool01"),
9+
TemporalBool02 BOOLEAN GENERATED ALWAYS AS (search_attributes->"$.TemporalBool02"),
10+
TemporalDatetime01 DATETIME(6) GENERATED ALWAYS AS (
11+
CONVERT_TZ(
12+
REGEXP_REPLACE(search_attributes->>"$.TemporalDatetime01", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''),
13+
SUBSTR(REPLACE(search_attributes->>"$.TemporalDatetime01", 'Z', '+00:00'), -6, 6),
14+
'+00:00'
15+
)
16+
),
17+
TemporalDatetime02 DATETIME(6) GENERATED ALWAYS AS (
18+
CONVERT_TZ(
19+
REGEXP_REPLACE(search_attributes->>"$.TemporalDatetime02", 'Z|[+-][0-9]{2}:[0-9]{2}$', ''),
20+
SUBSTR(REPLACE(search_attributes->>"$.TemporalDatetime02", 'Z', '+00:00'), -6, 6),
21+
'+00:00'
22+
)
23+
),
24+
TemporalDouble01 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.TemporalDouble01"),
25+
TemporalDouble02 DECIMAL(20, 5) GENERATED ALWAYS AS (search_attributes->"$.TemporalDouble02"),
26+
TemporalInt01 BIGINT GENERATED ALWAYS AS (search_attributes->"$.TemporalInt01"),
27+
TemporalInt02 BIGINT GENERATED ALWAYS AS (search_attributes->"$.TemporalInt02"),
28+
TemporalKeyword01 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalKeyword01"),
29+
TemporalKeyword02 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalKeyword02"),
30+
TemporalKeyword03 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalKeyword03"),
31+
TemporalKeyword04 VARCHAR(255) GENERATED ALWAYS AS (search_attributes->>"$.TemporalKeyword04"),
32+
TemporalKeywordList01 JSON GENERATED ALWAYS AS (search_attributes->"$.TemporalKeywordList01"),
33+
TemporalKeywordList02 JSON GENERATED ALWAYS AS (search_attributes->"$.TemporalKeywordList02"),
34+
35+
PRIMARY KEY (namespace_id, run_id)
36+
);
37+
38+
CREATE INDEX by_temporal_bool_01 ON chasm_search_attributes (namespace_id, TemporalBool01);
39+
CREATE INDEX by_temporal_bool_02 ON chasm_search_attributes (namespace_id, TemporalBool02);
40+
CREATE INDEX by_temporal_datetime_01 ON chasm_search_attributes (namespace_id, TemporalDatetime01);
41+
CREATE INDEX by_temporal_datetime_02 ON chasm_search_attributes (namespace_id, TemporalDatetime02);
42+
CREATE INDEX by_temporal_double_01 ON chasm_search_attributes (namespace_id, TemporalDouble01);
43+
CREATE INDEX by_temporal_double_02 ON chasm_search_attributes (namespace_id, TemporalDouble02);
44+
CREATE INDEX by_temporal_int_01 ON chasm_search_attributes (namespace_id, TemporalInt01);
45+
CREATE INDEX by_temporal_int_02 ON chasm_search_attributes (namespace_id, TemporalInt02);
46+
CREATE INDEX by_temporal_keyword_01 ON chasm_search_attributes (namespace_id, TemporalKeyword01);
47+
CREATE INDEX by_temporal_keyword_02 ON chasm_search_attributes (namespace_id, TemporalKeyword02);
48+
CREATE INDEX by_temporal_keyword_03 ON chasm_search_attributes (namespace_id, TemporalKeyword03);
49+
CREATE INDEX by_temporal_keyword_04 ON chasm_search_attributes (namespace_id, TemporalKeyword04);
50+
CREATE INDEX by_temporal_keyword_list_01 ON chasm_search_attributes (namespace_id, (CAST(TemporalKeywordList01 AS CHAR(255) ARRAY)));
51+
CREATE INDEX by_temporal_keyword_list_02 ON chasm_search_attributes (namespace_id, (CAST(TemporalKeywordList02 AS CHAR(255) ARRAY)));

‎schema/mysql/v8/visibility/versioned/v1.11/manifest.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"CurrVersion": "1.11",
3+
"MinCompatibleVersion": "0.1",
4+
"Description": "add CHASM search attributes table and indices",
5+
"SchemaUpdateCqlFiles": [
6+
"add_chasm_search_attributes.sql"
7+
]
8+
}

‎schema/postgresql/v12/version.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ const Version = "1.18"
88

99
// VisibilityVersion is the Postgres visibility database release version
1010
// Temporal supports both MySQL and Postgres officially, so upgrade should be performed for both MySQL and Postgres
11-
const VisibilityVersion = "1.10"
11+
const VisibilityVersion = "1.11"

0 commit comments

Comments
 (0)