Skip to content

Commit 6b07085

Browse files
olegsavinolegsavin
committed
Trino v1.1: remove phoenix connector, HMS config simplified
1 parent b0a25f9 commit 6b07085

File tree

2 files changed

+93
-24
lines changed

2 files changed

+93
-24
lines changed

‎articles/hdinsight-aks/trino/trino-connect-to-metastore.md

Lines changed: 93 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ The following example covers the addition of Hive catalog and metastore database
3535

3636
**There are few important sections you need to add to your cluster ARM template to configure the Hive catalog and Hive metastore database:**
3737

38-
- `secretsProfile` – It specifies Azure Key Vault and list of secrets to be used in Trino cluster, required to connect to external Hive metastore.
39-
- `serviceConfigsProfiles` - It includes configuration for Trino catalogs. For more information, see [Add catalogs to existing cluster](trino-add-catalogs.md).
40-
- `trinoProfile.catalogOptions.hive` – List of Hive or iceberg or delta catalogs with parameters of external Hive metastore database for each catalog. To use external metastore database, catalog must be present in this list.
41-
38+
- `secretsProfile` with Azure Key Vault and list of secrets to be used in Trino cluster, required to connect to external Hive metastore.
39+
- `serviceConfigsProfiles` with Trino catalogs. For more information, see [Add catalogs to existing cluster](trino-add-catalogs.md).
40+
- `hive.metastore=hdi` catalog property indicating to use external Hive metastores database with in-cluster hive metastore service, for each catalog where hive metastore is required.
41+
- `config.properties` with external Hive metastore database parameters. Alternatively same parameters specified in `trinoProfile.catalogOptions.hive` section.
4242

4343
| Property| Description| Example|
4444
|---|---|---|
@@ -48,10 +48,10 @@ The following example covers the addition of Hive catalog and metastore database
4848
|secretsProfile.secrets[*].keyVaultObjectName|Name of secret object in Azure Key Vault containing actual secret value.|secret1|
4949
|trinoProfile.catalogOptions.hive|List of Hive or iceberg or delta catalogs with parameters of external Hive metastore database, require parameters for each. To use external metastore database, catalog must be present in this list.
5050
|trinoProfile.catalogOptions.hive[*].catalogName|Name of Trino catalog configured in `serviceConfigsProfiles`, which configured to use external Hive metastore database.|hive1|
51-
|trinoProfile.catalogOptions.hive[*].metastoreDbConnectionURL|JDBC connection string to database.|jdbc:sqlserver://mysqlserver1.database.windows.net;database=myhmsdb1;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=30|
52-
|trinoProfile.catalogOptions.hive[*].metastoreDbConnectionUserName|SQL user name to connect to database.|trinoadmin|
53-
|trinoProfile.catalogOptions.hive[*].metastoreDbConnectionPasswordSecret|Secret referenceName configured in secretsProfile with password.|hms-db-pwd|
54-
|trinoProfile.catalogOptions.hive[*].metastoreWarehouseDir|ABFS URI to location in storage where data is stored.|`abfs://[email protected]/hive/warehouse`|
51+
|config.properties/hive.metastore.hdi.metastoreDbConnectionURL (or trinoProfile.catalogOptions.hive[*].metastoreDbConnectionURL)|JDBC connection string to database.|jdbc:sqlserver://mysqlserver1.database.windows.net;database=myhmsdb1;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=30|
52+
|hive.metastore.hdi.metastoreDbConnectionUserName (or trinoProfile.catalogOptions.hive[*].metastoreDbConnectionUserName)|SQL user name to connect to database.|trinoadmin|
53+
|hive.metastore.hdi.metastoreDbConnectionPasswordSecret (or trinoProfile.catalogOptions.hive[*].metastoreDbConnectionPasswordSecret)|Secret referenceName configured in secretsProfile with password.|hms-db-pwd|
54+
|hive.metastore.hdi.metastoreWarehouseDir (or trinoProfile.catalogOptions.hive[*].metastoreWarehouseDir)|ABFS URI to location in storage where data is stored.|`abfs://[email protected]/hive/warehouse`|
5555

5656
To configure external Hive metastore to an existing Trino cluster, add the required sections in your cluster ARM template by referring to the following example:
5757

@@ -85,33 +85,35 @@ To configure external Hive metastore to an existing Trino cluster, add the requi
8585
{
8686
"serviceName": "trino",
8787
"configs": [
88+
{
89+
"component": "common",
90+
"files": [
91+
{
92+
"fileName": "config.properties",
93+
"values": {
94+
"hive.metastore.hdi.metastoreDbConnectionURL": "jdbc:sqlserver://mysqlserver1.database.windows.net;database=myhmsdb1;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=30",
95+
"hive.metastore.hdi.metastoreDbConnectionUserName": "trinoadmin",
96+
"hive.metastore.hdi.metastoreDbConnectionPasswordSecret": "hms-db-pwd",
97+
"hive.metastore.hdi.metastoreWarehouseDir": "abfs://[email protected]/hive/warehouse"
98+
}
99+
}
100+
]
101+
},
88102
{
89103
"component": "catalogs",
90104
"files": [
91105
{
92106
"fileName": "hive1.properties",
93107
"values": {
94-
"connector.name": "hive"
108+
"connector.name": "hive",
109+
"hive.metastore": "hdi"
95110
}
96111
}
97112
]
98113
}
99114
]
100115
}
101-
],
102-
"trinoProfile": {
103-
"catalogOptions": {
104-
"hive": [
105-
{
106-
"catalogName": "hive1",
107-
"metastoreDbConnectionURL": "jdbc:sqlserver://mysqlserver1.database.windows.net;database=myhmsdb1;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=30",
108-
"metastoreDbConnectionUserName": "trinoadmin",
109-
"metastoreDbConnectionPasswordSecret": "hms-db-pwd",
110-
"metastoreWarehouseDir": "abfs://[email protected]/hive/warehouse"
111-
}
112-
]
113-
}
114-
}
116+
]
115117
}
116118
}
117119
}
@@ -135,3 +137,71 @@ create schema hive1.schema1;
135137
create table hive1.schema1.tpchorders as select * from tpch.tiny.orders;
136138
select * from hive1.schema1.tpchorders limit 100;
137139
```
140+
141+
## Alternative configuration
142+
Alternatively external Hive metastore database parameters can be specified in trinoProfile together with `hive.metastore=hdi` catalog property:
143+
144+
145+
```json
146+
{
147+
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
148+
"contentVersion": "1.0.0.0",
149+
"parameters": {},
150+
"resources": [
151+
{
152+
"type": "microsoft.hdinsight/clusterpools/clusters",
153+
"apiVersion": "<api-version>",
154+
"name": "<cluster-pool-name>/<cluster-name>",
155+
"location": "<region, e.g. westeurope>",
156+
"tags": {},
157+
"properties": {
158+
"clusterType": "Trino",
159+
160+
"clusterProfile": {
161+
"secretsProfile": {
162+
"keyVaultResourceId": "/subscriptions/{USER_SUBSCRIPTION_ID}/resourceGroups/{USER_RESOURCE_GROUP}/providers/Microsoft.KeyVault/vaults/{USER_KEYVAULT_NAME}",
163+
"secrets": [
164+
{
165+
"referenceName": "hms-db-pwd",
166+
"type": "secret",
167+
"keyVaultObjectName": "hms-db-pwd"
168+
} ]
169+
},
170+
"serviceConfigsProfiles": [
171+
{
172+
"serviceName": "trino",
173+
"configs": [
174+
{
175+
"component": "catalogs",
176+
"files": [
177+
{
178+
"fileName": "hive1.properties",
179+
"values": {
180+
"connector.name": "hive",
181+
"hive.metastore": "hdi"
182+
}
183+
}
184+
]
185+
}
186+
]
187+
}
188+
],
189+
"trinoProfile": {
190+
"catalogOptions": {
191+
"hive": [
192+
{
193+
"catalogName": "hive1",
194+
"metastoreDbConnectionURL": "jdbc:sqlserver://mysqlserver1.database.windows.net;database=myhmsdb1;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=30",
195+
"metastoreDbConnectionUserName": "trinoadmin",
196+
"metastoreDbConnectionPasswordSecret": "hms-db-pwd",
197+
"metastoreWarehouseDir": "abfs://[email protected]/hive/warehouse"
198+
}
199+
]
200+
}
201+
}
202+
}
203+
}
204+
}
205+
]
206+
}
207+
```

‎articles/hdinsight-aks/trino/trino-connectors.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ Trino in HDInsight on AKS enables seamless integration with data sources. You ca
3232
* [MongoDB](https://trino.io/docs/410/connector/mongodb.html)
3333
* [MySQL](https://trino.io/docs/410/connector/mysql.html)
3434
* [Oracle](https://trino.io/docs/410/connector/oracle.html)
35-
* [Phoenix](https://trino.io/docs/410/connector/phoenix.html)
3635
* [PostgreSQL](https://trino.io/docs/410/connector/postgresql.html)
3736
* [Prometheus](https://trino.io/docs/410/connector/prometheus.html)
3837
* [Redis](https://trino.io/docs/410/connector/redis.html)

0 commit comments

Comments
 (0)