Mount Nielsen UDFs to Plugin Directory

ethanyzhang · ethanyzhang · commit c30b29c001fd · 2024-08-20T09:42:10.000-05:00
diff --git a/clusters/2xlarge/README.md b/clusters/2xlarge/README.md
@@ -24,3 +24,4 @@ r6i.16xlarge (vCPU: 64, Memory: 490 GB) * 48
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 488 - 25 - 2 = 461`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(461 * 0.95) = 437`
+### For Spark clusters: TODO
diff --git a/clusters/2xlarge/docker-stack-native.yaml b/clusters/2xlarge/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:
diff --git a/clusters/large-ssd/README.md b/clusters/large-ssd/README.md
@@ -25,3 +25,4 @@ r6id.8xlarge (vCPU: 32, Memory: 248 GB) * 16
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 246 - 13 - 2 = 231`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(231 * 0.95) = 219`
+### For Spark clusters: TODO
diff --git a/clusters/large-ssd/docker-stack-native.yaml b/clusters/large-ssd/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:
diff --git a/clusters/large/README.md b/clusters/large/README.md
@@ -24,3 +24,4 @@ r6i.8xlarge (vCPU: 32, Memory: 248 GB) * 16
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 246 - 13 - 2 = 231`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(231 * 0.95) = 219`
+### For Spark clusters: TODO
diff --git a/clusters/large/docker-stack-native.yaml b/clusters/large/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:
diff --git a/clusters/medium-spill/README.md b/clusters/medium-spill/README.md
@@ -25,3 +25,4 @@ r6i.4xlarge (vCPU: 16, Memory: 124 GB) * 8
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 122 - 7 - 1 = 114`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(114 * 0.95) = 108`
+### For Spark clusters: TODO
diff --git a/clusters/medium-spill/docker-stack-native.yaml b/clusters/medium-spill/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,6 +90,7 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     # spilling is on
     - /home/centos/presto/spilled_data:/opt/presto-server/spilled_data
     networks:
diff --git a/clusters/medium-ssd/README.md b/clusters/medium-ssd/README.md
@@ -25,3 +25,4 @@ r6id.4xlarge (vCPU: 16, Memory: 124 GB) * 8
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 122 - 7 - 1 = 114`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(114 * 0.95) = 108`
+### For Spark clusters: TODO
diff --git a/clusters/medium-ssd/docker-stack-native.yaml b/clusters/medium-ssd/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:
diff --git a/clusters/medium/README.md b/clusters/medium/README.md
@@ -24,3 +24,4 @@ r6i.4xlarge (vCPU: 16, Memory: 124 GB) * 8
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 122 - 7 - 1 = 114`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(114 * 0.95) = 108`
+### For Spark clusters: TODO
diff --git a/clusters/medium/docker-stack-native.yaml b/clusters/medium/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:
diff --git a/clusters/small/README.md b/clusters/small/README.md
@@ -24,3 +24,4 @@ r6i.2xlarge (vCPU: 8, Memory: 62 GB) * 4
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 60 - 3 - 1 = 56`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(56 * 0.95) = 53`
+### For Spark clusters: TODO
diff --git a/clusters/small/docker-stack-native.yaml b/clusters/small/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:
diff --git a/clusters/templates/README.md b/clusters/templates/README.md
@@ -30,3 +30,4 @@
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = {{ .ContainerMemoryGb }} - {{ .NativeBufferMemGb }} - {{ .NativeProxygenMemGb }} = {{ .NativeSystemMemGb }}`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor({{ .NativeSystemMemGb }} * {{ .GeneratorParameters.NativeQueryMemPercentOfSysMem }}) = {{ .NativeQueryMemGb }}`
+### For Spark clusters: TODO
diff --git a/clusters/templates/docker-stack-native.yaml b/clusters/templates/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,6 +90,7 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     {{ if .SpillEnabled -}}
     # spilling is on
     - /home/centos/presto/spilled_data:/opt/presto-server/spilled_data
diff --git a/clusters/xlarge-ssd/README.md b/clusters/xlarge-ssd/README.md
@@ -25,3 +25,4 @@ r6id.16xlarge (vCPU: 64, Memory: 490 GB) * 16
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 488 - 25 - 2 = 461`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(461 * 0.95) = 437`
+### For Spark clusters: TODO
diff --git a/clusters/xlarge-ssd/docker-stack-native.yaml b/clusters/xlarge-ssd/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:
diff --git a/clusters/xlarge/README.md b/clusters/xlarge/README.md
@@ -24,3 +24,4 @@ r6i.16xlarge (vCPU: 64, Memory: 490 GB) * 16
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 488 - 25 - 2 = 461`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(461 * 0.95) = 437`
+### For Spark clusters: TODO
diff --git a/clusters/xlarge/docker-stack-native.yaml b/clusters/xlarge/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:
diff --git a/clusters/xsmall/README.md b/clusters/xsmall/README.md
@@ -24,3 +24,4 @@ r6i.xlarge (vCPU: 4, Memory: 30 GB) * 2
 
 * `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 28 - 2 - 1 = 25`
 * `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(25 * 0.95) = 23`
+### For Spark clusters: TODO
diff --git a/clusters/xsmall/docker-stack-native.yaml b/clusters/xsmall/docker-stack-native.yaml
@@ -47,6 +47,7 @@ services:
     - /etc/telegraf:/etc/telegraf
     - /var/run/docker.sock:/var/run/docker.sock
     - /home/centos/presto/plugin/event-listener/:/opt/presto-server/plugin/event-listener/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
     networks:
       presto-swarm:
@@ -89,5 +90,6 @@ services:
     - /home/centos/.ssh/authorized_keys:/root/.ssh/authorized_keys2
     - /etc/ssh:/etc/ssh
     - /home/centos/presto/log/presto-server/:/var/log/presto-server/
+    - /home/centos/presto/plugin/presto-udf-nielsen/:/opt/presto-server/plugin/udfs/
     networks:
       presto-swarm:

Original file line number	Diff line number	Diff line change
`@@ -24,3 +24,4 @@ r6i.16xlarge (vCPU: 64, Memory: 490 GB) * 48`
`24`	`24`
`25`	`25`	* `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 488 - 25 - 2 = 461`
`26`	`26`	* `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(461 * 0.95) = 437`
	`27`	`+### For Spark clusters: TODO`
Original file line number	Diff line number	Diff line change
`@@ -25,3 +25,4 @@ r6id.8xlarge (vCPU: 32, Memory: 248 GB) * 16`
`25`	`25`
`26`	`26`	* `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 246 - 13 - 2 = 231`
`27`	`27`	* `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(231 * 0.95) = 219`
	`28`	`+### For Spark clusters: TODO`
Original file line number	Diff line number	Diff line change
`@@ -24,3 +24,4 @@ r6i.8xlarge (vCPU: 32, Memory: 248 GB) * 16`
`24`	`24`
`25`	`25`	* `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 246 - 13 - 2 = 231`
`26`	`26`	* `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(231 * 0.95) = 219`
	`27`	`+### For Spark clusters: TODO`
Original file line number	Diff line number	Diff line change
`@@ -25,3 +25,4 @@ r6i.4xlarge (vCPU: 16, Memory: 124 GB) * 8`
`25`	`25`
`26`	`26`	* `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 122 - 7 - 1 = 114`
`27`	`27`	* `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(114 * 0.95) = 108`
	`28`	`+### For Spark clusters: TODO`
Original file line number	Diff line number	Diff line change
`@@ -25,3 +25,4 @@ r6id.4xlarge (vCPU: 16, Memory: 124 GB) * 8`
`25`	`25`
`26`	`26`	* `system-memory-gb = ContainerMemory - NativeBufferMemGb - NativeProxygenMemGb = 122 - 7 - 1 = 114`
`27`	`27`	* `query-memory-gb = query.max-memory-per-node = floor(system-memory-gb * NativeQueryMemPercentOfSysMem) = floor(114 * 0.95) = 108`
	`28`	`+### For Spark clusters: TODO`