Skip to content

Commit b24978b

Browse files
Aggarwal-RaghavAggarwal-Raghav
committed
Build fast + setup
1 parent 25e5830 commit b24978b

File tree

11 files changed

+221
-116
lines changed

11 files changed

+221
-116
lines changed

‎bin/hive‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ done
328328

329329
# to initialize logging for all services
330330

331-
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=hive-log4j2.properties "
331+
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=${HIVE_CONF_DIR}/hive-log4j2.properties"
332332

333333
if [ -f "${HIVE_CONF_DIR}/parquet-logging.properties" ]; then
334334
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Djava.util.logging.config.file=${HIVE_CONF_DIR}/parquet-logging.properties "

‎conf/hive-env.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
export HADOOP_HOME=$HOME/Desktop/setup/hadoop
2+
export HIVE_HOME=$HOME/Desktop/setup/hive
3+
export HIVE_CONF_DIR=$HOME/Desktop/setup/hive/conf
4+
export HIVE_LOG_DIR=/tmp/hadoop/hive
5+
6+
7+
if [ "$SERVICE" = "beeline" ]; then
8+
export HADOOP_HEAPSIZE=128
9+
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Djava.io.tmpdir=/tmp -Dhive.log.dir=$HIVE_LOG_DIR \
10+
-XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$HIVE_LOG_DIR \
11+
-Dhive.log.file=beeline.log -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
12+
13+
fi
14+
if [ "$SERVICE" = "metastore" ]; then
15+
export HADOOP_HEAPSIZE=2048
16+
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Djava.io.tmpdir=/tmp -Dhive.log.dir=$HIVE_LOG_DIR \
17+
-Dhive.log.file=hivemetastore.log -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$HIVE_LOG_DIR \
18+
-Xlog:gc*=info:file=$HIVE_LOG_DIR/gc-${SERVICE}-%p-%t.log:time,uptime,level,tags:filecount=5,filesize=16M \
19+
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006 \
20+
-agentpath:$HOME/Desktop/software/async-profiler-4.0-macos/lib/libasyncProfiler.dylib=start,event=cpu,file=$HIVE_LOG_DIR/hms_profile.html"
21+
fi
22+
23+
if [ "$SERVICE" = "hiveserver2" ]; then
24+
export HADOOP_HEAPSIZE=2048
25+
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Djava.io.tmpdir=/tmp -Dhive.log.dir=$HIVE_LOG_DIR \
26+
-Dhive.log.file=hiveserver2.log -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$HIVE_LOG_DIR \
27+
-Xlog:gc*=info:file=$HIVE_LOG_DIR/gc-${SERVICE}-%p-%t.log:time,uptime,level,tags:filecount=5,filesize=16M \
28+
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5007 \
29+
-agentpath:$HOME/Desktop/software/async-profiler-4.0-macos/lib/libasyncProfiler.dylib=start,event=cpu,file=$HIVE_LOG_DIR/hs2_profile.html"
30+
fi
31+

‎conf/hive-site.xml‎

Lines changed: 148 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,153 @@
1-
<?xml version="1.0"?>
2-
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3-
<!--
4-
Licensed to the Apache Software Foundation (ASF) under one or more
5-
contributor license agreements. See the NOTICE file distributed with
6-
this work for additional information regarding copyright ownership.
7-
The ASF licenses this file to You under the Apache License, Version 2.0
8-
(the "License"); you may not use this file except in compliance with
9-
the License. You may obtain a copy of the License at
10-
11-
http://www.apache.org/licenses/LICENSE-2.0
12-
13-
Unless required by applicable law or agreed to in writing, software
14-
distributed under the License is distributed on an "AS IS" BASIS,
15-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16-
See the License for the specific language governing permissions and
17-
limitations under the License.
18-
-->
19-
201
<configuration>
212

223
<property>
23-
<name>hive.lock.sleep.between.retries</name>
24-
<value>2</value>
4+
<name>javax.jdo.option.ConnectionURL</name>
5+
<value>jdbc:mysql://localhost:3307/hive</value>
6+
</property>
7+
8+
<property>
9+
<name>javax.jdo.option.ConnectionDriverName</name>
10+
<value>com.mysql.jdbc.Driver</value>
11+
</property>
12+
13+
<property>
14+
<name>javax.jdo.option.ConnectionUserName</name>
15+
<value>root</value>
16+
</property>
17+
18+
<property>
19+
<name>javax.jdo.option.ConnectionPassword</name>
20+
<value>qwerty@123</value>
2521
</property>
22+
23+
<property>
24+
<name>javax.jdo.PersistenceManagerFactoryClass</name>
25+
<value>org.datanucleus.api.jdo.JDOPersistenceManagerFactory</value>
26+
</property>
27+
28+
<property>
29+
<name>hive.metastore.warehouse.dir</name>
30+
<value>/user/hive/warehouse</value>
31+
</property>
32+
33+
<property>
34+
<name>hive.metastore.uris</name>
35+
<value>thrift://localhost:9083</value>
36+
</property>
37+
38+
<property>
39+
<name>hive.metastore.event.db.notification.api.auth</name>
40+
<value>false</value>
41+
</property>
42+
43+
<property>
44+
<name>hive.server2.thrift.port</name>
45+
<value>10000</value>
46+
</property>
47+
48+
<property>
49+
<name>hive.server2.enable.doAs</name>
50+
<value>false</value>
51+
</property>
52+
53+
<property>
54+
<name>hive.execution.engine</name>
55+
<value>tez</value>
56+
</property>
57+
58+
<property>
59+
<name>hive.metastore.port</name>
60+
<value>9083</value>
61+
</property>
62+
63+
<property>
64+
<name>hive.tez.container.size</name>
65+
<value>1024</value>
66+
</property>
67+
68+
<property>
69+
<name>hive.tez.exec.print.summary</name>
70+
<value>true</value>
71+
</property>
72+
73+
<property>
74+
<name>hive.txn.manager</name>
75+
<value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
76+
</property>
77+
78+
<property>
79+
<name>hive.support.concurrency</name>
80+
<value>true</value>
81+
</property>
82+
83+
<property>
84+
<name>hive.resultset.use.unique.column.names</name>
85+
<value>true</value>
86+
</property>
87+
88+
<property>
89+
<name>hive.security.authorization.enabled</name>
90+
<value>true</value>
91+
</property>
92+
93+
<property>
94+
<name>hive.security.authorization.manager</name>
95+
<value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider</value>
96+
</property>
97+
98+
<property>
99+
<name>hive.support.concurrency</name>
100+
<value>true</value>
101+
</property>
102+
103+
<property>
104+
<name>hive.txn.manager</name>
105+
<value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
106+
</property>
107+
108+
<property>
109+
<name>hive.metastore.failure.retries</name>
110+
<value>10</value>
111+
</property>
112+
113+
<property>
114+
<name>hive.metastore.client.socket.timeout</name>
115+
<value>1800</value>
116+
</property>
117+
118+
<property>
119+
<name>hive.exec.post.hooks</name>
120+
<value>org.apache.hadoop.hive.ql.hooks.HiveProtoLoggingHook</value>
121+
</property>
122+
123+
<property>
124+
<name>hive.exec.pre.hooks</name>
125+
<value>org.apache.hadoop.hive.ql.hooks.HiveProtoLoggingHook</value>
126+
</property>
127+
128+
<property>
129+
<name>hive.exec.failure.hooks</name>
130+
<value>org.apache.hadoop.hive.ql.hooks.HiveProtoLoggingHook</value>
131+
</property>
132+
133+
<property>
134+
<name>hive.hook.proto.base-directory</name>
135+
<value>/user/hive/warehouse/proto/</value>
136+
</property>
137+
138+
<property>
139+
<name>hive.server2.thrift.max.worker.threads</name>
140+
<value>10000</value>
141+
</property>
142+
143+
<property>
144+
<name>hive.notification.event.poll.interval</name>
145+
<value>1800</value>
146+
</property>
147+
148+
<property>
149+
<name>hive.scheduled.queries.executor.enabled</name>
150+
<value>false</value>
151+
</property>
152+
26153
</configuration>

‎itests/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
<module>hive-jmh</module>
4242
<module>hive-unit-hadoop2</module>
4343
<module>hive-minikdc</module>
44-
<module>qtest-druid</module>
44+
<!-- <module>qtest-druid</module> -->
4545
<module>qtest-kudu</module>
4646
<module>qtest-iceberg</module>
4747
<module>hive-iceberg</module>

‎itests/qtest-iceberg/pom.xml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -97,17 +97,17 @@
9797
<artifactId>hadoop-yarn-registry</artifactId>
9898
<optional>true</optional>
9999
</dependency>
100-
<dependency>
101-
<groupId>org.apache.hive</groupId>
102-
<artifactId>hive-druid-handler</artifactId>
103-
<scope>test</scope>
104-
</dependency>
105-
<dependency>
106-
<groupId>org.apache.hive</groupId>
107-
<artifactId>hive-druid-handler</artifactId>
108-
<classifier>tests</classifier>
109-
<scope>test</scope>
110-
</dependency>
100+
<!-- <dependency> -->
101+
<!-- <groupId>org.apache.hive</groupId> -->
102+
<!-- <artifactId>hive-druid-handler</artifactId> -->
103+
<!-- <scope>test</scope> -->
104+
<!-- </dependency> -->
105+
<!-- <dependency> -->
106+
<!-- <groupId>org.apache.hive</groupId> -->
107+
<!-- <artifactId>hive-druid-handler</artifactId> -->
108+
<!-- <classifier>tests</classifier> -->
109+
<!-- <scope>test</scope> -->
110+
<!-- </dependency> -->
111111
<dependency>
112112
<groupId>org.apache.hive</groupId>
113113
<artifactId>hive-jdbc-handler</artifactId>

‎itests/qtest/pom.xml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -95,17 +95,17 @@
9595
<artifactId>hadoop-yarn-registry</artifactId>
9696
<optional>true</optional>
9797
</dependency>
98-
<dependency>
99-
<groupId>org.apache.hive</groupId>
100-
<artifactId>hive-druid-handler</artifactId>
101-
<scope>test</scope>
102-
</dependency>
103-
<dependency>
104-
<groupId>org.apache.hive</groupId>
105-
<artifactId>hive-druid-handler</artifactId>
106-
<classifier>tests</classifier>
107-
<scope>test</scope>
108-
</dependency>
98+
<!-- <dependency> -->
99+
<!-- <groupId>org.apache.hive</groupId> -->
100+
<!-- <artifactId>hive-druid-handler</artifactId> -->
101+
<!-- <scope>test</scope> -->
102+
<!-- </dependency> -->
103+
<!-- <dependency> -->
104+
<!-- <groupId>org.apache.hive</groupId> -->
105+
<!-- <artifactId>hive-druid-handler</artifactId> -->
106+
<!-- <classifier>tests</classifier> -->
107+
<!-- <scope>test</scope> -->
108+
<!-- </dependency> -->
109109
<dependency>
110110
<groupId>org.apache.hive</groupId>
111111
<artifactId>hive-jdbc-handler</artifactId>

‎packaging/pom.xml‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@
119119
</execution>
120120
</executions>
121121
</plugin>
122+
<!--
122123
<plugin>
123124
<groupId>org.codehaus.mojo</groupId>
124125
<artifactId>license-maven-plugin</artifactId>
@@ -193,6 +194,7 @@
193194
</licenseUrlFileNameSanitizers>
194195
</configuration>
195196
</plugin>
197+
-->
196198
</plugins>
197199
</build>
198200
</profile>
@@ -343,11 +345,11 @@
343345
<artifactId>hive-hbase-handler</artifactId>
344346
<version>${project.version}</version>
345347
</dependency>
346-
<dependency>
347-
<groupId>org.apache.hive</groupId>
348-
<artifactId>hive-druid-handler</artifactId>
349-
<version>${project.version}</version>
350-
</dependency>
348+
<!-- <dependency> -->
349+
<!-- <groupId>org.apache.hive</groupId> -->
350+
<!-- <artifactId>hive-druid-handler</artifactId> -->
351+
<!-- <version>${project.version}</version> -->
352+
<!-- </dependency> -->
351353
<dependency>
352354
<groupId>org.apache.hive</groupId>
353355
<artifactId>hive-iceberg-handler</artifactId>
@@ -472,5 +474,10 @@
472474
<artifactId>hive-kudu-handler</artifactId>
473475
<version>${project.version}</version>
474476
</dependency>
477+
<dependency>
478+
<groupId>com.mysql</groupId>
479+
<artifactId>mysql-connector-j</artifactId>
480+
<version>9.4.0</version>
481+
</dependency>
475482
</dependencies>
476483
</project>

‎packaging/src/main/assembly/bin.xml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
<exclude>org.apache.hadoop:*</exclude>
4444
<exclude>org.apache.hive.hcatalog:*</exclude>
4545
<exclude>org.slf4j:*</exclude>
46-
<exclude>log4j:*</exclude>
46+
<!-- <exclude>log4j:*</exclude> -->
4747
<exclude>ch.qos.reload4j:*</exclude>
4848
<exclude>junit:*</exclude>
4949
<exclude>commons-configuration:commons-configuration</exclude>
@@ -103,7 +103,7 @@
103103
<excludes>
104104
<exclude>org.apache.hadoop:*</exclude>
105105
<exclude>org.apache.hive:hive-jdbc:jar:standalone</exclude>
106-
<exclude>org.apache.httpcomponents:*</exclude>
106+
<!-- <exclude>org.apache.httpcomponents:*</exclude> -->
107107
</excludes>
108108
<includes>
109109
<include>org.apache.hive.hcatalog:hive-webhcat:*</include>
@@ -240,6 +240,8 @@
240240
<includes>
241241
<include>*.template</include>
242242
<include>ivysettings.xml</include>
243+
<include>hive-env.sh</include>
244+
<include>hive-site.xml</include>
243245
</includes>
244246
<outputDirectory>conf</outputDirectory>
245247
</fileSet>
@@ -417,7 +419,7 @@
417419
<file>
418420
<source>${project.parent.basedir}/common/src/main/resources/hive-log4j2.properties</source>
419421
<outputDirectory>conf</outputDirectory>
420-
<destName>hive-log4j2.properties.template</destName>
422+
<destName>hive-log4j2.properties</destName>
421423
</file>
422424
<file>
423425
<source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j2.properties</source>

‎pom.xml‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
<module>cli</module>
3636
<module>common</module>
3737
<module>contrib</module>
38-
<module>druid-handler</module>
38+
<!-- <module>druid-handler</module> -->
3939
<module>hbase-handler</module>
4040
<module>jdbc-handler</module>
4141
<module>hcatalog</module>
@@ -2219,25 +2219,5 @@
22192219
<module>itests</module>
22202220
</modules>
22212221
</profile>
2222-
<profile>
2223-
<id>dist</id>
2224-
<build>
2225-
<plugins>
2226-
<plugin>
2227-
<groupId>org.cyclonedx</groupId>
2228-
<artifactId>cyclonedx-maven-plugin</artifactId>
2229-
<version>${maven.cyclonedx.plugin.version}</version>
2230-
<executions>
2231-
<execution>
2232-
<phase>package</phase>
2233-
<goals>
2234-
<goal>makeBom</goal>
2235-
</goals>
2236-
</execution>
2237-
</executions>
2238-
</plugin>
2239-
</plugins>
2240-
</build>
2241-
</profile>
22422222
</profiles>
22432223
</project>

0 commit comments

Comments
 (0)