Skip to content
This repository was archived by the owner on Aug 29, 2023. It is now read-only.

Commit 32e4182

Browse files
joshuayaodevaraj-kavali
joshuayao
authored and
devaraj-kavali
committed
Build CDH parcel when mvn package. Support Hadoop and Spark module (#4)
* Build CDH parcel when mvn package. Support Hadoop and Spark module only so far. * Generate el6 & el7 parcels in a single release package as part of 'mvn package'. * Remove profiles el6 and el7. * Create a new module for building parcel. * Build a compete release package: 1. Build CDH parcels. 2. Build the binaries for non-CDH installation. 3. Create checksum for the release package. 4. Sign the package. * fix parent version
1 parent 7da14fa commit 32e4182

File tree

9 files changed

+408
-0
lines changed

9 files changed

+408
-0
lines changed

‎assembly/pom.xml‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
18+
http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
20+
<modelVersion>4.0.0</modelVersion>
21+
22+
<parent>
23+
<groupId>com.intel.qat</groupId>
24+
<artifactId>qat-parent</artifactId>
25+
<version>1.0.0</version>
26+
<relativePath>../pom.xml</relativePath>
27+
</parent>
28+
29+
<artifactId>assembly</artifactId>
30+
<description>QATCodec Release Package Builder</description>
31+
<name>QATCodec Package Builder</name>
32+
<packaging>pom</packaging>
33+
34+
<properties>
35+
<parcel.version>${project.version}</parcel.version>
36+
</properties>
37+
38+
<build>
39+
<plugins>
40+
<plugin>
41+
<artifactId>maven-resources-plugin</artifactId>
42+
<configuration>
43+
<encoding>UTF-8</encoding>
44+
</configuration>
45+
<executions>
46+
<execution>
47+
<id>create-parcel-meta-data</id>
48+
<phase>compile</phase>
49+
<goals>
50+
<goal>copy-resources</goal>
51+
</goals>
52+
<configuration>
53+
<outputDirectory>${project.build.directory}/meta</outputDirectory>
54+
<resources>
55+
<resource>
56+
<directory>${project.parent.basedir}/assembly/src/main/assembly/meta</directory>
57+
<filtering>true</filtering>
58+
<includes>
59+
<include>filelist.json</include>
60+
<include>qatcodec_env.sh</include>
61+
<include>parcel.json</include>
62+
</includes>
63+
</resource>
64+
</resources>
65+
</configuration>
66+
</execution>
67+
</executions>
68+
</plugin>
69+
<plugin>
70+
<groupId>org.apache.maven.plugins</groupId>
71+
<artifactId>maven-assembly-plugin</artifactId>
72+
<configuration>
73+
<finalName>QATCODEC-${parcel.version}</finalName>
74+
<descriptor>src/main/assembly/assembly.xml</descriptor>
75+
</configuration>
76+
<executions>
77+
<execution>
78+
<id>make-parcels</id>
79+
<phase>package</phase>
80+
<goals>
81+
<goal>single</goal>
82+
</goals>
83+
</execution>
84+
</executions>
85+
</plugin>
86+
<plugin>
87+
<groupId>com.coderplus.maven.plugins</groupId>
88+
<artifactId>copy-rename-maven-plugin</artifactId>
89+
<executions>
90+
<execution>
91+
<id>rename-parcel</id>
92+
<phase>package</phase>
93+
<goals>
94+
<goal>copy</goal>
95+
</goals>
96+
<configuration>
97+
<fileSets>
98+
<fileSet>
99+
<sourceFile>${project.build.directory}/QATCODEC-${parcel.version}.tar.gz</sourceFile>
100+
<destinationFile>${project.build.directory}/QATCODEC-${parcel.version}-el6.parcel</destinationFile>
101+
</fileSet>
102+
<fileSet>
103+
<sourceFile>${project.build.directory}/QATCODEC-${parcel.version}.tar.gz</sourceFile>
104+
<destinationFile>${project.build.directory}/QATCODEC-${parcel.version}-el7.parcel</destinationFile>
105+
</fileSet>
106+
</fileSets>
107+
</configuration>
108+
</execution>
109+
</executions>
110+
</plugin>
111+
<plugin>
112+
<groupId>com.googlecode.maven-download-plugin</groupId>
113+
<artifactId>maven-download-plugin</artifactId>
114+
<executions>
115+
<execution>
116+
<id>download-cm_ext</id>
117+
<phase>package</phase>
118+
<goals>
119+
<goal>wget</goal>
120+
</goals>
121+
<configuration>
122+
<url>https://github.com/cloudera/cm_ext/archive/master.zip</url>
123+
<unpack>true</unpack>
124+
<targetDirectory>${project.build.directory}</targetDirectory>
125+
</configuration>
126+
</execution>
127+
</executions>
128+
</plugin>
129+
<plugin>
130+
<groupId>org.codehaus.mojo</groupId>
131+
<artifactId>exec-maven-plugin</artifactId>
132+
<executions>
133+
<execution>
134+
<id>make-parcel-manifests</id>
135+
<phase>package</phase>
136+
<goals>
137+
<goal>exec</goal>
138+
</goals>
139+
<configuration>
140+
<executable>python</executable>
141+
<workingDirectory>${project.build.directory}</workingDirectory>
142+
<arguments>
143+
<argument>${project.build.directory}/cm_ext-master/make_manifest/make_manifest.py</argument>
144+
<argument>${project.build.directory}</argument>
145+
</arguments>
146+
</configuration>
147+
</execution>
148+
<execution>
149+
<id>make-checksum-signatures</id>
150+
<phase>verify</phase>
151+
<goals>
152+
<goal>exec</goal>
153+
</goals>
154+
<configuration>
155+
<executable>bash</executable>
156+
<workingDirectory>${project.build.directory}</workingDirectory>
157+
<arguments>
158+
<argument>${project.parent.basedir}/assembly/src/main/bash/create-release</argument>
159+
<argument>${project.build.directory}/QATCODEC-${parcel.version}.tar.gz</argument>
160+
</arguments>
161+
</configuration>
162+
</execution>
163+
</executions>
164+
</plugin>
165+
<plugin>
166+
<groupId>org.apache.maven.plugins</groupId>
167+
<artifactId>maven-antrun-plugin</artifactId>
168+
<executions>
169+
<execution>
170+
<id>make-release-package</id>
171+
<phase>package</phase>
172+
<goals>
173+
<goal>run</goal>
174+
</goals>
175+
<configuration>
176+
<tasks>
177+
<move file="${project.build.directory}/QATCODEC-${parcel.version}-el6.parcel"
178+
todir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/parcel" />
179+
<move file="${project.build.directory}/QATCODEC-${parcel.version}-el7.parcel"
180+
todir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/parcel" />
181+
<move file="${project.build.directory}/manifest.json"
182+
todir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/parcel" />
183+
<delete file="${project.build.directory}/QATCODEC-${parcel.version}.tar.gz" />
184+
<copy file="src/main/resources/ReadMe-nonCDH.txt"
185+
tofile="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/ReadMe.txt" />
186+
<copy file="src/main/resources/ReadMe.txt"
187+
todir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}" />
188+
<move todir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}">
189+
<fileset dir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/lib/hadoop/lib">
190+
<include name="*.jar"/>
191+
</fileset>
192+
<fileset dir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/lib/hadoop/lib/native">
193+
<include name="*.so"/>
194+
</fileset>
195+
<fileset dir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/lib/spark/lib">
196+
<include name="*.jar"/>
197+
</fileset>
198+
</move>
199+
<delete dir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/lib" />
200+
<delete dir="${project.build.directory}/QATCODEC-${parcel.version}/QATCODEC-${parcel.version}/meta" />
201+
<tar destfile="${project.build.directory}/QATCODEC-${parcel.version}.tar.gz"
202+
basedir="${project.build.directory}/QATCODEC-${parcel.version}"
203+
compression="gzip" />
204+
</tasks>
205+
</configuration>
206+
</execution>
207+
</executions>
208+
</plugin>
209+
</plugins>
210+
</build>
211+
</project>
212+

‎assembly/src/main/assembly/assembly.xml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<assembly>
2+
<formats>
3+
<format>tar.gz</format>
4+
<format>dir</format>
5+
</formats>
6+
<includeBaseDirectory>true</includeBaseDirectory>
7+
<fileSets>
8+
<fileSet>
9+
<directory>${project.parent.basedir}/hadoop_qat_wrapper/target</directory>
10+
<outputDirectory>lib/hadoop/lib</outputDirectory>
11+
<includes>
12+
<include>*.jar</include>
13+
</includes>
14+
<excludes>
15+
<exclude>*tests.jar</exclude>
16+
<exclude>*doc.jar</exclude>
17+
</excludes>
18+
<fileMode>0644</fileMode>
19+
</fileSet>
20+
<fileSet>
21+
<directory>${project.parent.basedir}/hadoop_qat_wrapper/target</directory>
22+
<outputDirectory>lib/hadoop/lib/native</outputDirectory>
23+
<includes>
24+
<include>*.so</include>
25+
</includes>
26+
<fileMode>755</fileMode>
27+
</fileSet>
28+
<fileSet>
29+
<directory>${project.parent.basedir}/spark_qat_wrapper/target</directory>
30+
<outputDirectory>lib/spark/lib</outputDirectory>
31+
<includes>
32+
<include>*.jar</include>
33+
</includes>
34+
<excludes>
35+
<exclude>*tests.jar</exclude>
36+
<exclude>*doc.jar</exclude>
37+
</excludes>
38+
<fileMode>0644</fileMode>
39+
</fileSet>
40+
<fileSet>
41+
<directory>${project.build.directory}/meta</directory>
42+
<outputDirectory>meta</outputDirectory>
43+
<includes>
44+
<include>*</include>
45+
</includes>
46+
<fileMode>0644</fileMode>
47+
</fileSet>
48+
</fileSets>
49+
<dependencySets>
50+
<dependencySet>
51+
<outputDirectory>lib</outputDirectory>
52+
</dependencySet>
53+
</dependencySets>
54+
</assembly>

‎assembly/src/main/assembly/meta/filelist.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"hadoop-qat-codec" : {
3+
"name": "hadoop-qat-codec",
4+
"version": "${parcel.version}",
5+
"files" : {
6+
"lib/hadoop/lib" : {},
7+
"lib/hadoop/lib/hadoop_qat_codec-${parcel.version}.jar" : {},
8+
"lib/hadoop/lib/native" : {},
9+
"lib/hadoop/lib/native/libqatcodec.so" : {}
10+
}
11+
}
12+
, "spark-qat-codec" : {
13+
"name": "spark-qat-codec",
14+
"version": "${parcel.version}",
15+
"files" : {
16+
"lib/spark" : {},
17+
"lib/spark/lib" : {},
18+
"lib/spark/lib/spark_qat_codec-${parcel.version}.jar" : {}
19+
}
20+
}
21+
}

‎assembly/src/main/assembly/meta/parcel.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"schema_version": 1,
3+
"name": "QATCODEC",
4+
"version": "${parcel.version}",
5+
"extraVersionInfo": {
6+
"fullVersion": "${parcel.version}-el",
7+
"baseVersion": "qatcodec${parcel.version}",
8+
"patchCount": ""
9+
},
10+
11+
"setActiveSymlink": true,
12+
13+
"scripts": {
14+
"defines": "qatcodec_env.sh"
15+
},
16+
17+
"packages": [
18+
{ "name": "hadoop-qat-codec",
19+
"version": "${parcel.version}"
20+
}
21+
,{ "name": "spark-qat-codec",
22+
"version": "${parcel.version}"
23+
}
24+
],
25+
26+
"components": [
27+
{ "name": "hadoop-qat-codec",
28+
"version": "${parcel.version}",
29+
"pkg_version": "${parcel.version}"
30+
}
31+
,{ "name": "spark-qat-codec",
32+
"version": "${parcel.version}",
33+
"pkg_version": "${parcel.version}"
34+
}
35+
],
36+
37+
"provides": [
38+
"cdh-plugin",
39+
"spark-plugin"
40+
],
41+
42+
"users": { },
43+
44+
"groups": [ ]
45+
}

‎assembly/src/main/assembly/meta/qatcodec_env.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
QATCODEC_VERSION=${parcel.version}
3+
QATCODEC_DIRNAME=${PARCEL_DIRNAME:-"QATCODEC-${QATCODEC_VERSION}"}
4+
5+
if [ -n "${HADOOP_CLASSPATH}" ]; then
6+
export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/*"
7+
else
8+
export HADOOP_CLASSPATH="$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/*"
9+
fi
10+
11+
if [ -n "${MR2_CLASSPATH}" ]; then
12+
export MR2_CLASSPATH="${MR2_CLASSPATH}:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/*"
13+
else
14+
export MR2_CLASSPATH="$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/*"
15+
fi
16+
17+
if [ -n "${JAVA_LIBRARY_PATH}" ]; then
18+
export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/native"
19+
else
20+
export JAVA_LIBRARY_PATH="$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/native"
21+
fi
22+
23+
if [ -n "${LD_LIBRARY_PATH}" ]; then
24+
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/native"
25+
else
26+
export LD_LIBRARY_PATH="$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/native"
27+
fi
28+
29+
if [ -n "${CDH_SPARK_CLASSPATH}" ]; then
30+
export CDH_SPARK_CLASSPATH="${CDH_SPARK_CLASSPATH}:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/spark/lib/*:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/*"
31+
else
32+
export CDH_SPARK_CLASSPATH="$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/spark/lib/*:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/*"
33+
fi
34+
35+
if [ -n "${SPARK_LIBRARY_PATH}" ]; then
36+
export SPARK_LIBRARY_PATH="${SPARK_LIBRARY_PATH}:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/native"
37+
else
38+
export SPARK_LIBRARY_PATH="$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hadoop/lib/native"
39+
fi
40+
41+
if [ -n "${AUX_CLASSPATH}" ]; then
42+
export AUX_CLASSPATH="$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/parquet/lib/*:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hive/lib/*:${AUX_CLASSPATH}"
43+
else
44+
export AUX_CLASSPATH="$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/parquet/lib/*:$PARCELS_ROOT/$QATCODEC_DIRNAME/lib/hive/lib/*"
45+
fi
46+

‎assembly/src/main/bash/create-release‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
3+
GPG=$(command -v gpg)
4+
${GPG} --use-agent --armor --default-key "0xE6D69F76" --output "${1}.asc" --detach-sig "${1}"
5+
${GPG} --print-mds "${1}" > "${1}.mds"

0 commit comments

Comments
 (0)