Skip to content
This repository was archived by the owner on Oct 8, 2020. It is now read-only.

Commit 7a6bca0

Browse files
LorenzBuehmannLorenzBuehmann
committed
CLI script generation for Flink and Spark
1 parent 2694b2b commit 7a6bca0

File tree

5 files changed

+251
-24
lines changed

5 files changed

+251
-24
lines changed

‎pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,16 @@
432432
<artifactId>maven-surefire-plugin</artifactId>
433433
<version>2.20</version>
434434
</plugin>
435+
436+
<plugin>
437+
<artifactId>maven-resources-plugin</artifactId>
438+
<version>3.0.2</version>
439+
</plugin>
440+
441+
<plugin>
442+
<artifactId>maven-assembly-plugin</artifactId>
443+
<version>3.0.0</version>
444+
</plugin>
435445
</plugins>
436446
</pluginManagement>
437447

‎sansa-inference-flink/pom.xml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,5 +386,70 @@ under the License.
386386
</plugins>
387387
</build>
388388
</profile>
389+
390+
<profile>
391+
<id>dist-package</id>
392+
<build>
393+
<plugins>
394+
<plugin>
395+
<artifactId>maven-assembly-plugin</artifactId>
396+
<configuration>
397+
<descriptors>
398+
<descriptor>src/assembly/assembly-bin.xml</descriptor>
399+
</descriptors>
400+
</configuration>
401+
<executions>
402+
<execution>
403+
<id>make-assembly</id> <!-- this is used for inheritance merges -->
404+
<phase>package</phase> <!-- bind to the packaging phase -->
405+
<goals>
406+
<goal>single</goal>
407+
</goals>
408+
</execution>
409+
</executions>
410+
</plugin>
411+
<plugin>
412+
<artifactId>maven-resources-plugin</artifactId>
413+
<executions>
414+
<execution>
415+
<id>copy-resources</id>
416+
<!-- here the phase you need -->
417+
<phase>prepare-package</phase>
418+
<goals>
419+
<goal>copy-resources</goal>
420+
</goals>
421+
<configuration>
422+
<outputDirectory>${basedir}/target/scripts</outputDirectory>
423+
<resources>
424+
<resource>
425+
<directory>scripts</directory>
426+
<filtering>true</filtering>
427+
</resource>
428+
</resources>
429+
</configuration>
430+
</execution>
431+
</executions>
432+
</plugin>
433+
<plugin>
434+
<artifactId>maven-antrun-plugin</artifactId>
435+
<executions>
436+
<execution>
437+
<phase>prepare-package</phase>
438+
<configuration>
439+
<tasks>
440+
<replace token="VERSION_TMP" value="dist-${project.artifactId}-${project.version}.jar" dir="target/scripts">
441+
<include name="cli"/>
442+
</replace>
443+
</tasks>
444+
</configuration>
445+
<goals>
446+
<goal>run</goal>
447+
</goals>
448+
</execution>
449+
</executions>
450+
</plugin>
451+
</plugins>
452+
</build>
453+
</profile>
389454
</profiles>
390455
</project>

‎sansa-inference-flink/scripts/cli‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
# version will be overwritten my Maven-Antrun-Plugin
21+
INFERENCE_LIB_JAR=VERSION_TMP
22+
23+
# the path to the Inference distribution Jar file relative to the current
24+
INFERENCE_LIB=./lib/$INFERENCE_LIB_JAR
25+
26+
# check whether environment variable FLINK_HOME was set
27+
if [ -z "${FLINK_HOME}" ]; then
28+
echo "You have to set the environment variable FLINK_HOME, which maps to the home directory of the Flink distribution."
29+
exit 1
30+
fi
31+
32+
# submit the Flink job
33+
exec "${FLINK_HOME}"/bin/flink run "$INFERENCE_LIB" "$@"

‎sansa-inference-spark/pom.xml‎

Lines changed: 138 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -396,30 +396,12 @@
396396
</profile>
397397

398398
<profile>
399-
<id>package</id>
399+
<id>dist-package</id>
400400
<build>
401401
<plugins>
402-
<plugin>
403-
<artifactId>maven-assembly-plugin</artifactId>
404-
<version>3.0.0</version>
405-
<configuration>
406-
<descriptors>
407-
<descriptor>src/assembly/assembly-bin.xml</descriptor>
408-
</descriptors>
409-
</configuration>
410-
<executions>
411-
<execution>
412-
<id>make-assembly</id> <!-- this is used for inheritance merges -->
413-
<phase>package</phase> <!-- bind to the packaging phase -->
414-
<goals>
415-
<goal>single</goal>
416-
</goals>
417-
</execution>
418-
</executions>
419-
</plugin>
402+
420403
<plugin>
421404
<artifactId>maven-resources-plugin</artifactId>
422-
<version>3.0.2</version>
423405
<executions>
424406
<execution>
425407
<id>copy-resources</id>
@@ -458,9 +440,145 @@
458440
</execution>
459441
</executions>
460442
</plugin>
443+
<plugin>
444+
<groupId>org.apache.maven.plugins</groupId>
445+
<artifactId>maven-shade-plugin</artifactId>
446+
<executions>
447+
<execution>
448+
<phase>package</phase>
449+
<goals>
450+
<goal>shade</goal>
451+
</goals>
452+
<configuration>
453+
<minimizeJar>true</minimizeJar>
454+
<artifactSet>
455+
<excludes>
456+
<exclude>org.apache.spark:spark-core_${scala.binary.version}</exclude>
457+
<exclude>org.apache.spark:spark-sql_${scala.binary.version}</exclude>
458+
<exclude>org.apache.spark:spark-graphx_${scala.binary.version}</exclude>
459+
<exclude>org.apache.spark:*</exclude>
460+
<exclude>org.eclipse.jetty:jetty-server</exclude>
461+
<exclude>org.eclipse.jetty:jetty-continuation</exclude>
462+
<exclude>org.eclipse.jetty:jetty-http</exclude>
463+
<exclude>org.eclipse.jetty:jetty-io</exclude>
464+
<exclude>org.eclipse.jetty:jetty-util</exclude>
465+
<exclude>org.eclipse.jetty:jetty-security</exclude>
466+
<exclude>org.eclipse.jetty:jetty-servlet</exclude>
467+
<exclude>org.eclipse.jetty:*</exclude>
468+
<exclude>org.eclipse.*:*</exclude>
469+
<exclude>org.glassfish.*:*</exclude>
470+
<exclude>org.netbeans.api:*</exclude>
471+
<exclude>org.scala-lang:scala-library</exclude>
472+
<exclude>org.scala-lang:scala-compiler</exclude>
473+
<exclude>org.scala-lang:scala-reflect</exclude>
474+
<exclude>commons-cli:commons-cli</exclude>
475+
<exclude>commons-codec:commons-codec</exclude>
476+
<exclude>commons-collections:commons-collections</exclude>
477+
<exclude>commons-configuration:commons-configuration</exclude>
478+
<exclude>commons-digester:commons-digester</exclude>
479+
<exclude>commons-httpclient:commons-httpclient</exclude>
480+
<exclude>commons-io:commons-io</exclude>
481+
<exclude>commons-lang:commons-lang</exclude>
482+
<exclude>commons-logging:commons-logging</exclude>
483+
<exclude>commons-net:commons-net</exclude>
484+
<exclude>io.dropwizard.metrics:metrics*</exclude>
485+
<exckude>io.netty:netty*</exckude>
486+
<exclude>javax.activation:activation</exclude>
487+
<exclude>javax.annotation:javax.annotation-api</exclude>
488+
<exclude>javax.servlet:javax.servlet-api</exclude>
489+
<exclude>javax.servlet.jsp:jsp-api</exclude>
490+
<exclude>javax.servlet:servlet-api</exclude>
491+
<exclude>javax.validation:validation-api</exclude>
492+
<exclude>javax.ws.rs:javax.ws.rs-api</exclude>
493+
<exclude>javax.xml.bind:jaxb-api</exclude>
494+
<exclude>javax.xml.stream:stax-api</exclude>
495+
<exclude>jdk.tools:jdk.tools</exclude>
496+
<exclude>net.java.dev.jets3t:jets3t</exclude>
497+
<exclude>net.jpountz.lz4:lz4</exclude>
498+
<exclude>net.razorvine:pyrolite</exclude>
499+
<exclude>net.sf.py4j:py4j</exclude>
500+
<exclude>org.antlr:antlr4-runtime</exclude>
501+
<exclude>org.apache.avro:avro*</exclude>
502+
<exclude>org.apache.commons:commons-lang3</exclude>
503+
<exclude>org.apache.commons:commons-math3</exclude>
504+
<exclude>org.apache.commons:commons-compress</exclude>
505+
<exclude>org.apache.curator:curator*</exclude>
506+
<exclude>org.apache.directory.api:*</exclude>
507+
<exclude>org.apache.directory.server:*</exclude>
508+
<exclude>org.apache.hadoop:*</exclude>
509+
<exclude>org.apache.htrace:htrace-core</exclude>
510+
<exclude>org.apache.httpcomponents:*</exclude>
511+
<exclude>org.apache.ivy:ivy</exclude>
512+
<exclude>org.apache.mesos:mesos</exclude>
513+
<exclude>org.apache.parquet:parquet*</exclude>
514+
<exclude>org.apache.xbean:xbean-asm5-shaded</exclude>
515+
<exclude>org.apache.zookeeper:zookeeper</exclude>
516+
<exclude>org.codehaus.jackson:jackson-*</exclude>
517+
<exclude>org.codehaus.janino:*</exclude>
518+
<exclude>org.codehaus.jettison:jettison</exclude>
519+
<exclude>org.fusesource.leveldbjni:leveldbjni-all</exclude>
520+
<exckude>org.glassfish.hk2*</exckude>
521+
<exclude>org.glassfish.jersey*</exclude>
522+
<exclude>org.javassist:javassist</exclude>
523+
<exclude>org.json4s:json4s*</exclude>
524+
<exclude>org.mortbay.jetty:jetty*</exclude>
525+
<exclude>org.objenesis:objenesis</exclude>
526+
<exclude>org.roaringbitmap:RoaringBitmap</exclude>
527+
<exclude>org.scala-lang:*</exclude>
528+
<exclude>org.slf4j:jul-to-slf4j</exclude>
529+
<exclude>org.slf4j:jcl-over-slf4j</exclude>
530+
<exclude>org.spark-project.spark:unused</exclude>
531+
<exclude>org.xerial.snappy:snappy-java</exclude>
532+
<exclude>oro:oro</exclude>
533+
<exclude>xmlenc:xmlenc</exclude>
534+
</excludes>
535+
536+
</artifactSet>
537+
<filters>
538+
<filter>
539+
<artifact>*:*</artifact>
540+
<excludes>
541+
<!-- Avoid a Spark error: Invalid signature file digest for Manifest main attributes-->
542+
<exclude>META-INF/*.SF</exclude>
543+
<exclude>META-INF/*.DSA</exclude>
544+
<exclude>META-INF/*.RSA</exclude>
545+
</excludes>
546+
</filter>
547+
</filters>
548+
<finalName>dist-${project.artifactId}-${project.version}</finalName>
549+
<outputDirectory>${project.build.directory}/shaded</outputDirectory>
550+
<transformers>
551+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
552+
<mainClass>net.sansa_stack.inference.spark.RDFGraphMaterializer</mainClass>
553+
</transformer>
554+
</transformers>
555+
<createDependencyReducedPom>false</createDependencyReducedPom>
556+
<!--<shadedArtifactAttached>true</shadedArtifactAttached>-->
557+
</configuration>
558+
</execution>
559+
</executions>
560+
</plugin>
561+
<plugin>
562+
<artifactId>maven-assembly-plugin</artifactId>
563+
<configuration>
564+
<descriptors>
565+
<descriptor>src/assembly/assembly-bin.xml</descriptor>
566+
</descriptors>
567+
</configuration>
568+
<executions>
569+
<execution>
570+
<id>make-assembly</id> <!-- this is used for inheritance merges -->
571+
<phase>package</phase> <!-- bind to the packaging phase -->
572+
<goals>
573+
<goal>single</goal>
574+
</goals>
575+
</execution>
576+
</executions>
577+
</plugin>
461578
</plugins>
462579
</build>
463580
</profile>
464581

465582
</profiles>
583+
466584
</project>

‎sansa-inference-spark/src/assembly/assembly-bin.xml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,26 +10,27 @@
1010
<fileSets>
1111
<fileSet>
1212
<directory>${project.basedir}</directory>
13-
<outputDirectory>/</outputDirectory>
13+
<outputDirectory></outputDirectory>
1414
<includes>
1515
<include>README*</include>
1616
<include>LICENSE*</include>
1717
<include>NOTICE*</include>
1818
</includes>
1919
</fileSet>
2020
<fileSet>
21-
<directory>${project.build.directory}</directory>
22-
<outputDirectory>/lib</outputDirectory>
21+
<directory>${project.build.directory}/shaded</directory>
22+
<outputDirectory>lib</outputDirectory>
2323
<includes>
2424
<include>*.jar</include>
2525
</includes>
2626
</fileSet>
2727
<fileSet>
2828
<directory>${project.build.directory}/scripts</directory>
29-
<outputDirectory>/bin</outputDirectory>
29+
<outputDirectory>bin</outputDirectory>
3030
<includes>
3131
<include>cli</include>
3232
</includes>
33+
<fileMode>0755</fileMode>
3334
</fileSet>
3435
<fileSet>
3536
<directory>${project.build.directory}/site</directory>

0 commit comments

Comments
 (0)