Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
pom-generated.xml
pom.xml
Manifest.txt
lein-hadoop*.jar
*.jar
lib
classes
classes
.lein-*
34 changes: 28 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,16 +1,38 @@
# lein-hadoop

This is a plugin for leiningen which generates a jar suitable to submission
as a hadoop job. Sepcifically, this is a runable jar with lib/ containing all
the project dependencies.
This is a plugin for [Leiningen][0] which generates a jar suitable for
submission as a [Hadoop][1] job. Specifically, this is a runable jar
with <tt>lib/</tt> containing all project dependencies. The contents
of <tt>lib/dev</tt> are excluded.

The name of the generated jar can be overridden by specifying a custom
value for <tt>:hadoop-name</tt> or <tt>:jar-name</tt> in project.clj.

You can use this plugin to build any Hadoop application which requires
a runable jar, including projects using [MapReduce][2],
[Cascading][3], and [Cascalog][4].

## Installation

Add to your project.clj <tt>:dev-dependencies [lein-hadoop "1.0.0"]]</tt>
Add to your project.clj

:dev-dependencies [[lein-hadoop "1.1.0"]]

## Usage

A new target is added which generates PROJECT-hadoop.jar:
$ lein clean, deps, hadoop

## License

Copyright © 2010-2012 Nick Dimiduk

Distributed under the Eclipse Public License, the same as Leiningen
uses.

$ lein hadoop
Thanks to the Leiningen team for a relatively pleasant build tool!

[0]: https://github.com/technomancy/leiningen "Hair Extinguisher"
[1]: http://hadoop.apache.org/ "Apache Hadoop"
[2]: http://hadoop.apache.org/mapreduce/ "Hadoop MapReduce"
[3]: http://www.cascading.org/ "Cascading"
[4]: http://www.cascalog.org/ "Cascalog"
7 changes: 3 additions & 4 deletions project.clj
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
(defproject lein-hadoop "1.0.0"
(defproject lein-hadoop "1.1.0"
:description "A leiningen plugin to build jars for hadoop."
:dev-dependencies [[org.clojure/clojure "1.1.0"]
[org.clojure/clojure-contrib "1.1.0"]
[lein-clojars "0.6.0"]])
:dependencies [[org.clojure/clojure "1.2.1"]]
:eval-in-leiningen true)
91 changes: 54 additions & 37 deletions src/leiningen/hadoop.clj
Original file line number Diff line number Diff line change
@@ -1,42 +1,59 @@
(ns leiningen.hadoop
"Create a jar for submission as a hadoop job."
(:require [leiningen.compile :as compile])
(:use [leiningen.pom :only [make-pom make-pom-properties]]
[leiningen.jar :only [write-jar]]
[clojure.contrib.java-utils :only [file]]))
(:use
[clojure.java.io :only [file]]
[leiningen.deps :only [deps]]
[leiningen.javac :only [javac]])
(:require
[leiningen.compile :as compile]
[leiningen.jar :as jar]))

(defn- jar
"Create a $PROJECT-hadoop.jar file containing the compiled .class files
as well as the source .clj files. If project.clj contains a :main symbol, it
will be used as the main-class for an executable jar."
([project jar-name]
(compile/compile project)
(let [jar-file (str (:root project) "/" jar-name)
filespecs [{:type :bytes
:path (format "meta-inf/maven/%s/%s/pom.xml"
(:group project)
(:name project))
:bytes (make-pom project)}
{:type :bytes
:path (format "meta-inf/maven/%s/%s/pom.properties"
(:group project)
(:name project))
:bytes (make-pom-properties project)}
(when (and (:resources-path project)
(.exists (file (:resources-path project))))
{:type :path :path (:resources-path project)})
{:type :path :path (:compile-path project)}
{:type :path :path (:source-path project)}
{:type :path :path (:library-path project)}
{:type :path :path (str (:root project) "/project.clj")}]]
;; TODO: support slim, etc
(write-jar project jar-file filespecs)
(println "Created" jar-file)
jar-file))
([project] (jar project (str (:name project) ".jar"))))
;; depends on private API :'(
(def filespecs #'jar/filespecs)

(defn hadoop
"Create a jar for submission to hadoop."
([project]
(jar project (str (:name project) "-hadoop.jar"))))
(defn add-lib-filespecs
"Add :library-path to the list of filespecs. To avoid inclusion of lib/dev,
use in conjunction with exclude-lib-dev."
[filespecs project]
(concat
(when (and (:library-path project)
(.exists (file (:library-path project))))
[{:type :path
:path (:library-path project)}])
filespecs))

(defn exclude-lib-dev
"Add (:library-path project)/dev.*.jar to :jar-exclusions"
[project]
(let [lib-base (-> (:library-path project)
(.replace (str (:root project) "/") ""))
dev-exclusions {:jar-exclusions
[(re-pattern (str lib-base "/dev/.*$"))]}]
(merge-with concat project dev-exclusions)))

(defn hadoopjar-name
"Determine a file name for the hadoop jar. Respects name definitions from
project in order: :hadoop-name, :jar-name"
[project]
(or (:hadoop-name project)
(-> (jar/get-default-jar-name project)
(.replaceAll ".jar$" "-hadoop.jar"))))

(defn ^{:help-arglists '([])} hadoop
"Create a jar for submission to Hadoop.

Create a $PROJECT-$VERSION-hadoop.jar file. This file is identical to the one
generated by jar, except it includes all project dependencies in the jar
under lib/. The contents of lib/dev are excluded. Filename can be overridden
via either :hadoop-name or :jar-name in project.clj."
[project]
(let [deps-fileset (deps project) ;; fetch deps
_ (javac project) ;; compile java
status (compile/compile project)] ;; compile clj
(if (zero? status)
(let [jar-path (jar/get-jar-filename project (hadoopjar-name project))
specs (add-lib-filespecs (filespecs project deps-fileset) project)]
(jar/write-jar (exclude-lib-dev project) jar-path specs)
(println "Created" jar-path)
jar-path)
status)))
27 changes: 0 additions & 27 deletions test/test_hadoop.clj

This file was deleted.