Versions

app/com/linkedin/drelephant/ElephantContext.java

@@ -50,7 +50,7 @@
 import org.apache.spark.SparkMetricsAggregator;
 import org.w3c.dom.Document;
 import play.api.Play;
-import play.api.templates.Html;
+import play.twirl.api.Html;
 
 
 /**

app/com/linkedin/drelephant/mapreduce/MapReduceMetricsAggregator.java

@@ -31,7 +31,7 @@ public class MapReduceMetricsAggregator implements HadoopMetricsAggregator {
   private static final String MAP_CONTAINER_CONFIG = "mapreduce.map.memory.mb";
   private static final String REDUCER_CONTAINER_CONFIG = "mapreduce.reduce.memory.mb";
   private static final String REDUCER_SLOW_START_CONFIG = "mapreduce.job.reduce.slowstart.completedmaps";
-  private static final long CONTAINER_MEMORY_DEFAULT_BYTES = 2048L * FileUtils.ONE_MB;
+  private static final long CONTAINER_MEMORY_DEFAULT_MBYTES = 2048L;
 
   private HadoopAggregatedData _hadoopAggregatedData = null;
   private TaskLevelAggregatedMetrics mapTasks;
@@ -83,17 +83,19 @@ public HadoopAggregatedData getResult() {
 
   private long getMapContainerSize(HadoopApplicationData data) {
     try {
-      return Long.parseLong(data.getConf().getProperty(MAP_CONTAINER_CONFIG));
+      long value = Long.parseLong(data.getConf().getProperty(MAP_CONTAINER_CONFIG));
+      return (value < 0) ? CONTAINER_MEMORY_DEFAULT_MBYTES : value;
     } catch ( NumberFormatException ex) {
-      return CONTAINER_MEMORY_DEFAULT_BYTES;
+      return CONTAINER_MEMORY_DEFAULT_MBYTES;
     }
   }
 
   private long getReducerContainerSize(HadoopApplicationData data) {
     try {
-      return Long.parseLong(data.getConf().getProperty(REDUCER_CONTAINER_CONFIG));
+      long value = Long.parseLong(data.getConf().getProperty(REDUCER_CONTAINER_CONFIG));
+      return (value < 0) ? CONTAINER_MEMORY_DEFAULT_MBYTES : value;
     } catch ( NumberFormatException ex) {
-      return CONTAINER_MEMORY_DEFAULT_BYTES;
+      return CONTAINER_MEMORY_DEFAULT_MBYTES;
     }
   }
 }

app/com/linkedin/drelephant/mapreduce/heuristics/GenericMemoryHeuristic.java

@@ -39,7 +39,7 @@
  */
 public abstract class GenericMemoryHeuristic implements Heuristic<MapReduceApplicationData> {
   private static final Logger logger = Logger.getLogger(GenericMemoryHeuristic.class);
-  private static final long CONTAINER_MEMORY_DEFAULT_BYTES = 2048L * FileUtils.ONE_MB;
+  private static final long CONTAINER_MEMORY_DEFAULT_MBYTES = 2048L;
 
   // Severity Parameters
   private static final String MEM_RATIO_SEVERITY = "memory_ratio_severity";
@@ -53,6 +53,20 @@ public abstract class GenericMemoryHeuristic implements Heuristic<MapReduceAppli
   private String _containerMemConf;
   private HeuristicConfigurationData _heuristicConfData;
 
+  private long getContainerMemDefaultMBytes() {
+    Map<String, String> paramMap = _heuristicConfData.getParamMap();
+    if (paramMap.containsKey(CONTAINER_MEM_DEFAULT_MB)) {
+      String strValue = paramMap.get(CONTAINER_MEM_DEFAULT_MB);
+      try {
+        return Long.valueOf(strValue);
+      }
+      catch (NumberFormatException e) {
+        logger.warn(CONTAINER_MEM_DEFAULT_MB + ": expected number [" + strValue + "]");
+      }
+    }
+    return CONTAINER_MEMORY_DEFAULT_MBYTES;
+  }
+
   private void loadParameters() {
     Map<String, String> paramMap = _heuristicConfData.getParamMap();
     String heuristicName = _heuristicConfData.getHeuristicName();
@@ -64,10 +78,7 @@ private void loadParameters() {
     logger.info(heuristicName + " will use " + MEM_RATIO_SEVERITY + " with the following threshold settings: "
         + Arrays.toString(memRatioLimits));
 
-    long containerMemDefaultBytes = CONTAINER_MEMORY_DEFAULT_BYTES;
-    if (paramMap.containsKey(CONTAINER_MEM_DEFAULT_MB)) {
-      containerMemDefaultBytes = Long.valueOf(paramMap.get(CONTAINER_MEM_DEFAULT_MB)) * FileUtils.ONE_MB;
-    }
+    long containerMemDefaultBytes = getContainerMemDefaultMBytes() * FileUtils.ONE_MB;
     logger.info(heuristicName + " will use " + CONTAINER_MEM_DEFAULT_MB + " with the following threshold setting: "
             + containerMemDefaultBytes);
 
@@ -104,23 +115,31 @@ public HeuristicResult apply(MapReduceApplicationData data) {
     }
 
     String containerSizeStr = data.getConf().getProperty(_containerMemConf);
-    if (containerSizeStr == null) {
-      return null;
-    }
-
-    long containerMem;
-    try {
-      containerMem = Long.parseLong(containerSizeStr);
-    } catch (NumberFormatException e) {
-      // Some job has a string var like "${VAR}" for this config.
-      if(containerSizeStr.startsWith("$")) {
-        String realContainerConf = containerSizeStr.substring(containerSizeStr.indexOf("{")+1,
-            containerSizeStr.indexOf("}"));
-        containerMem = Long.parseLong(data.getConf().getProperty(realContainerConf));
-      } else {
-        throw e;
+    long containerMem = -1L;
+
+    if (containerSizeStr != null) {
+      try {
+        containerMem = Long.parseLong(containerSizeStr);
+      } catch (NumberFormatException e0) {
+        // Some job has a string var like "${VAR}" for this config.
+        if(containerSizeStr.startsWith("$")) {
+          String realContainerConf = containerSizeStr.substring(containerSizeStr.indexOf("{")+1,
+              containerSizeStr.indexOf("}"));
+          String realContainerSizeStr = data.getConf().getProperty(realContainerConf);
+          try {
+            containerMem = Long.parseLong(realContainerSizeStr);
+          }
+          catch (NumberFormatException e1) {
+            logger.warn(realContainerConf + ": expected number [" + realContainerSizeStr + "]");
+          }
+        } else {
+          logger.warn(_containerMemConf + ": expected number [" + containerSizeStr + "]");
+        }
       }
     }
+    if (containerMem < 0) {
+      containerMem = getContainerMemDefaultMBytes();
+    }
     containerMem *= FileUtils.ONE_MB;
 
     MapReduceTaskData[] tasks = getTasks(data);

app/controllers/Application.java

@@ -45,7 +45,7 @@
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.log4j.Logger;
-import play.api.templates.Html;
+import play.twirl.api.Html;
 import play.data.DynamicForm;
 import play.data.Form;
 import play.libs.Json;
@@ -142,15 +142,15 @@ public static Result dashboard() {
 
     // Update statistics only after FETCH_DELAY
     if (now - _lastFetch > FETCH_DELAY) {
-      _numJobsAnalyzed = AppResult.find.where().gt(AppResult.TABLE.FINISH_TIME, finishDate).findRowCount();
+      _numJobsAnalyzed = AppResult.find.where().gt(AppResult.TABLE.FINISH_TIME, finishDate).findCount();
       _numJobsCritical = AppResult.find.where()
           .gt(AppResult.TABLE.FINISH_TIME, finishDate)
           .eq(AppResult.TABLE.SEVERITY, Severity.CRITICAL.getValue())
-          .findRowCount();
+          .findCount();
       _numJobsSevere = AppResult.find.where()
           .gt(AppResult.TABLE.FINISH_TIME, finishDate)
           .eq(AppResult.TABLE.SEVERITY, Severity.SEVERE.getValue())
-          .findRowCount();
+          .findCount();
       _lastFetch = now;
     }
 
@@ -291,7 +291,7 @@ public static Result search() {
       List<AppResult> resultsToDisplay = results.subList((currentPage - paginationBarStartIndex) * pageLength,
               Math.min(results.size(), (currentPage - paginationBarStartIndex + 1) * pageLength));
       return ok(searchPage.render(paginationStats, searchResults.render(
-              String.format("Results: Showing %,d of %,d", resultsToDisplay.size(), query.findRowCount()), resultsToDisplay)));
+              String.format("Results: Showing %,d of %,d", resultsToDisplay.size(), query.findCount()), resultsToDisplay)));
     }
   }
 

app/controllers/MetricsController.java

@@ -107,7 +107,7 @@ public Integer getValue() {
         if (now - _lastUpdate > UPDATE_DELAY) {
           _count = AppResult.find.where()
                   .gt(AppResult.TABLE.FINISH_TIME, now - DAY)
-                  .findRowCount();
+                  .findCount();
           _lastUpdate = now;
         }
         return _count;

app/controllers/api/v1/Web.java

@@ -84,27 +84,27 @@ public static Result restDashboardSummaries() {
 
     //Update statistics only after FETCH_DELAY
     if (now - _lastFetch > FETCH_DELAY) {
-      _numJobsAnalyzed = AppResult.find.where().gt(AppResult.TABLE.FINISH_TIME, finishDate).findRowCount();
+      _numJobsAnalyzed = AppResult.find.where().gt(AppResult.TABLE.FINISH_TIME, finishDate).findCount();
       _numJobsCritical = AppResult.find.where()
         .gt(AppResult.TABLE.FINISH_TIME, finishDate)
         .eq(AppResult.TABLE.SEVERITY, Severity.CRITICAL.getValue())
-        .findRowCount();
+        .findCount();
       _numJobsSevere = AppResult.find.where()
         .gt(AppResult.TABLE.FINISH_TIME, finishDate)
         .eq(AppResult.TABLE.SEVERITY, Severity.SEVERE.getValue())
-        .findRowCount();
+        .findCount();
       _numJobsModerate = AppResult.find.where()
         .gt(AppResult.TABLE.FINISH_TIME, finishDate)
         .eq(AppResult.TABLE.SEVERITY, Severity.MODERATE.getValue())
-        .findRowCount();
+        .findCount();
       _numJobsLow = AppResult.find.where()
         .gt(AppResult.TABLE.FINISH_TIME, finishDate)
         .eq(AppResult.TABLE.SEVERITY, Severity.LOW.getValue())
-        .findRowCount();
+        .findCount();
       _numJobsNone = AppResult.find.where()
         .gt(AppResult.TABLE.FINISH_TIME, finishDate)
         .eq(AppResult.TABLE.SEVERITY, Severity.NONE.getValue())
-        .findRowCount();
+        .findCount();
       _lastFetch = now;
     }
 
@@ -1401,7 +1401,7 @@ public static Result search() {
     Query<AppResult> query =
         Application.generateSearchQuery(AppResult.getSearchFields(), Application.getSearchParams());
 
-    total = query.findRowCount();
+    total = query.findCount();
 
     if (offset > total) {
       offset = total;

app/models/AppHeuristicResult.java

@@ -33,7 +33,7 @@
 import com.linkedin.drelephant.analysis.Severity;
 import com.linkedin.drelephant.util.Utils;
 
-import play.db.ebean.Model;
+import com.avaje.ebean.Model;
 
 
 @Entity
@@ -83,4 +83,4 @@ public static String getSearchFields() {
   @OneToMany(cascade = CascadeType.ALL, mappedBy = "yarnAppHeuristicResult")
   public List<AppHeuristicResultDetails> yarnAppHeuristicResultDetails;
 
-}
+}

app/models/AppHeuristicResultDetails.java

@@ -22,13 +22,14 @@
 import javax.persistence.Embeddable;
 import javax.persistence.EmbeddedId;
 import javax.persistence.Entity;
+import javax.persistence.Id;
 import javax.persistence.ManyToOne;
 
 import javax.persistence.Table;
 
 import com.fasterxml.jackson.annotation.JsonBackReference;
 
-import play.db.ebean.Model;
+import com.avaje.ebean.Model;
 
 
 @Entity
@@ -49,6 +50,10 @@ public static class TABLE {
     public static final String DETAILS = "details";
   }
 
+  @JsonIgnore
+  @Id
+  public int id;
+
   @JsonBackReference
   @ManyToOne(cascade = CascadeType.ALL)
   public AppHeuristicResult yarnAppHeuristicResult;

app/models/AppResult.java

@@ -21,7 +21,7 @@
 
 import com.linkedin.drelephant.util.Utils;
 import java.util.Date;
-import play.db.ebean.Model;
+import com.avaje.ebean.Model;
 
 import java.util.List;
 
@@ -163,5 +163,5 @@ public static String getSearchFields() {
   @OneToMany(cascade = CascadeType.ALL, mappedBy = "yarnAppResult")
   public List<AppHeuristicResult> yarnAppHeuristicResults;
 
-  public static Finder<String, AppResult> find = new Finder<String, AppResult>(String.class, AppResult.class);
+  public static Finder<String, AppResult> find = new Finder<String, AppResult>(AppResult.class);
 }

app/org/apache/spark/deploy/history/SparkFSFetcher.scala

@@ -149,7 +149,7 @@ class SparkFSFetcher(fetcherConfData: FetcherConfigurationData) extends Elephant
   def checkActivation(httpValue: String): Boolean = {
     val url: URL = new URL("http://" + httpValue + "/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus");
     val rootNode: JsonNode = readJsonNode(url);
-    val status: String = rootNode.path("beans").get(0).path("State").getValueAsText();
+    val status: String = rootNode.path("beans").get(0).path("State").asText();
     if (status.equals("active")) {
       return true;
     }

build.sbt

@@ -14,23 +14,36 @@
 // the License.
 //
 
-import play.Project._
 import Dependencies._
 
+scalaVersion := "2.11.8"
+
 name := "dr-elephant"
 
 version := "2.0.6"
 
+lazy val root = (project in file(".")).enablePlugins(PlayJava, PlayEbean, SbtTwirl)
+
 organization := "com.linkedin.drelephant"
 
+retrieveManaged := true
+
 javacOptions in Compile ++= Seq("-source", "1.6", "-target", "1.6")
 
+javaOptions in Test += "-javaagent:lib_managed/jars/org.jmockit/jmockit/jmockit-1.23.jar"
+
 libraryDependencies ++= dependencies
 
+excludeDependencies ++= Seq(
+  "javax.persistence" % "persistence-api",
+  "org.avaje.ebeanorm" % "avaje-ebeanorm",
+  "org.avaje.ebeanorm" % "avaje-ebeanorm-agent"
+)
+
 // Create a new custom configuration called compileonly
 ivyConfigurations += config("compileonly").hide
 
 // Append all dependencies with 'compileonly' configuration to unmanagedClasspath in Compile.
 unmanagedClasspath in Compile ++= update.value.select(configurationFilter("compileonly"))
 
-playJavaSettings
+routesGenerator := StaticRoutesGenerator

compile.sh

@@ -29,7 +29,7 @@ function play_command() {
 }
 
 # Default configurations
-HADOOP_VERSION="2.3.0"
+HADOOP_VERSION="2.7.3"
 SPARK_VERSION="1.4.0"
 
 # User should pass an optional argument which is a path to config file

conf/application.conf

@@ -26,7 +26,7 @@ application.secret=${?APPLICATION_SECRET}
 
 # The application languages
 # ~~~~~
-application.langs="en"
+play.i18n.langs=[ "en" ]
 
 # Global object class
 # ~~~~~
@@ -60,20 +60,12 @@ db.default.driver=com.mysql.jdbc.Driver
 # ~~~~~
 # You can disable evolutions if needed
 # evolutionplugin=disabled
-applyEvolutions.default=true
+play.evolutions.enabled=true
+play.evolutions.autoApply=true
 
 # Ebean configuration
 # ~~~~~
 # You can declare as many Ebean servers as you want.
 # By convention, the default server is named `default`
 #
 ebean.default= "models.*"
-
-# Root logger:
-logger.root=ERROR
-
-# Logger used by the framework:
-logger.play=INFO
-
-# Logger provided to your application:
-logger.application=DEBUG

conf/evolutions/default/4.sql

@@ -0,0 +1,14 @@
+# --- Simplify primary key for yarn_app_heuristic_result_details
+# --- !Ups
+
+alter table yarn_app_heuristic_result_details add id int auto_increment unique not null first;
+create unique index yarn_app_heuristic_result_details_i2 on yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name);
+alter table yarn_app_heuristic_result_details drop primary key, add primary key(id);
+drop index id on yarn_app_heuristic_result_details;
+
+# --- !Downs
+
+create index id on yarn_app_heuristic_result_details (id);
+alter table yarn_app_heuristic_result_details drop primary key, add primary key(yarn_app_heuristic_result_id,name);
+drop index yarn_app_heuristic_result_details_i2 on yarn_app_heuristic_result_details;
+alter table yarn_app_heuristic_result_details drop id;