Resolve conflicts from merge with apache

Author: ash211

common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java

@@ -17,20 +17,16 @@
 
 package org.apache.spark.util.kvstore;
 
-import com.google.common.base.Preconditions;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Stream;
 
-<<<<<<< HEAD:common/kvstore/src/main/java/org/apache/spark/kvstore/KVTypeInfo.java
-=======
 import com.google.common.base.Preconditions;
 
 import org.apache.spark.annotation.Private;
 
->>>>>>> origin/master:common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java
 /**
  * Wrapper around types managed in a KVStore, providing easy access to their indexed fields.
  */

core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java

@@ -121,23 +121,15 @@ final class ShuffleExternalSorter extends MemoryConsumer {
     this.numPartitions = numPartitions;
     // Use getSizeAsKb (not bytes) to maintain backwards compatibility if no units are provided
     this.fileBufferSizeBytes =
-<<<<<<< HEAD
-            (int) (long) conf.get(package$.MODULE$.SHUFFLE_FILE_BUFFER_SIZE()) * 1024;
-=======
         (int) (long) conf.get(package$.MODULE$.SHUFFLE_FILE_BUFFER_SIZE()) * 1024;
->>>>>>> origin/master
     this.numElementsForSpillThreshold =
       conf.getLong("spark.shuffle.spill.numElementsForceSpillThreshold", 1024 * 1024 * 1024);
     this.writeMetrics = writeMetrics;
     this.inMemSorter = new ShuffleInMemorySorter(
       this, initialSize, conf.getBoolean("spark.shuffle.sort.useRadixSort", true));
     this.peakMemoryUsedBytes = getMemoryUsage();
     this.diskWriteBufferSize =
-<<<<<<< HEAD
-            (int) (long) conf.get(package$.MODULE$.SHUFFLE_DISK_WRITE_BUFFER_SIZE());
-=======
         (int) (long) conf.get(package$.MODULE$.SHUFFLE_DISK_WRITE_BUFFER_SIZE());
->>>>>>> origin/master
   }
 
   /**

core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala

@@ -304,11 +304,8 @@ object SparkSubmit extends CommandLineUtils {
     }
     val isYarnCluster = clusterManager == YARN && deployMode == CLUSTER
     val isMesosCluster = clusterManager == MESOS && deployMode == CLUSTER
-<<<<<<< HEAD
     val isKubernetesCluster = clusterManager == KUBERNETES && deployMode == CLUSTER
-=======
     val isStandAloneCluster = clusterManager == STANDALONE && deployMode == CLUSTER
->>>>>>> origin/master
 
     if (!isMesosCluster && !isStandAloneCluster) {
       // Resolve maven dependencies if there are any and add classpath to jars. Add them to py-files

pom.xml

@@ -133,7 +133,6 @@
     <!-- Version used for internal directory structure -->
     <hive.version.short>1.2.1</hive.version.short>
     <derby.version>10.12.1.1</derby.version>
-<<<<<<< HEAD
     <parquet.version>1.9.1-palantir3</parquet.version>
     <feign.version>8.18.0</feign.version>
     <okhttp3.version>3.8.0</okhttp3.version>
@@ -142,13 +141,8 @@
     <retrofit.version>2.2.0</retrofit.version>
     <bouncycastle.version>1.54</bouncycastle.version>
     <jetty.version>9.4.6.v20170531</jetty.version>
-=======
-    <parquet.version>1.8.2</parquet.version>
     <orc.version>1.4.0</orc.version>
     <orc.classifier>nohive</orc.classifier>
-    <hive.parquet.version>1.6.0</hive.parquet.version>
-    <jetty.version>9.3.20.v20170531</jetty.version>
->>>>>>> origin/master
     <javaxservlet.version>3.1.0</javaxservlet.version>
     <chill.version>0.8.0</chill.version>
     <ivy.version>2.4.0</ivy.version>

project/plugins.sbt

@@ -1,9 +1,6 @@
-<<<<<<< HEAD
 addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.8.5")
 
-=======
 // need to make changes to uptake sbt 1.0 support in "com.eed3si9n" % "sbt-assembly" % "1.14.5"
->>>>>>> origin/master
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
 
 // sbt 1.0.0 support: https://github.com/typesafehub/sbteclipse/issues/343

sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java

@@ -206,21 +206,11 @@ private static final class RowComparator extends RecordComparator {
 
     @Override
     public int compare(Object baseObj1, long baseOff1, Object baseObj2, long baseOff2) {
-<<<<<<< HEAD
-      // TODO: Why are the sizes -1?
-      row1.pointTo(baseObj1, baseOff1, -1);
-      row2.pointTo(baseObj2, baseOff2, -1);
-      int comparison = ordering.compare(row1, row2);
-      row1.pointTo(null, 0L, -1);
-      row2.pointTo(null, 0L, -1);
-      return comparison;
-=======
       // Note that since ordering doesn't need the total length of the record, we just pass 0
       // into the row.
       row1.pointTo(baseObj1, baseOff1, 0);
       row2.pointTo(baseObj2, baseOff2, 0);
       return ordering.compare(row1, row2);
->>>>>>> origin/master
     }
   }
 }

sql/core/pom.xml

@@ -111,10 +111,6 @@
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
-<<<<<<< HEAD
-      <version>${fasterxml.jackson.databind.version}</version>
-=======
->>>>>>> origin/master
     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>

sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java

@@ -244,18 +244,9 @@ private static final class KVComparator extends RecordComparator {
     public int compare(Object baseObj1, long baseOff1, Object baseObj2, long baseOff2) {
       // Note that since ordering doesn't need the total length of the record, we just pass 0
       // into the row.
-<<<<<<< HEAD
-      row1.pointTo(baseObj1, baseOff1 + 4, -1);
-      row2.pointTo(baseObj2, baseOff2 + 4, -1);
-      int comparison = ordering.compare(row1, row2);
-      row1.pointTo(null, 0L, -1);
-      row2.pointTo(null, 0L, -1);
-      return comparison;
-=======
       row1.pointTo(baseObj1, baseOff1 + 4, 0);
       row2.pointTo(baseObj2, baseOff2 + 4, 0);
       return ordering.compare(row1, row2);
->>>>>>> origin/master
     }
   }
 

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SQLHadoopMapReduceCommitProtocol.scala

@@ -33,22 +33,15 @@ class SQLHadoopMapReduceCommitProtocol(jobId: String, path: String)
   extends HadoopMapReduceCommitProtocol(jobId, path) with Serializable with Logging {
 
   override protected def setupCommitter(context: TaskAttemptContext): OutputCommitter = {
-    val clazz = context.getConfiguration
-      .getClass(SQLConf.OUTPUT_COMMITTER_CLASS.key, null, classOf[OutputCommitter])
+    var committer = context.getOutputFormatClass.newInstance().getOutputCommitter(context)
 
-<<<<<<< HEAD
-    if (clazz != null) {
-      logInfo(s"Using user defined output committer class ${clazz.getCanonicalName}")
-
-=======
     val configuration = context.getConfiguration
     val clazz =
       configuration.getClass(SQLConf.OUTPUT_COMMITTER_CLASS.key, null, classOf[OutputCommitter])
 
     if (clazz != null) {
       logInfo(s"Using user defined output committer class ${clazz.getCanonicalName}")
 
->>>>>>> origin/master
       // Every output format based on org.apache.hadoop.mapreduce.lib.output.OutputFormat
       // has an associated output committer. To override this output committer,
       // we will first try to use the output committer set in SQLConf.OUTPUT_COMMITTER_CLASS.
@@ -58,25 +51,15 @@ class SQLHadoopMapReduceCommitProtocol(jobId: String, path: String)
         // The specified output committer is a FileOutputCommitter.
         // So, we will use the FileOutputCommitter-specified constructor.
         val ctor = clazz.getDeclaredConstructor(classOf[Path], classOf[TaskAttemptContext])
-<<<<<<< HEAD
-        ctor.newInstance(new Path(path), context)
-=======
         committer = ctor.newInstance(new Path(path), context)
->>>>>>> origin/master
       } else {
         // The specified output committer is just an OutputCommitter.
         // So, we will use the no-argument constructor.
         val ctor = clazz.getDeclaredConstructor()
-<<<<<<< HEAD
-        ctor.newInstance()
-=======
         committer = ctor.newInstance()
->>>>>>> origin/master
       }
-    } else {
-      val committer = context.getOutputFormatClass.newInstance().getOutputCommitter(context)
-      logInfo(s"Using output committer class ${committer.getClass.getCanonicalName}")
-      committer
     }
+    logInfo(s"Using output committer class ${committer.getClass.getCanonicalName}")
+    committer
   }
 }

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala

@@ -35,11 +35,8 @@ import org.apache.spark.sql.catalyst.expressions.Literal
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.datasources.{PartitionPath => Partition}
-<<<<<<< HEAD
-=======
 import org.apache.spark.sql.execution.streaming.MemoryStream
 import org.apache.spark.sql.functions._
->>>>>>> origin/master
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSQLContext
 import org.apache.spark.sql.types._