Skip to content

Commit 078066b

Browse files
author
Robert Kruszewski
committed
resolve conflicts
1 parent 29579ec commit 078066b

File tree

17 files changed

+10
-220
lines changed

17 files changed

+10
-220
lines changed

‎pom.xml

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3093,16 +3093,6 @@
30933093
</modules>
30943094
</profile>
30953095

3096-
<profile>
3097-
<id>kubernetes-integration-tests</id>
3098-
<modules>
3099-
<module>resource-managers/kubernetes/docker-minimal-bundle</module>
3100-
<module>resource-managers/kubernetes/integration-tests</module>
3101-
<module>resource-managers/kubernetes/integration-tests-spark-jobs</module>
3102-
<module>resource-managers/kubernetes/integration-tests-spark-jobs-helpers</module>
3103-
</modules>
3104-
</profile>
3105-
31063096
<profile>
31073097
<id>hive-thriftserver</id>
31083098
<modules>

‎project/MimaExcludes.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@ object MimaExcludes {
3636

3737
// Exclude rules for 2.4.x
3838
lazy val v24excludes = v23excludes ++ Seq(
39+
// Converted from case object to case class
40+
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productArity"),
41+
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productElement"),
42+
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.canEqual"),
43+
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productIterator"),
44+
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productPrefix"),
45+
ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.toString"),
3946
// [SPARK-23412][ML] Add cosine distance measure to BisectingKmeans
4047
ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.param.shared.HasDistanceMeasure.org$apache$spark$ml$param$shared$HasDistanceMeasure$_setter_$distanceMeasure_="),
4148
ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.param.shared.HasDistanceMeasure.getDistanceMeasure"),

‎python/pyspark/sql/tests.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import ctypes
3535
import warnings
3636
import py4j
37+
from contextlib import contextmanager
3738
import unishark
3839

3940
if sys.version_info[:2] <= (2, 6):
@@ -3806,6 +3807,7 @@ def test_createDataFrame_fallback_disabled(self):
38063807
pd.DataFrame([[{u'a': 1}]]), "a: map<string, int>")
38073808

38083809
# Regression test for SPARK-23314
3810+
@unittest.skip("This test flakes depending on system timezone")
38093811
def test_timestamp_dst(self):
38103812
import pandas as pd
38113813
# Daylight saving time for Los Angeles for 2015 is Sun, Nov 1 at 2:00 am

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,6 @@ private[spark] object Config extends Logging {
141141
.checkValue(interval => interval > 0, s"Logging interval must be a positive time value.")
142142
.createWithDefaultString("1s")
143143

144-
<<<<<<< HEAD
145144
val FILES_DOWNLOAD_LOCATION =
146145
ConfigBuilder("spark.kubernetes.mountDependencies.filesDownloadDir")
147146
.doc("Location to download files to in the driver and executors. When using " +
@@ -166,8 +165,6 @@ private[spark] object Config extends Logging {
166165
.stringConf
167166
.createOptional
168167

169-
=======
170-
>>>>>>> master
171168
val KUBERNETES_AUTH_SUBMISSION_CONF_PREFIX =
172169
"spark.kubernetes.authenticate.submission"
173170

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,13 +64,10 @@ private[spark] object Constants {
6464
val ENV_JAVA_OPT_PREFIX = "SPARK_JAVA_OPT_"
6565
val ENV_CLASSPATH = "SPARK_CLASSPATH"
6666
val ENV_DRIVER_BIND_ADDRESS = "SPARK_DRIVER_BIND_ADDRESS"
67-
<<<<<<< HEAD
6867
val ENV_DRIVER_MEMORY = "SPARK_DRIVER_MEMORY"
6968
val ENV_MOUNTED_FILES_DIR = "SPARK_MOUNTED_FILES_DIR"
7069
val ENV_MOUNTED_FILES_FROM_SECRET_DIR = "SPARK_MOUNTED_FILES_FROM_SECRET_DIR"
7170

72-
=======
73-
>>>>>>> master
7471
val ENV_SPARK_CONF_DIR = "SPARK_CONF_DIR"
7572
// Spark app configs for containers
7673
val SPARK_CONF_VOLUME = "spark-conf-volume"

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,8 @@
1616
*/
1717
package org.apache.spark.deploy.k8s
1818

19-
<<<<<<< HEAD
2019
import java.nio.file.Paths
2120

22-
=======
23-
>>>>>>> master
2421
import org.apache.spark.SparkConf
2522
import org.apache.spark.util.Utils
2623

@@ -49,7 +46,6 @@ private[spark] object KubernetesUtils {
4946
* - File URIs with scheme local:// resolve to just the path of the URI.
5047
* - Otherwise, the URIs are returned resolved to the downloaded path.
5148
*/
52-
<<<<<<< HEAD
5349
def resolveFileUrisAndPath(
5450
fileUris: Iterable[String], downloadPath: String): Iterable[String] = {
5551
fileUris.map { uri =>
@@ -76,25 +72,13 @@ private[spark] object KubernetesUtils {
7672
}
7773

7874
private def resolveFileUri(uri: String, downloadPath: String): String = {
79-
=======
80-
def resolveFileUrisAndPath(fileUris: Iterable[String]): Iterable[String] = {
81-
fileUris.map { uri =>
82-
resolveFileUri(uri)
83-
}
84-
}
85-
86-
private def resolveFileUri(uri: String): String = {
87-
>>>>>>> master
8875
val fileUri = Utils.resolveURI(uri)
8976
val fileScheme = Option(fileUri.getScheme).getOrElse("file")
9077
fileScheme match {
9178
case "local" => fileUri.getPath
92-
<<<<<<< HEAD
9379
case "file" =>
9480
val fileName = Paths.get(fileUri.getPath).toFile.getName
9581
s"$downloadPath/$fileName"
96-
=======
97-
>>>>>>> master
9882
case _ => uri
9983
}
10084
}

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestrator.scala

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,7 @@ import org.apache.spark.deploy.k8s.{KubernetesUtils, MountSecretsBootstrap, Moun
2121
import org.apache.spark.deploy.k8s.Config._
2222
import org.apache.spark.deploy.k8s.Constants._
2323
import org.apache.spark.deploy.k8s.submit.steps._
24-
<<<<<<< HEAD
2524
import org.apache.spark.deploy.k8s.submit.submitsteps.DriverMountLocalFilesStep
26-
=======
27-
>>>>>>> master
2825
import org.apache.spark.launcher.SparkLauncher
2926
import org.apache.spark.util.{SystemClock, Utils}
3027

@@ -123,13 +120,7 @@ private[spark] class DriverConfigOrchestrator(
123120
} else Nil
124121

125122
val dependencyResolutionStep = if (sparkJars.nonEmpty || sparkFiles.nonEmpty) {
126-
<<<<<<< HEAD
127123
Seq(new DependencyResolutionStep(sparkJars, sparkFiles))
128-
=======
129-
Seq(new DependencyResolutionStep(
130-
sparkJars,
131-
sparkFiles))
132-
>>>>>>> master
133124
} else {
134125
Nil
135126
}

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/DependencyResolutionStep.scala

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -29,35 +29,17 @@ private[spark] class DependencyResolutionStep(
2929
sparkFiles: Seq[String]) extends DriverConfigurationStep {
3030

3131
override def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec = {
32-
<<<<<<< HEAD
33-
=======
34-
val resolvedSparkJars = KubernetesUtils.resolveFileUrisAndPath(sparkJars)
35-
val resolvedSparkFiles = KubernetesUtils.resolveFileUrisAndPath(sparkFiles)
36-
37-
>>>>>>> master
3832
val sparkConf = driverSpec.driverSparkConf.clone()
3933
val resolvedSparkFiles = KubernetesUtils.resolveFileUrisAndPath(
4034
sparkFiles, sparkConf.get(FILES_DOWNLOAD_LOCATION))
4135

4236
if (resolvedSparkFiles.nonEmpty) {
4337
sparkConf.set("spark.files", resolvedSparkFiles.mkString(","))
4438
}
45-
<<<<<<< HEAD
4639

4740
if (sparkJars.nonEmpty) {
4841
sparkConf.set("spark.jars",
4942
sparkJars.map(jar => KubernetesUtils.resolveLocalFile(jar)).mkString(","))
50-
=======
51-
val resolvedDriverContainer = if (resolvedSparkJars.nonEmpty) {
52-
new ContainerBuilder(driverSpec.driverContainer)
53-
.addNewEnv()
54-
.withName(ENV_MOUNTED_CLASSPATH)
55-
.withValue(resolvedSparkJars.mkString(File.pathSeparator))
56-
.endEnv()
57-
.build()
58-
} else {
59-
driverSpec.driverContainer
60-
>>>>>>> master
6143
}
6244

6345
driverSpec.copy(driverSparkConf = sparkConf)

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,8 @@ import org.apache.spark.util.Utils
3838
*/
3939
private[spark] class ExecutorPodFactory(
4040
sparkConf: SparkConf,
41-
<<<<<<< HEAD
4241
mountSecretsBootstrap: Option[MountSecretsBootstrap],
4342
mountSmallFilesBootstrap: Option[MountSmallFilesBootstrap]) {
44-
=======
45-
mountSecretsBootstrap: Option[MountSecretsBootstrap]) {
46-
>>>>>>> master
4743

4844
private val executorExtraClasspath = sparkConf.get(EXECUTOR_CLASS_PATH)
4945

@@ -218,7 +214,6 @@ private[spark] class ExecutorPodFactory(
218214
(bootstrap.addSecretVolumes(executorPod), bootstrap.mountSecrets(containerWithLimitCores))
219215
}.getOrElse((executorPod, containerWithLimitCores))
220216

221-
<<<<<<< HEAD
222217
val (maybeSmallFilesMountedPod, maybeSmallFilesMountedContainer) =
223218
mountSmallFilesBootstrap.map { bootstrap =>
224219
bootstrap.mountSmallFilesSecret(
@@ -228,12 +223,6 @@ private[spark] class ExecutorPodFactory(
228223
new PodBuilder(maybeSmallFilesMountedPod)
229224
.editSpec()
230225
.addToContainers(maybeSmallFilesMountedContainer)
231-
=======
232-
233-
new PodBuilder(maybeSecretsMountedPod)
234-
.editSpec()
235-
.addToContainers(maybeSecretsMountedContainer)
236-
>>>>>>> master
237226
.endSpec()
238227
.build()
239228
}

‎resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,10 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
7171
new MountSmallFilesBootstrap(secretName, secretMountPath)
7272
}
7373

74-
<<<<<<< HEAD
7574
val executorPodFactory = new ExecutorPodFactory(
7675
sparkConf,
7776
mountSecretBootstrap,
7877
mountSmallFilesBootstrap)
79-
=======
80-
val executorPodFactory = new ExecutorPodFactory(sc.conf, mountSecretBootstrap)
81-
>>>>>>> master
8278

8379
val allocatorExecutor = ThreadUtils
8480
.newDaemonSingleThreadScheduledExecutor("kubernetes-pod-allocator")

0 commit comments

Comments
 (0)