|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | + * contributor license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright ownership. |
| 5 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | + * (the "License"); you may not use this file except in compliance with |
| 7 | + * the License. You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | + |
| 18 | +package org.apache.spark.shuffle.sort |
| 19 | + |
| 20 | +import java.io.{BufferedInputStream, Closeable, File, FileInputStream, FileOutputStream} |
| 21 | +import java.util.UUID |
| 22 | + |
| 23 | +import org.apache.commons.io.FileUtils |
| 24 | +import org.mockito.{Mock, MockitoAnnotations} |
| 25 | +import org.mockito.Answers.RETURNS_SMART_NULLS |
| 26 | +import org.mockito.ArgumentMatchers.any |
| 27 | +import org.mockito.Mockito.when |
| 28 | +import scala.collection.mutable |
| 29 | +import scala.collection.mutable.ArrayBuffer |
| 30 | +import scala.util.Random |
| 31 | + |
| 32 | +import org.apache.spark.{HashPartitioner, ShuffleDependency, SparkConf, TaskContext} |
| 33 | +import org.apache.spark.benchmark.{Benchmark, BenchmarkBase} |
| 34 | +import org.apache.spark.executor.TaskMetrics |
| 35 | +import org.apache.spark.memory.{MemoryManager, TaskMemoryManager, TestMemoryManager} |
| 36 | +import org.apache.spark.rpc.{RpcEndpoint, RpcEndpointRef, RpcEnv} |
| 37 | +import org.apache.spark.serializer.{KryoSerializer, Serializer, SerializerManager} |
| 38 | +import org.apache.spark.shuffle.IndexShuffleBlockResolver |
| 39 | +import org.apache.spark.storage.{BlockManager, DiskBlockManager, TempShuffleBlockId} |
| 40 | +import org.apache.spark.util.Utils |
| 41 | + |
| 42 | +abstract class ShuffleWriterBenchmarkBase extends BenchmarkBase { |
| 43 | + |
| 44 | + protected val DEFAULT_DATA_STRING_SIZE = 5 |
| 45 | + |
| 46 | + // This is only used in the writer constructors, so it's ok to mock |
| 47 | + @Mock(answer = RETURNS_SMART_NULLS) protected var dependency: |
| 48 | + ShuffleDependency[String, String, String] = _ |
| 49 | + // This is only used in the stop() function, so we can safely mock this without affecting perf |
| 50 | + @Mock(answer = RETURNS_SMART_NULLS) protected var taskContext: TaskContext = _ |
| 51 | + @Mock(answer = RETURNS_SMART_NULLS) protected var rpcEnv: RpcEnv = _ |
| 52 | + @Mock(answer = RETURNS_SMART_NULLS) protected var rpcEndpointRef: RpcEndpointRef = _ |
| 53 | + |
| 54 | + protected val defaultConf: SparkConf = new SparkConf(loadDefaults = false) |
| 55 | + protected val serializer: Serializer = new KryoSerializer(defaultConf) |
| 56 | + protected val partitioner: HashPartitioner = new HashPartitioner(10) |
| 57 | + protected val serializerManager: SerializerManager = |
| 58 | + new SerializerManager(serializer, defaultConf) |
| 59 | + protected val shuffleMetrics: TaskMetrics = new TaskMetrics |
| 60 | + |
| 61 | + protected val tempFilesCreated: ArrayBuffer[File] = new ArrayBuffer[File] |
| 62 | + protected val filenameToFile: mutable.Map[String, File] = new mutable.HashMap[String, File] |
| 63 | + |
| 64 | + class TestDiskBlockManager(tempDir: File) extends DiskBlockManager(defaultConf, false) { |
| 65 | + override def getFile(filename: String): File = { |
| 66 | + if (filenameToFile.contains(filename)) { |
| 67 | + filenameToFile(filename) |
| 68 | + } else { |
| 69 | + val outputFile = File.createTempFile("shuffle", null, tempDir) |
| 70 | + filenameToFile(filename) = outputFile |
| 71 | + outputFile |
| 72 | + } |
| 73 | + } |
| 74 | + |
| 75 | + override def createTempShuffleBlock(): (TempShuffleBlockId, File) = { |
| 76 | + var blockId = new TempShuffleBlockId(UUID.randomUUID()) |
| 77 | + val file = getFile(blockId) |
| 78 | + tempFilesCreated += file |
| 79 | + (blockId, file) |
| 80 | + } |
| 81 | + } |
| 82 | + |
| 83 | + class TestBlockManager(tempDir: File, memoryManager: MemoryManager) extends BlockManager("0", |
| 84 | + rpcEnv, |
| 85 | + null, |
| 86 | + serializerManager, |
| 87 | + defaultConf, |
| 88 | + memoryManager, |
| 89 | + null, |
| 90 | + null, |
| 91 | + null, |
| 92 | + null, |
| 93 | + 1) { |
| 94 | + override val diskBlockManager = new TestDiskBlockManager(tempDir) |
| 95 | + override val remoteBlockTempFileManager = null |
| 96 | + } |
| 97 | + |
| 98 | + protected var tempDir: File = _ |
| 99 | + |
| 100 | + protected var blockManager: BlockManager = _ |
| 101 | + protected var blockResolver: IndexShuffleBlockResolver = _ |
| 102 | + |
| 103 | + protected var memoryManager: TestMemoryManager = _ |
| 104 | + protected var taskMemoryManager: TaskMemoryManager = _ |
| 105 | + |
| 106 | + MockitoAnnotations.initMocks(this) |
| 107 | + when(dependency.partitioner).thenReturn(partitioner) |
| 108 | + when(dependency.serializer).thenReturn(serializer) |
| 109 | + when(dependency.shuffleId).thenReturn(0) |
| 110 | + when(taskContext.taskMetrics()).thenReturn(shuffleMetrics) |
| 111 | + when(rpcEnv.setupEndpoint(any[String], any[RpcEndpoint])).thenReturn(rpcEndpointRef) |
| 112 | + |
| 113 | + def setup(): Unit = { |
| 114 | + memoryManager = new TestMemoryManager(defaultConf) |
| 115 | + memoryManager.limit(PackedRecordPointer.MAXIMUM_PAGE_SIZE_BYTES) |
| 116 | + taskMemoryManager = new TaskMemoryManager(memoryManager, 0) |
| 117 | + tempDir = Utils.createTempDir() |
| 118 | + blockManager = new TestBlockManager(tempDir, memoryManager) |
| 119 | + blockResolver = new IndexShuffleBlockResolver( |
| 120 | + defaultConf, |
| 121 | + blockManager) |
| 122 | + } |
| 123 | + |
| 124 | + def addBenchmarkCase(benchmark: Benchmark, name: String)(func: Benchmark.Timer => Unit): Unit = { |
| 125 | + benchmark.addTimerCase(name) { timer => |
| 126 | + setup() |
| 127 | + func(timer) |
| 128 | + teardown() |
| 129 | + } |
| 130 | + } |
| 131 | + |
| 132 | + def teardown(): Unit = { |
| 133 | + FileUtils.deleteDirectory(tempDir) |
| 134 | + tempFilesCreated.clear() |
| 135 | + filenameToFile.clear() |
| 136 | + } |
| 137 | + |
| 138 | + protected class DataIterator (size: Int) |
| 139 | + extends Iterator[Product2[String, String]] { |
| 140 | + val random = new Random(123) |
| 141 | + var count = 0 |
| 142 | + override def hasNext: Boolean = { |
| 143 | + count < size |
| 144 | + } |
| 145 | + |
| 146 | + override def next(): Product2[String, String] = { |
| 147 | + count+=1 |
| 148 | + val string = random.alphanumeric.take(5).mkString |
| 149 | + (string, string) |
| 150 | + } |
| 151 | + } |
| 152 | + |
| 153 | + |
| 154 | + def createDataIterator(size: Int): DataIterator = { |
| 155 | + new DataIterator(size) |
| 156 | + } |
| 157 | + |
| 158 | +} |
0 commit comments