|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | + * contributor license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright ownership. |
| 5 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | + * (the "License"); you may not use this file except in compliance with |
| 7 | + * the License. You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | + |
| 18 | +package org.apache.spark.sql.catalyst.util |
| 19 | + |
| 20 | +import scala.util.Random |
| 21 | + |
| 22 | +import org.apache.spark.SparkFunSuite |
| 23 | +import org.apache.spark.sql.RandomDataGenerator |
| 24 | +import org.apache.spark.sql.catalyst.encoders.{ExamplePointUDT, RowEncoder} |
| 25 | +import org.apache.spark.sql.catalyst.expressions.{FromUnsafeProjection, UnsafeArrayData, UnsafeProjection} |
| 26 | +import org.apache.spark.sql.types._ |
| 27 | + |
| 28 | +class ArrayDataIndexedSeqSuite extends SparkFunSuite { |
| 29 | + private def compArray(arrayData: ArrayData, elementDt: DataType, array: Array[Any]): Unit = { |
| 30 | + assert(arrayData.numElements == array.length) |
| 31 | + array.zipWithIndex.map { case (e, i) => |
| 32 | + if (e != null) { |
| 33 | + elementDt match { |
| 34 | + // For NaN, etc. |
| 35 | + case FloatType | DoubleType => assert(arrayData.get(i, elementDt).equals(e)) |
| 36 | + case _ => assert(arrayData.get(i, elementDt) === e) |
| 37 | + } |
| 38 | + } else { |
| 39 | + assert(arrayData.isNullAt(i)) |
| 40 | + } |
| 41 | + } |
| 42 | + |
| 43 | + val seq = arrayData.toSeq[Any](elementDt) |
| 44 | + array.zipWithIndex.map { case (e, i) => |
| 45 | + if (e != null) { |
| 46 | + elementDt match { |
| 47 | + // For Nan, etc. |
| 48 | + case FloatType | DoubleType => assert(seq(i).equals(e)) |
| 49 | + case _ => assert(seq(i) === e) |
| 50 | + } |
| 51 | + } else { |
| 52 | + assert(seq(i) == null) |
| 53 | + } |
| 54 | + } |
| 55 | + |
| 56 | + intercept[IndexOutOfBoundsException] { |
| 57 | + seq(-1) |
| 58 | + }.getMessage().contains("must be between 0 and the length of the ArrayData.") |
| 59 | + |
| 60 | + intercept[IndexOutOfBoundsException] { |
| 61 | + seq(seq.length) |
| 62 | + }.getMessage().contains("must be between 0 and the length of the ArrayData.") |
| 63 | + } |
| 64 | + |
| 65 | + private def testArrayData(): Unit = { |
| 66 | + val elementTypes = Seq(BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, |
| 67 | + DoubleType, DecimalType.USER_DEFAULT, StringType, BinaryType, DateType, TimestampType, |
| 68 | + CalendarIntervalType, new ExamplePointUDT()) |
| 69 | + val arrayTypes = elementTypes.flatMap { elementType => |
| 70 | + Seq(ArrayType(elementType, containsNull = false), ArrayType(elementType, containsNull = true)) |
| 71 | + } |
| 72 | + val random = new Random(100) |
| 73 | + arrayTypes.foreach { dt => |
| 74 | + val schema = StructType(StructField("col_1", dt, nullable = false) :: Nil) |
| 75 | + val row = RandomDataGenerator.randomRow(random, schema) |
| 76 | + val rowConverter = RowEncoder(schema) |
| 77 | + val internalRow = rowConverter.toRow(row) |
| 78 | + |
| 79 | + val unsafeRowConverter = UnsafeProjection.create(schema) |
| 80 | + val safeRowConverter = FromUnsafeProjection(schema) |
| 81 | + |
| 82 | + val unsafeRow = unsafeRowConverter(internalRow) |
| 83 | + val safeRow = safeRowConverter(unsafeRow) |
| 84 | + |
| 85 | + val genericArrayData = safeRow.getArray(0).asInstanceOf[GenericArrayData] |
| 86 | + val unsafeArrayData = unsafeRow.getArray(0).asInstanceOf[UnsafeArrayData] |
| 87 | + |
| 88 | + val elementType = dt.elementType |
| 89 | + test("ArrayDataIndexedSeq - UnsafeArrayData - " + dt.toString) { |
| 90 | + compArray(unsafeArrayData, elementType, unsafeArrayData.toArray[Any](elementType)) |
| 91 | + } |
| 92 | + |
| 93 | + test("ArrayDataIndexedSeq - GenericArrayData - " + dt.toString) { |
| 94 | + compArray(genericArrayData, elementType, genericArrayData.toArray[Any](elementType)) |
| 95 | + } |
| 96 | + } |
| 97 | + } |
| 98 | + |
| 99 | + testArrayData() |
| 100 | +} |
0 commit comments