diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index ebccc8a8ec..82e1c6f997 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -39,7 +39,6 @@ body: - Avro - Pig - Protobuf - - Scala - Thrift - CLI - Benchmark diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 4b5519d2d0..e1f580ed32 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -42,9 +42,8 @@ body: - Avro - Pig - Protobuf - - Scala - Thrift - CLI - Benchmark validations: - required: false \ No newline at end of file + required: false diff --git a/.github/ISSUE_TEMPLATE/usage_question.yaml b/.github/ISSUE_TEMPLATE/usage_question.yaml index 32c00d17d8..f8d1216e9c 100644 --- a/.github/ISSUE_TEMPLATE/usage_question.yaml +++ b/.github/ISSUE_TEMPLATE/usage_question.yaml @@ -62,9 +62,8 @@ body: - Avro - Pig - Protobuf - - Scala - Thrift - CLI - Benchmark validations: - required: false \ No newline at end of file + required: false diff --git a/README.md b/README.md index bb6372ecce..7bf70067ef 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Parquet is a very active project, and new features are being added quickly. Here * Column stats * Delta encoding * Index pages +* Scala DSL (deprecated) * Java Vector API support (experimental) ## Java Vector API support diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml deleted file mode 100644 index b7a180dd7d..0000000000 --- a/parquet-scala/pom.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - org.apache.parquet - parquet - ../pom.xml - 1.15.0-SNAPSHOT - - - 4.0.0 - - parquet-scala_${scala.binary.version} - jar - - Apache Parquet Scala - https://parquet.apache.org - - - - scala-tools.org - Scala-tools Maven2 Repository - https://scala-tools.org/repo-releases - - - - - scala-tools.org - Scala-tools Maven2 Repository - https://scala-tools.org/repo-releases - - - - - - org.apache.parquet - parquet-column - ${project.version} - - - org.scala-lang - scala-library - ${scala.version} - - - org.scalatest - scalatest_${scala.binary.version} - 3.0.1 - test - - - org.scalactic - scalactic_${scala.binary.version} - 3.0.1 - test - - - org.slf4j - slf4j-simple - ${slf4j.version} - test - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.9.1 - - - - compile - testCompile - - - - - - - diff --git a/parquet-scala/src/main/scala/org/apache/parquet/filter2/dsl/Dsl.scala b/parquet-scala/src/main/scala/org/apache/parquet/filter2/dsl/Dsl.scala deleted file mode 100644 index c26dca7232..0000000000 --- a/parquet-scala/src/main/scala/org/apache/parquet/filter2/dsl/Dsl.scala +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.filter2.dsl - -import java.lang.{Boolean => JBoolean, Double => JDouble, Float => JFloat, Integer => JInt, Long => JLong} -import java.io.Serializable - -import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate, Operators, UserDefinedPredicate} -import org.apache.parquet.io.api.Binary - -/** - * Instead of using the methods in [[FilterApi]] directly in scala code, - * use this Dsl instead. Example usage: - * - * {{{ - * import parquet.filter2.dsl.Dsl._ - * - * val abc = IntColumn("a.b.c") - * val xyz = DoubleColumn("x.y.z") - * - * val myPredicate = !(abc > 10 && (xyz === 17 || ((xyz !== 13) && (xyz <= 20)))) - * - * }}} - * - * Note that while the operators >, >=, <, <= all work, the == and != operators do not. - * Using == or != will result in a runtime exception. Instead use === and !== - * - * This is due to a limitation in overriding the the equals method. - */ -object Dsl { - - private[Dsl] trait Column[T <: Comparable[T], C <: Operators.Column[T]] { - val javaColumn: C - - def filterBy[U <: UserDefinedPredicate[T]](clazz: Class[U]) = FilterApi.userDefined(javaColumn, clazz) - - def filterBy[U <: UserDefinedPredicate[T] with Serializable](udp: U) = FilterApi.userDefined(javaColumn, udp) - - // this is not supported because it allows for easy mistakes. For example: - // val pred = IntColumn("foo") == "hello" - // will compile, but pred will be of type boolean instead of FilterPredicate - override def equals(x: Any) = - throw new UnsupportedOperationException("You probably meant to use === or !==") - } - - case class IntColumn(columnPath: String) extends Column[JInt, Operators.IntColumn] { - override val javaColumn = FilterApi.intColumn(columnPath) - } - - case class LongColumn(columnPath: String) extends Column[JLong, Operators.LongColumn] { - override val javaColumn = FilterApi.longColumn(columnPath) - } - - case class FloatColumn(columnPath: String) extends Column[JFloat, Operators.FloatColumn] { - override val javaColumn = FilterApi.floatColumn(columnPath) - } - - case class DoubleColumn(columnPath: String) extends Column[JDouble, Operators.DoubleColumn] { - override val javaColumn = FilterApi.doubleColumn(columnPath) - } - - case class BooleanColumn(columnPath: String) extends Column[JBoolean, Operators.BooleanColumn] { - override val javaColumn = FilterApi.booleanColumn(columnPath) - } - - case class BinaryColumn(columnPath: String) extends Column[Binary, Operators.BinaryColumn] { - override val javaColumn = FilterApi.binaryColumn(columnPath) - } - - implicit def enrichEqNotEq[T <: Comparable[T], C <: Operators.Column[T] with Operators.SupportsEqNotEq](column: Column[T, C]): SupportsEqNotEq[T,C] = new SupportsEqNotEq(column) - - class SupportsEqNotEq[T <: Comparable[T], C <: Operators.Column[T] with Operators.SupportsEqNotEq](val column: Column[T, C]) { - def ===(v: T) = FilterApi.eq(column.javaColumn, v) - def !== (v: T) = FilterApi.notEq(column.javaColumn, v) - } - - implicit def enrichLtGt[T <: Comparable[T], C <: Operators.Column[T] with Operators.SupportsLtGt](column: Column[T, C]): SupportsLtGt[T,C] = new SupportsLtGt(column) - - class SupportsLtGt[T <: Comparable[T], C <: Operators.Column[T] with Operators.SupportsLtGt](val column: Column[T, C]) { - def >(v: T) = FilterApi.gt(column.javaColumn, v) - def >=(v: T) = FilterApi.gtEq(column.javaColumn, v) - def <(v: T) = FilterApi.lt(column.javaColumn, v) - def <=(v: T) = FilterApi.ltEq(column.javaColumn, v) - } - - implicit def enrichPredicate(pred: FilterPredicate): RichPredicate = new RichPredicate(pred) - - class RichPredicate(val pred: FilterPredicate) { - def &&(other: FilterPredicate) = FilterApi.and(pred, other) - def ||(other: FilterPredicate) = FilterApi.or(pred, other) - def unary_! = FilterApi.not(pred) - } - - implicit def stringToBinary(s: String): Binary = Binary.fromString(s) - -} diff --git a/parquet-scala/src/test/scala/org/apache/parquet/filter2/dsl/DslTest.scala b/parquet-scala/src/test/scala/org/apache/parquet/filter2/dsl/DslTest.scala deleted file mode 100644 index 652717d299..0000000000 --- a/parquet-scala/src/test/scala/org/apache/parquet/filter2/dsl/DslTest.scala +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.filter2.dsl - -import java.lang.{Double => JDouble, Integer => JInt} -import java.io.Serializable - -import org.junit.runner.RunWith -import org.scalatest.FlatSpec -import org.scalatest.junit.JUnitRunner -import org.apache.parquet.filter2.predicate.Operators.{Or, UserDefined, UserDefinedByClass, DoubleColumn => JDoubleColumn, IntColumn => JIntColumn} -import org.apache.parquet.filter2.predicate.{FilterApi, Statistics, UserDefinedPredicate} - -class DummyFilter extends UserDefinedPredicate[JInt] with Serializable { - override def keep(value: JInt): Boolean = false - - override def canDrop(statistics: Statistics[JInt]): Boolean = false - - override def inverseCanDrop(statistics: Statistics[JInt]): Boolean = false -} - -@RunWith(classOf[JUnitRunner]) -class DslTest extends FlatSpec{ - import org.apache.parquet.filter2.dsl.Dsl._ - - "predicates" should "be correctly constructed using the dsl" in { - val abc = IntColumn("a.b.c") - val xyz = DoubleColumn("x.y.z") - - val complexPredicate = !(abc > 10 && (xyz === 17 || ((xyz !== 13) && (xyz <= 20)))) - val abcGt = FilterApi.gt[JInt, JIntColumn](abc.javaColumn, 10) - val xyzAnd = FilterApi.and(FilterApi.notEq[JDouble, JDoubleColumn](xyz.javaColumn, 13.0), - FilterApi.ltEq[JDouble, JDoubleColumn](xyz.javaColumn, 20.0)) - val xyzEq = FilterApi.eq[JDouble, JDoubleColumn](xyz.javaColumn, 17.0) - val xyzPred = FilterApi.or(xyzEq, xyzAnd) - val expected = FilterApi.not(FilterApi.and(abcGt, xyzPred)) - - assert(complexPredicate === expected) - } - - "user defined predicates" should "be correctly constructed" in { - val abc = IntColumn("a.b.c") - val predByClass = (abc > 10) || abc.filterBy(classOf[DummyFilter]) - val instance = new DummyFilter - val predByInstance = (abc > 10) || abc.filterBy(instance) - - val expectedByClass = FilterApi.or(FilterApi.gt[JInt, JIntColumn](abc.javaColumn, 10), FilterApi.userDefined(abc.javaColumn, classOf[DummyFilter])) - val expectedByInstance = FilterApi.or(FilterApi.gt[JInt, JIntColumn](abc.javaColumn, 10), FilterApi.userDefined(abc.javaColumn, instance)) - assert(predByClass === expectedByClass) - assert(predByInstance === expectedByInstance) - - val intUserDefinedByClass = predByClass.asInstanceOf[Or].getRight.asInstanceOf[UserDefinedByClass[JInt, DummyFilter]] - assert(intUserDefinedByClass.getUserDefinedPredicateClass === classOf[DummyFilter]) - assert(intUserDefinedByClass.getUserDefinedPredicate.isInstanceOf[DummyFilter]) - - val intUserDefinedByInstance = predByInstance.asInstanceOf[Or].getRight.asInstanceOf[UserDefined[JInt, DummyFilter]] - assert(intUserDefinedByInstance.getUserDefinedPredicate === instance) - } - - "Column == and != " should "throw a helpful warning" in { - val abc = IntColumn("a.b.c") - - intercept[UnsupportedOperationException] { - abc == 10 - } - - intercept[UnsupportedOperationException] { - abc != 10 - } - } -} diff --git a/pom.xml b/pom.xml index d8abf0c40f..b6688ee808 100644 --- a/pom.xml +++ b/pom.xml @@ -148,7 +148,6 @@ parquet-pig parquet-pig-bundle parquet-protobuf - parquet-scala parquet-thrift parquet-hadoop-bundle