Merge pull request #45 from valencik/langs-pt2

Add yet more languages

Author: valencik

lucene/src/main/scala/textmogrify/lucene/AnalyzerBuilder.scala

@@ -20,12 +20,15 @@ import scala.jdk.CollectionConverters._
 
 import cats.effect.kernel.{Resource, Sync}
 import org.apache.lucene.analysis.Analyzer.TokenStreamComponents
+import org.apache.lucene.analysis.snowball.SnowballFilter
 import org.apache.lucene.analysis.standard.StandardTokenizer
 import org.apache.lucene.analysis.en.PorterStemFilter
 import org.apache.lucene.analysis.es.SpanishLightStemFilter
 import org.apache.lucene.analysis.fr.FrenchLightStemFilter
 import org.apache.lucene.analysis.it.ItalianLightStemFilter
 import org.apache.lucene.analysis.de.GermanLightStemFilter
+import org.apache.lucene.analysis.pt.PortugueseLightStemFilter
+import org.apache.lucene.analysis.br.BrazilianStemFilter
 import org.apache.lucene.analysis.LowerCaseFilter
 import org.apache.lucene.analysis.Analyzer
 import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter
@@ -37,6 +40,11 @@ import org.apache.lucene.analysis.fr.FrenchAnalyzer.{getDefaultStopSet => getFre
 import org.apache.lucene.analysis.es.SpanishAnalyzer.{getDefaultStopSet => getSpanishStopSet}
 import org.apache.lucene.analysis.it.ItalianAnalyzer.{getDefaultStopSet => getItalianStopSet}
 import org.apache.lucene.analysis.de.GermanAnalyzer.{getDefaultStopSet => getGermanStopSet}
+import org.apache.lucene.analysis.nl.DutchAnalyzer.{getDefaultStopSet => getDutchStopSet}
+import org.apache.lucene.analysis.pt.PortugueseAnalyzer.{getDefaultStopSet => getPortugueseStopSet}
+import org.apache.lucene.analysis.br.BrazilianAnalyzer.{
+  getDefaultStopSet => getBrazilianPortugueseStopSet
+}
 
 final case class Config(
     lowerCase: Boolean,
@@ -122,6 +130,12 @@ object AnalyzerBuilder {
     new FrenchAnalyzerBuilder(Config.empty, false)
   def german: GermanAnalyzerBuilder =
     new GermanAnalyzerBuilder(Config.empty, false)
+  def dutch: DutchAnalyzerBuilder =
+    new DutchAnalyzerBuilder(Config.empty, false)
+  def brazilianPortuguese: BrazilianPortugueseAnalyzerBuilder =
+    new BrazilianPortugueseAnalyzerBuilder(Config.empty, false)
+  def portuguese: PortugueseAnalyzerBuilder =
+    new PortugueseAnalyzerBuilder(Config.empty, false)
   def italian: ItalianAnalyzerBuilder =
     new ItalianAnalyzerBuilder(Config.empty, false)
   def spanish: SpanishAnalyzerBuilder =
@@ -143,6 +157,21 @@ final class DefaultAnalyzerBuilder private[lucene] (config: Config)
   def french: FrenchAnalyzerBuilder =
     new FrenchAnalyzerBuilder(config, false)
 
+  def german: GermanAnalyzerBuilder =
+    new GermanAnalyzerBuilder(config, false)
+
+  def dutch: DutchAnalyzerBuilder =
+    new DutchAnalyzerBuilder(config, false)
+
+  def brazilianPortuguese: BrazilianPortugueseAnalyzerBuilder =
+    new BrazilianPortugueseAnalyzerBuilder(config, false)
+
+  def portuguese: PortugueseAnalyzerBuilder =
+    new PortugueseAnalyzerBuilder(config, false)
+
+  def italian: ItalianAnalyzerBuilder =
+    new ItalianAnalyzerBuilder(config, false)
+
   def spanish: SpanishAnalyzerBuilder =
     new SpanishAnalyzerBuilder(config, false)
 
@@ -174,7 +203,7 @@ final class EnglishAnalyzerBuilder private[lucene] (
 
   /** Adds the Porter Stemmer to the end of the analyzer pipeline and enables lowercasing.
     * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
-    * NOTE: Lowercasing is forced as it is required for the Lucene PorterStemFilter.
+    * NOTE: Lowercasing is forced as it is required by most Lucene stemmers.
     */
   def withPorterStemmer: EnglishAnalyzerBuilder =
     copy(config.copy(lowerCase = true), stemmer = true)
@@ -211,7 +240,7 @@ final class FrenchAnalyzerBuilder private[lucene] (
 
   /** Adds the FrenchLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
     * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
-    * NOTE: Lowercasing is forced as it is required for the Lucene FrenchLightStemFilter.
+    * NOTE: Lowercasing is forced as it is required by most Lucene stemmers.
     */
   def withFrenchLightStemmer: FrenchAnalyzerBuilder =
     copy(config.copy(lowerCase = true), stemmer = true)
@@ -247,7 +276,7 @@ final class SpanishAnalyzerBuilder private[lucene] (
 
   /** Adds the SpanishLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
     * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
-    * NOTE: Lowercasing is forced as it is required for the Lucene SpanishLightStemFilter.
+    * NOTE: Lowercasing is forced as it is required by most Lucene stemmers.
     */
   def withSpanishLightStemmer: SpanishAnalyzerBuilder =
     copy(config.copy(lowerCase = true), stemmer = true)
@@ -283,7 +312,7 @@ final class ItalianAnalyzerBuilder private[lucene] (
 
   /** Adds the ItalianLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
     * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
-    * NOTE: Lowercasing is forced as it is required for the Lucene ItalianLightStemFilter.
+    * NOTE: Lowercasing is forced as it is required by most Lucene stemmers.
     */
   def withItalianLightStemmer: ItalianAnalyzerBuilder =
     copy(config.copy(lowerCase = true), stemmer = true)
@@ -319,7 +348,7 @@ final class GermanAnalyzerBuilder private[lucene] (
 
   /** Adds the GermanLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
     * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
-    * NOTE: Lowercasing is forced as it is required for the Lucene GermanLightStemFilter.
+    * NOTE: Lowercasing is forced as it is required by most Lucene stemmers.
     */
   def withGermanLightStemmer: GermanAnalyzerBuilder =
     copy(config.copy(lowerCase = true), stemmer = true)
@@ -330,3 +359,118 @@ final class GermanAnalyzerBuilder private[lucene] (
       if (self.stemmer) new GermanLightStemFilter(tokens) else tokens
     }
 }
+
+final class DutchAnalyzerBuilder private[lucene] (
+    config: Config,
+    stemmer: Boolean,
+) extends AnalyzerBuilder(config) { self =>
+  type Builder = DutchAnalyzerBuilder
+
+  private def copy(
+      newConfig: Config,
+      stemmer: Boolean = self.stemmer,
+  ): DutchAnalyzerBuilder =
+    new DutchAnalyzerBuilder(newConfig, stemmer)
+
+  def withConfig(newConfig: Config): DutchAnalyzerBuilder =
+    copy(newConfig = newConfig)
+
+  /** A convenience value for debugging or investigating, to inspect the Lucene default stop words.
+    * This set is immutable, and unused; it is the underlying Lucene `CharArraySet` that we use to
+    * build the default StopFilter
+    */
+  lazy val defaultStopWords: Set[String] =
+    getDutchStopSet().asScala.map(ca => String.valueOf(ca.asInstanceOf[Array[Char]])).toSet
+
+  /** Adds the Dutch Snowball Stemmer to the end of the analyzer pipeline and enables lowercasing.
+    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
+    * NOTE: Lowercasing is forced as it is required by most Lucene stemmers.
+    */
+  def withDutchStemmer: DutchAnalyzerBuilder =
+    copy(config.copy(lowerCase = true), stemmer = true)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config) { ts =>
+      val tokens = if (self.config.defaultStopWords) new StopFilter(ts, getDutchStopSet()) else ts
+      if (self.stemmer) {
+        new SnowballFilter(ts, new org.tartarus.snowball.ext.DutchStemmer())
+      } else tokens
+    }
+}
+
+final class PortugueseAnalyzerBuilder private[lucene] (
+    config: Config,
+    stemmer: Boolean,
+) extends AnalyzerBuilder(config) { self =>
+  type Builder = PortugueseAnalyzerBuilder
+
+  private def copy(
+      newConfig: Config,
+      stemmer: Boolean = self.stemmer,
+  ): PortugueseAnalyzerBuilder =
+    new PortugueseAnalyzerBuilder(newConfig, stemmer)
+
+  def withConfig(newConfig: Config): PortugueseAnalyzerBuilder =
+    copy(newConfig = newConfig)
+
+  /** A convenience value for debugging or investigating, to inspect the Lucene default stop words.
+    * This set is immutable, and unused; it is the underlying Lucene `CharArraySet` that we use to
+    * build the default StopFilter
+    */
+  lazy val defaultStopWords: Set[String] =
+    getPortugueseStopSet().asScala.map(ca => String.valueOf(ca.asInstanceOf[Array[Char]])).toSet
+
+  /** Adds the PortugueseLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
+    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
+    * NOTE: Lowercasing is forced as it is required by most Lucene stemmers.
+    */
+  def withPortugueseLightStemmer: PortugueseAnalyzerBuilder =
+    copy(config.copy(lowerCase = true), stemmer = true)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config) { ts =>
+      val tokens =
+        if (self.config.defaultStopWords) new StopFilter(ts, getPortugueseStopSet()) else ts
+      if (self.stemmer) new PortugueseLightStemFilter(tokens) else tokens
+    }
+}
+
+final class BrazilianPortugueseAnalyzerBuilder private[lucene] (
+    config: Config,
+    stemmer: Boolean,
+) extends AnalyzerBuilder(config) { self =>
+  type Builder = BrazilianPortugueseAnalyzerBuilder
+
+  private def copy(
+      newConfig: Config,
+      stemmer: Boolean = self.stemmer,
+  ): BrazilianPortugueseAnalyzerBuilder =
+    new BrazilianPortugueseAnalyzerBuilder(newConfig, stemmer)
+
+  def withConfig(newConfig: Config): BrazilianPortugueseAnalyzerBuilder =
+    copy(newConfig = newConfig)
+
+  /** A convenience value for debugging or investigating, to inspect the Lucene default stop words.
+    * This set is immutable, and unused; it is the underlying Lucene `CharArraySet` that we use to
+    * build the default StopFilter
+    */
+  lazy val defaultStopWords: Set[String] =
+    getBrazilianPortugueseStopSet().asScala
+      .map(ca => String.valueOf(ca.asInstanceOf[Array[Char]]))
+      .toSet
+
+  /** Adds the Brazilian Stemmer to the end of the analyzer pipeline and enables lowercasing.
+    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
+    * NOTE: Lowercasing is forced as it is required by most Lucene stemmers.
+    */
+  def withBrazilianStemmer: BrazilianPortugueseAnalyzerBuilder =
+    copy(config.copy(lowerCase = true), stemmer = true)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config) { ts =>
+      val tokens =
+        if (self.config.defaultStopWords) new StopFilter(ts, getBrazilianPortugueseStopSet())
+        else ts
+      if (self.stemmer) new BrazilianStemFilter(tokens) else tokens
+    }
+}

lucene/src/test/scala/textmogrify/lucene/AnalyzerBuilderSuite.scala

@@ -316,3 +316,162 @@ class GermanAnalyzerBuilderSuite extends CatsEffectSuite {
   }
 
 }
+
+class DutchAnalyzerBuilderSuite extends CatsEffectSuite {
+
+  val jalapenos = "Ik hou van Jalapeños"
+  val jumping = "Neeko springt graag op balies"
+
+  test("dutch analyzer default should tokenize without any transformations") {
+    val analyzer = AnalyzerBuilder.dutch
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Ik", "hou", "van", "Jalapeños"))
+  }
+
+  test("dutch analyzer withLowerCasing should lowercase all letters") {
+    val analyzer = AnalyzerBuilder.dutch.withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("ik", "hou", "van", "jalapeños"))
+  }
+
+  test("dutch analyzer withASCIIFolding should fold 'ñ' to 'n'") {
+    val analyzer = AnalyzerBuilder.dutch.withASCIIFolding
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Ik", "hou", "van", "Jalapenos"))
+  }
+
+  test("dutch analyzer withCustomStopWords should filter them out") {
+    val analyzer = AnalyzerBuilder.dutch.withCustomStopWords(Set("Ik"))
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("hou", "van", "Jalapeños"))
+  }
+
+  test("dutch analyzer withDefaultStopWords should filter them out") {
+    val analyzer = AnalyzerBuilder.dutch.withDefaultStopWords
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("Neeko", "springt", "graag", "balies"))
+  }
+
+  test("dutch analyzer withDutchLightStemmer should lowercase and stem words") {
+    val analyzer = AnalyzerBuilder.dutch.withDutchStemmer
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("neeko", "springt", "grag", "op", "balies"))
+  }
+
+  test("dutch analyzer builder settings can be chained") {
+    val analyzer = AnalyzerBuilder.dutch.withDutchStemmer
+      .withCustomStopWords(Set("Neeko"))
+      .withDefaultStopWords
+      .withASCIIFolding
+      .withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("springt", "grag", "op", "balies"))
+  }
+
+}
+
+class PortugueseAnalyzerBuilderSuite extends CatsEffectSuite {
+
+  val jalapenos = "Eu gosto de jalapeños"
+  val jumping = "Neeko gosta de saltar em balcões"
+
+  test("portuguese analyzer default should tokenize without any transformations") {
+    val analyzer = AnalyzerBuilder.portuguese
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Eu", "gosto", "de", "jalapeños"))
+  }
+
+  test("portuguese analyzer withLowerCasing should lowercase all letters") {
+    val analyzer = AnalyzerBuilder.portuguese.withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("eu", "gosto", "de", "jalapeños"))
+  }
+
+  test("portuguese analyzer withASCIIFolding should fold 'ñ' to 'n'") {
+    val analyzer = AnalyzerBuilder.portuguese.withASCIIFolding
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Eu", "gosto", "de", "jalapenos"))
+  }
+
+  test("portuguese analyzer withCustomStopWords should filter them out") {
+    val analyzer = AnalyzerBuilder.portuguese.withCustomStopWords(Set("eu", "de"))
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("gosto", "jalapeños"))
+  }
+
+  test("portuguese analyzer withDefaultStopWords should filter them out") {
+    val analyzer = AnalyzerBuilder.portuguese.withDefaultStopWords
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("Neeko", "gosta", "saltar", "balcões"))
+  }
+
+  test("portuguese analyzer withPortugueseLightStemmer should lowercase and stem words") {
+    val analyzer = AnalyzerBuilder.portuguese.withPortugueseLightStemmer
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("neek", "gost", "de", "saltar", "em", "balca"))
+  }
+
+  test("portuguese analyzer builder settings can be chained") {
+    val analyzer = AnalyzerBuilder.portuguese.withPortugueseLightStemmer
+      .withCustomStopWords(Set("Neeko"))
+      .withDefaultStopWords
+      .withASCIIFolding
+      .withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("gost", "saltar", "balco"))
+  }
+
+}
+
+class BrazilianPortugueseAnalyzerBuilderSuite extends CatsEffectSuite {
+
+  val jalapenos = "Eu gosto de jalapeños"
+  val jumping = "Neeko gosta de pular em balcões"
+
+  test("brazilianPortuguese analyzer default should tokenize without any transformations") {
+    val analyzer = AnalyzerBuilder.brazilianPortuguese
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Eu", "gosto", "de", "jalapeños"))
+  }
+
+  test("brazilianPortuguese analyzer withLowerCasing should lowercase all letters") {
+    val analyzer = AnalyzerBuilder.brazilianPortuguese.withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("eu", "gosto", "de", "jalapeños"))
+  }
+
+  test("brazilianPortuguese analyzer withASCIIFolding should fold 'ñ' to 'n'") {
+    val analyzer = AnalyzerBuilder.brazilianPortuguese.withASCIIFolding
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("Eu", "gosto", "de", "jalapenos"))
+  }
+
+  test("brazilianPortuguese analyzer withCustomStopWords should filter them out") {
+    val analyzer = AnalyzerBuilder.brazilianPortuguese.withCustomStopWords(Set("eu", "de"))
+    val actual = analyzer.tokenizer[IO].use(f => f(jalapenos))
+    assertIO(actual, Vector("gosto", "jalapeños"))
+  }
+
+  test("brazilianPortuguese analyzer withDefaultStopWords should filter them out") {
+    val analyzer = AnalyzerBuilder.brazilianPortuguese.withDefaultStopWords
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("Neeko", "gosta", "pular", "balcões"))
+  }
+
+  test("brazilianPortuguese analyzer withPortugueseLightStemmer should lowercase and stem words") {
+    val analyzer = AnalyzerBuilder.brazilianPortuguese.withBrazilianStemmer
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("neek", "gost", "de", "pul", "em", "balco"))
+  }
+
+  test("brazilianPortuguese analyzer builder settings can be chained") {
+    val analyzer = AnalyzerBuilder.brazilianPortuguese.withBrazilianStemmer
+      .withCustomStopWords(Set("Neeko"))
+      .withDefaultStopWords
+      .withASCIIFolding
+      .withLowerCasing
+    val actual = analyzer.tokenizer[IO].use(f => f(jumping))
+    assertIO(actual, Vector("gost", "pul", "balco"))
+  }
+
+}