Merge pull request #25 from valencik/other-langs

valencik · web-flow · commit 0dccf13628b1 · 2022-09-13T06:54:29.000-04:00
Add Support For Multiple Languages
diff --git a/docs/index.md b/docs/index.md
@@ -4,7 +4,7 @@ Textmogrify is a pre-alpha text manipulation library that hopefully works well w
 
 ## Usage
 
-This library is currently available for Scala binary versions 2.13 and 3.1.
+This library is currently available for Scala binary versions 2.13 and 3.2.
 
 To use the latest version, include the following in your `build.sbt`:
 
@@ -26,9 +26,10 @@ libraryDependencies ++= Seq(
 
 The Lucene module lets you use a Lucene [Analyzer][analyzer] to modify text, additionally it provides helpers to use `Analyzer`s with an fs2 [Stream][stream].
 
+
 ### Basics
 
-Typical usage is to use the `AnalyzerBuilder` to configure an `Analyzer` and call `.tokenizer` to get a `Resource[F, String => F[Vector[String]]]`:
+Typical usage is to use the `AnalyzerBuilder` to configure an `Analyzer` and call `.tokenizer[F]` to get a `Resource[F, String => F[Vector[String]]]`:
 
 ```scala mdoc:silent
 import textmogrify.lucene.AnalyzerBuilder
@@ -52,9 +53,26 @@ tokens.unsafeRunSync()
 We can see that our text was lowercased and the unicode `ñ` replaced with an ASCII `n`.
 
 
+### Languages
+
+Textmogrify comes with support for multiple languages.
+When setting up an `AnalyzerBuilder` you'll have access to language specific options once you call one of the helper language methods like `english` or `french`.
+Specifying a language preserves the configuration set beforehand.
+
+```scala mdoc:silent
+val base = AnalyzerBuilder.default.withLowerCasing.withASCIIFolding
+
+val en = base.english.withPorterStemmer.tokenizer[IO]
+val fr = base.french.withFrenchLightStemmer.tokenizer[IO]
+val es = base.spanish.withSpanishLightStemmer.tokenizer[IO]
+```
+
+All of `en`, `fr`, and `es` will both lowercase and asciifold their inputs in addition to using their language specific stemmers.
+
+
 ### Pipelines
 
-Another common use is to construct a `Pipe`, or `Stream` to `Stream` function.
+Another common use is to construct a `Pipe`, or `Stream` to `Stream` function using an `Analyzer`.
 Let's say we have some messages we want to analyze and index as part of some search component.
 Given a raw `Msg` type and an analyzed `Doc` type, we want to transform a `Stream[F, Msg]` into a `Stream[F, Doc]`.
 
@@ -75,7 +93,7 @@ val input = Stream(
 import fs2.Pipe
 
 val normalizeMsgs: Pipe[IO, Msg, Doc] = msgs => {
-  val tokenizer = AnalyzerBuilder.default
+  val tokenizer = AnalyzerBuilder.english
     .withLowerCasing
     .withStopWords(Set("how", "do", "i", "my"))
     .withPorterStemmer
diff --git a/example/src/main/scala/textmogrify/MultiLingualPipeline.scala b/example/src/main/scala/textmogrify/MultiLingualPipeline.scala
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2022 Pig.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package textmogrify
+
+import textmogrify.lucene.AnalyzerBuilder
+import cats.syntax.all._
+import cats.effect.{IO, IOApp, Resource}
+import fs2.{Pipe, Stream}
+
+object MultiLingualPipeline extends IOApp.Simple {
+
+  sealed trait Lang extends Product with Serializable
+  case object En extends Lang
+  case object Fr extends Lang
+  case object Es extends Lang
+
+  case class Msg(lang: Lang, msg: String)
+  case class Doc(lang: Lang, tokens: Vector[String])
+
+  val input = Stream(
+    Msg(En, "I Like Jalapeños"),
+    Msg(En, "Neeko likes jumping on counters"),
+    Msg(Fr, "J'aime Les Jalapeños"),
+    Msg(Fr, "Neeko aime sauter sur les compteurs"),
+    Msg(Es, "Me gustan los jalapeños"),
+    Msg(Es, "A Neeko le gusta saltar sobre los mostradores"),
+  )
+
+  def multiTokenizer: Resource[IO, Msg => IO[Vector[String]]] = {
+    val base = AnalyzerBuilder.default.withLowerCasing.withASCIIFolding
+
+    val englishA = base.english.withPorterStemmer.tokenizer[IO]
+    val frenchA = base.french.withFrenchLightStemmer.tokenizer[IO]
+    val spanishA = base.spanish.withSpanishLightStemmer.tokenizer[IO]
+
+    (englishA, frenchA, spanishA).parTupled.map { case (en, fr, es) =>
+      (msg: Msg) =>
+        msg.lang match {
+          case En => en(msg.msg)
+          case Fr => fr(msg.msg)
+          case Es => es(msg.msg)
+        }
+    }
+  }
+
+  val tokenizeMsgs: Pipe[IO, Msg, Doc] = msgs =>
+    Stream
+      .resource(multiTokenizer)
+      .flatMap(f => msgs.evalMap(m => f(m).map(ts => Doc(m.lang, ts))))
+
+  val docs: Stream[IO, Doc] = input.through(tokenizeMsgs)
+  val run = docs.compile.toList.flatMap(IO.println)
+
+}
diff --git a/example/src/main/scala/textmogrify/Pipeline.scala b/example/src/main/scala/textmogrify/Pipeline.scala
@@ -32,7 +32,7 @@ object Pipeline extends IOApp.Simple {
   )
 
   val tokenizeMsgs: Pipe[IO, Msg, Doc] = msgs => {
-    val tokenizer = AnalyzerBuilder.default.withLowerCasing
+    val tokenizer = AnalyzerBuilder.english.withLowerCasing
       .withStopWords(Set("how", "do", "i", "my"))
       .withPorterStemmer
       .tokenizer[IO]
diff --git a/lucene/src/main/scala/textmogrify/lucene/AnalyzerBuilder.scala b/lucene/src/main/scala/textmogrify/lucene/AnalyzerBuilder.scala
@@ -20,87 +20,187 @@ import cats.effect.kernel.{Resource, Sync}
 import org.apache.lucene.analysis.Analyzer.TokenStreamComponents
 import org.apache.lucene.analysis.standard.StandardTokenizer
 import org.apache.lucene.analysis.en.PorterStemFilter
+import org.apache.lucene.analysis.es.SpanishLightStemFilter
+import org.apache.lucene.analysis.fr.FrenchLightStemFilter
 import org.apache.lucene.analysis.LowerCaseFilter
 import org.apache.lucene.analysis.Analyzer
 import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter
 import org.apache.lucene.analysis.CharArraySet
 import org.apache.lucene.analysis.StopFilter
+import org.apache.lucene.analysis.TokenStream
 
-/** Build an Analyzer or tokenizer function
-  */
-final class AnalyzerBuilder private (
-    val lowerCase: Boolean,
-    val foldASCII: Boolean,
-    val stopWords: Set[String],
-    val stemmer: Boolean,
-) { self =>
+final case class Config(
+    lowerCase: Boolean,
+    foldASCII: Boolean,
+    stopWords: Set[String],
+) {
+  def withLowerCasing: Config =
+    copy(lowerCase = true)
 
-  private def copy(
-      lowerCase: Boolean = self.lowerCase,
-      foldASCII: Boolean = self.foldASCII,
-      stemmer: Boolean = self.stemmer,
-      stopWords: Set[String] = self.stopWords,
-  ): AnalyzerBuilder =
-    new AnalyzerBuilder(
-      lowerCase = lowerCase,
-      foldASCII = foldASCII,
-      stemmer = stemmer,
-      stopWords = stopWords,
-    )
+  def withASCIIFolding: Config =
+    copy(foldASCII = true)
+
+  def withStopWords(words: Set[String]): Config =
+    copy(stopWords = words)
+}
+object Config {
+  def empty: Config = Config(false, false, Set.empty)
+}
+
+/** Build an Analyzer or tokenizer function */
+sealed abstract class AnalyzerBuilder private[lucene] (config: Config) {
+  type Builder <: AnalyzerBuilder
+
+  def withConfig(config: Config): Builder
 
   /** Adds a lowercasing stage to the analyzer pipeline */
-  def withLowerCasing: AnalyzerBuilder =
-    copy(lowerCase = true)
+  def withLowerCasing: Builder =
+    withConfig(config.withLowerCasing)
 
   /** Adds an ASCII folding stage to the analyzer pipeline
     * ASCII folding converts alphanumeric and symbolic Unicode characters into
     * their ASCII equivalents, if one exists.
     */
-  def withASCIIFolding: AnalyzerBuilder =
-    copy(foldASCII = true)
+  def withASCIIFolding: Builder =
+    withConfig(config.withASCIIFolding)
 
-  /** Adds the Porter Stemmer to the end of the analyzer pipeline and enables lowercasing.
-    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
-    * NOTE: Lowercasing is forced as it is required for the Lucene PorterStemFilter.
-    */
-  def withPorterStemmer: AnalyzerBuilder =
-    copy(stemmer = true, lowerCase = true)
+  /** Adds a stop filter stage to analyzer pipeline for non-empty sets. */
+  def withStopWords(words: Set[String]): Builder =
+    withConfig(config.withStopWords(words))
 
-  /** Adds a stop filter stage to analyzer pipeline for non-empty sets.
-    */
-  def withStopWords(words: Set[String]): AnalyzerBuilder =
-    copy(stopWords = words)
+  /** Build the Analyzer wrapped inside a Resource. */
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer]
 
-  /** Build the Analyzer wrapped inside a Resource.
+  /** Directly construct a tokenizing function
     */
-  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+  def tokenizer[F[_]](implicit F: Sync[F]): Resource[F, String => F[Vector[String]]] =
+    build.map(a => Tokenizer.vectorTokenizer(a))
+
+  private[lucene] def mkFromStandardTokenizer[F[_]](
+      config: Config
+  )(extras: TokenStream => TokenStream)(implicit F: Sync[F]): Resource[F, Analyzer] =
     Resource.make(F.delay(new Analyzer {
       protected def createComponents(fieldName: String): TokenStreamComponents = {
         val source = new StandardTokenizer()
-        var tokens = if (self.lowerCase) new LowerCaseFilter(source) else source
-        tokens = if (self.foldASCII) new ASCIIFoldingFilter(tokens) else tokens
+        var tokens = if (config.lowerCase) new LowerCaseFilter(source) else source
+        tokens = if (config.foldASCII) new ASCIIFoldingFilter(tokens) else tokens
         tokens =
-          if (self.stopWords.isEmpty) tokens
+          if (config.stopWords.isEmpty) tokens
           else {
-            val stopSet = new CharArraySet(self.stopWords.size, true)
-            stopWords.foreach(w => stopSet.add(w))
+            val stopSet = new CharArraySet(config.stopWords.size, true)
+            config.stopWords.foreach(w => stopSet.add(w))
             new StopFilter(tokens, stopSet)
           }
-        tokens = if (self.stemmer) new PorterStemFilter(tokens) else tokens
-        new TokenStreamComponents(source, tokens)
+        new TokenStreamComponents(source, extras(tokens))
       }
     }))(analyzer => F.delay(analyzer.close()))
 
-  /** Directly construct a tokenizing function
-    */
-  def tokenizer[F[_]](implicit F: Sync[F]): Resource[F, String => F[Vector[String]]] =
-    self.build.map(a => Tokenizer.vectorTokenizer(a))
 }
 object AnalyzerBuilder {
-  def default: AnalyzerBuilder = new AnalyzerBuilder(
-    lowerCase = false,
-    foldASCII = false,
-    stemmer = false,
-    stopWords = Set.empty,
-  )
+  def default: DefaultAnalyzerBuilder =
+    new DefaultAnalyzerBuilder(Config.empty)
+  def english: EnglishAnalyzerBuilder =
+    new EnglishAnalyzerBuilder(Config.empty, false)
+  def french: FrenchAnalyzerBuilder =
+    new FrenchAnalyzerBuilder(Config.empty, false)
+  def spanish: SpanishAnalyzerBuilder =
+    new SpanishAnalyzerBuilder(Config.empty, false)
+}
+
+final class DefaultAnalyzerBuilder private[lucene] (config: Config)
+    extends AnalyzerBuilder(config) { self =>
+  type Builder = DefaultAnalyzerBuilder
+
+  def withConfig(newConfig: Config): DefaultAnalyzerBuilder =
+    new DefaultAnalyzerBuilder(newConfig)
+
+  def english: EnglishAnalyzerBuilder =
+    new EnglishAnalyzerBuilder(config, false)
+
+  def french: FrenchAnalyzerBuilder =
+    new FrenchAnalyzerBuilder(config, false)
+
+  def spanish: SpanishAnalyzerBuilder =
+    new SpanishAnalyzerBuilder(config, false)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config)(identity)
+}
+
+final class EnglishAnalyzerBuilder private[lucene] (
+    config: Config,
+    stemmer: Boolean,
+) extends AnalyzerBuilder(config) { self =>
+  type Builder = EnglishAnalyzerBuilder
+
+  private def copy(
+      newConfig: Config,
+      stemmer: Boolean = self.stemmer,
+  ): EnglishAnalyzerBuilder =
+    new EnglishAnalyzerBuilder(newConfig, stemmer)
+
+  def withConfig(newConfig: Config): EnglishAnalyzerBuilder =
+    copy(newConfig = newConfig)
+
+  /** Adds the Porter Stemmer to the end of the analyzer pipeline and enables lowercasing.
+    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
+    * NOTE: Lowercasing is forced as it is required for the Lucene PorterStemFilter.
+    */
+  def withPorterStemmer: EnglishAnalyzerBuilder =
+    copy(config.copy(lowerCase = true), stemmer = true)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config)(ts => if (self.stemmer) new PorterStemFilter(ts) else ts)
+}
+
+final class FrenchAnalyzerBuilder private[lucene] (
+    config: Config,
+    stemmer: Boolean,
+) extends AnalyzerBuilder(config) { self =>
+  type Builder = FrenchAnalyzerBuilder
+
+  private def copy(
+      newConfig: Config,
+      stemmer: Boolean = self.stemmer,
+  ): FrenchAnalyzerBuilder =
+    new FrenchAnalyzerBuilder(newConfig, stemmer)
+
+  def withConfig(newConfig: Config): FrenchAnalyzerBuilder =
+    copy(newConfig = newConfig)
+
+  /** Adds the FrenchLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
+    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
+    * NOTE: Lowercasing is forced as it is required for the Lucene FrenchLightStemFilter.
+    */
+  def withFrenchLightStemmer: FrenchAnalyzerBuilder =
+    copy(config.copy(lowerCase = true), stemmer = true)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config)(ts => if (self.stemmer) new FrenchLightStemFilter(ts) else ts)
+}
+
+final class SpanishAnalyzerBuilder private[lucene] (
+    config: Config,
+    stemmer: Boolean,
+) extends AnalyzerBuilder(config) { self =>
+  type Builder = SpanishAnalyzerBuilder
+
+  private def copy(
+      newConfig: Config,
+      stemmer: Boolean = self.stemmer,
+  ): SpanishAnalyzerBuilder =
+    new SpanishAnalyzerBuilder(newConfig, stemmer)
+
+  def withConfig(newConfig: Config): SpanishAnalyzerBuilder =
+    copy(newConfig = newConfig)
+
+  /** Adds the SpanishLight Stemmer to the end of the analyzer pipeline and enables lowercasing.
+    * Stemming reduces words like `jumping` and `jumps` to their root word `jump`.
+    * NOTE: Lowercasing is forced as it is required for the Lucene SpanishLightStemFilter.
+    */
+  def withSpanishLightStemmer: SpanishAnalyzerBuilder =
+    copy(config.copy(lowerCase = true), stemmer = true)
+
+  def build[F[_]](implicit F: Sync[F]): Resource[F, Analyzer] =
+    mkFromStandardTokenizer(config)(ts => if (self.stemmer) new SpanishLightStemFilter(ts) else ts)
 }
diff --git a/lucene/src/test/scala/textmogrify/lucene/AnalyzerBuilderSuite.scala b/lucene/src/test/scala/textmogrify/lucene/AnalyzerBuilderSuite.scala

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ object Pipeline extends IOApp.Simple {`
`32`	`32`	`)`
`33`	`33`
`34`	`34`	`val tokenizeMsgs: Pipe[IO, Msg, Doc] = msgs => {`
`35`		`- val tokenizer = AnalyzerBuilder.default.withLowerCasing`
	`35`	`+ val tokenizer = AnalyzerBuilder.english.withLowerCasing`
`36`	`36`	`.withStopWords(Set("how", "do", "i", "my"))`
`37`	`37`	`.withPorterStemmer`
`38`	`38`	`.tokenizer[IO]`