fix(lucene): add info about diacritics analyzer

duncdrum · duncdrum · commit 23895a83585e · 2020-01-27T14:41:55.000+01:00
close #83
diff --git a/src/main/xar-resources/data/lucene/listings/listing-11.xml b/src/main/xar-resources/data/lucene/listings/listing-11.xml
@@ -1,4 +1,4 @@
 <lucene>
-    <text qname="p"/>
-    <text qname="placeName"/>
-</lucene>
+  <text qname="p"/>
+  <text qname="placeName"/>
+</lucene>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-19.xml b/src/main/xar-resources/data/lucene/listings/listing-19.xml
@@ -1,3 +1,3 @@
 <text qname="p">
-    <inline qname="em"/>
-</text>
+  <inline qname="em"/>
+</text>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-20.xml b/src/main/xar-resources/data/lucene/listings/listing-20.xml
@@ -1,2 +1,3 @@
 <p>This is a paragraph
-<note>containing an inline note</note>.</p>
+  <note>containing an inline note</note>.
+</p>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-22.xml b/src/main/xar-resources/data/lucene/listings/listing-22.xml
@@ -1,5 +1,5 @@
 <lucene>
-    <text qname="p"/>
-    <text qname="note"/>
-    <ignore qname="note"/>
-</lucene>
+  <text qname="p"/>
+  <text qname="note"/>
+  <ignore qname="note"/>
+</lucene>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-23.xml b/src/main/xar-resources/data/lucene/listings/listing-23.xml
@@ -0,0 +1,12 @@
+<collection xmlns="http://exist-db.org/collection-config/1.0">
+  <index xmlns:xs="http://www.w3.org/2001/XMLSchema">
+    <lucene>
+      <analyzer class="org.exist.indexing.lucene.analyzers.NoDiacriticsStandardAnalyzer" id="nodiacritics"/>
+      <text qname="letter" analyzer="nodiacritics">
+        <field name="place" expression="place" analyzer="nodiacritics"/>
+        <field name="from" expression="from" store="no"/>
+        <field name="to" expression="to"/>
+      </text>
+    </lucene>
+  </index>
+</collection>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-24.xml b/src/main/xar-resources/data/lucene/listings/listing-24.xml
@@ -0,0 +1,12 @@
+<collection xmlns="http://exist-db.org/collection-config/1.0">
+  <index>
+    <!-- Lucene indexes -->
+    <lucene diacritics='no'>
+      <analyzer class='org.apache.lucene.analysis.standard.StandardAnalyzer'/>
+      <text match="//title[@xml:lang='Sa-Ltn']"/>
+      <text match="/TEI/text">
+        <ignore qname="text"/>
+      </text>
+    </lucene>
+  </index>
+</collection>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-25.xml b/src/main/xar-resources/data/lucene/listings/listing-25.xml
@@ -1,12 +1,12 @@
 <lucene>
-    <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
-    <text qname="section">
-        <ignore qname="title"/>
-        <ignore qname="programlisting"/>
-        <ignore qname="screen"/>
-        <ignore qname="synopsis"/>
-    </text>
-    <text qname="para"/>
-    <text qname="title" boost="2.0"/>
+  <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
+  <text qname="section">
     <ignore qname="title"/>
-</lucene>
+    <ignore qname="programlisting"/>
+    <ignore qname="screen"/>
+    <ignore qname="synopsis"/>
+  </text>
+  <text qname="para"/>
+  <text qname="title" boost="2.0"/>
+  <ignore qname="title"/>
+</lucene>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-27.xml b/src/main/xar-resources/data/lucene/listings/listing-27.xml
@@ -1,3 +1,3 @@
 <text qname="@val">
-    <match-sibling-attr boost="25" qname="att" value="writtenForm"/>
+  <match-sibling-attr boost="25" qname="att" value="writtenForm"/>
 </text>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-28.xml b/src/main/xar-resources/data/lucene/listings/listing-28.xml
@@ -1,3 +1,3 @@
 <text qname="feat">
-    <has-attr boost="0" qname="xml:lang"/>
+  <has-attr boost="0" qname="xml:lang"/>
 </text>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-29.xml b/src/main/xar-resources/data/lucene/listings/listing-29.xml
@@ -1,6 +1,6 @@
 <lucene>
-    <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
-    <analyzer id="ws" class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
-    <text match="//SPEECH//*"/>
-    <text qname="TITLE" analyzer="ws"/>
-</lucene>
+  <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
+  <analyzer id="ws" class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
+  <text match="//SPEECH//*"/>
+  <text qname="TITLE" analyzer="ws"/>
+</lucene>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-3.xml b/src/main/xar-resources/data/lucene/listings/listing-3.xml
@@ -1,20 +1,19 @@
 <collection xmlns="http://exist-db.org/collection-config/1.0">
-    <index xmlns:wiki="http://exist-db.org/xquery/wiki" xmlns:html="http://www.w3.org/1999/xhtml" xmlns:atom="http://www.w3.org/2005/Atom">
-        <!-- Disable the old full text index -->
-        <fulltext default="none" attributes="false"/>
-	<!-- Lucene index is configured below -->
-        <lucene>
-	        <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
-            <analyzer id="ws" class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
-	        <text qname="TITLE" analyzer="ws"/>
-	        <text qname="p">
-	            <inline qname="em"/>
-            </text>
-            <text match="//foo/*"/>
-            <!-- "inline" and "ignore" can be specified globally or per-index as
-                 shown above -->
-	        <inline qname="b"/>
-	        <ignore qname="note"/>
-        </lucene>
-    </index>
-</collection>
+  <index xmlns:wiki="http://exist-db.org/xquery/wiki" xmlns:html="http://www.w3.org/1999/xhtml" xmlns:atom="http://www.w3.org/2005/Atom">
+    <!-- Disable the old full text index -->
+    <fulltext default="none" attributes="false"/>
+    <!-- Lucene index is configured below -->
+    <lucene>
+      <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
+      <analyzer id="ws" class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
+      <text qname="TITLE" analyzer="ws"/>
+      <text qname="p">
+        <inline qname="em"/>
+      </text>
+      <text match="//foo/*"/>
+      <!-- "inline" and "ignore" can be specified globally or per-index as shown above -->
+      <inline qname="b"/>
+      <ignore qname="note"/>
+    </lucene>
+  </index>
+</collection>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-30.xml b/src/main/xar-resources/data/lucene/listings/listing-30.xml
@@ -1,3 +1,3 @@
 <analyzer id="stdstops" class="org.apache.lucene.analysis.standard.StandardAnalyzer">
-    <param name="stopwords" type="java.io.FileReader" value="/tmp/stop.txt"/>
-</analyzer>
+  <param name="stopwords" type="java.io.FileReader" value="/tmp/stop.txt"/>
+</analyzer>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-31.xml b/src/main/xar-resources/data/lucene/listings/listing-31.xml
@@ -1,8 +1,8 @@
 <analyzer id="stdstops" class="org.apache.lucene.analysis.standard.StandardAnalyzer">
-    <param name="stopwords" type="org.apache.lucene.analysis.util.CharArraySet">
-        <value>the</value>
-        <value>this</value>
-        <value>and</value>
-        <value>that</value>
-    </param>
-</analyzer>
+  <param name="stopwords" type="org.apache.lucene.analysis.util.CharArraySet">
+    <value>the</value>
+    <value>this</value>
+    <value>and</value>
+    <value>that</value>
+  </param>
+</analyzer>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-32.xml b/src/main/xar-resources/data/lucene/listings/listing-32.xml
@@ -1,9 +1,9 @@
 <analyzer id="sbstops" class="org.apache.lucene.analysis.snowball.SnowballAnalyzer">
-    <param name="name" value="English"/>
-    <param name="stopwords" type="org.apache.lucene.analysis.util.CharArraySet">
-        <value>the</value>
-        <value>this</value>
-        <value>and</value>
-        <value>that</value>
-    </param>
-</analyzer>
+  <param name="name" value="English"/>
+  <param name="stopwords" type="org.apache.lucene.analysis.util.CharArraySet">
+    <value>the</value>
+    <value>this</value>
+    <value>and</value>
+    <value>that</value>
+  </param>
+</analyzer>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-4.xml b/src/main/xar-resources/data/lucene/listings/listing-4.xml
@@ -1,18 +1,17 @@
 <collection xmlns="http://exist-db.org/collection-config/1.0">
-    <index xmlns:wiki="http://exist-db.org/xquery/wiki" xmlns:html="http://www.w3.org/1999/xhtml" xmlns:atom="http://www.w3.org/2005/Atom">
-        <!-- Lucene index is configured below -->
-        <lucene>
-	        <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
-            <analyzer id="ws" class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
-	        <text qname="TITLE" analyzer="ws"/>
-	        <text qname="p">
-	            <inline qname="em"/>
-            </text>
-            <text match="//foo/*"/>
-            <!-- "inline" and "ignore" can be specified globally or per-index as
-                 shown above -->
-	        <inline qname="b"/>
-	        <ignore qname="note"/>
-        </lucene>
-    </index>
-</collection>
+  <index xmlns:wiki="http://exist-db.org/xquery/wiki" xmlns:html="http://www.w3.org/1999/xhtml" xmlns:atom="http://www.w3.org/2005/Atom">
+    <!-- Lucene index is configured below -->
+    <lucene>
+      <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
+      <analyzer id="ws" class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
+      <text qname="TITLE" analyzer="ws"/>
+      <text qname="p">
+        <inline qname="em"/>
+      </text>
+      <text match="//foo/*"/>
+      <!-- "inline" and "ignore" can be specified globally or per-index as shown above -->
+      <inline qname="b"/>
+      <ignore qname="note"/>
+    </lucene>
+  </index>
+</collection>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-520.xml b/src/main/xar-resources/data/lucene/listings/listing-520.xml
@@ -1,13 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <collection xmlns="http://exist-db.org/collection-config/1.0">
-    <index xmlns:db="http://docbook.org/ns/docbook" xmlns:xs="http://www.w3.org/2001/XMLSchema">
-        <lucene>
-            <module uri="http://exist-db.org/lucene/test/" prefix="idx" at="module.xql"/>
-            <text qname="db:article">
-                <facet dimension="keyword" expression="db:info/db:keywordset/db:keyword"/>
-                <facet dimension="date" expression="tokenize(db:info/db:pubdate, '-')" hierarchical="yes"/>
-                <facet dimension="subject" expression="idx:subject-hierarchy(db:info/db:subjectset/db:subject/db:subjectterm)" hierarchical="yes"/>
-            </text>
-        </lucene>
-    </index>
-</collection>
+  <index xmlns:db="http://docbook.org/ns/docbook" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+    <lucene>
+      <module uri="http://exist-db.org/lucene/test/" prefix="idx" at="module.xql"/>
+      <text qname="db:article">
+        <facet dimension="keyword" expression="db:info/db:keywordset/db:keyword"/>
+        <facet dimension="date" expression="tokenize(db:info/db:pubdate, '-')" hierarchical="yes"/>
+        <facet dimension="subject" expression="idx:subject-hierarchy(db:info/db:subjectset/db:subject/db:subjectterm)" hierarchical="yes"/>
+      </text>
+    </lucene>
+  </index>
+</collection>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-522.xml b/src/main/xar-resources/data/lucene/listings/listing-522.xml
@@ -1,11 +1,11 @@
 <subject>
-    <subject name="science">
-        <subject name="math"/>
-        <subject name="physics"/>
-    </subject>
-    <subject name="humanities">
-        <subject name="art"/>
-        <subject name="sociology"/>
-        <subject name="history"/>
-    </subject>
-</subject>
+  <subject name="science">
+    <subject name="math"/>
+    <subject name="physics"/>
+  </subject>
+  <subject name="humanities">
+    <subject name="art"/>
+    <subject name="sociology"/>
+    <subject name="history"/>
+  </subject>
+</subject>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-58.xml b/src/main/xar-resources/data/lucene/listings/listing-58.xml
@@ -1,13 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <collection xmlns="http://exist-db.org/collection-config/1.0">
-    <index xmlns:db="http://docbook.org/ns/docbook" xmlns:xs="http://www.w3.org/2001/XMLSchema">
-        <lucene>
-            <module uri="http://exist-db.org/lucene/test/" prefix="idx" at="module.xql"/>
-            <text qname="db:article">
-                <facet dimension="keyword" expression="db:info/db:keywordset/db:keyword"/>
-                <field name="title" expression="db:info/db:title"/>
-                <field name="author" expression="idx:author(db:info/db:author)"/>
-            </text>
-        </lucene>
-    </index>
-</collection>
+  <index xmlns:db="http://docbook.org/ns/docbook" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+    <lucene>
+      <module uri="http://exist-db.org/lucene/test/" prefix="idx" at="module.xql"/>
+      <text qname="db:article">
+        <facet dimension="keyword" expression="db:info/db:keywordset/db:keyword"/>
+        <field name="title" expression="db:info/db:title"/>
+        <field name="author" expression="idx:author(db:info/db:author)"/>
+      </text>
+    </lucene>
+  </index>
+</collection>
diff --git a/src/main/xar-resources/data/lucene/listings/listing-59.xml b/src/main/xar-resources/data/lucene/listings/listing-59.xml
@@ -1,13 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <collection xmlns="http://exist-db.org/collection-config/1.0">
-    <index xmlns:db="http://docbook.org/ns/docbook" xmlns:xs="http://www.w3.org/2001/XMLSchema">
-        <lucene>
-            <analyzer class="org.apache.lucene.analysis.de.GermanAnalyzer" id="german"/>
-            <analyzer class="org.apache.lucene.analysis.en.EnglishAnalyzer" id="english"/>
-            <text qname="db:article" index="no">
-                <field name="english" if="@xml:lang='en'" analyzer="english"/>
-                <field name="german" if="@xml:lang='de'" analyzer="german"/>
-            </text>
-        </lucene>
-    </index>
-</collection>
+  <index xmlns:db="http://docbook.org/ns/docbook" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+    <lucene>
+      <analyzer class="org.apache.lucene.analysis.de.GermanAnalyzer" id="german"/>
+      <analyzer class="org.apache.lucene.analysis.en.EnglishAnalyzer" id="english"/>
+      <text qname="db:article" index="no">
+        <field name="english" if="@xml:lang='en'" analyzer="english"/>
+        <field name="german" if="@xml:lang='de'" analyzer="german"/>
+      </text>
+    </lucene>
+  </index>
+</collection>
diff --git a/src/main/xar-resources/data/lucene/lucene.xml b/src/main/xar-resources/data/lucene/lucene.xml
@@ -288,7 +288,10 @@
                 the text at white space characters, but treats all other characters - including
                 punctuation - as part of the token. The tokens are not converted to lower case and
                 there's no stopword filter applied.</para>
-
+            <para>eXist-db provides a special analyzer for characters with diacrits based on the <literal>StandardAnalyzer</literal>. The <literal>NoDiacriticsStandardAnalyzer</literal> can be switched on and off by setting the <literal>diacritics</literal> attribute on the <tag>lucene</tag> element of your index configuration file.</para>
+            <programlisting language="xml" xlink:href="listings/listing-24.xml"/>
+            <para>Without diacrits <literal>ä</literal>, <literal>å</literal>, <literal>ā</literal>, etc will all be indexed as <literal>a</literal>. Alternatively, this analyzer can also be called by its full name. </para>
+            <programlisting language="xml" xlink:href="listings/listing-23.xml"/>
             <sect3 xml:id="conf">
                 <title>Configuring the Analyzer</title>
 
