SOLR-10808: Enable docValues by default on specific fields (#2606)

HoustonPutman · web-flow · commit b44bdc198d83 · 2024-08-02T12:29:18.000-04:00
Fields:
- PrimitiveField (Numbers/Dates/String/UUID/Enum/Bool) - Without DenseVectors
- SortableTextField
- SortableBinaryField
- ICUCollationField
- CollationField
- LatLonPointSpacialField
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
@@ -153,6 +153,14 @@ New Features
 
 Improvements
 ---------------------
+* SOLR-10808, SOLR-12963: The Solr schema version has been increased to 1.7.
+  Starting in schema version 1.7, most fields/fieldTypes that support docValues will have them enabled by default.
+  These field types include primitive (Numeric, Date, Bool, String, Enum, UUID), sorting (SortableTextField, SortableBinaryField, CollationField, ICUCollationField) and LatLonPointSpacialField.
+  This behavior can be reverted by setting the 'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below.
+  Also in schema version 1.7, all fields/fieldTypes will be unable to be uninverted by default.
+  This behavior can be reverted by setting the 'uninvertible' parameter for a field or a field type to true, the default for schema versions 1.6 and below.
+  (Houston Putman, hossman)
+
 * SOLR-17137: Enable Prometheus exporter to communicate with SSL protected Solr. (Eivind Bergstøl via Eric Pugh)
 
 * SOLR-16921: use -solrUrl to derive the zk host connection for bin/solr zk subcommands (Eric Pugh)
@@ -183,13 +191,6 @@ Improvements
   Statuses are now removed 5 minutes after the read of a completed/failed status.  Helps collection
   async backup/restore and other operations scale to 100+ shards.  (Pierre Salagnac, David Smiley)
 
-* SOLR-10808, SOLR-12963: The Solr schema version has been increased to 1.7.
-  Starting in schema version 1.7, all fields/fieldTypes that support docValues will have them enabled by default.
-  This behavior can be reverted by setting the 'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below.
-  Also in schema version 1.7, all fields/fieldTypes will be unable to be uninverted by default.
-  This behavior can be reverted by setting the 'uninvertible' parameter for a field or a field type to true, the default for schema versions 1.6 and below.
-  (Houston Putman, hossman)
-
 * SOLR-10808 : The Solr schema version has been increased to 1.7. Since schema version 1.7, all fields/fieldTypes that
   support docValues will have them enabled by default. This behavior can be reverted by setting
   'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below. (Houston Putman)
diff --git a/solr/core/src/java/org/apache/solr/schema/BinaryField.java b/solr/core/src/java/org/apache/solr/schema/BinaryField.java
@@ -152,11 +152,6 @@ public List<IndexableField> createFields(SchemaField field, Object val) {
   protected void checkSupportsDocValues() { // we support DocValues
   }
 
-  @Override
-  protected boolean doesTypeSupportDocValues() {
-    return true;
-  }
-
   @Override
   public Object toNativeType(Object val) {
     if (val instanceof byte[]) {
diff --git a/solr/core/src/java/org/apache/solr/schema/CollationField.java b/solr/core/src/java/org/apache/solr/schema/CollationField.java
@@ -246,7 +246,7 @@ protected void checkSupportsDocValues() { // we support DocValues
   }
 
   @Override
-  protected boolean doesTypeSupportDocValues() {
+  protected boolean enableDocValuesByDefault() {
     return true;
   }
 
diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
@@ -179,7 +179,7 @@ public VectorEncoding getVectorEncoding() {
   }
 
   @Override
-  protected boolean doesTypeSupportDocValues() {
+  protected boolean enableDocValuesByDefault() {
     return false;
   }
 
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -188,7 +188,7 @@ protected void setArgs(IndexSchema schema, Map<String, String> args) {
       args.remove("compressThreshold");
     }
     if (schemaVersion >= 1.6f) properties |= USE_DOCVALUES_AS_STORED;
-    if (schemaVersion >= 1.7f && doesTypeSupportDocValues()) properties |= DOC_VALUES;
+    if (schemaVersion >= 1.7f && enableDocValuesByDefault()) properties |= DOC_VALUES;
 
     if (schemaVersion < 1.7f) properties |= UNINVERTIBLE;
 
@@ -1161,15 +1161,17 @@ protected void checkSupportsDocValues() {
         ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values");
   }
 
-  /** Returns whether this field type supports docValues. By default none do. */
-  protected boolean doesTypeSupportDocValues() {
-    try {
-      // TODO: In Solr 10.0 change this such that checkSupportsDocValues() calls this method instead
-      checkSupportsDocValues();
-      return true;
-    } catch (Exception ignored) {
-      return false;
-    }
+  /**
+   * Returns whether this field type should enable docValues by default for schemaVersion &gt;= 1.7.
+   * This should not be enabled for fields that did not have docValues implemented by Solr 9.7, as
+   * users may have indexed documents without docValues (since they weren't supported). Flipping the
+   * default docValues values when they upgrade to a new version will break their index
+   * compatibility.
+   *
+   * <p>New field types can enable this without issue, as long as they support docValues.
+   */
+  protected boolean enableDocValuesByDefault() {
+    return false;
   }
 
   public static final String TYPE = "type";
diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java
@@ -70,7 +70,7 @@ protected void checkSupportsDocValues() { // we support DocValues
   }
 
   @Override
-  protected boolean doesTypeSupportDocValues() {
+  protected boolean enableDocValuesByDefault() {
     return true;
   }
 
diff --git a/solr/core/src/java/org/apache/solr/schema/PointType.java b/solr/core/src/java/org/apache/solr/schema/PointType.java
@@ -172,11 +172,6 @@ protected void checkSupportsDocValues() {
     }
   }
 
-  @Override
-  protected boolean doesTypeSupportDocValues() {
-    return true;
-  }
-
   /**
    * Calculates the range and creates a RangeQuery (bounding box) wrapped in a BooleanQuery (unless
    * the dimension is 1, one range for every dimension, AND'd together by a Boolean
diff --git a/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java b/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java
@@ -39,7 +39,7 @@ protected void checkSupportsDocValues() { // primitive types support DocValues
   }
 
   @Override
-  protected boolean doesTypeSupportDocValues() {
+  protected boolean enableDocValuesByDefault() {
     return true;
   }
 
diff --git a/solr/core/src/java/org/apache/solr/schema/SortableTextField.java b/solr/core/src/java/org/apache/solr/schema/SortableTextField.java
@@ -151,7 +151,7 @@ protected void checkSupportsDocValues() {
   }
 
   @Override
-  protected boolean doesTypeSupportDocValues() {
+  protected boolean enableDocValuesByDefault() {
     return true;
   }
 
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml
@@ -33,7 +33,7 @@
 
   <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="true"/>
   <field name="data" type="binary" stored="true"/>
-  <field name="data_dv" type="binary" stored="false" />
+  <field name="data_dv" type="binary" stored="false" docValues="true" />
 
 
   <uniqueKey>id</uniqueKey>
diff --git a/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
@@ -316,7 +316,7 @@ protected void checkSupportsDocValues() { // we support DocValues
   }
 
   @Override
-  protected boolean doesTypeSupportDocValues() {
+  protected boolean enableDocValuesByDefault() {
     return true;
   }
 
diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/docvalues.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/docvalues.adoc
@@ -35,7 +35,21 @@ This approach promises to relieve some of the memory requirements of the fieldCa
 == Enabling DocValues
 
 [IMPORTANT]
-DocValues are enabled by default for all field types that support them (when using a schemaVersion >= 1.7).
+DocValues are enabled by default for most field types that support them *when using a schemaVersion >= 1.7*.
+
+* Primitive Fields
+** `Numeric` (not `DenseVectorField`)
+** `Boolean`
+** `String`
+** `Date`
+** `UUID`
+** `Enum`
+* Sorting Fields
+** `CollationField`
+** `ICUCollationField`
+** `SortableTextField`
+** `SortableBinaryField`
+* `LatLonPointSpacialField` (not `PointType`, which is different than `PointField`)
 
 When using an earlier schemaVersion (\<= 1.6), you only need to enable docValues for a field that you will use it with.
 As with all schema design, you need to define a field type and then define fields of that type with docValues enabled.
diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/field-type-definitions-and-properties.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/field-type-definitions-and-properties.adoc
@@ -198,7 +198,7 @@ The table below includes the default value for most `FieldType` implementations
 |Property |Description |Implicit Default
 |`indexed` |If `true`, the value of the field can be used in queries to retrieve matching documents. |`true`
 |`stored` |If `true`, the actual value of the field can be retrieved by queries.  |`true`
-|`docValues` |If `true`, the value of the field will be put in a column-oriented xref:docvalues.adoc[] structure. |`true`
+|`docValues` |If `true`, the value of the field will be put in a column-oriented xref:docvalues.adoc[] structure. |`true` for xref:indexing-guide:docvalues.adoc#enabling-docvalues[_most_ fields]
 |`sortMissingFirst`, `sortMissingLast` |Control the placement of documents when a sort field is not present. |`false`
 |`multiValued` |If `true`, indicates that a single document might contain multiple values for this field type. |`false`
 |`uninvertible` |If `true`, indicates that an `indexed="true" docValues="false"` field can be "un-inverted" at query time to build up large in memory data structure to serve in place of xref:docvalues.adoc[]. |`false`
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/spatial-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/spatial-search.adoc
@@ -27,7 +27,7 @@ Using spatial search, you can:
 
 There are four main field types available for spatial search:
 
-* `LatLonPointSpatialField`
+* `LatLonPointSpatialField` (has `docValues` enabled by default)
 * `PointType`
 * `SpatialRecursivePrefixTreeFieldType` (RPT for short), including `RptWithGeometrySpatialField`, a derivative
 * `BBoxField`
diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -70,8 +70,9 @@ Due to changes in Lucene 9, that isn't possible any more.
 == Solr 9.7
 === SchemaVersion upgraded to 1.7
 The default schemaVersion is now 1.7.
-With the new schema version, all fields and fieldTypes that support docValues (StrField, *PointField, BoolField, etc.)
-will have docValues enabled by default.
+With the new schema version, most fields and fieldTypes that support docValues will have docValues enabled by default.
+See the xref:indexing-guide:docvalues.adoc#enabling-docvalues[Enabling Doc Values Section] for a complete list.
+
 Also, fields will be unable to be uninverted by default. (`uninvertible=false`)
 
 In order to upgrade from schemaVersion 1.6 to 1.7, a re-index of all data will generally be required.
@@ -81,7 +82,7 @@ https://issues.apache.org/jira/browse/LUCENE-9334[Lucene does not allow adding d
 However, if any of the following is true, a re-index is not required:
 
 * All applicable fields already have docValues enabled in the original schema
-* The schema is updated to explicitly default `docValues="false"` for all fields and fieldTypes that did not have an explicit `docValues` default provided.
+* The schema is updated to xref:indexing-guide:docvalues.adoc#disabling-docvalues[explicitly disable docValues] for all fields and fieldTypes that did not have an explicit `docValues` default provided.
 
 The `uninvertible=false` default has no impact on the index, so is unrelated to re-indexing concerns.
 
diff --git a/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java b/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java
@@ -37,7 +37,7 @@ protected void checkSupportsDocValues() { // we support DocValues
   }
 
   @Override
-  protected boolean doesTypeSupportDocValues() {
+  protected boolean enableDocValuesByDefault() {
     return true;
   }
 

Original file line number	Diff line number	Diff line change
`@@ -246,7 +246,7 @@ protected void checkSupportsDocValues() { // we support DocValues`
`246`	`246`	`}`
`247`	`247`
`248`	`248`	`@Override`
`249`		`- protected boolean doesTypeSupportDocValues() {`
	`249`	`+ protected boolean enableDocValuesByDefault() {`
`250`	`250`	`return true;`
`251`	`251`	`}`
`252`	`252`
Original file line number	Diff line number	Diff line change
`@@ -179,7 +179,7 @@ public VectorEncoding getVectorEncoding() {`
`179`	`179`	`}`
`180`	`180`
`181`	`181`	`@Override`
`182`		`- protected boolean doesTypeSupportDocValues() {`
	`182`	`+ protected boolean enableDocValuesByDefault() {`
`183`	`183`	`return false;`
`184`	`184`	`}`
`185`	`185`
Original file line number	Diff line number	Diff line change
`@@ -70,7 +70,7 @@ protected void checkSupportsDocValues() { // we support DocValues`
`70`	`70`	`}`
`71`	`71`
`72`	`72`	`@Override`
`73`		`- protected boolean doesTypeSupportDocValues() {`
	`73`	`+ protected boolean enableDocValuesByDefault() {`
`74`	`74`	`return true;`
`75`	`75`	`}`
`76`	`76`
Original file line number	Diff line number	Diff line change
`@@ -172,11 +172,6 @@ protected void checkSupportsDocValues() {`
`172`	`172`	`}`
`173`	`173`	`}`
`174`	`174`
`175`		`- @Override`
`176`		`- protected boolean doesTypeSupportDocValues() {`
`177`		`- return true;`
`178`		`- }`
`179`		`-`
`180`	`175`	`/**`
`181`	`176`	`* Calculates the range and creates a RangeQuery (bounding box) wrapped in a BooleanQuery (unless`
`182`	`177`	`* the dimension is 1, one range for every dimension, AND'd together by a Boolean`
Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ protected void checkSupportsDocValues() { // primitive types support DocValues`
`39`	`39`	`}`
`40`	`40`
`41`	`41`	`@Override`
`42`		`- protected boolean doesTypeSupportDocValues() {`
	`42`	`+ protected boolean enableDocValuesByDefault() {`
`43`	`43`	`return true;`
`44`	`44`	`}`
`45`	`45`
Original file line number	Diff line number	Diff line change
`@@ -151,7 +151,7 @@ protected void checkSupportsDocValues() {`
`151`	`151`	`}`
`152`	`152`
`153`	`153`	`@Override`
`154`		`- protected boolean doesTypeSupportDocValues() {`
	`154`	`+ protected boolean enableDocValuesByDefault() {`
`155`	`155`	`return true;`
`156`	`156`	`}`
`157`	`157`