SOLR-18071: Support stored fields in ExportWriter (#4053)

 via a new includeStoredFields parameter

Author: kotman12

CHANGELOG.md

@@ -33,6 +33,7 @@ This file lists Solr's raw release notes with details of every change to Solr. M
 - Added efSearch parameter to knn query, exposed efSearchScaleFactor that is used to calculate efSearch internally #17928 [SOLR-17928](https://issues.apache.org/jira/browse/SOLR-17928) (Puneet Ahuja) (Elia Porciani)
 - Support indexing primitive float[] values for DenseVectorField via JavaBin [SOLR-17948](https://issues.apache.org/jira/browse/SOLR-17948) (Puneet Ahuja) (Noble Paul)
 - Enable MergeOnFlushMergePolicy in Solr [SOLR-17984](https://issues.apache.org/jira/browse/SOLR-17984) ([Houston Putman](https://home.apache.org/phonebook.html?uid=houston) @HoustonPutman)
+- Add support for stored-only fields in ExportWriter with includeStoredFields=true. The default is false because it can negatively impact performance. [SOLR-18071](https://issues.apache.org/jira/browse/SOLR-18071) (Luke Kot-Zaniewski)
 
 ### Changed (30 changes)
 

changelog/unreleased/SOLR-18071-support-stored-fields-export-writer.yml

@@ -0,0 +1,8 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Support including stored fields in Export Writer output.
+type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other
+authors:
+  - name: Luke Kot-Zaniewski
+links:
+  - name: SOLR-18071
+    url: https://issues.apache.org/jira/browse/SOLR-18071

solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java

@@ -34,16 +34,15 @@ public DoubleFieldWriter(
   }
 
   @Override
-  public boolean write(
-      SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex)
+  public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew)
       throws IOException {
     double val;
     SortValue sortValue = sortDoc.getSortValue(this.field);
     if (sortValue != null) {
       if (sortValue.isPresent()) {
         val = (double) sortValue.getCurrentValue();
       } else { // empty-value
-        return false;
+        return;
       }
     } else {
       // field is not part of 'sort' param, but part of 'fl' param
@@ -53,10 +52,9 @@ public boolean write(
       if (vals != null) {
         val = Double.longBitsToDouble(vals.longValue());
       } else {
-        return false;
+        return;
       }
     }
     ew.put(this.field, val);
-    return true;
   }
 }

solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java

@@ -28,8 +28,11 @@
 import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeSet;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
@@ -99,15 +102,15 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
 
   public static final String BATCH_SIZE_PARAM = "batchSize";
   public static final String QUEUE_SIZE_PARAM = "queueSize";
+  public static final String INCLUDE_STORED_FIELDS_PARAM = "includeStoredFields";
 
   public static final int DEFAULT_BATCH_SIZE = 30000;
   public static final int DEFAULT_QUEUE_SIZE = 150000;
   private static final FieldWriter EMPTY_FIELD_WRITER =
       new FieldWriter() {
         @Override
-        public boolean write(
-            SortDoc sortDoc, LeafReaderContext readerContext, EntryWriter out, int fieldIndex) {
-          return false;
+        public void write(SortDoc sortDoc, LeafReaderContext readerContext, EntryWriter out) {
+          // do nothing
         }
       };
 
@@ -482,45 +485,72 @@ void writeDoc(
       throws IOException {
     int ord = sortDoc.ord;
     LeafReaderContext context = leaves.get(ord);
-    int fieldIndex = 0;
     for (FieldWriter fieldWriter : writers) {
-      if (fieldWriter.write(sortDoc, context, ew, fieldIndex)) {
-        ++fieldIndex;
-      }
+      fieldWriter.write(sortDoc, context, ew);
     }
   }
 
   public List<FieldWriter> getFieldWriters(String[] fields, SolrQueryRequest req)
       throws IOException {
     DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(req.getSearcher(), false);
-
     SolrReturnFields solrReturnFields = new SolrReturnFields(fields, req);
+    boolean includeStoredFields = req.getParams().getBool(INCLUDE_STORED_FIELDS_PARAM, false);
 
     List<FieldWriter> writers = new ArrayList<>();
+    Set<SchemaField> docValueFields = new LinkedHashSet<>();
+    Map<String, SchemaField> storedFields = new LinkedHashMap<>();
+
     for (String field : req.getSearcher().getFieldNames()) {
       if (!solrReturnFields.wantsField(field)) {
         continue;
       }
       SchemaField schemaField = req.getSchema().getField(field);
-      if (!schemaField.hasDocValues()) {
-        throw new IOException(schemaField + " must have DocValues to use this feature.");
-      }
-      boolean multiValued = schemaField.multiValued();
       FieldType fieldType = schemaField.getType();
-      FieldWriter writer;
 
-      if (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) {
-        if (solrReturnFields.getRequestedFieldNames() != null
-            && solrReturnFields.getRequestedFieldNames().contains(field)) {
-          // Explicitly requested field cannot be used due to not having useDocValuesAsStored=true,
-          // throw exception
+      Set<String> requestFieldNames =
+          solrReturnFields.getRequestedFieldNames() == null
+              ? Set.of()
+              : solrReturnFields.getRequestedFieldNames();
+
+      if (canUseDocValues(schemaField, fieldType)) {
+        // Prefer DocValues when available
+        docValueFields.add(schemaField);
+      } else if (schemaField.stored()) {
+        // Field is stored-only (no usable DocValues)
+        if (includeStoredFields) {
+          storedFields.put(field, schemaField);
+        } else if (requestFieldNames.contains(field)) {
+          // Explicitly requested field without DocValues and includeStoredFields=false
+          throw new IOException(
+              schemaField
+                  + " must have DocValues to use this feature. "
+                  + "Try setting includeStoredFields=true to retrieve this field from stored values.");
+        }
+        // Else: glob matched stored-only field without includeStoredFields - silently skip
+      } else if (requestFieldNames.contains(field)) {
+        // Explicitly requested field that has neither DocValues nor stored
+        if (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) {
           throw new IOException(
               schemaField + " Must have useDocValuesAsStored='true' to be used with export writer");
         } else {
-          // Glob pattern matched field cannot be used due to not having useDocValuesAsStored=true
-          continue;
+          throw new IOException(
+              schemaField + " must have DocValues or be stored to use this feature.");
         }
       }
+      // Else: glob matched field with neither DocValues nor stored - silently skip
+    }
+
+    for (SchemaField schemaField : docValueFields) {
+      String field = schemaField.getName();
+      boolean multiValued = schemaField.multiValued();
+      FieldType fieldType = schemaField.getType();
+      FieldWriter writer;
+
+      if (schemaField.stored() && !storedFields.isEmpty()) {
+        // if we're reading StoredFields *anyway*, then we might as well avoid this extra DV lookup
+        storedFields.put(field, schemaField);
+        continue;
+      }
 
       DocValuesIteratorCache.FieldDocValuesSupplier docValuesCache = dvIterCache.getSupplier(field);
 
@@ -574,9 +604,24 @@ public List<FieldWriter> getFieldWriters(String[] fields, SolrQueryRequest req)
       }
       writers.add(writer);
     }
+
+    if (!storedFields.isEmpty()) {
+      writers.add(new StoredFieldsWriter(storedFields));
+    }
+
     return writers;
   }
 
+  private static boolean canUseDocValues(SchemaField schemaField, FieldType fieldType) {
+    return schemaField.hasDocValues()
+        // Special handling for SortableTextField: unlike other field types, it requires
+        // useDocValuesAsStored=true to be included via glob patterns in /export. This
+        // matches the behavior of /select (which requires useDocValuesAsStored=true for
+        // all globbed fields) and avoids performance issues. The requirement cannot be
+        // extended to other field types in /export for backward compatibility reasons.
+        && (!(fieldType instanceof SortableTextField) || schemaField.useDocValuesAsStored());
+  }
+
   SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
     SortValue[] sortValues = new SortValue[sortFields.length];
     IndexSchema schema = searcher.getSchema();
@@ -591,7 +636,7 @@ SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IO
         throw new IOException(field + " must have DocValues to use this feature.");
       }
 
-      if (ft instanceof SortableTextField && schemaField.useDocValuesAsStored() == false) {
+      if (ft instanceof SortableTextField && !schemaField.useDocValuesAsStored()) {
         throw new IOException(
             schemaField + " Must have useDocValuesAsStored='true' to be used with export writer");
       }

solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java

@@ -22,7 +22,15 @@
 import org.apache.solr.common.MapWriter;
 
 abstract class FieldWriter {
-  public abstract boolean write(
-      SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter out, int fieldIndex)
+  /**
+   * Writes field values from the document to the output.
+   *
+   * @param sortDoc the document being exported
+   * @param readerContext the leaf reader context for accessing field values
+   * @param out the output writer to write field values to
+   * @throws IOException if an I/O error occurs while reading or writing field values
+   */
+  public abstract void write(
+      SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter out)
       throws IOException;
 }

solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java

@@ -34,16 +34,15 @@ public FloatFieldWriter(
   }
 
   @Override
-  public boolean write(
-      SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex)
+  public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew)
       throws IOException {
     float val;
     SortValue sortValue = sortDoc.getSortValue(this.field);
     if (sortValue != null) {
       if (sortValue.isPresent()) {
         val = (float) sortValue.getCurrentValue();
       } else { // empty-value
-        return false;
+        return;
       }
     } else {
       // field is not part of 'sort' param, but part of 'fl' param
@@ -53,10 +52,9 @@ public boolean write(
       if (vals != null) {
         val = Float.intBitsToFloat((int) vals.longValue());
       } else {
-        return false;
+        return;
       }
     }
     ew.put(this.field, val);
-    return true;
   }
 }

solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java

@@ -34,16 +34,15 @@ public IntFieldWriter(
   }
 
   @Override
-  public boolean write(
-      SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex)
+  public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew)
       throws IOException {
     int val;
     SortValue sortValue = sortDoc.getSortValue(this.field);
     if (sortValue != null) {
       if (sortValue.isPresent()) {
         val = (int) sortValue.getCurrentValue();
       } else { // empty-value
-        return false;
+        return;
       }
     } else {
       // field is not part of 'sort' param, but part of 'fl' param
@@ -53,10 +52,9 @@ public boolean write(
       if (vals != null) {
         val = (int) vals.longValue();
       } else {
-        return false;
+        return;
       }
     }
     ew.put(this.field, val);
-    return true;
   }
 }

solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java

@@ -35,16 +35,15 @@ public LongFieldWriter(
   }
 
   @Override
-  public boolean write(
-      SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew, int fieldIndex)
+  public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter ew)
       throws IOException {
     long val;
     SortValue sortValue = sortDoc.getSortValue(this.field);
     if (sortValue != null) {
       if (sortValue.isPresent()) {
         val = (long) sortValue.getCurrentValue();
       } else { // empty-value
-        return false;
+        return;
       }
     } else {
       // field is not part of 'sort' param, but part of 'fl' param
@@ -54,11 +53,10 @@ public boolean write(
       if (vals != null) {
         val = vals.longValue();
       } else {
-        return false;
+        return;
       }
     }
     doWrite(ew, val);
-    return true;
   }
 
   protected void doWrite(MapWriter.EntryWriter ew, long val) throws IOException {

solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java

@@ -61,15 +61,14 @@ public MultiFieldWriter(
   }
 
   @Override
-  public boolean write(
-      SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter out, int fieldIndex)
+  public void write(SortDoc sortDoc, LeafReaderContext readerContext, MapWriter.EntryWriter out)
       throws IOException {
     if (this.fieldType.isPointField()) {
       SortedNumericDocValues vals =
           docValuesCache.getSortedNumericDocValues(
               sortDoc.docId, readerContext.reader(), readerContext.ord);
       if (vals == null) {
-        return false;
+        return;
       }
 
       final SortedNumericDocValues docVals = vals;
@@ -82,13 +81,12 @@ public boolean write(
                   w.add(bitsToValue.apply(docVals.nextValue()));
                 }
               });
-      return true;
     } else {
       SortedSetDocValues vals =
           docValuesCache.getSortedSetDocValues(
               sortDoc.docId, readerContext.reader(), readerContext.ord);
       if (vals == null) {
-        return false;
+        return;
       }
 
       final SortedSetDocValues docVals = vals;
@@ -105,7 +103,6 @@ public boolean write(
                   else w.add(fieldType.toObject(f));
                 }
               });
-      return true;
     }
   }