Merge pull request #36043 from vespa-engine/toregge/add-limited-support-for-hiding-match-data-from-ranking-framework

toregge · web-flow · commit 5e836a5f398c · 2026-02-27T12:50:14.000+01:00
Add limited support for hiding match data from ranking framework.
diff --git a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp
@@ -71,6 +71,10 @@ class Verifier : public SearchIteratorVerifier {
         (void) strict;
         return SearchIterator::UP(_fp->createIterator(_tfmda));
     }
+    std::unique_ptr<SearchIterator> create(bool, const TermFieldMatchDataArray& tfmda) const override {
+        return _fp->createIterator(tfmda);
+    }
+    void verify_hidden_from_ranking() { SearchIteratorVerifier::verify_hidden_from_ranking(_tfmda); }
 private:
     TermFieldMatchData _tfmd;
     TermFieldMatchDataArray _tfmda;
@@ -197,6 +201,7 @@ DiskIndexTest::requireThatSearchIteratorsConforms()
             FakePosting::SP f(ff->make(fw));
             Verifier verifier(f);
             verifier.verify();
+            verifier.verify_hidden_from_ranking();
         }
     }
 }
diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp
@@ -19,13 +19,16 @@ template <typename FieldIndexType>
 class Verifier : public SearchIteratorVerifier {
 private:
     mutable TermFieldMatchData _tfmd;
+    TermFieldMatchDataArray    _tfmda;
     FieldIndexType _field_index;
 
 public:
     Verifier(const Schema& schema)
         : _tfmd(),
+          _tfmda(),
           _field_index(schema, 0)
     {
+        _tfmda.add(&_tfmd);
         WrapInserter inserter(_field_index);
         inserter.word("a");
         for (uint32_t docId : getExpectedDocIds()) {
@@ -37,10 +40,12 @@ class Verifier : public SearchIteratorVerifier {
 
     SearchIterator::UP create(bool strict) const override {
         (void) strict;
-        TermFieldMatchDataArray match_data;
-        match_data.add(&_tfmd);
-        return _field_index.make_search_iterator("a", 0, match_data);
+        return _field_index.make_search_iterator("a", 0, _tfmda);
     }
+    std::unique_ptr<SearchIterator> create(bool, const TermFieldMatchDataArray& tfmda) const override {
+        return _field_index.make_search_iterator("a", 0, tfmda);
+    }
+    void verify_hidden_from_ranking() { SearchIteratorVerifier::verify_hidden_from_ranking(_tfmda); }
 };
 
 template <typename FieldIndexType>
@@ -69,12 +74,14 @@ TEST(FieldIndexIteratorTest, require_that_normal_posting_iterator_conforms)
 {
     Fixture<FieldIndex<false>> f;
     f.verifier.verify();
+    f.verifier.verify_hidden_from_ranking();
 }
 
 TEST(FieldIndexIteratorTest, require_that_interleaved_posting_iterator_conforms)
 {
     Fixture<FieldIndex<true>> f;
     f.verifier.verify();
+    f.verifier.verify_hidden_from_ranking();
 }
 
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp
@@ -237,6 +237,7 @@ unpackFeatures(const search::fef::TermFieldMatchDataArray &matchData,
     }
     TermFieldMatchData *tfmd = matchData[0];
     tfmd->reset(docId);
+    tfmd->clear_hidden_from_ranking();
     uint32_t elementId = 0;
     for (uint32_t elementDone = 0; elementDone < numElements; ++elementDone, ++elementId) {
         int32_t elementWeight = 1;
@@ -627,6 +628,7 @@ unpackFeatures(const search::fef::TermFieldMatchDataArray &matchData,
     }
     TermFieldMatchData *tfmd = matchData[0];
     tfmd->reset(docId);
+    tfmd->clear_hidden_from_ranking();
     uint32_t elementId = 0;
     for (uint32_t elementDone = 0; elementDone < numElements; ++elementDone, ++elementId) {
         int32_t elementWeight = 1;
diff --git a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp
@@ -132,7 +132,11 @@ template <bool bigEndian>
 void
 ZcRareWordPostingIteratorBase<bigEndian>::doUnpack(uint32_t docId)
 {
-    if (!_matchData.valid() || getUnpacked()) {
+    if (!_matchData.valid()) {
+        return;
+    }
+    if (getUnpacked()) {
+        _matchData[0]->clear_hidden_from_ranking();
         return;
     }
     assert(docId == getDocId());
@@ -142,9 +146,11 @@ ZcRareWordPostingIteratorBase<bigEndian>::doUnpack(uint32_t docId)
         } else {
             _decodeContext->skipFeatures(1);
             _matchData[0]->reset(docId);
+            _matchData[0]->clear_hidden_from_ranking();
         }
     } else {
         _matchData[0]->reset(docId);
+        _matchData[0]->clear_hidden_from_ranking();
     }
     if (_decode_interleaved_features && _unpack_interleaved_features) {
         TermFieldMatchData *tfmd = _matchData[0];
@@ -604,7 +610,11 @@ template <bool bigEndian>
 void
 ZcPostingIterator<bigEndian>::doUnpack(uint32_t docId)
 {
-    if (!_matchData.valid() || getUnpacked()) {
+    if (!_matchData.valid()) {
+        return;
+    }
+    if (getUnpacked()) {
+        _matchData[0]->clear_hidden_from_ranking();
         return;
     }
     assert(docId == getDocId());
@@ -621,6 +631,7 @@ ZcPostingIterator<bigEndian>::doUnpack(uint32_t docId)
         _decodeContext->unpackFeatures(_matchData, docId);
     } else {
         _matchData[0]->reset(docId);
+        _matchData[0]->clear_hidden_from_ranking();
     }
     if (_decode_interleaved_features && _unpack_interleaved_features) {
         TermFieldMatchData *tfmd = _matchData[0];
diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h
@@ -56,6 +56,7 @@ class TermFieldMatchData
     static constexpr uint16_t UNPACK_NORMAL_FEATURES_FLAG = 4;
     static constexpr uint16_t UNPACK_INTERLEAVED_FEATURES_FLAG = 8;
     static constexpr uint16_t UNPACK_ALL_FEATURES_MASK = UNPACK_NORMAL_FEATURES_FLAG | UNPACK_INTERLEAVED_FEATURES_FLAG;
+    static constexpr uint16_t HIDDEN_FROM_RANKING = 16;
 
     uint32_t  _docId;
     uint16_t  _fieldId;
@@ -205,7 +206,7 @@ class TermFieldMatchData
     }
 
     // Returns true if this instance has match data for docId that is visible to the ranking framework.
-    bool has_ranking_data(uint32_t docId) const noexcept { return docId == _docId; }
+    bool has_ranking_data(uint32_t docId) const noexcept { return docId == _docId && !is_hidden_from_ranking(); }
 
     // Returns true if this instance has match data for docId.
     bool has_data(uint32_t docId) const noexcept { return docId == _docId; }
@@ -304,6 +305,16 @@ class TermFieldMatchData
         }
     }
 
+    void set_hidden_from_ranking() noexcept {
+        _flags |= HIDDEN_FROM_RANKING;
+    }
+
+    void clear_hidden_from_ranking() noexcept {
+        _flags &= ~HIDDEN_FROM_RANKING;
+    }
+
+    bool is_hidden_from_ranking() const noexcept { return (_flags & HIDDEN_FROM_RANKING) != 0; }
+
     void filter_elements(uint32_t docid, std::span<const uint32_t> element_ids);
 
     /**
diff --git a/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp
@@ -119,7 +119,11 @@ template <bool interleaved_features, bool unpack_normal_features, bool unpack_in
 void
 PostingIterator<interleaved_features, unpack_normal_features, unpack_interleaved_features>::doUnpack(uint32_t docId)
 {
-    if (!_matchData.valid() || getUnpacked()) {
+    if (!_matchData.valid()) {
+        return;
+    }
+    if (getUnpacked()) {
+        _matchData[0]->clear_hidden_from_ranking();
         return;
     }
     assert(docId == getDocId());
@@ -131,6 +135,7 @@ PostingIterator<interleaved_features, unpack_normal_features, unpack_interleaved
         _feature_decoder.unpackFeatures(_matchData, docId);
     } else {
         _matchData[0]->reset(docId);
+        _matchData[0]->clear_hidden_from_ranking();
     }
     if (interleaved_features && unpack_interleaved_features) {
         auto* tfmd = _matchData[0];
diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp
@@ -38,6 +38,7 @@ FakeSearch::doSeek(uint32_t docid)
 void
 FakeSearch::doUnpack(uint32_t docid)
 {
+    _tfmda[0]->clear_hidden_from_ranking();
     if (_unpacked_docid == docid) {
         return;
     }
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp
@@ -708,7 +708,11 @@ template <bool bigEndian>
 void
 FakeFilterOccEGCompressed64ArrayIterator<bigEndian>::doUnpack(uint32_t docId)
 {
-    if (_matchData.size() != 1 || getUnpacked()) {
+    if (_matchData.size() != 1) {
+        return;
+    }
+    _matchData[0]->clear_hidden_from_ranking();
+    if (getUnpacked()) {
         return;
     }
     assert(docId == getDocId());
@@ -1433,7 +1437,11 @@ template <bool doSkip>
 void
 FakeFilterOccEGCompressed64SkipArrayIterator<doSkip>::doUnpack(uint32_t docId)
 {
-    if (_matchData.size() != 1 || getUnpacked()) {
+    if (_matchData.size() != 1) {
+        return;
+    }
+    _matchData[0]->clear_hidden_from_ranking();
+    if (getUnpacked()) {
         return;
     }
     assert(docId == getDocId());
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp
@@ -163,7 +163,11 @@ FakeFilterOccArrayIterator::~FakeFilterOccArrayIterator() = default;
 void
 FakeFilterOccArrayIterator::doUnpack(uint32_t docId)
 {
-    if (_matchData.size() != 1 || getUnpacked()) {
+    if (_matchData.size() != 1) {
+        return;
+    }
+    _matchData[0]->clear_hidden_from_ranking();
+    if (getUnpacked()) {
         return;
     }
     assert(docId == getDocId());
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp
@@ -228,7 +228,11 @@ FakeFilterOccZCBArrayIterator::doSeek(uint32_t docId)
 void
 FakeFilterOccZCBArrayIterator::doUnpack(uint32_t docId)
 {
-    if (_matchData.size() != 1 || getUnpacked()) {
+    if (_matchData.size() != 1) {
+        return;
+    }
+    _matchData[0]->clear_hidden_from_ranking();
+    if (getUnpacked()) {
         return;
     }
     assert(docId == getDocId());
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp
@@ -467,7 +467,11 @@ FakeFilterOccZCArrayIterator::doSeek(uint32_t docId)
 void
 FakeFilterOccZCArrayIterator::doUnpack(uint32_t docId)
 {
-    if (_matchData.size() != 1 || getUnpacked()) {
+    if (_matchData.size() != 1) {
+        return;
+    }
+    _matchData[0]->clear_hidden_from_ranking();
+    if (getUnpacked()) {
         return;
     }
     assert(docId == getDocId());
diff --git a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp
@@ -4,6 +4,7 @@
 #include "docid_iterator.h"
 #include "initrange.h"
 #include <vespa/vespalib/gtest/gtest.h>
+#include <vespa/searchlib/fef/termfieldmatchdataarray.h>
 #include <vespa/searchlib/queryeval/emptysearch.h>
 #include <vespa/searchlib/queryeval/truesearch.h>
 #include <vespa/searchlib/queryeval/termwise_search.h>
@@ -13,6 +14,8 @@
 #include <vespa/searchlib/common/bitvectoriterator.h>
 #include <set>
 
+using search::fef::TermFieldMatchData;
+using search::fef::TermFieldMatchDataArray;
 
 namespace search::test {
 using namespace search::queryeval;
@@ -275,4 +278,36 @@ SearchIteratorVerifier::searchStrict(SearchIterator & it, Range range)
     return result;
 }
 
+std::unique_ptr<SearchIterator>
+SearchIteratorVerifier::create(bool, const fef::TermFieldMatchDataArray&) const
+{
+    return {};
+}
+
+void
+SearchIteratorVerifier::verify_hidden_from_ranking(const fef::TermFieldMatchDataArray& tfmda) const
+{
+    auto* tfmd = tfmda[0];
+    auto iterator = create(true, tfmda);
+    ASSERT_TRUE(iterator);
+    iterator->initFullRange();
+    iterator->seek(_docIds[0]);
+    EXPECT_FALSE(iterator->isAtEnd());
+    EXPECT_EQ(_docIds[0], iterator->getDocId());
+    EXPECT_FALSE(tfmd->has_data(iterator->getDocId()));
+    iterator->unpack(iterator->getDocId());
+    EXPECT_TRUE(tfmd->has_ranking_data(iterator->getDocId()));
+    tfmd->set_hidden_from_ranking();
+    EXPECT_FALSE(tfmd->has_ranking_data(iterator->getDocId()));
+    iterator->unpack(iterator->getDocId());
+    EXPECT_TRUE(tfmd->has_ranking_data(iterator->getDocId()));
+    tfmd->set_hidden_from_ranking();
+    EXPECT_FALSE(tfmd->has_ranking_data(iterator->getDocId()));
+    iterator->seek(_docIds[1]);
+    EXPECT_EQ(_docIds[1], iterator->getDocId());
+    EXPECT_FALSE(tfmd->has_ranking_data(iterator->getDocId()));
+    iterator->unpack(iterator->getDocId());
+    EXPECT_TRUE(tfmd->has_ranking_data(iterator->getDocId()));
+}
+
 }
diff --git a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.h b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.h
@@ -5,6 +5,8 @@
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include <vector>
 
+namespace search::fef { class TermFieldMatchDataArray; }
+
 namespace search::test {
 
 class SearchIteratorVerifier {
@@ -21,7 +23,9 @@ class SearchIteratorVerifier {
     SearchIteratorVerifier();
     virtual ~SearchIteratorVerifier();
     void verify() const;
+    void verify_hidden_from_ranking(const fef::TermFieldMatchDataArray& tfmda) const;
     virtual SearchIterator::UP create(bool strict) const = 0;
+    virtual std::unique_ptr<SearchIterator> create(bool strict, const fef::TermFieldMatchDataArray& tfmda) const;
     const DocIds & getExpectedDocIds() const { return _docIds; }
     static uint32_t getDocIdLimit() { return 207; }
 private:

Original file line number	Diff line number	Diff line change
`@@ -71,6 +71,10 @@ class Verifier : public SearchIteratorVerifier {`
`71`	`71`	`(void) strict;`
`72`	`72`	`return SearchIterator::UP(_fp->createIterator(_tfmda));`
`73`	`73`	`}`
	`74`	`+ std::unique_ptr<SearchIterator> create(bool, const TermFieldMatchDataArray& tfmda) const override {`
	`75`	`+ return _fp->createIterator(tfmda);`
	`76`	`+ }`
	`77`	`+ void verify_hidden_from_ranking() { SearchIteratorVerifier::verify_hidden_from_ranking(_tfmda); }`
`74`	`78`	`private:`
`75`	`79`	`TermFieldMatchData _tfmd;`
`76`	`80`	`TermFieldMatchDataArray _tfmda;`
`@@ -197,6 +201,7 @@ DiskIndexTest::requireThatSearchIteratorsConforms()`
`197`	`201`	`FakePosting::SP f(ff->make(fw));`
`198`	`202`	`Verifier verifier(f);`
`199`	`203`	`verifier.verify();`
	`204`	`+ verifier.verify_hidden_from_ranking();`
`200`	`205`	`}`
`201`	`206`	`}`
`202`	`207`	`}`
Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@ FakeSearch::doSeek(uint32_t docid)`
`38`	`38`	`void`
`39`	`39`	`FakeSearch::doUnpack(uint32_t docid)`
`40`	`40`	`{`
	`41`	`+ _tfmda[0]->clear_hidden_from_ranking();`
`41`	`42`	`if (_unpacked_docid == docid) {`
`42`	`43`	`return;`
`43`	`44`	`}`
Original file line number	Diff line number	Diff line change
`@@ -163,7 +163,11 @@ FakeFilterOccArrayIterator::~FakeFilterOccArrayIterator() = default;`
`163`	`163`	`void`
`164`	`164`	`FakeFilterOccArrayIterator::doUnpack(uint32_t docId)`
`165`	`165`	`{`
`166`		`- if (_matchData.size() != 1 \|\| getUnpacked()) {`
	`166`	`+ if (_matchData.size() != 1) {`
	`167`	`+ return;`
	`168`	`+ }`
	`169`	`+ _matchData[0]->clear_hidden_from_ranking();`
	`170`	`+ if (getUnpacked()) {`
`167`	`171`	`return;`
`168`	`172`	`}`
`169`	`173`	`assert(docId == getDocId());`
Original file line number	Diff line number	Diff line change
`@@ -228,7 +228,11 @@ FakeFilterOccZCBArrayIterator::doSeek(uint32_t docId)`
`228`	`228`	`void`
`229`	`229`	`FakeFilterOccZCBArrayIterator::doUnpack(uint32_t docId)`
`230`	`230`	`{`
`231`		`- if (_matchData.size() != 1 \|\| getUnpacked()) {`
	`231`	`+ if (_matchData.size() != 1) {`
	`232`	`+ return;`
	`233`	`+ }`
	`234`	`+ _matchData[0]->clear_hidden_from_ranking();`
	`235`	`+ if (getUnpacked()) {`
`232`	`236`	`return;`
`233`	`237`	`}`
`234`	`238`	`assert(docId == getDocId());`