Propagate scoring function through random sampler

[jan-elastic]

fixes: #110134

[elasticsearchmachine]

Pinging @elastic/ml-core (Team:ML)

[elasticsearchmachine]

Hi @jan-elastic, I've created a changelog YAML for you.

[jan-elastic]

Example request / response for the fix:

POST /test-index/_search
{
  "query": {
    "function_score": {
      "random_score": {
      }
    }
  },
  "aggs": {
    "random_sampler": {
      "random_sampler": {
        "probability": 0.5
      },
      "aggs": {
        "samples": {
          "top_hits": {
            "size": 2,
            "_source": [
              "text"
            ]
          }
        }
      }
    }
  }
}

{
  "took": 3,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 5,
      "relation": "eq"
    },
    "max_score": 0.96045524,
    "hits": [
      {
        "_index": "test-index",
        "_id": "ZjX1G5MBaC3tcnySK34-",
        "_score": 0.96045524,
        "_source": {
          "text": "text1"
        }
      },
      {
        "_index": "test-index",
        "_id": "ZTXrG5MBaC3tcnySAH4a",
        "_score": 0.82856405,
        "_source": {
          "text": [
            "text1",
            "text2"
          ]
        }
      },
      {
        "_index": "test-index",
        "_id": "aDX1G5MBaC3tcnySK34-",
        "_score": 0.33547562,
        "_source": {
          "text": [
            "text3",
            "text4"
          ]
        }
      },
      {
        "_index": "test-index",
        "_id": "ZDXrG5MBaC3tcnySAH4Z",
        "_score": 0.14092654,
        "_source": {
          "text": "text1"
        }
      },
      {
        "_index": "test-index",
        "_id": "ZzX1G5MBaC3tcnySK34-",
        "_score": 0.031006515,
        "_source": {
          "text": [
            "text1",
            "text2"
          ]
        }
      }
    ]
  },
  "aggregations": {
    "random_sampler": {
      "seed": -820777490,
      "probability": 0.5,
      "doc_count": 2,
      "samples": {
        "hits": {
          "total": {
            "value": 2,
            "relation": "eq"
          },
          "max_score": 0.96045524,
          "hits": [
            {
              "_index": "test-index",
              "_id": "ZjX1G5MBaC3tcnySK34-",
              "_score": 0.96045524,
              "_source": {
                "text": "text1"
              }
            },
            {
              "_index": "test-index",
              "_id": "ZDXrG5MBaC3tcnySAH4Z",
              "_score": 0.14092654,
              "_source": {
                "text": "text1"
              }
            }
          ]
        }
      }
    }
  }
}

Without the `random_score` you get similar results, all with scores of `1.0`.

[benwtrent]

@jan-elastic I wonder how it would be with multiple layers of nesting? The original issue that @dgieselaar ran into I think had multiple layers of aggregations (random_sampler -> categorize_text -> top hits)

[jan-elastic]

Regarding multiple layers of nesting: look like it works fine @benwtrent

{
  "query": {
    "function_score": {
      "random_score": {
      }
    }
  },
  "aggs": {
    "random_sampler": {
      "random_sampler": {
        "probability": 0.5
      },
      "aggs": {
        "message": {
          "categorize_text": {
            "size": 6,
            "field": "text"
          },
          "aggs": {
            "samples": {
              "top_hits": {
                "size": 1,
                "_source": [
                  "text"
                ]
              }
            }
          }
        }
      }
    }
  }
}

{
  "took": 14,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 5,
      "relation": "eq"
    },
    "max_score": 0.8408959,
    "hits": [
      {
        "_index": "test-index",
        "_id": "ZjX1G5MBaC3tcnySK34-",
        "_score": 0.8408959,
        "_source": {
          "text": "text1"
        }
      },
      {
        "_index": "test-index",
        "_id": "ZzX1G5MBaC3tcnySK34-",
        "_score": 0.72381663,
        "_source": {
          "text": [
            "text1",
            "text2"
          ]
        }
      },
      {
        "_index": "test-index",
        "_id": "ZDXrG5MBaC3tcnySAH4Z",
        "_score": 0.50730485,
        "_source": {
          "text": "text1"
        }
      },
      {
        "_index": "test-index",
        "_id": "ZTXrG5MBaC3tcnySAH4a",
        "_score": 0.24077094,
        "_source": {
          "text": [
            "text1",
            "text2"
          ]
        }
      },
      {
        "_index": "test-index",
        "_id": "aDX1G5MBaC3tcnySK34-",
        "_score": 0.17789865,
        "_source": {
          "text": [
            "text3",
            "text4"
          ]
        }
      }
    ]
  },
  "aggregations": {
    "random_sampler": {
      "seed": -685459259,
      "probability": 0.5,
      "doc_count": 2,
      "message": {
        "buckets": [
          {
            "doc_count": 4,
            "key": "text1",
            "regex": ".*?text1.*?",
            "max_matching_length": 5,
            "samples": {
              "hits": {
                "total": {
                  "value": 2,
                  "relation": "eq"
                },
                "max_score": 0.72381663,
                "hits": [
                  {
                    "_index": "test-index",
                    "_id": "ZzX1G5MBaC3tcnySK34-",
                    "_score": 0.72381663,
                    "_source": {
                      "text": [
                        "text1",
                        "text2"
                      ]
                    }
                  }
                ]
              }
            }
          },
          {
            "doc_count": 4,
            "key": "text2",
            "regex": ".*?text2.*?",
            "max_matching_length": 5,
            "samples": {
              "hits": {
                "total": {
                  "value": 2,
                  "relation": "eq"
                },
                "max_score": 0.72381663,
                "hits": [
                  {
                    "_index": "test-index",
                    "_id": "ZzX1G5MBaC3tcnySK34-",
                    "_score": 0.72381663,
                    "_source": {
                      "text": [
                        "text1",
                        "text2"
                      ]
                    }
                  }
                ]
              }
            }
          }
        ]
      }
    }
  }
}

[benwtrent]

I think for initial work, this is in a good place. Let's add some yaml testing and such.

[elasticsearchmachine]

Hi @jan-elastic, I've updated the changelog YAML for you.

[benwtrent]

[elasticsearchmachine]

💔 Backport failed

Status	Branch	Result
❌	8.16	Commit could not be cherrypicked due to conflicts
✅	8.x

You can use sqren/backport to manually backport by running backport --upstream elastic/elasticsearch --pr 116957