Merge pull request internetarchive#11156 from cdrini/feature/monitor-empty-homepage

Improve solr + nginx performance monitoring

Author: mekarpeles

compose.production.yaml

@@ -46,7 +46,8 @@ services:
         max-file: "4"
 
   solr_restarter:
-    profiles: ["ol-solr0"]
+    # Disabled for now ; seemed to be making things worse, and seems no longer needed
+    profiles: ["ol-never"]
     build:
       context: scripts/solr_restarter
       args:
@@ -316,7 +317,7 @@ services:
   # This job runs various monitoring/grafana checks across the entire cluster.
   # It has access to the other nodes via the docker socket.
   monitoring:
-    profiles: ["ol-web0", "ol-web1", "ol-web2", "ol-covers0", "ol-www0"]
+    profiles: ["ol-web0", "ol-web1", "ol-web2", "ol-covers0", "ol-www0", "ol-solr0", "ol-solr1"]
     build:
       context: .
       dockerfile: scripts/monitoring/Dockerfile

openlibrary/macros/RawQueryCarousel.html

@@ -56,7 +56,7 @@
       if has_fulltext_only:
         params['has_fulltext'] = 'true'
 
-      results = work_search(params, fields = ",".join(fields), sort=sort, limit=limit, facet=False)
+      results = work_search(params, fields = ",".join(fields), sort=sort, limit=limit, facet=False, query_label='BOOK_CAROUSEL')
       books = [storage(b) for b in (results.get('docs', []))]
       load_more = {
         "queryType": "SEARCH",

openlibrary/plugins/openlibrary/partials.py

@@ -232,7 +232,14 @@ def generate(self) -> dict:
         param = data.get('param', {})
 
         sort = None
-        search_response = do_search(param, sort, rows=0, spellcheck_count=3, facet=True)
+        search_response = do_search(
+            param,
+            sort,
+            rows=0,
+            spellcheck_count=3,
+            facet=True,
+            query_label='BOOK_SEARCH_FACETS',
+        )
 
         sidebar = render_template(
             'search/work_search_facets',

openlibrary/plugins/worksearch/code.py

@@ -1,4 +1,5 @@
 import copy
+import functools
 import itertools
 import json
 import logging
@@ -188,6 +189,24 @@ def read_cookie():
     return 'details'
 
 
+QueryLabel = Literal[
+    'UNLABELLED',
+    'BOOK_SEARCH',
+    'BOOK_SEARCH_API',
+    'BOOK_SEARCH_FACETS',
+    'BOOK_CAROUSEL',
+    # Used for the internal request made by solr to choose the best edition
+    # during a normal book search
+    'EDITION_MATCH',
+    'LIST_SEARCH',
+    'LIST_SEARCH_API',
+    'SUBJECT_SEARCH',
+    'SUBJECT_SEARCH_API',
+    'AUTHOR_SEARCH',
+    'AUTHOR_SEARCH_API',
+]
+
+
 def run_solr_query(  # noqa: PLR0912
     scheme: SearchScheme,
     param: dict | None = None,
@@ -200,6 +219,7 @@ def run_solr_query(  # noqa: PLR0912
     facet: bool | Iterable[str] = True,
     allowed_filter_params: set[str] | None = None,
     extra_params: list[tuple[str, Any]] | None = None,
+    query_label: QueryLabel = 'UNLABELLED',
 ):
     """
     :param param: dict of query parameters
@@ -219,6 +239,7 @@ def run_solr_query(  # noqa: PLR0912
         *(('fq', subquery) for subquery in scheme.universe),
         ('start', offset),
         ('rows', rows),
+        ('ol.label', query_label),
         ('wt', param.get('wt', 'json')),
     ] + (extra_params or [])
 
@@ -361,6 +382,7 @@ def do_search(
     rows=100,
     facet=False,
     spellcheck_count=None,
+    query_label: QueryLabel = 'UNLABELLED',
 ):
     """
     :param param: dict of search url parameters
@@ -391,6 +413,7 @@ def do_search(
         spellcheck_count,
         fields=list(fields),
         facet=facet,
+        query_label=query_label,
     )
 
 
@@ -585,7 +608,12 @@ def GET(self):
         rows = 20
         if param:
             search_response = do_search(
-                param, sort, page, rows=rows, spellcheck_count=3
+                param,
+                sort,
+                page,
+                rows=rows,
+                spellcheck_count=3,
+                query_label='BOOK_SEARCH',
             )
         else:
             search_response = SearchResponse(
@@ -709,18 +737,23 @@ def GET(self):  # referenced subject_search
         req = ListSearchRequest.from_web_input(web.input(api='new'))
         # Can't set fields when rendering html
         req.fields = 'key'
-        resp = self.get_results(req)
+        resp = self.get_results(req, 'LIST_SEARCH')
         lists = list(web.ctx.site.get_many([doc['key'] for doc in resp.docs]))
         return render_template('search/lists.html', req, resp, lists)
 
-    def get_results(self, req: ListSearchRequest):
+    def get_results(
+        self,
+        req: ListSearchRequest,
+        query_label: Literal['LIST_SEARCH', 'LIST_SEARCH_API'],
+    ):
         return run_solr_query(
             ListSearchScheme(),
             {'q': req.q},
             offset=req.offset,
             rows=req.limit,
             fields=req.fields,
             sort=req.sort,
+            query_label=query_label,
         )
 
 
@@ -732,7 +765,7 @@ class list_search_json(list_search):
 
     def GET(self):
         req = ListSearchRequest.from_web_input(web.input())
-        resp = self.get_results(req)
+        resp = self.get_results(req, 'LIST_SEARCH_API')
 
         web.header('Content-Type', 'application/json')
         if req.api == 'next':
@@ -769,15 +802,23 @@ class subject_search(delegate.page):
     path = '/search/subjects'
 
     def GET(self):
-        return render_template('search/subjects', self.get_results)
-
-    def get_results(self, q, offset=0, limit=100):
+        get_results = functools.partial(self.get_results, query_label='SUBJECT_SEARCH')
+        return render_template('search/subjects', get_results)
+
+    def get_results(
+        self,
+        q,
+        query_label: Literal['SUBJECT_SEARCH', 'SUBJECT_SEARCH_API'],
+        offset=0,
+        limit=100,
+    ):
         response = run_solr_query(
             SubjectSearchScheme(),
             {'q': q},
             offset=offset,
             rows=limit,
             sort='work_count desc',
+            query_label=query_label,
         )
 
         return response
@@ -793,7 +834,12 @@ def GET(self):
         limit = safeint(i.limit, 100)
         limit = min(1000, limit)  # limit limit to 1000.
 
-        response = self.get_results(i.q, offset=offset, limit=limit)
+        response = self.get_results(
+            i.q,
+            query_label='SUBJECT_SEARCH_API',
+            offset=offset,
+            limit=limit,
+        )
 
         # Backward compatibility :/
         raw_resp = response.raw_resp['response']
@@ -809,16 +855,26 @@ class author_search(delegate.page):
     path = '/search/authors'
 
     def GET(self):
-        return render_template('search/authors', self.get_results)
-
-    def get_results(self, q, offset=0, limit=100, fields='*', sort=''):
+        get_results = functools.partial(self.get_results, query_label='AUTHOR_SEARCH')
+        return render_template('search/authors', get_results)
+
+    def get_results(
+        self,
+        q,
+        query_label: Literal['AUTHOR_SEARCH', 'AUTHOR_SEARCH_API'],
+        offset=0,
+        limit=100,
+        fields='*',
+        sort='',
+    ):
         resp = run_solr_query(
             AuthorSearchScheme(),
             {'q': q},
             offset=offset,
             rows=limit,
             fields=fields,
             sort=sort,
+            query_label=query_label,
         )
 
         return resp
@@ -835,7 +891,12 @@ def GET(self):
         limit = min(1000, limit)  # limit limit to 1000.
 
         response = self.get_results(
-            i.q, offset=offset, limit=limit, fields=i.fields, sort=i.sort
+            i.q,
+            query_label='AUTHOR_SEARCH_API',
+            offset=offset,
+            limit=limit,
+            fields=i.fields,
+            sort=i.sort,
         )
         raw_resp = response.raw_resp['response']
         for doc in raw_resp['docs']:
@@ -901,6 +962,7 @@ def work_search(
     fields: str = '*',
     facet: bool = True,
     spellcheck_count: int | None = None,
+    query_label: QueryLabel = 'UNLABELLED',
 ) -> dict:
     """
     :param sort: key of SORTS dict at the top of this file
@@ -923,6 +985,7 @@ def work_search(
         fields=fields,
         facet=facet,
         spellcheck_count=spellcheck_count,
+        query_label=query_label,
     )
     response = resp.raw_resp['response']
 
@@ -996,6 +1059,7 @@ def GET(self):
             # so disable it. This makes it much faster.
             facet=False,
             spellcheck_count=spellcheck_count,
+            query_label='BOOK_SEARCH_API',
         )
         response['documentation_url'] = "https://openlibrary.org/dev/docs/api/search"
         response['q'] = q

openlibrary/plugins/worksearch/schemes/works.py

@@ -508,6 +508,7 @@ def convert_work_query_to_edition_query(work_query: str) -> str:
             # for `userEdQuery` to be available to `editions.q`, we will
             # need to specify it twice.
             new_params.append(('editions.userEdQuery', ed_q or '*:*'))
+            new_params.append(('editions.ol.label', 'EDITION_MATCH'))
 
             full_ed_query = '({{!edismax bq="{bq}" v={v} qf="{qf}"}})'.format(
                 # See qf in work_query

scripts/deployment/deploy.sh

@@ -28,7 +28,7 @@ DEPLOY_DIR="/tmp/openlibrary_deploy"
 mkdir -p $DEPLOY_DIR
 
 WEB_HOSTNAMES="ol-web0 ol-web1 ol-web2"
-ALL_HOSTNAMES="ol-home0 ol-covers0 ol-www0 $WEB_HOSTNAMES"
+ALL_HOSTNAMES="ol-home0 ol-covers0 ol-www0 ol-solr0 ol-solr1 $WEB_HOSTNAMES"
 SERVER_SUFFIX=${SERVER_SUFFIX:-".us.archive.org"}
 
 KILL_CRON=${KILL_CRON:-""}

scripts/monitoring/haproxy_monitor.py

@@ -10,7 +10,7 @@
 import time
 from collections.abc import Callable, Iterable
 from dataclasses import dataclass
-from typing import Literal
+from typing import Literal, cast
 
 import requests
 
@@ -32,6 +32,32 @@ class GraphiteEvent:
     def serialize(self):
         return (self.path, (self.timestamp, self.value))
 
+    def serialize_str(self) -> str:
+        return f"{self.path} {self.value} {self.timestamp}"
+
+    def submit(self, graphite_address: str | tuple[str, int]):
+        GraphiteEvent.submit_many([self], graphite_address)
+
+    @staticmethod
+    def submit_many(
+        events: 'list[GraphiteEvent]', graphite_address: str | tuple[str, int]
+    ):
+        if isinstance(graphite_address, str):
+            graphite_host, graphite_port = cast(
+                tuple[str, str], tuple(graphite_address.split(':', 1))
+            )
+            graphite_address_tuple = (graphite_host, int(graphite_port))
+        else:
+            graphite_address_tuple = graphite_address
+
+        payload = pickle.dumps([e.serialize() for e in events], protocol=2)
+        header = struct.pack("!L", len(payload))
+        message = header + payload
+
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            sock.connect(graphite_address_tuple)
+            sock.sendall(message)
+
 
 @dataclass
 class HaproxyCapture:
@@ -83,8 +109,8 @@ async def main(
     commit_freq=30,
     agg: Literal['max', 'min', 'sum', None] = None,
 ):
-    graphite_address = tuple(graphite_address.split(':', 1))
-    graphite_address = (graphite_address[0], int(graphite_address[1]))
+    graphite_address_tuple = tuple(graphite_address.split(':', 1))
+    graphite_address_tuple = (graphite_address_tuple[0], int(graphite_address_tuple[1]))
 
     agg_options: dict[str, Callable[[Iterable[float]], float]] = {
         'max': max,
@@ -127,15 +153,7 @@ async def main(
                 print(e.serialize())
 
             if not dry_run:
-                payload = pickle.dumps(
-                    [e.serialize() for e in events_buffer], protocol=2
-                )
-                header = struct.pack("!L", len(payload))
-                message = header + payload
-
-                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-                    sock.connect(graphite_address)
-                    sock.sendall(message)
+                GraphiteEvent.submit_many(events_buffer, graphite_address_tuple)
 
             events_buffer = []
             last_commit_ts = ts