Index 2026+ package pointers (#4683)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: sir-sigurd

lambdas/indexer/CHANGELOG.md

@@ -17,6 +17,7 @@ where verb is one of
 
 ## Changes
 
+- [Fixed] Process package pointers from year 2026+ ([#4683](https://github.com/quiltdata/quilt/pull/4683))
 - [Changed] Switch to uv ([#4616](https://github.com/quiltdata/quilt/pull/4616))
 - [Changed] Upgrade to Python 3.12 ([#4616](https://github.com/quiltdata/quilt/pull/4616))
 - [Changed] Manifest handling moved to `manifest_indexer` lambda ([#4422](https://github.com/quiltdata/quilt/pull/4422))

lambdas/indexer/src/t4_lambda_es_indexer/index.py

@@ -51,6 +51,7 @@
 import os
 import pathlib
 import re
+import time
 from os.path import split
 from typing import Optional, Tuple
 from urllib.parse import unquote_plus
@@ -301,9 +302,11 @@ def index_if_pointer(
         return False
     try:
         manifest_timestamp = int(pointer_file)
-        if not 1451631600 <= manifest_timestamp <= 1767250800:
+        if manifest_timestamp < 1451631600:
             logger_.warning("Unexpected manifest timestamp s3://%s/%s", bucket, key)
             return False
+        if manifest_timestamp > time.time():
+            logger_.warning("Manifest timestamp s3://%s/%s is in the future", bucket, key)
     except ValueError as err:
         logger_.debug("Non-integer manifest pointer: s3://%s/%s, %s", bucket, key, err)
 

lambdas/indexer/test/test_index.py

@@ -127,6 +127,13 @@
 }
 
 
+@pytest.fixture
+def s3_client(mocker):
+    client = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+    mocker.patch("t4_lambda_es_indexer.index.make_s3_client", return_value=client)
+    return client
+
+
 def _check_event(synthetic, organic):
     # Ensure that synthetic events have the same shape as actual organic ones,
     # and that overridden properties like bucket, key, eTag are properly set
@@ -913,64 +920,10 @@ def test_index_if_pointer_not_exists(self, append_mock, es_mock):
         )
         append_mock.assert_not_called()
 
-    @patch.object(index, "es")
-    @patch.object(index.DocumentQueue, 'append_document')
-    def test_index_if_pointer(self, append_mock, es_mock):
-        bucket = "quilt-example"
-        handle = "author/semantic"
-        pointer_file = "1610412903"
-        key = f"{NAMED_PACKAGES_PREFIX}{handle}/{pointer_file}"
-        pkg_hash = "a" * 64
-        last_modified = datetime.datetime(2021, 1, 1, 0, 0, tzinfo=tzutc())
-
-        self.s3_stubber.add_response(
-            method="get_object",
-            expected_params={
-                "Bucket": bucket,
-                "Key": key,
-            },
-            service_response={
-                "Body": BytesIO(pkg_hash.encode()),
-                "LastModified": last_modified,
-            },
-        )
-
-        index.index_if_pointer(
-            self.s3_client,
-            index.DocumentQueue(None),
-            bucket=bucket,
-            key=key,
-        )
-
-        es_mock.delete_by_query.assert_called_once_with(
-            index=bucket + PACKAGE_INDEX_SUFFIX,
-            body={
-                "query": {
-                    "bool": {
-                        "filter": [{"term": {"_id": get_ptr_doc_id("author/semantic", "1610412903")}}],
-                        "must_not": [{"term": {"join_field#mnfst": get_manifest_doc_id("a" * 64)}}],
-                    }
-                }
-            },
-        )
-        append_mock.assert_called_once_with({
-            "_index": bucket + PACKAGE_INDEX_SUFFIX,
-            "_op_type": "index",
-            "_id": get_ptr_doc_id(handle, pointer_file),
-            "join_field": {
-                "name": "ptr",
-                "parent": get_manifest_doc_id(pkg_hash),
-            },
-            "routing": get_manifest_doc_id(pkg_hash),
-            "ptr_name": handle,
-            "ptr_tag": pointer_file,
-            "ptr_last_modified": last_modified,
-        })
-
     def test_index_if_pointer_skip(self):
         """test cases where index_if_pointer ignores input for different reasons"""
         # none of these should index due to out-of-range timestamp or non-integer name
-        for file_name in [1451631500, 1767250801]:
+        for file_name in [1451631500]:
             key = f".quilt/named_packages/foo/bar/{file_name}"
             assert not index.index_if_pointer(
                 self.s3_client,
@@ -1695,6 +1648,70 @@ def test_get_object_tagging_no_such_key(self):
             )
 
 
+@pytest.mark.parametrize(
+    "pointer_file",
+    [
+        "1610412903",
+        "9999999999",
+    ],
+)
+@patch("t4_lambda_es_indexer.index.es")
+@patch("t4_lambda_es_indexer.document_queue.DocumentQueue.append_document")
+def test_index_if_pointer(append_mock, es_mock, s3_client, pointer_file):
+    bucket = "quilt-example"
+    handle = "author/semantic"
+    key = f"{NAMED_PACKAGES_PREFIX}{handle}/{pointer_file}"
+    pkg_hash = "a" * 64
+    last_modified = datetime.datetime(2021, 1, 1, 0, 0, tzinfo=tzutc())
+
+    with Stubber(s3_client) as s3_stubber:
+        s3_stubber.add_response(
+            method="get_object",
+            expected_params={
+                "Bucket": bucket,
+                "Key": key,
+            },
+            service_response={
+                "Body": BytesIO(pkg_hash.encode()),
+                "LastModified": last_modified,
+            },
+        )
+
+        index.index_if_pointer(
+            s3_client,
+            index.DocumentQueue(None),
+            bucket=bucket,
+            key=key,
+        )
+
+        es_mock.delete_by_query.assert_called_once_with(
+            index=bucket + PACKAGE_INDEX_SUFFIX,
+            body={
+                "query": {
+                    "bool": {
+                        "filter": [{"term": {"_id": get_ptr_doc_id("author/semantic", pointer_file)}}],
+                        "must_not": [{"term": {"join_field#mnfst": get_manifest_doc_id("a" * 64)}}],
+                    }
+                }
+            },
+        )
+        append_mock.assert_called_once_with({
+            "_index": bucket + PACKAGE_INDEX_SUFFIX,
+            "_op_type": "index",
+            "_id": get_ptr_doc_id(handle, pointer_file),
+            "join_field": {
+                "name": "ptr",
+                "parent": get_manifest_doc_id(pkg_hash),
+            },
+            "routing": get_manifest_doc_id(pkg_hash),
+            "ptr_name": handle,
+            "ptr_tag": pointer_file,
+            "ptr_last_modified": last_modified,
+        })
+
+    s3_stubber.assert_no_pending_responses()
+
+
 def test_extract_pptx():
     lorem = "Lorem ipsum dolor sit amet, consectetur"
 

lambdas/pkgevents/CHANGELOG.md

@@ -17,6 +17,7 @@ where verb is one of
 
 ## Changes
 
+- [Fixed] Process package pointers from year 2026+ ([#4683](https://github.com/quiltdata/quilt/pull/4683))
 - [Changed] Migrate to proper package structure ([#4647](https://github.com/quiltdata/quilt/pull/4647))
 - [Changed] Switch to uv ([#4647](https://github.com/quiltdata/quilt/pull/4647))
 - [Changed] Upgrade to Python 3.13 ([#4647](https://github.com/quiltdata/quilt/pull/4647))

lambdas/pkgevents/src/t4_lambda_pkgevents/__init__.py

@@ -1,6 +1,7 @@
 import itertools
 import json
 import re
+import time
 
 import boto3
 
@@ -53,16 +54,19 @@ def pkg_created_event(s3_event):
     if not s3_event['eventName'].startswith('ObjectCreated:'):
         return
     s3_event_obj = s3_event['s3']
+    bucket = s3_event_obj["bucket"]["name"]
     obj = s3_event_obj['object']
     key = obj['key']
     match = PKG_POINTER_REGEX.fullmatch(key)
     if not match:
         return
     pkg_name, pointer_name = match.groups()
-    if not '1451631600' <= pointer_name <= '1767250800':
+    pointer_timestamp = int(pointer_name)
+    if pointer_timestamp < 1451631600:
+        logger.warning("pointer %r in bucket %r at %r is too old, skipping", pointer_name, bucket, key)
         return
-    bucket_obj = s3_event_obj['bucket']
-    bucket = bucket_obj['name']
+    if pointer_timestamp > time.time():
+        logger.warning("pointer %r in bucket %r at %r is in the future", pointer_name, bucket, key)
     try:
         resp = s3.get_object(Bucket=bucket, Key=key, Range=f'bytes=0-{EXPECTED_POINTER_SIZE - 1}')
     except s3.exceptions.NoSuchKey:

lambdas/pkgevents/tests/test_index.py

@@ -31,7 +31,6 @@
             '.quilt/named_packages/a/b/1451631599',
             '.quilt/named_packages/a/b/1451631600/',
             '.quilt/named_packages/a/b/1767250800/',
-            '.quilt/named_packages/a/b/1767250801',
             '.quilt/named_packages//b/1451631600',
             '.quilt/named_packages/a//1451631600',
             '.quilt/named_packages/a/b/145163160߀',
@@ -46,15 +45,25 @@ def test_pkg_created_event_bad_key(key):
                 'object': {
                     'key': key,
                 },
+                "bucket": {
+                    "name": "test-bucket",
+                },
             },
         }
     ) is None
 
 
-def test_pkg_created_event():
+@pytest.mark.parametrize(
+    "pointer_file",
+    (
+        "1451631600",
+        "9999999999",
+    ),
+)
+def test_pkg_created_event(pointer_file):
     bucket_name = 'test-bucket'
     handle = 'a/b'
-    key = f'.quilt/named_packages/{handle}/1451631600'
+    key = f".quilt/named_packages/{handle}/{pointer_file}"
     event_time = '2021-03-11T14:29:19.277067Z'
     top_hash = b'a' * 64
     event = {