feat: Implement get_overlapping_features_for_region function

This function queries the Ensembl API with exponential backoff as needed, returning a list of features which overlap the passed region.

Author: bencap

settings/.env.dev

@@ -31,3 +31,9 @@ MAVEDB_API_KEY=
 ####################################################################################################
 
 SEQREPO_ROOT_DIR=/usr/local/share/seqrepo/2024-12-20
+
+####################################################################################################
+# Environment variables for ensembl
+####################################################################################################
+
+ENSEMBL_API_URL=https://rest.ensembl.org

src/dcd_mapping/lookup.py

@@ -11,6 +11,7 @@
 import logging
 import os
 from pathlib import Path
+from typing import Any
 
 import hgvs
 import polars as pl
@@ -50,6 +51,7 @@
 from gene.schemas import MatchType, SourceName
 
 from dcd_mapping.exceptions import DataLookupError
+from dcd_mapping.resource_utils import ENSEMBL_API_URL, request_with_backoff
 from dcd_mapping.schemas import (
     GeneLocation,
     ManeDescription,
@@ -645,6 +647,68 @@ def _sort_mane_result(description: ManeDescription) -> int:
     return mane_data
 
 
+# --------------------------------- Ensembl --------------------------------- #
+
+
+def get_overlapping_features_for_region(
+    chromosome: str, start: int, end: int, features: list[str] | None = None
+) -> list[dict[str, Any]]:
+    """Get genes overlapping a specific genomic region.
+
+    :param chromosome: Chromosome identifier
+    :param start: Start position of the region
+    :param end: End position of the region
+    :param features: List of features to retrieve (default is ["gene"])
+    :return: List of overlapping gene symbols
+    """
+    if not features:
+        features = ["gene"]
+        _logger.debug("No features specified, defaulting to %s", features)
+
+    chrom = get_chromosome_identifier(chromosome)
+
+    query = f"/{chrom}:{start}-{end}"
+    if features:
+        query += "?"
+    for feature in features:
+        query += f"feature={feature};"
+
+    try:
+        _logger.debug(
+            "Fetching overlapping features for region %s:%d-%d with features %s",
+            chromosome,
+            start,
+            end,
+            features,
+        )
+
+        url = f"{ENSEMBL_API_URL}/overlap/region/human{query}"
+        response = request_with_backoff(
+            url, headers={"Content-Type": "application/json"}
+        )
+        response.raise_for_status()
+    except requests.RequestException as e:
+        _logger.error(
+            "Failed to fetch overlapping features for region %s-%s on chromosome %s: %s",
+            start,
+            end,
+            chromosome,
+            e,
+        )
+        return []
+
+    overlapping_features = response.json()
+    _logger.debug(
+        "Successfully fetched %d overlapping features for region %s:%d-%d with features %s",
+        len(overlapping_features),
+        chromosome,
+        start,
+        end,
+        features,
+    )
+    return overlapping_features
+
+
 # ---------------------------------- Misc. ---------------------------------- #
 
 

src/dcd_mapping/resource_utils.py

@@ -1,5 +1,6 @@
 """Provide basic utilities for fetching and storing external data."""
 import os
+import time
 from pathlib import Path
 
 import click
@@ -8,6 +9,7 @@
 
 MAVEDB_API_KEY = os.environ.get("MAVEDB_API_KEY")
 MAVEDB_BASE_URL = os.environ.get("MAVEDB_BASE_URL")
+ENSEMBL_API_URL = os.environ.get("ENSEMBL_API_URL", "https://rest.ensembl.org")  # TODO
 
 LOCAL_STORE_PATH = Path(
     os.environ.get(
@@ -57,3 +59,67 @@ def http_download(url: str, out_path: Path, silent: bool = True) -> Path:
                     if chunk:
                         h.write(chunk)
     return out_path
+
+
+def request_with_backoff(
+    url: str, max_retries: int = 5, backoff_factor: float = 0.3, **kwargs
+) -> requests.Response:
+    """HTTP GET with exponential backoff only for retryable errors.
+
+    Retries on:
+    - Connection timeout or connection errors
+    - HTTP 5xx server errors
+    - HTTP 429 rate limiting (respecting Retry-After when present)
+
+    Immediately raises on other HTTP errors (e.g., 4xx client errors).
+    """
+    attempt = 0
+    while attempt < max_retries:
+        try:
+            response = requests.get(url, timeout=60, **kwargs)
+        except (requests.Timeout, requests.ConnectionError):
+            # Retry on transient network failures
+            if attempt == max_retries - 1:
+                raise
+            sleep_time = backoff_factor * (2**attempt)
+            time.sleep(sleep_time)
+            attempt += 1
+            continue
+
+        # If we have a response, decide retry based on status code
+        status = response.status_code
+        if 200 <= status < 300:
+            return response
+
+        # 429: Too Many Requests — optionally use Retry-After
+        if status == 429:
+            if attempt == max_retries - 1:
+                response.raise_for_status()
+            retry_after = response.headers.get("Retry-After")
+            try:
+                sleep_time = (
+                    float(retry_after)
+                    if retry_after is not None
+                    else backoff_factor * (2**attempt)
+                )
+            except ValueError:
+                sleep_time = backoff_factor * (2**attempt)
+            time.sleep(sleep_time)
+            attempt += 1
+            continue
+
+        # 5xx: server errors — retry
+        if 500 <= status < 600:
+            if attempt == max_retries - 1:
+                response.raise_for_status()
+            sleep_time = backoff_factor * (2**attempt)
+            time.sleep(sleep_time)
+            attempt += 1
+            continue
+
+        # Non-retryable (e.g., 4xx other than 429): raise immediately
+        response.raise_for_status()
+
+    # Exhausted retries without success
+    msg = f"Failed to fetch {url} after {max_retries} attempts"
+    raise Exception(msg)

tests/test_lookup.py

@@ -0,0 +1,107 @@
+"""Tests for dcd_mapping.lookup"""
+
+from unittest.mock import patch
+
+import requests
+
+from dcd_mapping.lookup import get_overlapping_features_for_region
+
+RAW_OVERLAP_RESPONSE = [
+    {
+        "seq_region_name": "22",
+        "version": 1,
+        "biotype": "protein_coding",
+        "feature_type": "gene",
+        "description": "novel transcript",
+        "logic_name": "havana_homo_sapiens",
+        "start": 19717220,
+        "id": "ENSG00000284874",
+        "source": "havana",
+        "canonical_transcript": "ENST00000455843.5",
+        "assembly_name": "GRCh38",
+        "end": 19724772,
+        "gene_id": "ENSG00000284874",
+        "strand": 1,
+    },
+    {
+        "source": "ensembl_havana",
+        "canonical_transcript": "ENST00000366425.4",
+        "assembly_name": "GRCh38",
+        "end": 19724776,
+        "gene_id": "ENSG00000203618",
+        "strand": 1,
+        "external_name": "GP1BB",
+        "seq_region_name": "22",
+        "version": 7,
+        "biotype": "protein_coding",
+        "logic_name": "ensembl_havana_gene_homo_sapiens",
+        "feature_type": "gene",
+        "start": 19723539,
+        "description": "glycoprotein Ib platelet subunit beta [Source:HGNC Symbol;Acc:HGNC:4440]",
+        "id": "ENSG00000203618",
+    },
+    {
+        "end": 19724224,
+        "gene_id": "ENSG00000184702",
+        "strand": 1,
+        "canonical_transcript": "ENST00000455784.7",
+        "source": "ensembl_havana",
+        "assembly_name": "GRCh38",
+        "seq_region_name": "22",
+        "version": 21,
+        "biotype": "protein_coding",
+        "description": "septin 5 [Source:HGNC Symbol;Acc:HGNC:9164]",
+        "feature_type": "gene",
+        "logic_name": "ensembl_havana_gene_homo_sapiens",
+        "start": 19714467,
+        "id": "ENSG00000184702",
+        "external_name": "SEPTIN5",
+    },
+]
+
+
+class _FakeResponse:
+    def __init__(self, data):
+        self._data = data
+        self.status_code = 200
+
+    def json(self):
+        return self._data
+
+    def raise_for_status(self):
+        return None
+
+
+def test_get_overlapping_features_for_region_success():
+    with (
+        patch(
+            "dcd_mapping.lookup.request_with_backoff",
+            return_value=_FakeResponse(RAW_OVERLAP_RESPONSE),
+        ),
+        patch("dcd_mapping.lookup.get_chromosome_identifier", side_effect=lambda c: c),
+    ):
+        result = get_overlapping_features_for_region(
+            "NC_000022.11", 19714000, 19725000, features=["gene"]
+        )
+        assert isinstance(result, list)
+        assert result == RAW_OVERLAP_RESPONSE
+
+
+def test_get_overlapping_features_for_region_error():
+    class ErrorResponse(_FakeResponse):
+        def __init__(self):
+            super().__init__(None)
+            self.status_code = 500
+
+        def raise_for_status(self):
+            msg = f"HTTP {self.status_code} Error"
+            raise requests.RequestException(msg)
+
+    with (
+        patch("dcd_mapping.lookup.request_with_backoff", return_value=ErrorResponse()),
+        patch("dcd_mapping.lookup.get_chromosome_identifier", side_effect=lambda c: c),
+    ):
+        result = get_overlapping_features_for_region(
+            "NC_000022.11", 19714000, 19725000, features=["gene"]
+        )
+        assert result == []