Skip to content

Commit 5ab7ae2

Browse files
F4everF4ever
authored
Merge pull request #799 from lidofinance/feature/orc-460-pinata-issues-fix
feat(orc-460): Add pinata dedicated gateways support
2 parents 1070513 + fc721b9 commit 5ab7ae2

File tree

7 files changed

+191
-54
lines changed

7 files changed

+191
-54
lines changed

‎README.md‎

Lines changed: 41 additions & 39 deletions
Large diffs are not rendered by default.

‎src/main.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,10 +144,12 @@ def ipfs_providers() -> Iterator[IPFSProvider]:
144144
timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS,
145145
)
146146

147-
if variables.PINATA_JWT:
147+
if variables.PINATA_JWT and variables.PINATA_DEDICATED_GATEWAY_URL and variables.PINATA_DEDICATED_GATEWAY_TOKEN:
148148
yield Pinata(
149149
variables.PINATA_JWT,
150150
timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS,
151+
dedicated_gateway_url=variables.PINATA_DEDICATED_GATEWAY_URL,
152+
dedicated_gateway_token=variables.PINATA_DEDICATED_GATEWAY_TOKEN,
151153
)
152154

153155
yield PublicIPFS(timeout=variables.HTTP_REQUEST_TIMEOUT_IPFS)

‎src/providers/ipfs/pinata.py‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import logging
22
from json import JSONDecodeError
3+
from urllib.parse import urljoin
34

45
import requests
6+
from requests.adapters import HTTPAdapter
7+
from urllib3 import Retry
58

69
from src.utils.jwt import validate_jwt
710

@@ -15,17 +18,48 @@ class Pinata(IPFSProvider):
1518
"""pinata.cloud IPFS provider"""
1619

1720
API_ENDPOINT = "https://api.pinata.cloud"
18-
GATEWAY = "https://gateway.pinata.cloud"
21+
PUBLIC_GATEWAY = "https://gateway.pinata.cloud"
22+
MAX_DEDICATED_GATEWAY_RETRIES = 1
1923

20-
def __init__(self, jwt_token: str, *, timeout: int) -> None:
24+
def __init__(self, jwt_token: str, *, timeout: int, dedicated_gateway_url: str, dedicated_gateway_token: str) -> None:
2125
super().__init__()
2226
validate_jwt(jwt_token)
2327
self.timeout = timeout
28+
2429
self.session = requests.Session()
2530
self.session.headers["Authorization"] = f"Bearer {jwt_token}"
2631

32+
dedicated_adapter = HTTPAdapter(max_retries=Retry(
33+
total=self.MAX_DEDICATED_GATEWAY_RETRIES,
34+
status_forcelist=list(range(400, 600)),
35+
backoff_factor=3.0,
36+
))
37+
self.dedicated_session = requests.Session()
38+
self.dedicated_session.headers["x-pinata-gateway-token"] = dedicated_gateway_token
39+
self.dedicated_session.mount("https://", dedicated_adapter)
40+
self.dedicated_session.mount("http://", dedicated_adapter)
41+
42+
self.dedicated_gateway_url = dedicated_gateway_url
43+
self.dedicated_gateway_token = dedicated_gateway_token
44+
2745
def fetch(self, cid: CID) -> bytes:
28-
url = f"{self.GATEWAY}/ipfs/{cid}"
46+
try:
47+
return self._fetch_from_dedicated_gateway(cid)
48+
except requests.RequestException as ex:
49+
logger.warning({
50+
"msg": "Dedicated gateway failed after retries, trying public gateway",
51+
"error": str(ex)
52+
})
53+
return self._fetch_from_public_gateway(cid)
54+
55+
def _fetch_from_dedicated_gateway(self, cid: CID) -> bytes:
56+
url = urljoin(self.dedicated_gateway_url, f"/ipfs/{cid}")
57+
resp = self.dedicated_session.get(url, timeout=self.timeout)
58+
resp.raise_for_status()
59+
return resp.content
60+
61+
def _fetch_from_public_gateway(self, cid: CID) -> bytes:
62+
url = urljoin(self.PUBLIC_GATEWAY, f'/ipfs/{cid}')
2963
try:
3064
resp = requests.get(url, timeout=self.timeout)
3165
resp.raise_for_status()
@@ -40,8 +74,7 @@ def publish(self, content: bytes, name: str | None = None) -> CID:
4074

4175
def _upload(self, content: bytes, name: str | None = None) -> str:
4276
"""Pinata has no dedicated endpoint for uploading, so pinFileToIPFS is used"""
43-
44-
url = f"{self.API_ENDPOINT}/pinning/pinFileToIPFS"
77+
url = urljoin(self.API_ENDPOINT, '/pinning/pinFileToIPFS')
4578
try:
4679
with self.session as s:
4780
resp = s.post(url, files={"file": content})

‎src/variables.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
KEYS_API_URI: Final = os.getenv('KEYS_API_URI', '').split(',')
1414

1515
PINATA_JWT: Final = from_file_or_env('PINATA_JWT')
16+
PINATA_DEDICATED_GATEWAY_URL: Final = os.getenv('PINATA_DEDICATED_GATEWAY_URL')
17+
PINATA_DEDICATED_GATEWAY_TOKEN: Final = from_file_or_env('PINATA_DEDICATED_GATEWAY_TOKEN')
1618
KUBO_HOST: Final = os.getenv('KUBO_HOST')
1719
KUBO_GATEWAY_PORT: Final = int(os.getenv('KUBO_GATEWAY_PORT', 8080))
1820
KUBO_RPC_PORT: Final = int(os.getenv('KUBO_RPC_PORT', 5001))
@@ -157,6 +159,7 @@ def raise_from_errors(errors):
157159
'CONSENSUS_CLIENT_URI': CONSENSUS_CLIENT_URI,
158160
'KEYS_API_URI': KEYS_API_URI,
159161
'PINATA_JWT': PINATA_JWT,
162+
'PINATA_DEDICATED_GATEWAY_TOKEN': PINATA_DEDICATED_GATEWAY_TOKEN,
160163
'MEMBER_PRIV_KEY': MEMBER_PRIV_KEY,
161164
'OPSGENIE_API_KEY': OPSGENIE_API_KEY,
162165
'OPSGENIE_API_URL': OPSGENIE_API_URL,

‎tests/integration/contracts/test_cs_parameters_registry.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ def test_cs_parameters_registry(cs_params_contract, caplog):
1515
check_contract(
1616
cs_params_contract,
1717
[
18-
("get_performance_coefficients", None, check_is_instance_of(PerformanceCoefficients)),
19-
("get_reward_share_data", None, check_is_instance_of(KeyNumberValueIntervalList)),
20-
("get_performance_leeway_data", None, check_is_instance_of(KeyNumberValueIntervalList)),
21-
("get_strikes_params", None, check_is_instance_of(StrikesParams)),
18+
("get_performance_coefficients", (0,), check_is_instance_of(PerformanceCoefficients)),
19+
("get_reward_share_data", (0,), check_is_instance_of(KeyNumberValueIntervalList)),
20+
("get_performance_leeway_data", (0,), check_is_instance_of(KeyNumberValueIntervalList)),
21+
("get_strikes_params", (0,), check_is_instance_of(StrikesParams)),
2222
],
2323
caplog,
2424
)

‎tests/integration/contracts/test_lido.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,14 @@ def test_lido_contract_call(lido_contract, accounting_oracle_contract, burner_co
1717
(
1818
1746275159, # timestamp
1919
86400,
20-
389746,
21-
9190764598468942000000000,
20+
403105, # Updated to match current beacon_validators count
21+
8462132592019028000000000, # Updated to match current beacon_balance
2222
13771995248000000000,
2323
478072602914417566,
2424
0,
2525
accounting_oracle_contract.address,
26-
11620928,
27-
# Call depends on contract state
28-
'0xffa34bcc5a08c92272a62e591f7afb9cb839134aa08c091ae0c95682fba35da9',
26+
11620928, # ref_slot
27+
'0x9bad2cb4e0ef017912b8c77e9ce1c6ec52a6b79013fe8d0d099a65a51ee4a66e', # block_identifier
2928
),
3029
lambda response: check_value_type(response, LidoReportRebase),
3130
),

‎tests/providers/test_pinata.py‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import pytest
2+
import responses
3+
4+
from src.providers.ipfs.pinata import Pinata
5+
from src.providers.ipfs.types import FetchError
6+
7+
8+
@pytest.fixture
9+
def pinata_provider():
10+
return Pinata(
11+
jwt_token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyX2lkIjoxLCJ1c2VybmFtZSI6InRlc3QiLCJleHAiOjk5OTk5OTk5OTl9.Ps6jFKniFhNMYr_4WgETZP_LcXEfSzg3yUhNBn6Xgok",
12+
timeout=30,
13+
dedicated_gateway_url="https://dedicated.gateway.com",
14+
dedicated_gateway_token="dedicated_token_123",
15+
)
16+
17+
18+
@pytest.mark.unit
19+
@responses.activate
20+
def test_fetch__dedicated_gateway_available__returns_content_from_dedicated(pinata_provider):
21+
responses.add(responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", body=b'test content', status=200)
22+
23+
result = pinata_provider.fetch("QmTest123")
24+
25+
assert result == b'test content'
26+
assert len(responses.calls) == 1
27+
request_headers = responses.calls[0].request.headers
28+
assert request_headers.get("x-pinata-gateway-token") == "dedicated_token_123"
29+
30+
31+
@pytest.mark.unit
32+
@responses.activate
33+
def test_fetch__dedicated_gateway_fails_max_attempts__falls_back_to_public(pinata_provider):
34+
responses.add(
35+
responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Gateway error"}, status=500
36+
)
37+
responses.add(
38+
responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Gateway error"}, status=500
39+
)
40+
responses.add(responses.GET, "https://gateway.pinata.cloud/ipfs/QmTest123", body=b'public content', status=200)
41+
42+
result = pinata_provider.fetch("QmTest123")
43+
44+
assert result == b'public content'
45+
assert len(responses.calls) == 3
46+
assert responses.calls[0].request.headers.get("x-pinata-gateway-token") == "dedicated_token_123"
47+
assert responses.calls[1].request.headers.get("x-pinata-gateway-token") == "dedicated_token_123"
48+
assert "x-pinata-gateway-token" not in responses.calls[2].request.headers
49+
50+
51+
@pytest.mark.unit
52+
@responses.activate
53+
def test_fetch__dedicated_gateway_fails_once__retries_and_succeeds(pinata_provider):
54+
responses.add(
55+
responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "First failure"}, status=500
56+
)
57+
responses.add(responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", body=b'dedicated success', status=200)
58+
59+
result = pinata_provider.fetch("QmTest123")
60+
61+
assert result == b'dedicated success'
62+
assert len(responses.calls) == 2
63+
64+
65+
@pytest.mark.unit
66+
@responses.activate
67+
def test_fetch__both_gateways_fail__raises_fetch_error(pinata_provider):
68+
responses.add(
69+
responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Dedicated error"}, status=500
70+
)
71+
responses.add(
72+
responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Dedicated error"}, status=500
73+
)
74+
responses.add(
75+
responses.GET, "https://gateway.pinata.cloud/ipfs/QmTest123", json={"error": "Public error"}, status=500
76+
)
77+
78+
with pytest.raises(FetchError):
79+
pinata_provider.fetch("QmTest123")
80+
81+
assert len(responses.calls) == 3
82+
83+
84+
@pytest.mark.unit
85+
@responses.activate
86+
def test_fetch__dedicated_gateway_429_rate_limit__retries_and_falls_back_to_public(pinata_provider):
87+
responses.add(
88+
responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Rate limit exceeded"}, status=429
89+
)
90+
responses.add(
91+
responses.GET, "https://dedicated.gateway.com/ipfs/QmTest123", json={"error": "Rate limit exceeded"}, status=429
92+
)
93+
responses.add(responses.GET, "https://gateway.pinata.cloud/ipfs/QmTest123", body=b'public content', status=200)
94+
95+
result = pinata_provider.fetch("QmTest123")
96+
97+
assert result == b'public content'
98+
assert len(responses.calls) == 3

0 commit comments

Comments
 (0)