Skip to content

Commit 3c7c0c5

Browse files
kdermekderme
committed
Use ipfs gateways
Fixes #1792
1 parent 182cdb4 commit 3c7c0c5

File tree

8 files changed

+56
-15
lines changed

8 files changed

+56
-15
lines changed

‎cardano-db-sync/app/http-get-json-metadata.hs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,7 @@ runHttpGetVote voteUrl mHash vtype =
126126
reportSuccess =<< runOrThrowIO (runExceptT httpGet)
127127
where
128128
httpGet :: ExceptT OffChainFetchError IO SimplifiedOffChainVoteData
129-
httpGet = do
130-
request <- parseOffChainUrl $ OffChainVoteUrl voteUrl
131-
manager <- liftIO $ Http.newManager tlsManagerSettings
132-
httpGetOffChainVoteData manager request voteUrl mHash vtype
129+
httpGet = httpGetOffChainVoteData [] voteUrl mHash vtype
133130

134131
reportSuccess :: SimplifiedOffChainVoteData -> IO ()
135132
reportSuccess spod = do

‎cardano-db-sync/app/test-http-get-json-metadata.hs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ data TestFailure = TestFailure
9696
, tfIOException :: !Word
9797
, tfTimeout :: !Word
9898
, tfConnectionFailure :: !Word
99+
, tfOtherError :: !Word
99100
}
100101

101102
classifyFetchError :: TestFailure -> OffChainFetchError -> TestFailure
@@ -112,9 +113,10 @@ classifyFetchError tf fe =
112113
OCFErrIOException {} -> tf {tfIOException = tfIOException tf + 1}
113114
OCFErrTimeout {} -> tf {tfTimeout = tfTimeout tf + 1}
114115
OCFErrConnectionFailure {} -> tf {tfConnectionFailure = tfConnectionFailure tf + 1}
116+
_otherwise -> tf {tfOtherError = tfOtherError tf + 1}
115117

116118
emptyTestFailure :: TestFailure
117-
emptyTestFailure = TestFailure 0 0 0 0 0 0 0 0 0 0 0
119+
emptyTestFailure = TestFailure 0 0 0 0 0 0 0 0 0 0 0 0
118120

119121
reportTestFailures :: TestFailure -> IO ()
120122
reportTestFailures tf = do

‎cardano-db-sync/src/Cardano/DbSync/Config.hs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import Cardano.DbSync.Config.Node (NodeConfig (..), parseNodeConfig, parseSyncPr
3232
import Cardano.DbSync.Config.Shelley
3333
import Cardano.DbSync.Config.Types
3434
import Cardano.Prelude
35+
import qualified Data.Text as Text
3536
import System.FilePath (takeDirectory, (</>))
3637

3738
configureLogging :: SyncNodeConfig -> Text -> IO (Trace IO Text)
@@ -91,7 +92,11 @@ coalesceConfig pcfg ncfg adjustGenesisPath = do
9192
, dncBabbageHardFork = ncBabbageHardFork ncfg
9293
, dncConwayHardFork = ncConwayHardFork ncfg
9394
, dncInsertOptions = extractInsertOptions pcfg
95+
, dncIpfsGateway = endsInSlash <$> pcIpfsGateway pcfg
9496
}
9597

9698
mkAdjustPath :: SyncPreConfig -> (FilePath -> FilePath)
9799
mkAdjustPath cfg fp = takeDirectory (pcNodeConfigFilePath cfg) </> fp
100+
101+
endsInSlash :: Text -> Text
102+
endsInSlash txt = if Text.isSuffixOf "/" txt then txt else txt <> "/"

‎cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ data SyncNodeConfig = SyncNodeConfig
142142
, dncBabbageHardFork :: !TriggerHardFork
143143
, dncConwayHardFork :: !TriggerHardFork
144144
, dncInsertOptions :: !SyncInsertOptions
145+
, dncIpfsGateway :: [Text]
145146
}
146147

147148
data SyncPreConfig = SyncPreConfig
@@ -153,6 +154,7 @@ data SyncPreConfig = SyncPreConfig
153154
, pcEnableMetrics :: !Bool
154155
, pcPrometheusPort :: !Int
155156
, pcInsertConfig :: !SyncInsertConfig
157+
, pcIpfsGateway :: ![Text]
156158
}
157159
deriving (Show)
158160

@@ -390,6 +392,7 @@ parseGenSyncNodeConfig o =
390392
<*> o .: "EnableLogMetrics"
391393
<*> fmap (fromMaybe 8080) (o .:? "PrometheusPort")
392394
<*> o .:? "insert_options" .!= def
395+
<*> o .:? "ipfs_gateway" .!= ["https://ipfs.io/ipfs"]
393396

394397
instance FromJSON SyncProtocol where
395398
parseJSON o =

‎cardano-db-sync/src/Cardano/DbSync/OffChain.hs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import Cardano.Db (runIohkLogging)
2222
import qualified Cardano.Db as DB
2323
import Cardano.DbSync.Api
2424
import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..))
25+
import Cardano.DbSync.Config.Types
2526
import Cardano.DbSync.OffChain.Http
2627
import Cardano.DbSync.OffChain.Query
2728
import qualified Cardano.DbSync.OffChain.Vote.Types as Vote
@@ -212,12 +213,12 @@ runFetchOffChainVoteThread syncEnv = do
212213
-- load the offChain vote work queue using the db
213214
_ <- runReaderT (loadOffChainVoteWorkQueue trce (envOffChainVoteWorkQueue syncEnv)) backendVote
214215
voteq <- atomically $ flushTBQueue (envOffChainVoteWorkQueue syncEnv)
215-
manager <- Http.newManager tlsManagerSettings
216216
now <- liftIO Time.getPOSIXTime
217-
mapM_ (queueVoteInsert <=< fetchOffChainVoteData trce manager now) voteq
217+
mapM_ (queueVoteInsert <=< fetchOffChainVoteData gateways now) voteq
218218
where
219219
trce = getTrace syncEnv
220220
iopts = getInsertOptions syncEnv
221+
gateways = dncIpfsGateway $ envSyncNodeConfig syncEnv
221222

222223
queueVoteInsert :: OffChainVoteResult -> IO ()
223224
queueVoteInsert = atomically . writeTBQueue (envOffChainVoteResultQueue syncEnv)
@@ -260,13 +261,12 @@ fetchOffChainPoolData _tracer manager time oPoolWorkQ =
260261
, DB.offChainPoolFetchErrorRetryCount = retryCount (oPoolWqRetry oPoolWorkQ)
261262
}
262263

263-
fetchOffChainVoteData :: Trace IO Text -> Http.Manager -> Time.POSIXTime -> OffChainVoteWorkQueue -> IO OffChainVoteResult
264-
fetchOffChainVoteData _tracer manager time oVoteWorkQ =
264+
fetchOffChainVoteData :: [Text] -> Time.POSIXTime -> OffChainVoteWorkQueue -> IO OffChainVoteResult
265+
fetchOffChainVoteData gateways time oVoteWorkQ =
265266
convert <<$>> runExceptT $ do
266267
let url = oVoteWqUrl oVoteWorkQ
267268
metaHash = oVoteWqMetaHash oVoteWorkQ
268-
request <- parseOffChainUrl $ OffChainVoteUrl url
269-
httpGetOffChainVoteData manager request url (Just metaHash) (oVoteWqType oVoteWorkQ)
269+
httpGetOffChainVoteData gateways url (Just metaHash) (oVoteWqType oVoteWorkQ)
270270
where
271271
convert :: Either OffChainFetchError SimplifiedOffChainVoteData -> OffChainVoteResult
272272
convert eres =

‎cardano-db-sync/src/Cardano/DbSync/OffChain/Http.hs

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import qualified Data.Text.Encoding as Text
3838
import GHC.Show (show)
3939
import Network.HTTP.Client (HttpException (..))
4040
import qualified Network.HTTP.Client as Http
41+
import Network.HTTP.Client.TLS (tlsManagerSettings)
4142
import qualified Network.HTTP.Types as Http
4243

4344
-------------------------------------------------------------------------------------
@@ -78,13 +79,33 @@ httpGetOffChainPoolData manager request purl expectedMetaHash = do
7879
url = OffChainPoolUrl purl
7980

8081
httpGetOffChainVoteData ::
81-
Http.Manager ->
82-
Http.Request ->
82+
[Text] ->
8383
VoteUrl ->
8484
Maybe VoteMetaHash ->
8585
DB.AnchorType ->
8686
ExceptT OffChainFetchError IO SimplifiedOffChainVoteData
87-
httpGetOffChainVoteData manager request vurl metaHash anchorType = do
87+
httpGetOffChainVoteData gateways vurl metaHash anchorType = do
88+
case useIpfsGatewayMaybe vurl gateways of
89+
Nothing -> httpGetOffChainVoteDataSingle vurl metaHash anchorType
90+
Just [] -> left $ OCFErrNoIpfsGateway (OffChainVoteUrl vurl)
91+
Just urls -> tryAllGatewaysRec urls []
92+
where
93+
tryAllGatewaysRec [] acc = left $ OCFErrIpfsGatewayFailures (OffChainVoteUrl vurl) (reverse acc)
94+
tryAllGatewaysRec (url : rest) acc = do
95+
msocd <- liftIO $ runExceptT $ httpGetOffChainVoteDataSingle url metaHash anchorType
96+
case msocd of
97+
Right socd -> pure socd
98+
Left err -> tryAllGatewaysRec rest (err : acc)
99+
100+
httpGetOffChainVoteDataSingle ::
101+
VoteUrl ->
102+
Maybe VoteMetaHash ->
103+
DB.AnchorType ->
104+
ExceptT OffChainFetchError IO SimplifiedOffChainVoteData
105+
httpGetOffChainVoteDataSingle vurl metaHash anchorType = do
106+
manager <- liftIO $ Http.newManager tlsManagerSettings
107+
request <- parseOffChainUrl url
108+
let req = httpGetBytes manager request 10000 30000 url
88109
httpRes <- handleExceptT (convertHttpException url) req
89110
(respBS, respLBS, mContentType) <- hoistEither httpRes
90111
(ocvd, decodedValue, metadataHash, mWarning) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl)
@@ -98,7 +119,6 @@ httpGetOffChainVoteData manager request vurl metaHash anchorType = do
98119
, sovaWarning = mWarning
99120
}
100121
where
101-
req = httpGetBytes manager request 10000 30000 url
102122
url = OffChainVoteUrl vurl
103123

104124
parseAndValidateVoteData :: ByteString -> LBS.ByteString -> Maybe VoteMetaHash -> DB.AnchorType -> Maybe OffChainUrlType -> ExceptT OffChainFetchError IO (Vote.OffChainVoteData, Aeson.Value, ByteString, Maybe Text)
@@ -217,3 +237,9 @@ convertHttpException url he =
217237
case url of
218238
OffChainPoolUrl _ -> OCFErrUrlParseFail (OffChainPoolUrl $ PoolUrl $ Text.pack urlx) (Text.pack err)
219239
OffChainVoteUrl _ -> OCFErrUrlParseFail (OffChainVoteUrl $ VoteUrl $ Text.pack urlx) (Text.pack err)
240+
241+
useIpfsGatewayMaybe :: VoteUrl -> [Text] -> Maybe [VoteUrl]
242+
useIpfsGatewayMaybe vu gateways =
243+
case Text.stripPrefix "ipfs://" (unVoteUrl vu) of
244+
Just sf -> Just $ VoteUrl . (<> sf) <$> gateways
245+
Nothing -> Nothing

‎cardano-db-sync/src/Cardano/DbSync/Types.hs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,8 @@ data OffChainFetchError
231231
| OCFErrIOException !Text
232232
| OCFErrTimeout !OffChainUrlType !Text
233233
| OCFErrConnectionFailure !OffChainUrlType
234+
| OCFErrNoIpfsGateway !OffChainUrlType
235+
| OCFErrIpfsGatewayFailures !OffChainUrlType [OffChainFetchError]
234236
deriving (Eq, Generic)
235237

236238
instance Exception OffChainFetchError
@@ -271,6 +273,10 @@ instance Show OffChainFetchError where
271273
mconcat
272274
[fetchUrlToString url, "Connection failure error when fetching metadata from ", show url, "."]
273275
OCFErrIOException err -> "IO Exception: " <> show err
276+
OCFErrNoIpfsGateway url ->
277+
mconcat [fetchUrlToString url, "No ipfs_gateway provided in the db-sync config"]
278+
OCFErrIpfsGatewayFailures url errs ->
279+
mconcat $ [fetchUrlToString url, "List of errors for each ipfs gateway: "] <> fmap show errs
274280

275281
showMUrl :: Maybe OffChainUrlType -> String
276282
showMUrl = \case

‎cardano-db-sync/test/Cardano/DbSync/Gen.hs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ syncPreConfig =
5656
<*> Gen.bool
5757
<*> Gen.int (Range.linear 0 10000)
5858
<*> syncInsertConfig
59+
<*> Gen.list (Range.linear 0 10) (Gen.text (Range.linear 0 100) Gen.unicode)
5960

6061
syncNodeParams :: MonadGen m => m SyncNodeParams
6162
syncNodeParams =
@@ -103,6 +104,7 @@ syncNodeConfig loggingCfg =
103104
<*> triggerHardFork
104105
<*> triggerHardFork
105106
<*> syncInsertOptions
107+
<*> pure []
106108

107109
syncInsertConfig :: Gen SyncInsertConfig
108110
syncInsertConfig =

0 commit comments

Comments
 (0)