Skip to content

Commit 73da91e

Browse files
authored
Merge pull request #3961 from IntersectMBO/feat/govtool-saves-metadata
Feat/govtool saves metadata
2 parents 36bbb3c + ed197ab commit 73da91e

30 files changed

+948
-270
lines changed

govtool/backend/app/Main.hs

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import Control.Monad.Trans.Except
1616
import Control.Monad.Trans.Reader
1717

1818
import Data.Aeson hiding (Error)
19+
import Data.Aeson (encode)
1920
import qualified Data.ByteString as BS
2021
import Data.ByteString.Char8 (unpack)
2122
import qualified Data.Cache as Cache
@@ -35,7 +36,7 @@ import Data.Text.Encoding (encodeUtf8)
3536
import qualified Data.Text.IO as Text
3637
import qualified Data.Text.Lazy as LazyText
3738
import qualified Data.Text.Lazy.Encoding as LazyText
38-
39+
import qualified Data.ByteString.Lazy.Char8 as BS8
3940
import Database.PostgreSQL.Simple (close, connectPostgreSQL, Connection)
4041

4142
import Network.Wai
@@ -62,8 +63,10 @@ import VVA.API.Types
6263
import VVA.CommandLine
6364
import VVA.Config
6465
import VVA.Types (AppEnv (..),
65-
AppError (CriticalError, InternalError, NotFoundError, ValidationError),
66+
AppError (..),
6667
CacheEnv (..))
68+
import VVA.Ipfs (IpfsError(..))
69+
6770

6871
-- Function to create a connection pool with optimized settings
6972
createOptimizedConnectionPool :: BS.ByteString -> IO (Pool Connection)
@@ -288,10 +291,15 @@ liftServer appEnv =
288291
where
289292
handleErrors :: Either AppError a -> Handler a
290293
handleErrors (Right x) = pure x
291-
handleErrors (Left (ValidationError msg)) = throwError $ err400 { errBody = BS.fromStrict $ encodeUtf8 msg }
292-
handleErrors (Left (NotFoundError msg)) = throwError $ err404 { errBody = BS.fromStrict $ encodeUtf8 msg }
293-
handleErrors (Left (CriticalError msg)) = throwError $ err500 { errBody = BS.fromStrict $ encodeUtf8 msg }
294-
handleErrors (Left (InternalError msg)) = throwError $ err500 { errBody = BS.fromStrict $ encodeUtf8 msg }
294+
handleErrors (Left appError) = do
295+
let status = case appError of
296+
ValidationError _ -> err400
297+
NotFoundError _ -> err404
298+
CriticalError _ -> err500
299+
InternalError _ -> err500
300+
AppIpfsError (OtherIpfsError _) -> err400
301+
AppIpfsError _ -> err503
302+
throwError $ status { errBody = encode appError, errHeaders = [("Content-Type", "application/json")] }
295303
-- * Swagger
296304

297305
type SwaggerAPI = SwaggerSchemaUI "swagger-ui" "swagger.json"

govtool/backend/example-config.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"password" : "postgres",
77
"port" : 5432
88
},
9+
"pinataapijwt": "",
910
"port" : 9999,
1011
"host" : "localhost",
1112
"cachedurationseconds": 20,

govtool/backend/src/VVA/API.hs

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
{-# LANGUAGE DataKinds #-}
21
{-# LANGUAGE FlexibleContexts #-}
32
{-# LANGUAGE NamedFieldPuns #-}
43
{-# LANGUAGE OverloadedStrings #-}
54
{-# LANGUAGE RecordWildCards #-}
65
{-# LANGUAGE TypeOperators #-}
76
{-# LANGUAGE ViewPatterns #-}
7+
{-# LANGUAGE DataKinds #-}
88

99
module VVA.API where
1010

@@ -13,14 +13,16 @@ import Control.Exception (throw, throwIO)
1313
import Control.Monad.Except (runExceptT, throwError)
1414
import Control.Monad.Reader
1515

16-
import Data.Aeson (Value(..), Array, decode, encode, ToJSON, toJSON)
16+
import Data.Aeson (Value(..), Array, decode, ToJSON, toJSON)
1717
import Data.Bool (Bool)
1818
import Data.List (sortOn, sort)
1919
import qualified Data.Map as Map
2020
import Data.Maybe (Maybe (Nothing), catMaybes, fromMaybe, mapMaybe)
2121
import Data.Ord (Down (..))
2222
import Data.Text hiding (any, drop, elem, filter, length, map, null, take)
2323
import qualified Data.Text as Text
24+
import qualified Data.Text.Lazy as TL
25+
import qualified Data.Text.Lazy.Encoding as TL
2426
import qualified Data.Vector as V
2527
import Data.Time.LocalTime (TimeZone, getCurrentTimeZone)
2628

@@ -29,6 +31,7 @@ import Numeric.Natural (Natural)
2931

3032
import Servant.API
3133
import Servant.Server
34+
import Servant.Exception (Throws)
3235
import System.Random (randomRIO)
3336

3437
import Text.Read (readMaybe)
@@ -45,12 +48,18 @@ import qualified VVA.Proposal as Proposal
4548
import qualified VVA.Transaction as Transaction
4649
import qualified VVA.Types as Types
4750
import VVA.Types (App, AppEnv (..),
48-
AppError (CriticalError, InternalError, ValidationError),
51+
AppError (CriticalError, InternalError, ValidationError, AppIpfsError),
4952
CacheEnv (..))
5053
import Data.Time (TimeZone, localTimeToUTC)
54+
import qualified VVA.Ipfs as Ipfs
55+
import Data.ByteString.Lazy (ByteString)
56+
import qualified Data.ByteString.Lazy as BSL
57+
import Servant.Exception (Throws)
5158

5259
type VVAApi =
53-
"drep" :> "list"
60+
"ipfs"
61+
:> "upload" :> QueryParam "fileName" Text :> ReqBody '[PlainText] Text :> Post '[JSON] UploadResponse
62+
:<|> "drep" :> "list"
5463
:> QueryParam "search" Text
5564
:> QueryParams "status" DRepStatus
5665
:> QueryParam "sort" DRepSortMode
@@ -89,7 +98,8 @@ type VVAApi =
8998
:<|> "account" :> Capture "stakeKey" HexText :> Get '[JSON] GetAccountInfoResponse
9099

91100
server :: App m => ServerT VVAApi m
92-
server = drepList
101+
server = upload
102+
:<|> drepList
93103
:<|> getVotingPower
94104
:<|> getVotes
95105
:<|> drepInfo
@@ -107,6 +117,19 @@ server = drepList
107117
:<|> getNetworkTotalStake
108118
:<|> getAccountInfo
109119

120+
upload :: App m => Maybe Text -> Text -> m UploadResponse
121+
upload mFileName fileContentText = do
122+
AppEnv {vvaConfig} <- ask
123+
let fileContent = TL.encodeUtf8 $ TL.fromStrict fileContentText
124+
vvaPinataJwt = pinataApiJwt vvaConfig
125+
fileName = fromMaybe "data.txt" mFileName -- Default to data.txt if no filename is provided
126+
when (BSL.length fileContent > 1024 * 512) $
127+
throwError $ ValidationError "The uploaded file is larger than 500Kb"
128+
eIpfsHash <- liftIO $ Ipfs.ipfsUpload vvaPinataJwt fileName fileContent
129+
case eIpfsHash of
130+
Left err -> throwError $ AppIpfsError err
131+
Right ipfsHash -> return $ UploadResponse ipfsHash
132+
110133
mapDRepType :: Types.DRepType -> DRepType
111134
mapDRepType Types.DRep = NormalDRep
112135
mapDRepType Types.SoleVoter = SoleVoter

govtool/backend/src/VVA/API/Types.hs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1112,6 +1112,14 @@ data GetAccountInfoResponse
11121112
}
11131113
deriving (Generic, Show)
11141114
deriveJSON (jsonOptions "getAccountInfoResponse") ''GetAccountInfoResponse
1115+
1116+
data UploadResponse
1117+
= UploadResponse
1118+
{ uploadResponseIpfsCid :: Text
1119+
}
1120+
deriving (Generic, Show)
1121+
deriveJSON (jsonOptions "uploadResponse") ''UploadResponse
1122+
11151123
exampleGetAccountInfoResponse :: Text
11161124
exampleGetAccountInfoResponse =
11171125
"{\"stakeKey\": \"stake1u9\","
@@ -1125,3 +1133,14 @@ instance ToSchema GetAccountInfoResponse where
11251133
& description ?~ "GetAccountInfoResponse"
11261134
& example
11271135
?~ toJSON exampleGetAccountInfoResponse
1136+
1137+
exampleUploadResponse :: Text
1138+
exampleUploadResponse =
1139+
"{\"ipfsHash\": \"QmZKLGf2D3Z3F2J2K5J2L5J2L5J2L5J2L5J2L5J2L5J2L5\"}"
1140+
1141+
instance ToSchema UploadResponse where
1142+
declareNamedSchema _ = pure $ NamedSchema (Just "UploadResponse") $ mempty
1143+
& type_ ?~ OpenApiObject
1144+
& description ?~ "UploadResponse"
1145+
& example
1146+
?~ toJSON exampleUploadResponse

govtool/backend/src/VVA/Config.hs

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import qualified Conferer.Source.Env as Env
3232

3333
import Control.Monad.Reader
3434

35-
import Data.Aeson
35+
import Data.Aeson as Aeson
3636
import qualified Data.Aeson.Encode.Pretty as AP
3737
import Data.ByteString (ByteString, toStrict)
3838
import Data.Has (Has, getter)
@@ -58,7 +58,7 @@ data DBConfig
5858
-- | Port
5959
, dBConfigPort :: Int
6060
}
61-
deriving (FromConfig, Generic, Show)
61+
deriving (FromConfig, FromJSON, Generic, Show)
6262

6363
instance DefaultConfig DBConfig where
6464
configDef = DBConfig "localhost" "cexplorer" "postgres" "test" 9903
@@ -79,9 +79,12 @@ data VVAConfigInternal
7979
, vVAConfigInternalSentrydsn :: String
8080
-- | Sentry environment
8181
, vVAConfigInternalSentryEnv :: String
82+
-- | Pinata API JWT
83+
, vVAConfigInternalPinataApiJwt :: Maybe Text
8284
}
8385
deriving (FromConfig, Generic, Show)
8486

87+
8588
instance DefaultConfig VVAConfigInternal where
8689
configDef =
8790
VVAConfigInternal
@@ -90,7 +93,8 @@ instance DefaultConfig VVAConfigInternal where
9093
vVAConfigInternalHost = "localhost",
9194
vVaConfigInternalCacheDurationSeconds = 20,
9295
vVAConfigInternalSentrydsn = "https://username:[email protected]/id",
93-
vVAConfigInternalSentryEnv = "development"
96+
vVAConfigInternalSentryEnv = "development",
97+
vVAConfigInternalPinataApiJwt = Nothing
9498
}
9599

96100
-- | DEX configuration.
@@ -108,6 +112,8 @@ data VVAConfig
108112
, sentryDSN :: String
109113
-- | Sentry environment
110114
, sentryEnv :: String
115+
-- | Pinata API JWT
116+
, pinataApiJwt :: Maybe Text
111117
}
112118
deriving (Generic, Show, ToJSON)
113119

@@ -148,7 +154,8 @@ convertConfig VVAConfigInternal {..} =
148154
serverHost = vVAConfigInternalHost,
149155
cacheDurationSeconds = vVaConfigInternalCacheDurationSeconds,
150156
sentryDSN = vVAConfigInternalSentrydsn,
151-
sentryEnv = vVAConfigInternalSentryEnv
157+
sentryEnv = vVAConfigInternalSentryEnv,
158+
pinataApiJwt = vVAConfigInternalPinataApiJwt
152159
}
153160

154161
-- | Load configuration from a file specified on the command line. Load from
@@ -163,7 +170,7 @@ loadVVAConfig configFile = do
163170
where
164171
buildConfig :: IO Config
165172
buildConfig =
166-
Conferer.mkConfig'
173+
mkConfig'
167174
[]
168175
[ Env.fromConfig "vva",
169176
JSON.fromFilePath (fromMaybe "example-config.json" configFile)
@@ -185,4 +192,4 @@ getServerPort = asks (serverPort . getter)
185192
getServerHost ::
186193
(Has VVAConfig r, MonadReader r m) =>
187194
m Text
188-
getServerHost = asks (serverHost . getter)
195+
getServerHost = asks (serverHost . getter)

govtool/backend/src/VVA/Ipfs.hs

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
{-# LANGUAGE DeriveGeneric #-}
2+
{-# LANGUAGE OverloadedStrings #-}
3+
{-# LANGUAGE ScopedTypeVariables #-}
4+
5+
module VVA.Ipfs (ipfsUpload, IpfsError(..)) where
6+
7+
import Control.Exception (SomeException, try)
8+
import Control.Monad.IO.Class (liftIO)
9+
import qualified Data.Aeson as A
10+
import Data.Aeson (FromJSON(parseJSON), withObject, (.:), eitherDecode, ToJSON(..), encode,(.=),object)
11+
import qualified Data.ByteString.Lazy as LBS
12+
import Data.Text (Text)
13+
import qualified Data.Text.Encoding as TE
14+
import GHC.Generics (Generic)
15+
import Network.HTTP.Client (newManager, parseRequest, httpLbs, method, requestHeaders, RequestBody(..), Request, responseBody, responseStatus)
16+
import Network.HTTP.Client.TLS (tlsManagerSettings)
17+
import Network.HTTP.Client.MultipartFormData (formDataBody, partBS, partFileRequestBody)
18+
import Network.HTTP.Types.Status (statusIsSuccessful, Status, status503, status400)
19+
import qualified Data.ByteString.Lazy.Char8 as LBS8
20+
import qualified Data.Text.Lazy as TL
21+
import qualified Data.Text.Lazy.Encoding as TL
22+
import qualified Data.Text as T
23+
import Servant.Server (ServerError (errBody))
24+
import Servant.Exception (ToServantErr(..), Exception(..))
25+
26+
27+
data PinataData = PinataData
28+
{ cid :: Text
29+
, size :: Int
30+
, created_at :: Text
31+
, isDuplicate :: Maybe Bool
32+
} deriving (Show, Generic)
33+
34+
instance FromJSON PinataData
35+
36+
data PinataSuccessResponse = PinataSuccessResponse
37+
{ pinataData :: PinataData
38+
} deriving (Show)
39+
40+
instance FromJSON PinataSuccessResponse where
41+
parseJSON = withObject "PinataSuccessResponse" $ \v -> PinataSuccessResponse
42+
<$> v .: "data"
43+
44+
data IpfsError
45+
= PinataConnectionError String
46+
| PinataAPIError Status LBS.ByteString
47+
| PinataDecodingError String LBS.ByteString
48+
| IpfsUnconfiguredError
49+
| OtherIpfsError String
50+
deriving (Show, Generic)
51+
52+
instance ToJSON IpfsError where
53+
toJSON (PinataConnectionError msg) =
54+
object ["errorType" .= A.String "PinataConnectionError", "message" .= msg]
55+
56+
toJSON (PinataAPIError status body) =
57+
object
58+
[ "errorType" .= A.String "PinataAPIError"
59+
, "message" .= ("Pinata API returned error status : " ++ show status)
60+
, "pinataResponse" .= object
61+
[ "status" .= show status
62+
, "body" .= TL.unpack (TL.decodeUtf8 body)
63+
]
64+
]
65+
66+
toJSON (PinataDecodingError msg body) =
67+
object
68+
[ "errorType" .= A.String "PinataDecodingError"
69+
, "message" .= msg
70+
, "pinataResponse" .= object
71+
[ "status" .= ("unknown" :: String)
72+
, "body" .= TL.unpack (TL.decodeUtf8 body)
73+
]
74+
]
75+
76+
toJSON IpfsUnconfiguredError =
77+
object ["errorType" .= A.String "IpfsUnconfiguredError", "message" .= ("Backend is not configured for upfs upload" :: String)]
78+
79+
toJSON (OtherIpfsError msg) =
80+
object ["errorType" .= A.String "OtherIpfsError", "message" .= msg]
81+
82+
83+
instance Exception IpfsError
84+
85+
86+
87+
instance ToServantErr IpfsError where
88+
status (OtherIpfsError _) = status400
89+
status _ = status503
90+
91+
message (PinataConnectionError msg) = T.pack ("Pinata service connection error: " <> msg)
92+
message (PinataAPIError status body) = T.pack ("Pinata API error: " <> show status <> " - " <> LBS8.unpack body)
93+
message (PinataDecodingError msg body) = T.pack ("Pinata decoding error: " <> msg <> " - " <> LBS8.unpack body)
94+
message IpfsUnconfiguredError = T.pack ("Backend is not configured to support ipfs upload")
95+
message (OtherIpfsError msg) = T.pack msg
96+
97+
ipfsUpload :: Maybe Text -> Text -> LBS.ByteString -> IO (Either IpfsError Text)
98+
ipfsUpload maybeJwt fileName fileContent =
99+
case maybeJwt of
100+
Nothing -> pure $ Left $ IpfsUnconfiguredError
101+
Just "" -> pure $ Left $ IpfsUnconfiguredError
102+
Just jwt -> do
103+
manager <- newManager tlsManagerSettings
104+
initialRequest <- parseRequest "https://uploads.pinata.cloud/v3/files"
105+
let req = initialRequest
106+
{ method = "POST"
107+
, requestHeaders = [("Authorization", "Bearer " <> TE.encodeUtf8 jwt)]
108+
}
109+
result <- try $ flip httpLbs manager =<< formDataBody
110+
[ partBS "network" "public"
111+
, partFileRequestBody "file" (T.unpack fileName) $ RequestBodyLBS fileContent
112+
]
113+
req
114+
115+
case result of
116+
Left (e :: SomeException) -> do
117+
let errMsg = show e
118+
liftIO $ putStrLn errMsg
119+
pure $ Left $ PinataConnectionError errMsg
120+
Right response -> do
121+
let body = responseBody response
122+
let status = responseStatus response
123+
if statusIsSuccessful status
124+
then case eitherDecode body of
125+
Left err -> do
126+
let errMsg = "Failed to decode Pinata API reponse: " <> err
127+
liftIO $ putStrLn errMsg
128+
pure $ Left $ PinataDecodingError errMsg body
129+
Right (res :: PinataSuccessResponse) -> pure $ Right $ cid $ pinataData res
130+
else do
131+
let errMsg = "Pinata API request failed with status: " <> show status
132+
liftIO $ putStrLn errMsg
133+
pure $ Left $ PinataAPIError status body

0 commit comments

Comments
 (0)