Skip to content

Commit cfdd15a

Browse files
lazyninalazynina
authored
enforce limit on get nfts for user (#745)
* enforce limit on get nfts for user * temporarily log error in gs get * reuse shared client * remove err * add todo * pass filter values to filter on db entries * make all gs requests reuse client. json_data -> jsonData * remove todo
1 parent f8e22b7 commit cfdd15a

File tree

4 files changed

+93
-50
lines changed

4 files changed

+93
-50
lines changed

‎routes/global_state.go‎

Lines changed: 61 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"strings"
1010

1111
"github.com/deso-protocol/core/lib"
12-
1312
"github.com/dgraph-io/badger/v3"
1413
"github.com/nyaruka/phonenumbers"
1514
"github.com/pkg/errors"
@@ -29,6 +28,7 @@ type GlobalState struct {
2928
GlobalStateRemoteNode string
3029
GlobalStateRemoteSecret string
3130
GlobalStateDB *badger.DB
31+
SharedClient *http.Client
3232
}
3333

3434
// GlobalStateRoutes returns the routes for managing global state.
@@ -845,13 +845,13 @@ func (gs *GlobalState) PutRemote(ww http.ResponseWriter, rr *http.Request) {
845845
}
846846

847847
func (gs *GlobalState) CreatePutRequest(key []byte, value []byte) (
848-
_url string, _json_data []byte, _err error) {
848+
_url string, _jsonData []byte, _err error) {
849849

850850
req := PutRemoteRequest{
851851
Key: key,
852852
Value: value,
853853
}
854-
json_data, err := json.Marshal(req)
854+
jsonData, err := json.Marshal(req)
855855
if err != nil {
856856
return "", nil, fmt.Errorf("Put: Could not marshal JSON: %v", err)
857857
}
@@ -860,24 +860,27 @@ func (gs *GlobalState) CreatePutRequest(key []byte, value []byte) (
860860
gs.GlobalStateRemoteNode, RoutePathGlobalStatePutRemote,
861861
GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret)
862862

863-
return url, json_data, nil
863+
return url, jsonData, nil
864864
}
865865

866866
func (gs *GlobalState) Put(key []byte, value []byte) error {
867867
// If we have a remote node then use that node to fulfill this request.
868868
if gs.GlobalStateRemoteNode != "" {
869869
// TODO: This codepath is hard to exercise in a test.
870870

871-
url, json_data, err := gs.CreatePutRequest(key, value)
871+
url, jsonData, err := gs.CreatePutRequest(key, value)
872872
if err != nil {
873873
return fmt.Errorf("Put: Error constructing request: %v", err)
874874
}
875-
res, err := http.Post(
876-
url,
877-
"application/json", /*contentType*/
878-
bytes.NewBuffer(json_data))
875+
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
876+
if err != nil {
877+
return fmt.Errorf("Put: Error creating new request: ")
878+
}
879+
req.Header.Set("Content-Type", "application/json")
880+
881+
res, err := gs.SharedClient.Do(req)
879882
if err != nil {
880-
return fmt.Errorf("Put: Error processing remote request")
883+
return fmt.Errorf("Put: Error processing remote request: ")
881884
}
882885
res.Body.Close()
883886

@@ -931,12 +934,12 @@ func (gs *GlobalState) GetRemote(ww http.ResponseWriter, rr *http.Request) {
931934
}
932935

933936
func (gs *GlobalState) CreateGetRequest(key []byte) (
934-
_url string, _json_data []byte, _err error) {
937+
_url string, _jsonData []byte, _err error) {
935938

936939
req := GetRemoteRequest{
937940
Key: key,
938941
}
939-
json_data, err := json.Marshal(req)
942+
jsonData, err := json.Marshal(req)
940943
if err != nil {
941944
return "", nil, fmt.Errorf("Get: Could not marshal JSON: %v", err)
942945
}
@@ -945,26 +948,29 @@ func (gs *GlobalState) CreateGetRequest(key []byte) (
945948
gs.GlobalStateRemoteNode, RoutePathGlobalStateGetRemote,
946949
GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret)
947950

948-
return url, json_data, nil
951+
return url, jsonData, nil
949952
}
950953

951954
func (gs *GlobalState) Get(key []byte) (value []byte, _err error) {
952955
// If we have a remote node then use that node to fulfill this request.
953956
if gs.GlobalStateRemoteNode != "" {
954957
// TODO: This codepath is currently annoying to test.
955958

956-
url, json_data, err := gs.CreateGetRequest(key)
959+
url, jsonData, err := gs.CreateGetRequest(key)
957960
if err != nil {
958961
return nil, fmt.Errorf(
959962
"Get: Error constructing request: %v", err)
960963
}
961964

962-
resReturned, err := http.Post(
963-
url,
964-
"application/json", /*contentType*/
965-
bytes.NewBuffer(json_data))
965+
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
966966
if err != nil {
967-
return nil, fmt.Errorf("Get: Error processing remote request")
967+
return nil, fmt.Errorf("Get: Error creating new request: ")
968+
}
969+
req.Header.Set("Content-Type", "application/json")
970+
971+
resReturned, err := gs.SharedClient.Do(req)
972+
if err != nil {
973+
return nil, fmt.Errorf("Get: Error processing remote request: ")
968974
}
969975

970976
res := GetRemoteResponse{}
@@ -1035,12 +1041,12 @@ func (gs *GlobalState) BatchGetRemote(ww http.ResponseWriter, rr *http.Request)
10351041
}
10361042

10371043
func (gs *GlobalState) CreateBatchGetRequest(keyList [][]byte) (
1038-
_url string, _json_data []byte, _err error) {
1044+
_url string, _jsonData []byte, _err error) {
10391045

10401046
req := BatchGetRemoteRequest{
10411047
KeyList: keyList,
10421048
}
1043-
json_data, err := json.Marshal(req)
1049+
jsonData, err := json.Marshal(req)
10441050
if err != nil {
10451051
return "", nil, fmt.Errorf("BatchGet: Could not marshal JSON: %v", err)
10461052
}
@@ -1049,26 +1055,29 @@ func (gs *GlobalState) CreateBatchGetRequest(keyList [][]byte) (
10491055
gs.GlobalStateRemoteNode, RoutePathGlobalStateBatchGetRemote,
10501056
GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret)
10511057

1052-
return url, json_data, nil
1058+
return url, jsonData, nil
10531059
}
10541060

10551061
func (gs *GlobalState) BatchGet(keyList [][]byte) (value [][]byte, _err error) {
10561062
// If we have a remote node then use that node to fulfill this request.
10571063
if gs.GlobalStateRemoteNode != "" {
10581064
// TODO: This codepath is currently annoying to test.
10591065

1060-
url, json_data, err := gs.CreateBatchGetRequest(keyList)
1066+
url, jsonData, err := gs.CreateBatchGetRequest(keyList)
10611067
if err != nil {
10621068
return nil, fmt.Errorf(
10631069
"BatchGet: Error constructing request: %v", err)
10641070
}
10651071

1066-
resReturned, err := http.Post(
1067-
url,
1068-
"application/json", /*contentType*/
1069-
bytes.NewBuffer(json_data))
1072+
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
1073+
if err != nil {
1074+
return nil, fmt.Errorf("BatchGet: Error creating new request: ")
1075+
}
1076+
req.Header.Set("Content-Type", "application/json")
1077+
1078+
resReturned, err := gs.SharedClient.Do(req)
10701079
if err != nil {
1071-
return nil, fmt.Errorf("BatchGet: Error processing remote request")
1080+
return nil, fmt.Errorf("BatchGet: Error processing remote request: ")
10721081
}
10731082

10741083
res := BatchGetRemoteResponse{}
@@ -1113,12 +1122,12 @@ type DeleteRemoteResponse struct {
11131122
}
11141123

11151124
func (gs *GlobalState) CreateDeleteRequest(key []byte) (
1116-
_url string, _json_data []byte, _err error) {
1125+
_url string, _jsonData []byte, _err error) {
11171126

11181127
req := DeleteRemoteRequest{
11191128
Key: key,
11201129
}
1121-
json_data, err := json.Marshal(req)
1130+
jsonData, err := json.Marshal(req)
11221131
if err != nil {
11231132
return "", nil, fmt.Errorf("Delete: Could not marshal JSON: %v", err)
11241133
}
@@ -1127,7 +1136,7 @@ func (gs *GlobalState) CreateDeleteRequest(key []byte) (
11271136
gs.GlobalStateRemoteNode, RoutePathGlobalStateDeleteRemote,
11281137
GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret)
11291138

1130-
return url, json_data, nil
1139+
return url, jsonData, nil
11311140
}
11321141

11331142
func (gs *GlobalState) DeleteRemote(ww http.ResponseWriter, rr *http.Request) {
@@ -1159,17 +1168,20 @@ func (gs *GlobalState) Delete(key []byte) error {
11591168
if gs.GlobalStateRemoteNode != "" {
11601169
// TODO: This codepath is currently annoying to test.
11611170

1162-
url, json_data, err := gs.CreateDeleteRequest(key)
1171+
url, jsonData, err := gs.CreateDeleteRequest(key)
11631172
if err != nil {
11641173
return fmt.Errorf("Delete: Could not construct request: %v", err)
11651174
}
11661175

1167-
res, err := http.Post(
1168-
url,
1169-
"application/json", /*contentType*/
1170-
bytes.NewBuffer(json_data))
1176+
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
1177+
if err != nil {
1178+
return fmt.Errorf("Delete: Error creating new request: ")
1179+
}
1180+
req.Header.Set("Content-Type", "application/json")
1181+
1182+
res, err := gs.SharedClient.Do(req)
11711183
if err != nil {
1172-
return fmt.Errorf("Delete: Error processing remote request")
1184+
return fmt.Errorf("Delete: Error processing remote request: ")
11731185
}
11741186

11751187
res.Body.Close()
@@ -1202,7 +1214,7 @@ type SeekRemoteResponse struct {
12021214

12031215
func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []byte,
12041216
maxKeyLen int, numToFetch int, reverse bool, fetchValues bool) (
1205-
_url string, _json_data []byte, _err error) {
1217+
_url string, _jsonData []byte, _err error) {
12061218

12071219
req := SeekRemoteRequest{
12081220
StartPrefix: startPrefix,
@@ -1212,7 +1224,7 @@ func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []by
12121224
Reverse: reverse,
12131225
FetchValues: fetchValues,
12141226
}
1215-
json_data, err := json.Marshal(req)
1227+
jsonData, err := json.Marshal(req)
12161228
if err != nil {
12171229
return "", nil, fmt.Errorf("Seek: Could not marshal JSON: %v", err)
12181230
}
@@ -1221,7 +1233,7 @@ func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []by
12211233
gs.GlobalStateRemoteNode, RoutePathGlobalStateSeekRemote,
12221234
GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret)
12231235

1224-
return url, json_data, nil
1236+
return url, jsonData, nil
12251237
}
12261238

12271239
func (gs *GlobalState) GlobalStateSeekRemote(ww http.ResponseWriter, rr *http.Request) {
@@ -1266,7 +1278,7 @@ func (gs *GlobalState) Seek(startPrefix []byte, validForPrefix []byte,
12661278
// If we have a remote node then use that node to fulfill this request.
12671279
if gs.GlobalStateRemoteNode != "" {
12681280
// TODO: This codepath is currently annoying to test.
1269-
url, json_data, err := gs.CreateSeekRequest(
1281+
url, jsonData, err := gs.CreateSeekRequest(
12701282
startPrefix,
12711283
validForPrefix,
12721284
maxKeyLen,
@@ -1278,12 +1290,15 @@ func (gs *GlobalState) Seek(startPrefix []byte, validForPrefix []byte,
12781290
"Seek: Error constructing request: %v", err)
12791291
}
12801292

1281-
resReturned, err := http.Post(
1282-
url,
1283-
"application/json", /*contentType*/
1284-
bytes.NewBuffer(json_data))
1293+
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
1294+
if err != nil {
1295+
return nil, nil, fmt.Errorf("Seek: Error creating new request: ")
1296+
}
1297+
req.Header.Set("Content-Type", "application/json")
1298+
1299+
resReturned, err := gs.SharedClient.Do(req)
12851300
if err != nil {
1286-
return nil, nil, fmt.Errorf("Seek: Error processing remote request")
1301+
return nil, nil, fmt.Errorf("Seek: Error processing remote request: ")
12871302
}
12881303

12891304
res := SeekRemoteResponse{}

‎routes/nft.go‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -930,7 +930,9 @@ type GetNFTsForUserRequest struct {
930930
ReaderPublicKeyBase58Check string `safeForLogging:"true"`
931931
IsForSale *bool `safeForLogging:"true"`
932932
// Ignored if IsForSale is provided
933-
IsPending *bool `safeForLogging:"true"`
933+
IsPending *bool `safeForLogging:"true"`
934+
LastKeyHex string `safeForLogging:"true"`
935+
Limit int `safeForLogging:"true"`
934936
}
935937

936938
type NFTEntryAndPostEntryResponse struct {
@@ -939,7 +941,8 @@ type NFTEntryAndPostEntryResponse struct {
939941
}
940942

941943
type GetNFTsForUserResponse struct {
942-
NFTsMap map[string]*NFTEntryAndPostEntryResponse
944+
NFTsMap map[string]*NFTEntryAndPostEntryResponse
945+
LastKeyHex string
943946
}
944947

945948
func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) {
@@ -969,6 +972,20 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request)
969972
}
970973
}
971974

975+
limit := requestData.Limit
976+
if limit <= 0 || limit > 100 {
977+
limit = 100
978+
}
979+
980+
lastKeyBytes := []byte{}
981+
if requestData.LastKeyHex != "" {
982+
lastKeyBytes, err = hex.DecodeString(requestData.LastKeyHex)
983+
if err != nil {
984+
_AddBadRequestError(ww, fmt.Sprintf("GetNFTsForUser: Problem decoding LastKeyHex: %v", err))
985+
return
986+
}
987+
}
988+
972989
// Get the NFT bid so we can do a more hardcore validation of the request data.
973990
utxoView, err := fes.backendServer.GetMempool().GetAugmentedUniversalView()
974991
if err != nil {
@@ -987,7 +1004,8 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request)
9871004
readerPKID = readerPKIDEntry.PKID
9881005
}
9891006

990-
nftEntries := utxoView.GetNFTEntriesForPKID(pkid.PKID)
1007+
nftEntries, lastSeenKey := utxoView.GetNFTEntriesForPKID(
1008+
pkid.PKID, limit, lastKeyBytes, requestData.IsForSale, requestData.IsPending)
9911009

9921010
filteredNFTEntries := []*lib.NFTEntry{}
9931011
if requestData.IsForSale != nil {
@@ -1041,6 +1059,10 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request)
10411059
fes._nftEntryToResponse(nftEntry, nil, utxoView, true, readerPKID))
10421060
}
10431061

1062+
if len(lastSeenKey) > 0 {
1063+
res.LastKeyHex = hex.EncodeToString(lastSeenKey)
1064+
}
1065+
10441066
if err = json.NewEncoder(ww).Encode(res); err != nil {
10451067
_AddInternalServerError(ww, fmt.Sprintf("GetNFTsForUser: Problem serializing object to JSON: %v", err))
10461068
return

‎routes/server.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,13 @@ func NewAPIServer(
534534
GlobalStateRemoteSecret: config.GlobalStateRemoteSecret,
535535
GlobalStateRemoteNode: config.GlobalStateRemoteNode,
536536
GlobalStateDB: globalStateDB,
537+
SharedClient: &http.Client{
538+
Transport: &http.Transport{
539+
MaxIdleConns: 100,
540+
MaxIdleConnsPerHost: 100,
541+
IdleConnTimeout: 90 * time.Second,
542+
},
543+
},
537544
}
538545

539546
if globalStateDB == nil && globalState.GlobalStateRemoteNode == "" {

‎routes/user.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2351,7 +2351,6 @@ func (fes *APIServer) GetNotifications(ww http.ResponseWriter, req *http.Request
23512351
LastSeenIndex: lastSeenIndex,
23522352
}
23532353
if err = json.NewEncoder(ww).Encode(res); err != nil {
2354-
fmt.Printf("%#v\n", res)
23552354
_AddBadRequestError(ww, fmt.Sprintf(
23562355
"GetNotifications: Problem encoding response as JSON: %v", err))
23572356
return

0 commit comments

Comments
 (0)