MSP-change-till-2025-06-24

Load Bidders through Plugins (#50)

* Load Bidders through Plugins

* Separate core and msp bidders

add adapter nova (#51)

enforce fb bid floor (#52)

fix floor check bug (#53)

Add adapters to support facebook multicall (#54)

Co-authored-by: Peiling Ding <[email protected]>

Add nova multicall (#55)

Co-authored-by: Peiling Ding <[email protected]>

Load bidder based on so_path (#57)

fb dyn config (#60)

increase prometheus histogram buckets (#62)

Integrating Prometheus Metrics on MSP side (#64)

* Integrating Prometheus Metrics on MSP side

* Fixing error message

---------

Co-authored-by: Muhammad Ashhad Alam <[email protected]@Muhammads-MacBook-Pro.local>

add config for scylla access for nova (#70)

Signed-off-by: YifengW86 <[email protected]>

add fb app security id (#71)

mspai-config (#77)

implement file db (#76)

* implement file and db for prebid-server

* move new functions to a new file

fix msp analytics

add Moloco (#87)

fix fb auth id

Add Moloco Native For Multicall (#89)

Author: Peiling-Ding

adapters/audienceNetwork/facebook.go

@@ -38,8 +38,15 @@ type facebookAdMarkup struct {
 }
 
 type facebookReqExt struct {
-	PlatformID string `json:"platformid"`
-	AuthID     string `json:"authentication_id"`
+	PlatformID    string `json:"platformid"`
+	AuthID        string `json:"authentication_id"`
+	SecurityAppID string `json:"security_app_id,omitempty"`
+}
+
+type ExtImpFB struct {
+	AppSecret     string `json:"app_secret"`
+	PlatformID    string `json:"platform_id"`
+	SecurityAppID string `json:"security_app_id"`
 }
 
 func (a *adapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) {
@@ -118,8 +125,9 @@ func (a *adapter) buildRequests(request *openrtb2.BidRequest) ([]*adapters.Reque
 
 // The authentication ID is a sha256 hmac hash encoded as a hex string, based on
 // the app secret and the ID of the bid request
-func (a *adapter) makeAuthID(req *openrtb2.BidRequest) string {
-	h := hmac.New(sha256.New, []byte(a.appSecret))
+
+func (a *adapter) makeAuthID(req *openrtb2.BidRequest, appSecret string) string {
+	h := hmac.New(sha256.New, []byte(appSecret))
 	h.Write([]byte(req.ID))
 
 	return hex.EncodeToString(h.Sum(nil))
@@ -141,9 +149,26 @@ func (a *adapter) modifyRequest(out *openrtb2.BidRequest) error {
 	// ID *BEFORE* we generate the auth ID since its a hash based on the request ID
 	out.ID = imp.ID
 
+	platformId := a.platformID
+	appSecret := a.appSecret
+	securityAppId := ""
+
+	var bidderExt adapters.ExtImpBidder
+	err = json.Unmarshal(imp.Ext, &bidderExt)
+	if err == nil {
+		var impressionExt ExtImpFB
+		err = json.Unmarshal(bidderExt.Bidder, &impressionExt)
+		if err == nil && len(impressionExt.PlatformID) > 0 && len(impressionExt.AppSecret) > 0 && len(impressionExt.SecurityAppID) > 0 {
+			platformId = impressionExt.PlatformID
+			appSecret = impressionExt.AppSecret
+			securityAppId = impressionExt.SecurityAppID
+		}
+	}
+
 	reqExt := facebookReqExt{
-		PlatformID: a.platformID,
-		AuthID:     a.makeAuthID(out),
+		PlatformID:    platformId,
+		AuthID:        a.makeAuthID(out, appSecret),
+		SecurityAppID: securityAppId,
 	}
 
 	if out.Ext, err = json.Marshal(reqExt); err != nil {
@@ -371,6 +396,18 @@ func (a *adapter) MakeBids(request *openrtb2.BidRequest, adapterRequest *adapter
 				continue
 			}
 
+			// TODO: move to msp repo and apply for all bidders
+			var fbreq openrtb2.BidRequest
+			err := json.Unmarshal(adapterRequest.Body, &fbreq)
+			if err == nil {
+				if len(fbreq.Imp) > 0 && fbreq.Imp[0].BidFloor > bid.Price {
+					errs = append(errs, &errortypes.BadServerResponse{
+						Message: fmt.Sprintf("bid price %f less than floor %f", bid.Price, fbreq.Imp[0].BidFloor),
+					})
+					continue
+				}
+			}
+
 			bid.AdID = obj.BidID
 			bid.CrID = obj.BidID
 
@@ -454,7 +491,13 @@ func (a *adapter) MakeTimeoutNotification(req *adapters.RequestData) (*adapters.
 		return &adapters.RequestData{}, errors.New("path app.publisher.id not found in the request")
 	}
 
-	uri := fmt.Sprintf("https://www.facebook.com/audiencenetwork/nurl/?partner=%s&app=%s&auction=%s&ortb_loss_code=2", a.platformID, pubID, rID)
+	platformId := a.platformID
+	requestPlatformId, err := jsonparser.GetString(req.Body, "ext", "platformid")
+	if err == nil {
+		platformId = requestPlatformId
+	}
+
+	uri := fmt.Sprintf("https://www.facebook.com/audiencenetwork/nurl/?partner=%s&app=%s&auction=%s&ortb_loss_code=2", platformId, pubID, rID)
 	timeoutReq := adapters.RequestData{
 		Method:  "GET",
 		Uri:     uri,

analytics/build/build.go

@@ -44,6 +44,11 @@ func New(analytics *config.Analytics) analytics.Runner {
 		}
 	}
 
+	customAdapters := mspLoadAnalyticsAdapterPlugins(analytics.Custom)
+	for adapterName, adapter := range customAdapters {
+		modules[adapterName] = adapter
+	}
+
 	if analytics.Agma.Enabled {
 		agmaModule, err := agma.NewModule(
 			clients.GetDefaultHttpInstance(),

analytics/build/msp_plugin.go

@@ -0,0 +1,39 @@
+package build
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/golang/glog"
+	"github.com/prebid/prebid-server/v3/analytics"
+	mspPlugin "github.com/prebid/prebid-server/v3/msp/plugin"
+)
+
+type PluginBuilder interface {
+	Build(json.RawMessage) (analytics.Module, error)
+}
+
+func mspLoadAnalyticsAdapterPlugins(cfg map[string]interface{}) enabledAnalytics {
+	plugins := make(enabledAnalytics, 0)
+	for name, cfgData := range cfg {
+		builder, cfgJson, skip, err := mspPlugin.LoadBuilder[PluginBuilder](name, cfgData)
+
+		if skip {
+			continue
+		}
+
+		if err != nil {
+			panic(err)
+		}
+
+		plugin, err := builder.Build(cfgJson)
+		if err != nil {
+			panic(fmt.Sprintf("Failed to build Analytics Adapter plugin %s, error: %+v\n", name, err))
+		} else {
+			glog.Infof("Loaded Analytics Adapter plugin: %s\n", name)
+			plugins[name] = plugin
+		}
+	}
+
+	return plugins
+}

config/adapter.go

@@ -12,4 +12,7 @@ type Adapter struct {
 
 	// nededed for Facebook
 	AppSecret string
+
+	// needed for Nova
+	NovaScylla AdapterNovaScylla
 }

config/bidderinfo.go

@@ -48,7 +48,14 @@ type BidderInfo struct {
 	PlatformID string `yaml:"platform_id" mapstructure:"platform_id"`
 	AppSecret  string `yaml:"app_secret" mapstructure:"app_secret"`
 	// EndpointCompression determines, if set, the type of compression the bid request will undergo before being sent to the corresponding bid server
+
 	EndpointCompression string `yaml:"endpointCompression" mapstructure:"endpointCompression"`
+
+	// For MSP Plugin extension only.
+	MspSoPath string `mapstructure:"so_path,omitempty" json:"so_path,omitempty"`
+
+	// For Nova Adapter only
+	NovaScylla AdapterNovaScylla `yaml:"scylla" mapstructure:"scylla"`
 }
 
 type aliasNillableFields struct {
@@ -235,6 +242,13 @@ type SyncerEndpoint struct {
 	UserMacro string `yaml:"userMacro" mapstructure:"user_macro"`
 }
 
+// AdapterNovaScylla specifies the the scylla config for Nova Adapter
+type AdapterNovaScylla struct {
+	Cluster  string `yaml:"cluster" mapstructure:"cluster"`
+	KeySpace string `yaml:"keyspace" mapstructure:"keyspace"`
+	TimeOut  int32  `yaml:"query_timeout" mapstructure:"query_timeout"`
+}
+
 func (s *SyncerEndpoint) Equal(other *SyncerEndpoint) bool {
 	if s == nil && other == nil {
 		return true
@@ -705,6 +719,7 @@ func applyBidderInfoConfigOverrides(configBidderInfos nillableFieldBidderInfos,
 		if !exists {
 			return nil, fmt.Errorf("error setting configuration for bidder %s: unknown bidder", bidderName)
 		}
+
 		fsBidderInfo, exists := fsBidderInfos[string(normalizedBidderName)]
 		if !exists {
 			return nil, fmt.Errorf("error finding configuration for bidder %s: unknown bidder", bidderName)
@@ -757,6 +772,19 @@ func applyBidderInfoConfigOverrides(configBidderInfos nillableFieldBidderInfos,
 		if configBidderInfo.bidderInfo.EndpointCompression != "" {
 			mergedBidderInfo.EndpointCompression = configBidderInfo.bidderInfo.EndpointCompression
 		}
+		if configBidderInfo.bidderInfo.MspSoPath != "" {
+			mergedBidderInfo.MspSoPath = configBidderInfo.bidderInfo.MspSoPath
+		}
+		if configBidderInfo.bidderInfo.NovaScylla.Cluster != "" {
+			mergedBidderInfo.NovaScylla.Cluster = configBidderInfo.bidderInfo.NovaScylla.Cluster
+		}
+		if configBidderInfo.bidderInfo.NovaScylla.KeySpace != "" {
+			mergedBidderInfo.NovaScylla.KeySpace = configBidderInfo.bidderInfo.NovaScylla.KeySpace
+		}
+		if configBidderInfo.bidderInfo.NovaScylla.TimeOut != 0 {
+			mergedBidderInfo.NovaScylla.TimeOut = configBidderInfo.bidderInfo.NovaScylla.TimeOut
+		}
+
 		if configBidderInfo.bidderInfo.OpenRTB != nil {
 			mergedBidderInfo.OpenRTB = configBidderInfo.bidderInfo.OpenRTB
 		}

config/config.go

@@ -36,22 +36,25 @@ type Configuration struct {
 	GarbageCollectorThreshold int `mapstructure:"garbage_collector_threshold"`
 	// StatusResponse is the string which will be returned by the /status endpoint when things are OK.
 	// If empty, it will return a 204 with no content.
-	StatusResponse    string          `mapstructure:"status_response"`
-	AuctionTimeouts   AuctionTimeouts `mapstructure:"auction_timeouts_ms"`
-	TmaxAdjustments   TmaxAdjustments `mapstructure:"tmax_adjustments"`
-	TmaxDefault       int             `mapstructure:"tmax_default"`
-	CacheURL          Cache           `mapstructure:"cache"`
-	ExtCacheURL       ExternalCache   `mapstructure:"external_cache"`
-	RecaptchaSecret   string          `mapstructure:"recaptcha_secret"`
-	HostCookie        HostCookie      `mapstructure:"host_cookie"`
-	Metrics           Metrics         `mapstructure:"metrics"`
-	StoredRequests    StoredRequests  `mapstructure:"stored_requests"`
-	StoredRequestsAMP StoredRequests  `mapstructure:"stored_amp_req"`
-	CategoryMapping   StoredRequests  `mapstructure:"category_mapping"`
-	VTrack            VTrack          `mapstructure:"vtrack"`
-	Event             Event           `mapstructure:"event"`
-	Accounts          StoredRequests  `mapstructure:"accounts"`
-	UserSync          UserSync        `mapstructure:"user_sync"`
+
+	StatusResponse    string           `mapstructure:"status_response"`
+	AuctionTimeouts   AuctionTimeouts  `mapstructure:"auction_timeouts_ms"`
+	TmaxAdjustments   TmaxAdjustments  `mapstructure:"tmax_adjustments"`
+	TmaxDefault       int              `mapstructure:"tmax_default"`
+	CacheURL          Cache            `mapstructure:"cache"`
+	ExtCacheURL       ExternalCache    `mapstructure:"external_cache"`
+	RecaptchaSecret   string           `mapstructure:"recaptcha_secret"`
+	HostCookie        HostCookie       `mapstructure:"host_cookie"`
+	Metrics           Metrics          `mapstructure:"metrics"`
+	MSPMetricsConfig  MSPMetricsConfig `mapstructure:"msp_metrics"`
+	StoredRequests    StoredRequests   `mapstructure:"stored_requests"`
+	StoredRequestsAMP StoredRequests   `mapstructure:"stored_amp_req"`
+	CategoryMapping   StoredRequests   `mapstructure:"category_mapping"`
+	VTrack            VTrack           `mapstructure:"vtrack"`
+	Event             Event            `mapstructure:"event"`
+	Accounts          StoredRequests   `mapstructure:"accounts"`
+	UserSync          UserSync         `mapstructure:"user_sync"`
+
 	// Note that StoredVideo refers to stored video requests, and has nothing to do with caching video creatives.
 	StoredVideo     StoredRequests `mapstructure:"stored_video_req"`
 	StoredResponses StoredRequests `mapstructure:"stored_responses"`
@@ -479,6 +482,10 @@ type Analytics struct {
 	File     FileLogs      `mapstructure:"file"`
 	Agma     AgmaAnalytics `mapstructure:"agma"`
 	Pubstack Pubstack      `mapstructure:"pubstack"`
+
+	// Made by MSP. This field is for loading analytics adapter
+	// dynamically through Golang Plugin at runtime
+	Custom map[string]interface{} `mapstructure:"custom"`
 }
 
 type CurrencyConverter struct {
@@ -579,6 +586,12 @@ type Metrics struct {
 	Disabled   DisabledMetrics   `mapstructure:"disabled_metrics"`
 }
 
+type MSPMetricsConfig struct {
+	Enabled bool   `mapstructure:"enabled"`
+	Port    int    `mapstructure:"port"`
+	SoPath  string `mapstructure:"so_path"`
+}
+
 type DisabledMetrics struct {
 	// True if we want to stop collecting account-to-adapter metrics
 	AccountAdapterDetails bool `mapstructure:"account_adapter_details"`

endpoints/openrtb2/auction.go

@@ -1657,6 +1657,20 @@ func getJsonSyntaxError(testJSON []byte) (bool, string) {
 }
 
 func (deps *endpointDeps) getStoredRequests(ctx context.Context, requestJson []byte, impInfo []ImpExtPrebidData) (string, bool, map[string]json.RawMessage, map[string]json.RawMessage, []error) {
+	if isDbFetched(requestJson) {
+		dbReqMap, dbImpMap, parseErrs := parseDbStoredMaps(requestJson)
+		if len(parseErrs) > 0 {
+			return "", false, nil, nil, parseErrs
+		}
+
+		// same parameters as normal
+		storedBidRequestId, hasStoredBidRequest, err := getStoredRequestId(requestJson)
+		if err != nil {
+			return "", false, nil, nil, []error{err}
+		}
+
+		return storedBidRequestId, hasStoredBidRequest, dbReqMap, dbImpMap, nil
+	}
 	// Parse the Stored Request IDs from the BidRequest and Imps.
 	storedBidRequestId, hasStoredBidRequest, err := getStoredRequestId(requestJson)
 	if err != nil {

endpoints/openrtb2/db_helper.go

@@ -0,0 +1,41 @@
+package openrtb2
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/buger/jsonparser"
+)
+
+func isDbFetched(requestJson []byte) bool {
+	val, dataType, _, err := jsonparser.Get(requestJson, "ext", "db_fetched")
+	if err != nil {
+		return false
+	}
+	if dataType == jsonparser.Boolean && string(val) == "true" {
+		return true
+	}
+	return false
+}
+
+func parseDbStoredMaps(requestJson []byte) (map[string]json.RawMessage, map[string]json.RawMessage, []error) {
+	var errs []error
+	storedReqMap := make(map[string]json.RawMessage)
+	storedImpMap := make(map[string]json.RawMessage)
+
+	rawReqs, dt, _, err := jsonparser.Get(requestJson, "ext", "db_storedrequests")
+	if err == nil && dt == jsonparser.Object {
+		if e := json.Unmarshal(rawReqs, &storedReqMap); e != nil {
+			errs = append(errs, fmt.Errorf("failed to unmarshal db_storedrequests: %v", e))
+		}
+	}
+
+	rawImps, dt2, _, err2 := jsonparser.Get(requestJson, "ext", "db_storedimps")
+	if err2 == nil && dt2 == jsonparser.Object {
+		if e := json.Unmarshal(rawImps, &storedImpMap); e != nil {
+			errs = append(errs, fmt.Errorf("failed to unmarshal db_storedimps: %v", e))
+		}
+	}
+
+	return storedReqMap, storedImpMap, errs
+}

exchange/adapter_util.go

@@ -13,7 +13,8 @@ import (
 
 func BuildAdapters(client *http.Client, cfg *config.Configuration, infos config.BidderInfos, me metrics.MetricsEngine) (map[openrtb_ext.BidderName]AdaptedBidder, map[openrtb_ext.BidderName]struct{}, []error) {
 	server := config.Server{ExternalUrl: cfg.ExternalURL, GvlID: cfg.GDPR.HostVendorID, DataCenter: cfg.DataCenter}
-	bidders, singleFormatBidders, errs := buildBidders(infos, newAdapterBuilders(), server)
+	adapterBuilders := mspAddAdaptersFromPlugins(newAdapterBuilders(), infos)
+	bidders, singleFormatBidders, errs := buildBidders(infos, adapterBuilders, server)
 
 	if len(errs) > 0 {
 		return nil, nil, errs
@@ -92,6 +93,7 @@ func buildAdapterInfo(bidderInfo config.BidderInfo) config.Adapter {
 	adapter.PlatformID = bidderInfo.PlatformID
 	adapter.AppSecret = bidderInfo.AppSecret
 	adapter.XAPI = bidderInfo.XAPI
+	adapter.NovaScylla = bidderInfo.NovaScylla
 	return adapter
 }