11package handlers
22
33import (
4+ "crypto/sha256"
45 "encoding/json"
56 "fmt"
67 "github.com/golang/glog"
@@ -13,8 +14,10 @@ import (
1314 "github.com/mmcloughlin/geohash"
1415 "github.com/xeipuuv/gojsonschema"
1516 "io"
17+ "net"
1618 "net/http"
1719 "strconv"
20+ "time"
1821)
1922
2023const (
@@ -29,39 +32,62 @@ type AnalyticsLog struct {
2932 PageURL string `json:"page_url"`
3033 SourceURL string `json:"source_url"`
3134 Player string `json:"player"`
35+ Version string `json:"version"`
3236 UserAgent string `json:"user_agent"`
3337 UID string `json:"uid"`
3438 Events []AnalyticsLogEvent `json:"events"`
3539}
3640
3741type AnalyticsLogEvent struct {
38- Type string `json:"type"`
39- Timestamp int64 `json:"timestamp"`
40- Errors int `json:"errors,omitempty"`
41- PlaytimeMS int `json:"playtime_ms,omitempty"`
42- TTFFMS int `json:"ttff_ms,omitempty"`
43- PreloadTimeMS int `json:"preload_time_ms,omitempty"`
44- AutoplayStatus string `json:"autoplay_status,omitempty"`
45- BufferMS int `json:"buffer_ms,omitempty"`
46- ErrorMessage string `json:"error_message,omitempty"`
42+ // Shared fields by all events
43+ Type string `json:"type"`
44+ Timestamp int64 `json:"timestamp"`
45+
46+ // Heartbeat event
47+ Errors * int `json:"errors"`
48+ AutoplayStatus * string `json:"autoplay_status"`
49+ StalledCount * int `json:"stalled_count"`
50+ WaitingCount * int `json:"waiting_count"`
51+ TimeErroredMS * int `json:"time_errored_ms"`
52+ TimeStalledMS * int `json:"time_stalled_ms"`
53+ TimePlayingMS * int `json:"time_playing_ms"`
54+ TimeWaitingMS * int `json:"time_waiting_ms"`
55+ MountToPlayMS * int `json:"mount_to_play_ms"`
56+ MountToFirstFrameMS * int `json:"mount_to_first_frame_ms"`
57+ PlayToFirstFrameMS * int `json:"play_to_first_frame_ms"`
58+ DurationMS * int `json:"duration_ms"`
59+ OffsetMS * int `json:"offset_ms"`
60+ PlayerHeightPX * int `json:"player_height_px"`
61+ PlayerWidthPX * int `json:"player_width_px"`
62+ VideoHeightPX * int `json:"video_height_px"`
63+ VideoWidthPX * int `json:"video_width_px"`
64+ WindowHeightPX * int `json:"window_height_px"`
65+ WindowWidthPX * int `json:"window_width_px"`
66+
67+ // Error event
68+ ErrorMessage * string `json:"error_message"`
69+ Category * string `json:"category"`
4770}
4871
4972type AnalyticsGeo struct {
5073 GeoHash string
74+ Continent string
5175 Country string
76+ CountryCode string
5277 Subdivision string
5378 Timezone string
79+ IP string
5480}
5581
5682type AnalyticsHandlersCollection struct {
5783 extFetcher analytics.IExternalDataFetcher
5884 logProcessor analytics.ILogProcessor
5985}
6086
61- func NewAnalyticsHandlersCollection (streamCache mistapiconnector.IStreamCache , lapi * api.Client , metricsURL string , host string ) AnalyticsHandlersCollection {
87+ func NewAnalyticsHandlersCollection (streamCache mistapiconnector.IStreamCache , lapi * api.Client , lp analytics. ILogProcessor ) AnalyticsHandlersCollection {
6288 return AnalyticsHandlersCollection {
6389 extFetcher : analytics .NewExternalDataFetcher (streamCache , lapi ),
64- logProcessor : analytics . NewLogProcessor ( metricsURL , host ) ,
90+ logProcessor : lp ,
6591 }
6692}
6793
@@ -119,10 +145,11 @@ func parseAnalyticsLog(r *http.Request, schema *gojsonschema.Schema) (*Analytics
119145}
120146
121147func parseAnalyticsGeo (r * http.Request ) (AnalyticsGeo , error ) {
122- res := AnalyticsGeo {}
148+ res := AnalyticsGeo {IP : getIP ( r ) }
123149 var missingHeader []string
124150
125151 res .Country , missingHeader = getOrAddMissing ("X-City-Country-Name" , r .Header , missingHeader )
152+ res .CountryCode , missingHeader = getOrAddMissing ("X-City-Country-Code" , r .Header , missingHeader )
126153 res .Subdivision , missingHeader = getOrAddMissing ("X-Region-Name" , r .Header , missingHeader )
127154 res .Timezone , missingHeader = getOrAddMissing ("X-Time-Zone" , r .Header , missingHeader )
128155
@@ -146,6 +173,16 @@ func parseAnalyticsGeo(r *http.Request) (AnalyticsGeo, error) {
146173 return res , nil
147174}
148175
176+ func getIP (r * http.Request ) string {
177+ ip := r .RemoteAddr
178+ host , _ , err := net .SplitHostPort (ip )
179+ if err != nil {
180+ // If not possible to split, then just use RemoteAddr
181+ return ip
182+ }
183+ return host
184+ }
185+
149186func getOrAddMissing (key string , headers http.Header , missingHeaders []string ) (string , []string ) {
150187 if h := headers .Get (key ); h != "" {
151188 return h , missingHeaders
@@ -158,23 +195,70 @@ func toAnalyticsData(log *AnalyticsLog, geo AnalyticsGeo, extData analytics.Exte
158195 ua := useragent .Parse (log .UserAgent )
159196 var res []analytics.LogData
160197 for _ , e := range log .Events {
161- if e .Type == "heartbeat" {
162- res = append (res , analytics.LogData {
163- SessionID : log .SessionID ,
164- PlaybackID : log .PlaybackID ,
165- Browser : ua .Name ,
166- DeviceType : deviceTypeOf (ua ),
167- Country : geo .Country ,
168- UserID : extData .UserID ,
169- PlaytimeMs : e .PlaytimeMS ,
170- BufferMs : e .BufferMS ,
171- Errors : e .Errors ,
172- })
198+ if ! isSupportedEvent (e .Type ) {
199+ continue
173200 }
201+ res = append (res , analytics.LogData {
202+ SessionID : log .SessionID ,
203+ ServerTimestamp : time .Now ().UnixMilli (),
204+ PlaybackID : log .PlaybackID ,
205+ ViewerHash : hashViewer (log , geo ),
206+ Protocol : log .Protocol ,
207+ PageURL : log .PageURL ,
208+ SourceURL : log .SourceURL ,
209+ Player : log .Player ,
210+ Version : log .Version ,
211+ UserID : extData .UserID ,
212+ DStorageURL : extData .DStorageURL ,
213+ Source : extData .SourceType ,
214+ CreatorID : extData .CreatorID ,
215+ DeviceType : deviceTypeOf (ua ),
216+ DeviceModel : ua .Device ,
217+ Browser : ua .Name ,
218+ OS : ua .OS ,
219+ PlaybackGeoHash : geo .GeoHash ,
220+ PlaybackContinentName : geo .Continent ,
221+ PlaybackCountryCode : geo .CountryCode ,
222+ PlaybackCountryName : geo .Country ,
223+ PlaybackSubdivision : geo .Subdivision ,
224+ EventType : e .Type ,
225+ EventTimestamp : e .Timestamp ,
226+ EventData : analytics.LogDataEvent {
227+ Errors : e .Errors ,
228+ AutoplayStatus : e .AutoplayStatus ,
229+ StalledCount : e .StalledCount ,
230+ WaitingCount : e .WaitingCount ,
231+ TimeErroredMS : e .TimeErroredMS ,
232+ TimeStalledMS : e .TimeStalledMS ,
233+ TimePlayingMS : e .TimePlayingMS ,
234+ TimeWaitingMS : e .TimeWaitingMS ,
235+ MountToPlayMS : e .MountToPlayMS ,
236+ MountToFirstFrameMS : e .MountToFirstFrameMS ,
237+ PlayToFirstFrameMS : e .PlayToFirstFrameMS ,
238+ DurationMS : e .DurationMS ,
239+ OffsetMS : e .OffsetMS ,
240+ PlayerHeightPX : e .PlayerHeightPX ,
241+ PlayerWidthPX : e .PlayerWidthPX ,
242+ VideoHeightPX : e .VideoHeightPX ,
243+ VideoWidthPX : e .VideoWidthPX ,
244+ WindowHeightPX : e .WindowHeightPX ,
245+ WindowWidthPX : e .WindowWidthPX ,
246+
247+ ErrorMessage : e .ErrorMessage ,
248+ Category : e .Category ,
249+ },
250+ })
174251 }
175252 return res
176253}
177254
255+ func isSupportedEvent (eventType string ) bool {
256+ if eventType == "heartbeat" || eventType == "error" {
257+ return true
258+ }
259+ return false
260+ }
261+
178262func deviceTypeOf (ua useragent.UserAgent ) string {
179263 if ua .Mobile {
180264 return "mobile"
@@ -185,3 +269,12 @@ func deviceTypeOf(ua useragent.UserAgent) string {
185269 }
186270 return "unknown"
187271}
272+
273+ func hashViewer (log * AnalyticsLog , geo AnalyticsGeo ) string {
274+ if log .UID != "" {
275+ // If user defined the unique viewer ID, then we just use it
276+ return log .UID
277+ }
278+ // If user didn't define the unique viewer ID, then we hash IP and user agent data
279+ return fmt .Sprintf ("%x" , sha256 .Sum256 ([]byte (log .UserAgent + geo .IP )))
280+ }
0 commit comments