|
1 | | -package centiment |
2 | | - |
3 | | -import ( |
4 | | - "context" |
5 | | - |
6 | | - "github.com/go-kit/kit/log" |
7 | | - "github.com/pkg/errors" |
8 | | -) |
9 | | - |
10 | | -// Aggregator aggregates results from an analysis run. |
11 | | -type Aggregator struct { |
12 | | - logger log.Logger |
13 | | - db DB |
14 | | -} |
15 | | - |
16 | | -// NewAggregator creates a new Aggregator: call Run to collect results and save |
17 | | -// them to the given DB. |
18 | | -func NewAggregator(logger log.Logger, db DB) (*Aggregator, error) { |
19 | | - agg := &Aggregator{ |
20 | | - db: db, |
21 | | - logger: logger, |
22 | | - } |
23 | | - |
24 | | - return agg, nil |
25 | | -} |
26 | | - |
27 | | -// Run an aggregatation on the provided results. |
28 | | -func (ag *Aggregator) Run(ctx context.Context, results <-chan *AnalyzerResult) error { |
29 | | - var sentiments = make(map[string]*Sentiment) |
30 | | - |
31 | | - // TODO(matt): Handle cancellation. Use for-select here with two cases. |
32 | | - for res := range results { |
33 | | - topic := res.SearchTerm.Topic |
34 | | - if sentiments[topic] == nil { |
35 | | - sentiments[topic] = &Sentiment{} |
36 | | - } |
37 | | - |
38 | | - // Update the rolling aggregate for each topic. |
39 | | - sentiments[topic] = ag.updateAggregate( |
40 | | - res.Score, |
41 | | - res.Magnitude, |
42 | | - res.TweetID, |
43 | | - sentiments[topic], |
44 | | - ) |
45 | | - |
46 | | - sentiments[topic].populateWithSearch(res.SearchTerm) |
47 | | - } |
48 | | - |
49 | | - for topic, sentiment := range sentiments { |
50 | | - sentiment.finalize() |
51 | | - id, err := ag.db.SaveSentiment(ctx, *sentiment) |
52 | | - if err != nil { |
53 | | - // TODO(matt): Implement retry logic w/ back-off. |
54 | | - ag.logger.Log( |
55 | | - "err", errors.Wrap(err, "failed to save topic"), |
56 | | - "topic", topic, |
57 | | - ) |
58 | | - continue |
59 | | - } |
60 | | - |
61 | | - ag.logger.Log( |
62 | | - "state", "saved", |
63 | | - "topic", sentiment.Topic, |
64 | | - "slug", sentiment.Slug, |
65 | | - "id", id, |
66 | | - "score", sentiment.Score, |
67 | | - "count", sentiment.Count, |
68 | | - "stddev", sentiment.StdDev, |
69 | | - "variance", sentiment.Variance, |
70 | | - ) |
71 | | - } |
72 | | - |
73 | | - return nil |
74 | | -} |
75 | | - |
76 | | -func (ag *Aggregator) updateAggregate(score float32, magnitude float32, tweetID int64, sentiment *Sentiment) *Sentiment { |
77 | | - sentiment.Count++ |
78 | | - oldAverage := sentiment.Score |
79 | | - sentiment.Score = updateAverage(score, sentiment.Score, sentiment.Count) |
80 | | - sentiment.Variance = updateVariance( |
81 | | - score, |
82 | | - sentiment.Variance, |
83 | | - oldAverage, |
84 | | - sentiment.Score, |
85 | | - sentiment.Count, |
86 | | - ) |
87 | | - |
88 | | - // Record the largest (newest) Tweet ID we've seen across our results for this |
89 | | - // topic, as the checkpoint for future searches. |
90 | | - if tweetID > sentiment.LastSeenID { |
91 | | - sentiment.LastSeenID = tweetID |
92 | | - } |
93 | | - |
94 | | - return sentiment |
95 | | -} |
96 | | - |
97 | | -func updateAverage(value float32, currentAverage float64, count int64) float64 { |
98 | | - return currentAverage + ((float64(value) - currentAverage) / float64(count)) |
99 | | -} |
100 | | - |
101 | | -func updateVariance(value float32, variance float64, oldAverage float64, newAverage float64, count int64) float64 { |
102 | | - return variance + (float64(value)-oldAverage)*(float64(value)-newAverage) |
103 | | -} |
| 1 | +package centiment |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + |
| 6 | + "github.com/go-kit/kit/log" |
| 7 | + "github.com/pkg/errors" |
| 8 | +) |
| 9 | + |
| 10 | +// Aggregator aggregates results from an analysis run. |
| 11 | +type Aggregator struct { |
| 12 | + logger log.Logger |
| 13 | + db DB |
| 14 | +} |
| 15 | + |
| 16 | +// NewAggregator creates a new Aggregator: call Run to collect results and save |
| 17 | +// them to the given DB. |
| 18 | +func NewAggregator(logger log.Logger, db DB) (*Aggregator, error) { |
| 19 | + agg := &Aggregator{ |
| 20 | + db: db, |
| 21 | + logger: logger, |
| 22 | + } |
| 23 | + |
| 24 | + return agg, nil |
| 25 | +} |
| 26 | + |
| 27 | +// Run an aggregatation on the provided results. |
| 28 | +func (ag *Aggregator) Run(ctx context.Context, results <-chan *AnalyzerResult) error { |
| 29 | + var sentiments = make(map[string]*Sentiment) |
| 30 | + |
| 31 | + // TODO(matt): Handle cancellation. Use for-select here with two cases. |
| 32 | + for res := range results { |
| 33 | + topic := res.SearchTerm.Topic |
| 34 | + if sentiments[topic] == nil { |
| 35 | + sentiments[topic] = &Sentiment{} |
| 36 | + } |
| 37 | + |
| 38 | + // Update the rolling aggregate for each topic. |
| 39 | + sentiments[topic] = ag.updateAggregate( |
| 40 | + res.Score, |
| 41 | + res.Magnitude, |
| 42 | + res.TweetID, |
| 43 | + sentiments[topic], |
| 44 | + ) |
| 45 | + |
| 46 | + sentiments[topic].populateWithSearch(res.SearchTerm) |
| 47 | + } |
| 48 | + |
| 49 | + for topic, sentiment := range sentiments { |
| 50 | + sentiment.finalize() |
| 51 | + id, err := ag.db.SaveSentiment(ctx, *sentiment) |
| 52 | + if err != nil { |
| 53 | + // TODO(matt): Implement retry logic w/ back-off. |
| 54 | + ag.logger.Log( |
| 55 | + "err", errors.Wrap(err, "failed to save topic"), |
| 56 | + "topic", topic, |
| 57 | + ) |
| 58 | + continue |
| 59 | + } |
| 60 | + |
| 61 | + ag.logger.Log( |
| 62 | + "state", "saved", |
| 63 | + "topic", sentiment.Topic, |
| 64 | + "slug", sentiment.Slug, |
| 65 | + "id", id, |
| 66 | + "score", sentiment.Score, |
| 67 | + "count", sentiment.Count, |
| 68 | + "stddev", sentiment.StdDev, |
| 69 | + "variance", sentiment.Variance, |
| 70 | + ) |
| 71 | + } |
| 72 | + |
| 73 | + return nil |
| 74 | +} |
| 75 | + |
| 76 | +func (ag *Aggregator) updateAggregate(score float32, magnitude float32, tweetID int64, sentiment *Sentiment) *Sentiment { |
| 77 | + sentiment.Count++ |
| 78 | + oldAverage := sentiment.Score |
| 79 | + sentiment.Score = updateAverage(score, sentiment.Score, sentiment.Count) |
| 80 | + sentiment.Variance = updateVariance( |
| 81 | + score, |
| 82 | + sentiment.Variance, |
| 83 | + oldAverage, |
| 84 | + sentiment.Score, |
| 85 | + sentiment.Count, |
| 86 | + ) |
| 87 | + |
| 88 | + // Record the largest (newest) Tweet ID we've seen across our results for this |
| 89 | + // topic, as the checkpoint for future searches. |
| 90 | + if tweetID > sentiment.LastSeenID { |
| 91 | + sentiment.LastSeenID = tweetID |
| 92 | + } |
| 93 | + |
| 94 | + return sentiment |
| 95 | +} |
| 96 | + |
| 97 | +func updateAverage(value float32, currentAverage float64, count int64) float64 { |
| 98 | + return currentAverage + ((float64(value) - currentAverage) / float64(count)) |
| 99 | +} |
| 100 | + |
| 101 | +func updateVariance(value float32, variance float64, oldAverage float64, newAverage float64, count int64) float64 { |
| 102 | + return variance + (float64(value)-oldAverage)*(float64(value)-newAverage) |
| 103 | +} |
0 commit comments