Initial code commit, copied and modified from several percona/go-mysql pkgs @ 2a6037d

Author: daniel-nichter

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+*.out

Godeps/Godeps.json

@@ -0,0 +1,11 @@
+# Slow Log Parsing and Aggregation
+
+[![GoDoc](https://godoc.org/github.com/go-mysql/slowlog?status.svg)](https://godoc.org/github.com/go-mysql/slowlog)
+
+This package provides functions for working with the MySQL slow log.
+
+## Acknowledgement
+
+This code was originally copied from [percona/go-mysql](https://github.com/percona/go-mysql) @ `2a6037d7d809b18ebd6d735b397f2321879af611`. See that project for original contributors and copyright.
+
+This project is a fork to continue development of percona/go-mysql as separate packages. GitHub only allows forking a project once, which is why the code has been copied.

Godeps/Readme

@@ -0,0 +1,89 @@
+/*
+	Copyright 2017 Daniel Nichter
+	Copyright 2014-2016 Percona LLC and/or its affiliates
+*/
+
+package slowlog
+
+import (
+	"time"
+)
+
+// A Result contains a global class and per-ID classes with finalized metric
+// statistics. The classes are keyed on class ID.
+type Result struct {
+	Global    *Class            // all classes
+	Class     map[string]*Class // keyed on class ID
+	RateLimit uint
+	Error     string
+}
+
+// An Aggregator groups events by class ID. When there are no more events,
+// a call to Finalize computes all metric statistics and returns a Result.
+type Aggregator struct {
+	samples     bool
+	utcOffset   time.Duration
+	outlierTime float64
+	// --
+	global    *Class
+	classes   map[string]*Class
+	rateLimit uint
+}
+
+// NewAggregator returns a new Aggregator.
+func NewAggregator(samples bool, utcOffset time.Duration, outlierTime float64) *Aggregator {
+	a := &Aggregator{
+		samples:     samples,
+		utcOffset:   utcOffset,
+		outlierTime: outlierTime,
+		// --
+		global:  NewClass("", "", false),
+		classes: map[string]*Class{},
+	}
+	return a
+}
+
+// AddEvent adds the event to the aggregator, automatically creating new classes
+// as needed.
+func (a *Aggregator) AddEvent(event Event, id, fingerprint string) {
+	if a.rateLimit != event.RateLimit {
+		a.rateLimit = event.RateLimit
+	}
+
+	outlier := false
+	if a.outlierTime > 0 && event.TimeMetrics["Query_time"] > a.outlierTime {
+		outlier = true
+	}
+
+	a.global.AddEvent(event, outlier)
+
+	class, ok := a.classes[id]
+	if !ok {
+		class = NewClass(id, fingerprint, a.samples)
+		a.classes[id] = class
+	}
+	class.AddEvent(event, outlier)
+}
+
+// Finalize calculates all metric statistics and returns a Result.
+// Call this function when done adding events to the aggregator.
+func (a *Aggregator) Finalize() Result {
+	a.global.Finalize(a.rateLimit)
+	a.global.UniqueQueries = uint(len(a.classes))
+	for _, class := range a.classes {
+		class.Finalize(a.rateLimit)
+		class.UniqueQueries = 1
+		if class.Example != nil && class.Example.Ts != "" {
+			if t, err := time.Parse("060102 15:04:05", class.Example.Ts); err != nil {
+				class.Example.Ts = ""
+			} else {
+				class.Example.Ts = t.Add(a.utcOffset).Format("2006-01-02 15:04:05")
+			}
+		}
+	}
+	return Result{
+		Global:    a.global,
+		Class:     a.classes,
+		RateLimit: a.rateLimit,
+	}
+}

README.md

@@ -0,0 +1,153 @@
+/*
+	Copyright 2017 Daniel Nichter
+	Copyright 2014-2016 Percona LLC and/or its affiliates
+*/
+
+package slowlog_test
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/daniel-nichter/deep-equal"
+	"github.com/go-mysql/query"
+	"github.com/go-mysql/slowlog"
+)
+
+var examples = true
+
+func aggregateSlowLog(t *testing.T, input, output string, utcOffset time.Duration) (got slowlog.Result, expect slowlog.Result) {
+	bytes, err := ioutil.ReadFile(path.Join("test", "results", output))
+	if err != nil {
+		t.Fatal(err)
+	}
+	expect = slowlog.Result{}
+	if err := json.Unmarshal(bytes, &expect); err != nil {
+		t.Fatal(err)
+	}
+
+	file, err := os.Open(path.Join("test", "slow-logs", input))
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := slowlog.NewFileParser(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := p.Start(slowlog.Options{}); err != nil {
+		t.Fatal(err)
+	}
+	defer p.Stop()
+
+	a := slowlog.NewAggregator(examples, utcOffset, 10)
+	for e := range p.Events() {
+		f := query.Fingerprint(e.Query)
+		id := query.Id(f)
+		a.AddEvent(e, id, f)
+	}
+	got = a.Finalize()
+	return got, expect
+}
+
+func zeroPercentiles(r *slowlog.Result) {
+	for _, metrics := range r.Global.Metrics.TimeMetrics {
+		metrics.Med = 0
+		metrics.P95 = 0
+	}
+	for _, metrics := range r.Global.Metrics.NumberMetrics {
+		metrics.Med = 0
+		metrics.P95 = 0
+	}
+	for _, class := range r.Class {
+		for _, metrics := range class.Metrics.TimeMetrics {
+			metrics.Med = 0
+			metrics.P95 = 0
+		}
+		for _, metrics := range class.Metrics.NumberMetrics {
+			metrics.Med = 0
+			metrics.P95 = 0
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+
+func TestSlow001(t *testing.T) {
+	got, expect := aggregateSlowLog(t, "slow001.log", "slow001.json", 0)
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+}
+
+func TestSlow001WithTzOffset(t *testing.T) {
+	got, expect := aggregateSlowLog(t, "slow001.log", "slow001.json", -1*time.Hour)
+	// Use the same files as TestSlow001NoExamples but with a tz=-1
+	expect.Class["7F7D57ACDD8A346E"].Example.Ts = "2007-10-15 20:43:52"
+	expect.Class["3A99CC42AEDCCFCD"].Example.Ts = "2007-10-15 20:45:10"
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+}
+
+func TestSlow001NoExamples(t *testing.T) {
+	examples = false
+	defer func() { examples = true }()
+	got, expect := aggregateSlowLog(t, "slow001.log", "slow001-no-examples.json", 0)
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+}
+
+// Test p95 and median.
+func TestSlow010(t *testing.T) {
+	got, expect := aggregateSlowLog(t, "slow010.log", "slow010.json", 0)
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+}
+
+func TestSlow018(t *testing.T) {
+	got, expect := aggregateSlowLog(t, "slow018.log", "slow018.json", 0)
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+}
+
+// Tests for PCT-1006 & PCT-1085
+func TestUseDb(t *testing.T) {
+	// Test db is not inherited
+	got, expect := aggregateSlowLog(t, "slow020.log", "slow020.json", 0)
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+	// Test "use" is not case sensitive
+	got, expect = aggregateSlowLog(t, "slow021.log", "slow021.json", 0)
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+	// Test we are parsing db names in backticks
+	got, expect = aggregateSlowLog(t, "slow022.log", "slow022.json", 0)
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+}
+
+func TestOutlierSlow025(t *testing.T) {
+	got, expect := aggregateSlowLog(t, "slow025.log", "slow025.json", 0)
+	if diff, _ := deep.Equal(got, expect); diff != nil {
+		dump(got)
+		t.Error(diff)
+	}
+}

aggregator.go

@@ -0,0 +1,97 @@
+/*
+	Copyright 2017 Daniel Nichter
+	Copyright 2014-2016 Percona LLC and/or its affiliates
+*/
+
+package slowlog
+
+const (
+	MAX_EXAMPLE_BYTES = 1024 * 10
+)
+
+// A Class represents all events with the same fingerprint and class ID.
+// This is only enforced by convention, so be careful not to mix events from
+// different classes.
+type Class struct {
+	Id            string   // 32-character hex checksum of fingerprint
+	Fingerprint   string   // canonical form of query: values replaced with "?"
+	Metrics       Metrics  // statistics for each metric, e.g. max Query_time
+	TotalQueries  uint     // total number of queries in class
+	UniqueQueries uint     // unique number of queries in class
+	Example       *Example `json:",omitempty"` // sample query with max Query_time
+	// --
+	outliers uint
+	lastDb   string
+	sample   bool
+}
+
+// A Example is a real query and its database, timestamp, and Query_time.
+// If the query is larger than MAX_EXAMPLE_BYTES, it is truncated and "..."
+// is appended.
+type Example struct {
+	QueryTime float64 // Query_time
+	Db        string  // Schema: <db> or USE <db>
+	Query     string  // truncated to MAX_EXAMPLE_BYTES
+	Ts        string  `json:",omitempty"` // in MySQL time zone
+}
+
+// NewClass returns a new Class for the class ID and fingerprint.
+// If sample is true, the query with the greatest Query_time is saved.
+func NewClass(id, fingerprint string, sample bool) *Class {
+	return &Class{
+		Id:           id,
+		Fingerprint:  fingerprint,
+		Metrics:      NewMetrics(),
+		TotalQueries: 0,
+		Example:      &Example{},
+		sample:       sample,
+	}
+}
+
+// AddEvent adds an event to the query class.
+func (c *Class) AddEvent(e Event, outlier bool) {
+	if outlier {
+		c.outliers++
+	} else {
+		c.TotalQueries++
+	}
+
+	c.Metrics.AddEvent(e, outlier)
+
+	// Save last db seen for this query. This helps ensure the sample query
+	// has a db.
+	if e.Db != "" {
+		c.lastDb = e.Db
+	}
+	if c.sample {
+		if n, ok := e.TimeMetrics["Query_time"]; ok {
+			if float64(n) > c.Example.QueryTime {
+				c.Example.QueryTime = float64(n)
+				if e.Db != "" {
+					c.Example.Db = e.Db
+				} else {
+					c.Example.Db = c.lastDb
+				}
+				if len(e.Query) > MAX_EXAMPLE_BYTES {
+					c.Example.Query = e.Query[0:MAX_EXAMPLE_BYTES-3] + "..."
+				} else {
+					c.Example.Query = e.Query
+				}
+				c.Example.Ts = e.Ts
+			}
+		}
+	}
+}
+
+// Finalize calculates all metric statistics. Call this function when done
+// adding events to the class.
+func (c *Class) Finalize(rateLimit uint) {
+	if rateLimit == 0 {
+		rateLimit = 1
+	}
+	c.Metrics.Finalize(rateLimit)
+	c.TotalQueries = (c.TotalQueries * rateLimit) + c.outliers
+	if c.Example.QueryTime == 0 {
+		c.Example = nil
+	}
+}