Skip to content

Commit 405e1fc

Browse files
wonderflowwonderflow
authored
Merge pull request #780 from redHJ/schemaConvert
schema非法字符转换
2 parents 6efd78d + 7f5d5cf commit 405e1fc

File tree

7 files changed

+123
-109
lines changed

7 files changed

+123
-109
lines changed

‎reader/cloudwatch/cloudwatch.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717

1818
"github.com/qiniu/log"
1919
"github.com/qiniu/pandora-go-sdk/base/ratelimit"
20+
"github.com/qiniu/pandora-go-sdk/pipeline"
2021

2122
"github.com/qiniu/logkit/conf"
2223
"github.com/qiniu/logkit/reader"
@@ -488,7 +489,7 @@ func formatKey(metricName string, statistic string) string {
488489
}
489490

490491
func snakeCase(s string) string {
491-
s, _ = PandoraKey(s)
492+
s, _ = pipeline.PandoraKey(s)
492493
s = strings.Replace(s, "__", "_", -1)
493494
return s
494495
}

‎sender/pandora/pandora.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -613,6 +613,8 @@ func parseUserSchema(repoName, schema string) (us UserSchema) {
613613
if name == "" && alias == "" {
614614
continue
615615
}
616+
name, _ = pipeline.PandoraKey(name)
617+
alias, _ = pipeline.PandoraKey(alias)
616618
us.Fields[name] = alias
617619
}
618620
return

‎sender/pandora/pandora_test.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,32 @@ func TestPandoraSender(t *testing.T) {
174174
exp = "a1=1.1 ab=a ac=0 ax=b d=" + timeexp
175175
assert.Equal(t, exp, pandora.Body)
176176

177+
s.UserSchema = parseUserSchema("TestPandoraSenderRepo", "a-b b-c, a-b-c a1,a-b-c-d,d,e f-d,...")
178+
assert.Equal(t, UserSchema{
179+
DefaultAll: true,
180+
Fields: map[string]string{
181+
"a_b": "b_c",
182+
"a_b_c": "a1",
183+
"a_b_c_d": "a_b_c_d",
184+
"d": "d",
185+
"e": "f_d",
186+
},
187+
}, s.UserSchema)
188+
time.Sleep(2 * time.Second)
189+
d["a_b"] = "a"
190+
d["a_b_c"] = 1.1
191+
d["ac"] = 0
192+
d["d"] = 1477373632504888
193+
d["ax"] = "b"
194+
s.opt.updateInterval = 0
195+
err = s.Send([]Data{d})
196+
if st, ok := err.(*StatsError); ok {
197+
err = st.ErrorDetail
198+
}
199+
if err != nil {
200+
t.Error(err)
201+
}
202+
177203
}
178204

179205
func TestNestPandoraSender(t *testing.T) {

‎utils/models/utils.go‎

Lines changed: 4 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727

2828
"github.com/json-iterator/go"
2929

30+
"github.com/qiniu/pandora-go-sdk/pipeline"
31+
3032
"github.com/qiniu/log"
3133
"github.com/qiniu/logkit/times"
3234
)
@@ -805,72 +807,6 @@ func CheckPandoraKey(key string) bool {
805807
return true
806808
}
807809

808-
// 判断时只有数字和字母为合法字符，规则：
809-
// 1. 首字符为数字时，增加首字符 "K"
810-
// 2. 首字符为非法字符时，去掉首字符（例如，如果字符串全为非法字符，则转换后为空）
811-
// 3. 非首字符并且为非法字符时，使用 "_" 替代非法字符
812-
// 返回key，以及原始key是否合法，如果不合法，则后续需要用转换后的key进行替换
813-
func PandoraKey(key string) (string, bool) {
814-
// check
815-
valid := true
816-
size := 0
817-
if key == "" {
818-
return "KEmptyPandoraAutoAdd", false
819-
}
820-
821-
for idx, c := range key {
822-
if c >= '0' && c <= '9' {
823-
size++
824-
if idx == 0 {
825-
size++
826-
valid = false
827-
}
828-
continue
829-
}
830-
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') {
831-
size++
832-
continue
833-
}
834-
835-
if idx > 0 && size > 0 {
836-
size++
837-
}
838-
valid = false
839-
}
840-
if valid {
841-
return key, true
842-
}
843-
844-
if size <= 0 {
845-
return "", false
846-
}
847-
// set
848-
bytes := make([]byte, size)
849-
bp := 0
850-
for idx, c := range key {
851-
if c >= '0' && c <= '9' {
852-
if idx == 0 {
853-
bytes[bp] = 'K'
854-
bp++
855-
}
856-
bytes[bp] = byte(c)
857-
bp++
858-
continue
859-
}
860-
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') {
861-
bytes[bp] = byte(c)
862-
bp++
863-
continue
864-
}
865-
866-
if bp > 0 {
867-
bytes[bp] = '_'
868-
bp++
869-
}
870-
}
871-
return string(bytes), valid
872-
}
873-
874810
func DeepConvertKey(data map[string]interface{}) map[string]interface{} {
875811
for k, v := range data {
876812
switch nv := v.(type) {
@@ -882,7 +818,7 @@ func DeepConvertKey(data map[string]interface{}) map[string]interface{} {
882818
valid := CheckPandoraKey(k)
883819
if !valid {
884820
delete(data, k)
885-
k, _ := PandoraKey(k)
821+
k, _ := pipeline.PandoraKey(k)
886822
data[k] = v
887823
}
888824
}
@@ -899,7 +835,7 @@ func DeepConvertKeyWithCache(data map[string]interface{}, cache map[string]KeyIn
899835
}
900836
keyInfo, exist := cache[k]
901837
if !exist {
902-
keyInfo.NewKey, keyInfo.Valid = PandoraKey(k)
838+
keyInfo.NewKey, keyInfo.Valid = pipeline.PandoraKey(k)
903839
if cache == nil {
904840
cache = make(map[string]KeyInfo)
905841
}

‎utils/models/utils_test.go‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -723,17 +723,6 @@ func TestPickMapValue(t *testing.T) {
723723
assert.NotEqual(t, exp, pick)
724724
}
725725

726-
func TestPandoraKey(t *testing.T) {
727-
testKeys := []string{"", "@timestamp", ".dot", "percent%100", "^^^^^^^^^^", "timestamp"}
728-
expectKeys := []string{"KEmptyPandoraAutoAdd", "timestamp", "dot", "percent_100", "", "timestamp"}
729-
expectValid := []bool{false, false, false, false, false, true}
730-
for idx, key := range testKeys {
731-
actual, valid := PandoraKey(key)
732-
assert.Equal(t, expectKeys[idx], actual)
733-
assert.Equal(t, expectValid[idx], valid)
734-
}
735-
}
736-
737726
func TestCheckPandoraKey(t *testing.T) {
738727
testKeys := []string{"@timestamp", ".dot", "percent%100", "^^^^^^^^^^", "timestamp"}
739728
expectValid := []bool{false, false, false, false, true}
@@ -743,16 +732,6 @@ func TestCheckPandoraKey(t *testing.T) {
743732
}
744733
}
745734

746-
func BenchmarkPandoraKey(b *testing.B) {
747-
b.ReportAllocs()
748-
testKeys := []string{"@timestamp", ".dot", "percent%100", "^^^^^^^^^^", "timestamp", "aaa"}
749-
for i := 0; i < b.N; i++ {
750-
for _, key := range testKeys {
751-
PandoraKey(key)
752-
}
753-
}
754-
}
755-
756735
func BenchmarkCheckPandoraKey(b *testing.B) {
757736
b.ReportAllocs()
758737
testKeys := []string{"@timestamp", ".dot", "percent%100", "^^^^^^^^^^", "timestamp", "aaa"}

‎vendor/github.com/qiniu/pandora-go-sdk/pipeline/models.go‎

Lines changed: 70 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/vendor.json‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -501,56 +501,56 @@
501501
{
502502
"checksumSHA1": "0JEyusBC8nfE7dF2sKLylHTh93Y=",
503503
"path": "github.com/qiniu/pandora-go-sdk/base",
504-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
505-
"revisionTime": "2018-09-18T13:05:34Z"
504+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
505+
"revisionTime": "2018-09-27T09:58:43Z"
506506
},
507507
{
508508
"checksumSHA1": "bxhdCtUVijjJI4tzfMBipi5G4ko=",
509509
"path": "github.com/qiniu/pandora-go-sdk/base/config",
510-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
511-
"revisionTime": "2018-09-18T13:05:34Z"
510+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
511+
"revisionTime": "2018-09-27T09:58:43Z"
512512
},
513513
{
514514
"checksumSHA1": "k5JfcSQ5YsAFd6DSHpSeuNWbql8=",
515515
"path": "github.com/qiniu/pandora-go-sdk/base/models",
516-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
517-
"revisionTime": "2018-09-18T13:05:34Z"
516+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
517+
"revisionTime": "2018-09-27T09:58:43Z"
518518
},
519519
{
520520
"checksumSHA1": "lV2zb3SZ4BKaa1XpC4Q3ktbVgDo=",
521521
"path": "github.com/qiniu/pandora-go-sdk/base/ratelimit",
522-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
523-
"revisionTime": "2018-09-18T13:05:34Z"
522+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
523+
"revisionTime": "2018-09-27T09:58:43Z"
524524
},
525525
{
526526
"checksumSHA1": "sLeqqUJX9pa++wH3PtRSVbaCejs=",
527527
"path": "github.com/qiniu/pandora-go-sdk/base/reqerr",
528-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
529-
"revisionTime": "2018-09-18T13:05:34Z"
528+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
529+
"revisionTime": "2018-09-27T09:58:43Z"
530530
},
531531
{
532532
"checksumSHA1": "oEpRonb6KY/u9OWNOxSEjyyqqXk=",
533533
"path": "github.com/qiniu/pandora-go-sdk/base/request",
534-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
535-
"revisionTime": "2018-09-18T13:05:34Z"
534+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
535+
"revisionTime": "2018-09-27T09:58:43Z"
536536
},
537537
{
538538
"checksumSHA1": "BztKFnZ0NtYb14GWfUzWTWKOyAo=",
539539
"path": "github.com/qiniu/pandora-go-sdk/logdb",
540-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
541-
"revisionTime": "2018-09-18T13:05:34Z"
540+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
541+
"revisionTime": "2018-09-27T09:58:43Z"
542542
},
543543
{
544-
"checksumSHA1": "x9HBb8ckub8Qj7l1wBwllOfXIgY=",
544+
"checksumSHA1": "1XjsrAYKVCjg6TjDbfFSdoy2Ltg=",
545545
"path": "github.com/qiniu/pandora-go-sdk/pipeline",
546-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
547-
"revisionTime": "2018-09-18T13:05:34Z"
546+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
547+
"revisionTime": "2018-09-27T09:58:43Z"
548548
},
549549
{
550550
"checksumSHA1": "YeSUJIE3zLnpjK3g161wiMeIbmk=",
551551
"path": "github.com/qiniu/pandora-go-sdk/tsdb",
552-
"revision": "2b1d44771e6d07de7920288fa81d284cf21b4ac5",
553-
"revisionTime": "2018-09-18T13:05:34Z"
552+
"revision": "153674b0a84593cc63254b17d46818b64003daad",
553+
"revisionTime": "2018-09-27T09:58:43Z"
554554
},
555555
{
556556
"checksumSHA1": "KAzbLjI9MzW2tjfcAsK75lVRp6I=",

0 commit comments

Comments
 (0)