Skip to content

Commit f6bff8e

Browse files
author
Yongli Chen
authored
change telemetry to message queue and add npm (#366)
* change telemetry to message queue and add npm * remove [Azure-NPM] prefix * remove npmreport url * fair scheduling * holds up to 1k reports for each type * fix cap on reports
1 parent d986c92 commit f6bff8e

23 files changed

+546
-440
lines changed

‎Makefile‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ CNM_DIR = cnm/plugin
6262
CNI_NET_DIR = cni/network/plugin
6363
CNI_IPAM_DIR = cni/ipam/plugin
6464
CNI_TELEMETRY_DIR = cni/telemetry/service
65+
TELEMETRY_CONF_DIR = telemetry
6566
CNS_DIR = cns/service
6667
NPM_DIR = npm/plugin
6768
OUTPUT_DIR = output
@@ -71,6 +72,7 @@ CNI_BUILD_DIR = $(BUILD_DIR)/cni
7172
CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy
7273
CNS_BUILD_DIR = $(BUILD_DIR)/cns
7374
NPM_BUILD_DIR = $(BUILD_DIR)/npm
75+
NPM_TELEMETRY_DIR = $(NPM_BUILD_DIR)/telemetry
7476

7577
# Containerized build parameters.
7678
BUILD_CONTAINER_IMAGE = acn-build
@@ -97,6 +99,7 @@ CNI_MULTITENANCY_ARCHIVE_NAME = azure-vnet-cni-multitenancy-$(GOOS)-$(GOARCH)-$(
9799
CNS_ARCHIVE_NAME = azure-cns-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
98100
NPM_ARCHIVE_NAME = azure-npm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
99101
NPM_IMAGE_ARCHIVE_NAME = azure-npm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
102+
TELEMETRY_IMAGE_ARCHIVE_NAME = azure-vnet-telemetry-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
100103

101104
# Docker libnetwork (CNM) plugin v2 image parameters.
102105
CNM_PLUGIN_IMAGE ?= microsoft/azure-vnet-plugin
@@ -105,8 +108,10 @@ CNM_PLUGIN_ROOTFS = azure-vnet-plugin-rootfs
105108
# Azure network policy manager parameters.
106109
AZURE_NPM_IMAGE = containernetworking/azure-npm
107110

111+
# Azure vnet telemetry image parameters.
112+
AZURE_VNET_TELEMETRY_IMAGE = containernetworking/azure-vnet-telemetry
113+
108114
VERSION ?= $(shell git describe --tags --always --dirty)
109-
AZURE_NPM_VERSION = $(VERSION)
110115

111116
ENSURE_OUTPUT_DIR_EXISTS := $(shell mkdir -p $(OUTPUT_DIR))
112117

@@ -130,7 +135,7 @@ all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns
130135
endif
131136

132137
ifeq ($(GOOS),linux)
133-
all-images: azure-npm-image
138+
all-images: azure-npm-image azure-vnet-telemetry-image
134139
else
135140
all-images:
136141
@echo "Nothing to build. Skip."
@@ -153,7 +158,7 @@ $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT): $(CNIFILES)
153158
$(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT): $(CNIFILES)
154159
go build -v -o $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_IPAM_DIR)/*.go
155160

156-
# Build the Azure CNI IPAM plugin.
161+
# Build the Azure CNI telemetry plugin.
157162
$(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES)
158163
go build -v -o $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_TELEMETRY_DIR)/*.go
159164

@@ -163,6 +168,7 @@ $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES)
163168

164169
# Build the Azure NPM plugin.
165170
$(NPM_BUILD_DIR)/azure-npm$(EXE_EXT): $(NPMFILES)
171+
go build -v -o $(NPM_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_TELEMETRY_DIR)/*.go
166172
go build -v -o $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(NPM_DIR)/*.go
167173

168174
# Build all binaries in a container.
@@ -229,16 +235,32 @@ azure-npm-image: azure-npm
229235
ifeq ($(GOOS),linux)
230236
docker build \
231237
-f npm/Dockerfile \
232-
-t $(AZURE_NPM_IMAGE):$(AZURE_NPM_VERSION) \
238+
-t $(AZURE_NPM_IMAGE):$(VERSION) \
233239
--build-arg NPM_BUILD_DIR=$(NPM_BUILD_DIR) \
234240
.
235-
docker save $(AZURE_NPM_IMAGE):$(AZURE_NPM_VERSION) | gzip -c > $(NPM_BUILD_DIR)/$(NPM_ARCHIVE_NAME)
241+
docker save $(AZURE_NPM_IMAGE):$(VERSION) | gzip -c > $(NPM_BUILD_DIR)/$(NPM_IMAGE_ARCHIVE_NAME)
236242
endif
237243

238244
# Publish the Azure NPM image to a Docker registry
239245
.PHONY: publish-azure-npm-image
240246
publish-azure-npm-image:
241-
docker push $(AZURE_NPM_IMAGE):$(AZURE_NPM_VERSION)
247+
docker push $(AZURE_NPM_IMAGE):$(VERSION)
248+
249+
# Build the Azure vnet telemetry image
250+
.PHONY: azure-vnet-telemetry-image
251+
azure-vnet-telemetry-image: azure-vnet-telemetry
252+
docker build \
253+
-f cni/telemetry/Dockerfile \
254+
-t $(AZURE_VNET_TELEMETRY_IMAGE):$(VERSION) \
255+
--build-arg TELEMETRY_BUILD_DIR=$(NPM_BUILD_DIR) \
256+
--build-arg TELEMETRY_CONF_DIR=$(TELEMETRY_CONF_DIR) \
257+
.
258+
docker save $(AZURE_VNET_TELEMETRY_IMAGE):$(VERSION) | gzip -c > $(NPM_BUILD_DIR)/$(TELEMETRY_IMAGE_ARCHIVE_NAME)
259+
260+
# Publish the Azure vnet telemetry image to a Docker registry
261+
.PHONY: publish-azure-vnet-telemetry-image
262+
publish-azure-vnet-telemetry-image:
263+
docker push $(AZURE_VNET_TELEMETRY_IMAGE):$(VERSION)
242264

243265
# Create a CNI archive for the target platform.
244266
.PHONY: cni-archive
@@ -274,7 +296,8 @@ cns-archive:
274296
.PHONY: npm-archive
275297
npm-archive:
276298
ifeq ($(GOOS),linux)
277-
chmod 0755 $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT)
278-
cd $(NPM_BUILD_DIR) && $(ARCHIVE_CMD) $(NPM_ARCHIVE_NAME) azure-npm$(EXE_EXT)
299+
chmod 0755 $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT) $(NPM_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT)
300+
cp telemetry/azure-vnet-telemetry.config $(NPM_BUILD_DIR)/azure-vnet-telemetry.config
301+
cd $(NPM_BUILD_DIR) && $(ARCHIVE_CMD) $(NPM_ARCHIVE_NAME) azure-npm$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) azure-vnet-telemetry.config
279302
chown $(BUILD_USER):$(BUILD_USER) $(NPM_BUILD_DIR)/$(NPM_ARCHIVE_NAME)
280303
endif

‎cni/telemetry/Dockerfile‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Use a minimal image as a parent image
2+
FROM ubuntu:18.04
3+
ARG TELEMETRY_BUILD_DIR
4+
ARG TELEMETRY_CONF_DIR
5+
6+
# Install plugin
7+
COPY $TELEMETRY_BUILD_DIR/azure-vnet-telemetry /usr/bin
8+
COPY $TELEMETRY_CONF_DIR/azure-vnet-telemetry.config /usr/bin
9+
10+
WORKDIR /usr/bin
11+
12+
# Run below command by default when the container starts.
13+
ENTRYPOINT ["/usr/bin/azure-vnet-telemetry", "-d", "/usr/bin"]

‎cni/telemetry/service/telemetrymain.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -106,33 +106,33 @@ func main() {
106106
return
107107
}
108108

109-
log.Printf("args %+v", os.Args)
109+
log.Logf("args %+v", os.Args)
110110

111111
if runtime.GOOS == "linux" {
112112
configPath = fmt.Sprintf("%s/%s%s", configDirectory, azureVnetTelemetry, configExtension)
113113
} else {
114114
configPath = fmt.Sprintf("%s\\%s%s", configDirectory, azureVnetTelemetry, configExtension)
115115
}
116116

117-
log.Printf("[Telemetry] Config path: %s", configPath)
117+
log.Logf("[Telemetry] Config path: %s", configPath)
118118

119119
config, err = telemetry.ReadConfigFile(configPath)
120120
if err != nil {
121-
log.Printf("[Telemetry] Error reading telemetry config: %v", err)
121+
log.Logf("[Telemetry] Error reading telemetry config: %v", err)
122122
}
123123

124-
log.Printf("read config returned %+v", config)
124+
log.Logf("read config returned %+v", config)
125125

126126
for {
127127
tb = telemetry.NewTelemetryBuffer("")
128128

129-
log.Printf("[Telemetry] Starting telemetry server")
129+
log.Logf("[Telemetry] Starting telemetry server")
130130
err = tb.StartServer()
131131
if err == nil || tb.FdExists {
132132
break
133133
}
134134

135-
log.Printf("[Telemetry] Telemetry service starting failed: %v", err)
135+
log.Logf("[Telemetry] Telemetry service starting failed: %v", err)
136136
tb.Cleanup(telemetry.FdName)
137137
time.Sleep(time.Millisecond * 200)
138138
}
@@ -141,7 +141,7 @@ func main() {
141141
config.ReportToHostIntervalInSeconds = reportToHostIntervalInSeconds
142142
}
143143

144-
log.Printf("[Telemetry] Report to host for an interval of %d seconds", config.ReportToHostIntervalInSeconds)
144+
log.Logf("[Telemetry] Report to host for an interval of %d seconds", config.ReportToHostIntervalInSeconds)
145145
tb.BufferAndPushData(config.ReportToHostIntervalInSeconds * time.Second)
146146
log.Close()
147147
}

‎log/logger.go‎

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ func (logger *Logger) rotate() {
138138
fileName := logger.getLogFileName()
139139
fileInfo, err := os.Stat(fileName)
140140
if err != nil {
141-
logger.Printf("[log] Failed to query log file info %+v.", err)
141+
logger.Logf("[log] Failed to query log file info %+v.", err)
142142
return
143143
}
144144

@@ -185,7 +185,7 @@ func (logger *Logger) Response(tag string, response interface{}, returnCode int,
185185
}
186186
}
187187

188-
// Logf logs a formatted string.
188+
// logf logs a formatted string.
189189
func (logger *Logger) logf(format string, args ...interface{}) {
190190
if logger.callCount%rotationCheckFrq == 0 {
191191
logger.rotate()
@@ -195,27 +195,48 @@ func (logger *Logger) logf(format string, args ...interface{}) {
195195
logger.l.Printf(format, args...)
196196
}
197197

198+
// Logf wraps logf.
199+
func (logger *Logger) Logf(format string, args ...interface{}) {
200+
logger.mutex.Lock()
201+
logger.logf(format, args...)
202+
logger.mutex.Unlock()
203+
}
204+
198205
// Printf logs a formatted string at info level.
199206
func (logger *Logger) Printf(format string, args ...interface{}) {
200-
if logger.level >= LevelInfo {
201-
logger.mutex.Lock()
202-
logger.logf(format, args...)
203-
logger.mutex.Unlock()
207+
if logger.level < LevelInfo {
208+
return
204209
}
210+
211+
logger.mutex.Lock()
212+
logger.logf(format, args...)
213+
logger.mutex.Unlock()
214+
go func() {
215+
if logger.reports != nil {
216+
logger.reports <- fmt.Sprintf(format, args...)
217+
}
218+
}()
205219
}
206220

207-
// Debugf logs a formatted string at debug level.
221+
// Debugf logs a formatted string at info level.
208222
func (logger *Logger) Debugf(format string, args ...interface{}) {
209-
if logger.level >= LevelDebug {
210-
logger.mutex.Lock()
211-
logger.logf(format, args...)
212-
logger.mutex.Unlock()
223+
if logger.level < LevelDebug {
224+
return
213225
}
226+
227+
logger.mutex.Lock()
228+
logger.logf(format, args...)
229+
logger.mutex.Unlock()
230+
go func() {
231+
if logger.reports != nil {
232+
logger.reports <- fmt.Sprintf(format, args...)
233+
}
234+
}()
214235
}
215236

216237
// Errorf logs a formatted string at info level and sends the string to TelemetryBuffer.
217238
func (logger *Logger) Errorf(format string, args ...interface{}) {
218-
logger.Printf(format, args...)
239+
logger.Logf(format, args...)
219240
go func() {
220241
if logger.reports != nil {
221242
logger.reports <- fmt.Sprintf(format, args...)

‎log/logger_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func TestLogFileRotatesWhenSizeLimitIsReached(t *testing.T) {
2222
l.SetLogFileLimits(512, 2)
2323

2424
for i := 1; i <= 100; i++ {
25-
l.Printf("LogText %v", i)
25+
l.Logf("LogText %v", i)
2626
}
2727

2828
l.Close()

‎log/stdapi.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ func Response(tag string, response interface{}, returnCode int, returnStr string
4747
stdLog.Response(tag, response, returnCode, returnStr, err)
4848
}
4949

50+
// Logf logs to the local log.
51+
func Logf(format string, args ...interface{}) {
52+
stdLog.Logf(format, args...)
53+
}
54+
55+
// Printf logs to the local log and send the log through the channel.
5056
func Printf(format string, args ...interface{}) {
5157
stdLog.Printf(format, args...)
5258
}

‎npm/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Use a minimal image as a parent image
2-
FROM ubuntu:16.04
2+
FROM ubuntu:18.04
33
ARG NPM_BUILD_DIR
44

55
# Install dependencies.
@@ -9,6 +9,7 @@ RUN apt-get install -y ipset
99

1010
# Install plugin.
1111
COPY $NPM_BUILD_DIR/azure-npm /usr/bin
12+
1213
WORKDIR /usr/bin
1314

1415
# Run the npm command by default when the container starts.

‎npm/azure-npm.yaml‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ spec:
7676
beta.kubernetes.io/os: linux
7777
containers:
7878
- name: azure-npm
79-
image: containernetworking/azure-npm:v1.0.18
79+
image: containernetworking/azure-npm:v1.0.23
8080
securityContext:
8181
privileged: true
8282
env:
@@ -90,6 +90,17 @@ spec:
9090
mountPath: /run/xtables.lock
9191
- name: log
9292
mountPath: /var/log
93+
- name: socket-dir
94+
mountPath: /var/run
95+
- name: tmp
96+
mountPath: /tmp
97+
- name: azure-vnet-telemetry
98+
image: containernetworking/azure-vnet-telemetry:v1.0.23
99+
volumeMounts:
100+
- name: socket-dir
101+
mountPath: /var/run
102+
- name: tmp
103+
mountPath: /tmp
93104
hostNetwork: true
94105
volumes:
95106
- name: log
@@ -100,4 +111,10 @@ spec:
100111
hostPath:
101112
path: /run/xtables.lock
102113
type: File
114+
- name: tmp
115+
hostPath:
116+
path: /tmp
117+
type: Directory
118+
- name: socket-dir
119+
emptyDir: {}
103120
serviceAccountName: azure-npm

0 commit comments

Comments
 (0)