pkg/ansible: Add log aggregator sidecar for Ansible Operator (#1193)

* Add log aggregator sidecar for Ansible Operator

* add missing dependency

* rename operator container to operator, update CI

* accidentally mounted operator fs readOnly

* add scaffold to migrate command

* add docs on new logging container

Author: fabianvf

commands/operator-sdk/cmd/migrate.go

@@ -91,6 +91,7 @@ func migrateAnsible() error {
 		&ansible.Entrypoint{},
 		&ansible.UserSetup{},
 		&ansible.K8sStatus{},
+		&ansible.AoLogs{},
 	)
 	if err != nil {
 		return fmt.Errorf("migrate ansible scaffold failed: (%v)", err)

doc/ansible/dev/developer_guide.md

@@ -321,6 +321,21 @@ NAME                     DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
 foo-operator       1         1         1            1           1m
 ```
 
+#### Viewing the Ansible logs
+
+The `foo-operator` deployment creates a Pod with two containers, `operator` and `ansible`.
+The `ansible` container exists only to expose the standard Ansible stdout logs that most Ansible
+users will be familiar with. In order to see the logs from a particular container, you can run
+
+```sh
+kubectl logs deployment/foo-operator -c ansible
+kubectl logs deployment/foo-operator -c operator
+```
+
+The `ansible` logs contain all of the information about the Ansible run and will make it much easier to debug issues within your Ansible tasks, 
+whereas the `operator` logs will contain much more detailed information about the Ansible Operator's internals and interface with Kubernetes.
+
+
 ## Custom Resource Status Management
 The operator will automatically update the CR's `status` subresource with
 generic information about the previous Ansible run. This includes the number of

doc/ansible/user-guide.md

@@ -365,9 +365,23 @@ NAME                                  READY     STATUS    RESTARTS   AGE
 example-memcached-6fd7c98d8-7dqdr     1/1       Running   0          1m
 example-memcached-6fd7c98d8-g5k7v     1/1       Running   0          1m
 example-memcached-6fd7c98d8-m7vn7     1/1       Running   0          1m
-memcached-operator-7cc7cfdf86-vvjqk   1/1       Running   0          2m
+memcached-operator-7cc7cfdf86-vvjqk   2/2       Running   0          2m
 ```
 
+### View the Ansible logs
+
+The `memcached-operator` deployment creates a Pod with two containers, `operator` and `ansible`.
+The `ansible` container exists only to expose the standard Ansible stdout logs that most Ansible
+users will be familiar with. In order to see the logs from a particular container, you can run
+
+```sh
+kubectl logs deployment/memcached-operator -c ansible
+kubectl logs deployment/memcached-operator -c operator
+```
+
+The `ansible` logs contain all of the information about the Ansible run and will make it much easier to debug issues within your Ansible tasks, 
+whereas the `operator` logs will contain much more detailed information about the Ansible Operator's internals and interface with Kubernetes.
+
 ### Update the size
 
 Change the `spec.size` field in the memcached CR from 3 to 4 and apply the

hack/image/ansible/scaffold-ansible-image.go

@@ -39,6 +39,7 @@ func main() {
 		&ansible.Entrypoint{},
 		&ansible.UserSetup{},
 		&ansible.K8sStatus{},
+		&ansible.AoLogs{},
 	)
 	if err != nil {
 		log.Fatalf("Add scaffold failed: (%v)", err)

hack/tests/e2e-ansible.sh

@@ -35,7 +35,8 @@ test_operator() {
     if ! timeout 1m kubectl rollout status deployment/memcached-operator;
     then
         echo FAIL: operator failed to run
-        kubectl logs deployment/memcached-operator
+        kubectl logs deployment/memcached-operator -c operator
+        kubectl logs deployment/memcached-operator -c ansible
         exit 1
     fi
 
@@ -44,7 +45,8 @@ test_operator() {
     if ! timeout 20s bash -c -- 'until kubectl get deployment -l app=memcached | grep memcached; do sleep 1; done';
     then
         echo FAIL: operator failed to create memcached Deployment
-        kubectl logs deployment/memcached-operator
+        kubectl logs deployment/memcached-operator -c operator
+        kubectl logs deployment/memcached-operator -c ansible
         exit 1
     fi
     memcached_deployment=$(kubectl get deployment -l app=memcached -o jsonpath="{..metadata.name}")
@@ -65,23 +67,26 @@ test_operator() {
     if kubectl get configmap deleteme 2> /dev/null;
     then
         echo FAIL: the finalizer did not delete the configmap
-        kubectl logs deployment/memcached-operator
+        kubectl logs deployment/memcached-operator -c operator
+        kubectl logs deployment/memcached-operator -c ansible
         exit 1
     fi
 
     # The deployment should get garbage collected, so we expect to fail getting the deployment.
     if ! timeout 20s bash -c -- "while kubectl get deployment ${memcached_deployment} 2> /dev/null; do sleep 1; done";
     then
         echo FAIL: memcached Deployment did not get garbage collected
-        kubectl logs deployment/memcached-operator
+        kubectl logs deployment/memcached-operator -c operator
+        kubectl logs deployment/memcached-operator -c ansible
         exit 1
     fi
 
     # Ensure that no errors appear in the log
-    if kubectl logs deployment/memcached-operator | grep -i error;
+    if kubectl logs deployment/memcached-operator -c operator | grep -i error;
     then
         echo FAIL: the operator log includes errors
-        kubectl logs deployment/memcached-operator
+        kubectl logs deployment/memcached-operator -c operator
+        kubectl logs deployment/memcached-operator -c ansible
         exit 1
     fi
 }

pkg/scaffold/ansible/ao_logs.go

@@ -0,0 +1,50 @@
+// Copyright 2019 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ansible
+
+import (
+	"path/filepath"
+
+	"github.com/operator-framework/operator-sdk/pkg/scaffold/input"
+)
+
+//DockerfileHybrid - Dockerfile for a hybrid operator
+type AoLogs struct {
+	input.Input
+}
+
+// GetInput - gets the input
+func (a *AoLogs) GetInput() (input.Input, error) {
+	if a.Path == "" {
+		a.Path = filepath.Join("bin", "ao-logs")
+	}
+	a.TemplateBody = aoLogsTmpl
+	a.IsExec = true
+	return a.Input, nil
+}
+
+const aoLogsTmpl = `#!/bin/bash
+
+watch_dir=${1:-/tmp/ansible-operator/runner}
+filename=${2:-stdout}
+mkdir -p ${watch_dir}
+inotifywait -r -m -e close_write ${watch_dir} | while read dir op file
+do
+  if [[ "${file}" = "${filename}" ]] ; then
+    echo "${dir}/${file}"
+    cat ${dir}/${file}
+  fi
+done
+`

pkg/scaffold/ansible/deploy_operator.go

@@ -54,10 +54,25 @@ spec:
     spec:
       serviceAccountName: {{.ProjectName}}
       containers:
-        - name: {{.ProjectName}}
+        - name: ansible
+          command:
+          - /usr/local/bin/ao-logs
+          - /tmp/ansible-operator/runner
+          - stdout
           # Replace this with the built image name
           image: "{{ "{{ REPLACE_IMAGE }}" }}"
           imagePullPolicy: "{{ "{{ pull_policy|default('Always') }}"}}"
+          volumeMounts:
+          - mountPath: /tmp/ansible-operator/runner
+            name: runner
+            readOnly: true
+        - name: operator
+          # Replace this with the built image name
+          image: "{{ "{{ REPLACE_IMAGE }}" }}"
+          imagePullPolicy: "{{ "{{ pull_policy|default('Always') }}"}}"
+          volumeMounts:
+          - mountPath: /tmp/ansible-operator/runner
+            name: runner
           env:
             - name: WATCH_NAMESPACE
               {{- if .IsClusterScoped }}
@@ -73,4 +88,7 @@ spec:
                   fieldPath: metadata.name
             - name: OPERATOR_NAME
               value: "{{.ProjectName}}"
+      volumes:
+        - name: runner
+          emptyDir: {}
 `

pkg/scaffold/ansible/dockerfilehybrid.go

@@ -47,6 +47,7 @@ func (d *DockerfileHybrid) GetInput() (input.Input, error) {
 const dockerFileHybridAnsibleTmpl = `FROM ansible/ansible-runner
 
 RUN yum remove -y ansible python-idna
+RUN yum install -y inotify-tools && yum clean all
 RUN pip uninstall ansible-runner -y
 
 RUN pip install --upgrade setuptools

test/ansible-memcached/asserts.yml

@@ -179,13 +179,23 @@
       - name: get operator logs
         ignore_errors: yes
         failed_when: false
-        command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ namespace }}
+        command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ namespace }} -c operator
         vars:
           definition: "{{ lookup('file', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}"
         register: log
 
       - debug: var=log.stdout_lines
 
+      - name: get ansible logs
+        ignore_errors: yes
+        failed_when: false
+        command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ namespace }} -c ansible
+        vars:
+          definition: "{{ lookup('file', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}"
+        register: ansible_log
+
+      - debug: var=ansible_log.stdout_lines
+
       - fail:
           msg: "Failed in asserts.yml"