Merge "Clean up test process" into stable/stein

Author: Zuul

.zuul.d/base.yaml

@@ -46,9 +46,6 @@
     run: tests/playbooks/run.yml
     post-run: tests/playbooks/post.yml
     attempts: 1
-    required-projects:
-      - openstack/kolla-ansible
-      - openstack/requirements
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*
@@ -59,3 +56,6 @@
       - ^test-requirements.txt$
     vars:
       publisher: false
+    extra-vars:
+      kolla_logs_dir: "{{ zuul_output_dir }}/logs/kolla"
+      kolla_build_logs_dir: "{{ kolla_logs_dir }}/build"

tests/playbooks/post.yml

@@ -1,91 +1,88 @@
 ---
 - hosts: all
   tasks:
-    - shell:
+    - name: Dump host info to logs
+      command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+      args:
+        chdir: "{{ kolla_logs_dir }}"
+
+    - name: Collect various info to logs
+      shell:
         cmd: |
           set +o errexit
-          mkdir -p logs
-          # copy system log
+          set -o xtrace
 
-          sudo cp -r /var/log logs/system_log
+          # copy system log
+          sudo cp -r /var/log system_log
 
           if which journalctl ; then
               # the journal gives us syslog() and kernel output, so is like
               # a concatenation of the above.
-              sudo journalctl --no-pager | sudo tee logs/syslog.txt > /dev/null
-              sudo journalctl --no-pager -u docker.service | sudo tee logs/docker.log > /dev/null
+              sudo journalctl --no-pager > syslog.txt
+              sudo journalctl --no-pager -u docker.service > docker.log
           else
               # assume rsyslog
-              sudo cp /var/log/syslog logs/syslog.txt
-              sudo cp /var/log/kern.log logs/kern_log.txt
-              sudo cp /var/log/upstart/docker.log logs/docker.log
+              sudo cp /var/log/syslog syslog.txt
+              sudo cp /var/log/kern.log kern_log.txt
+              sudo cp /var/log/upstart/docker.log docker.log
           fi
 
           if sudo test -d /var/lib/docker/volumes/kolla_logs/_data; then
-              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data logs/kolla_logs
+              sudo cp -r /var/lib/docker/volumes/kolla_logs/_data kolla_logs
           fi
 
           # sudo config
-          sudo cp -r /etc/sudoers.d logs/
-          sudo cp /etc/sudoers logs/sudoers.txt
+          sudo cp -r /etc/sudoers.d ./
+          sudo cp /etc/sudoers sudoers.txt
 
-          df -h > logs/df.txt
-          free  > logs/free.txt
-          sudo parted -l | sudo tee logs/parted-l.txt > /dev/null
-          mount > logs/mount.txt
-          env > logs/env.txt
+          df -h > df.txt
+          free  > free.txt
+          sudo parted -l > parted-l.txt
+          mount > mount.txt
+          env > env.txt
 
           if [ `command -v dpkg` ]; then
-              dpkg -l | sudo tee logs/dpkg-l.txt > /dev/null
+              dpkg -l > dpkg-l.txt
           fi
           if [ `command -v rpm` ]; then
-              rpm -qa | sudo tee logs/rpm-qa.txt > /dev/null
+              rpm -qa > rpm-qa.txt
           fi
 
           # final memory usage and process list
-          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > logs/ps.txt
+          ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ps.txt
 
           # docker related information
-          (docker info && docker system df && docker system df -v) > logs/docker-info.txt
+          (docker info && docker system df && docker system df -v) > docker-info.txt
 
-          sudo cp -r /etc/kolla logs/kolla_configs
+          sudo cp -r /etc/kolla kolla_configs
 
-          # fix the permissions for logs folder
-          sudo chmod -R 777 logs
+          # last sudo to own all files
+          sudo chown -R {{ ansible_user_id }}: .
 
           # Parse build logs and extract pkg install info
-          tests/files/process_build_logs.py -l logs/build -b {{ base_distro }} > logs/packages-info.txt
+          {{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tests/files/process_build_logs.py -l build -b {{ base_distro }} > packages-info.txt
 
           # rename files to .txt; this is so that when displayed via
           # logs.openstack.org clicking results in the browser shows the
           # files, rather than trying to send it to another app or make you
           # download it, etc.
 
           # firstly, rename all .log files to .txt files
-          for f in $(find logs -name "*.log"); do
-              sudo mv $f ${f/.log/.txt}
+          for f in $(find . -name "*.log"); do
+              mv $f ${f/.log/.txt}
           done
 
           # Update symlinks to new file names
-          for f in $(find logs -name "*FAILED*"); do
-              sudo mv ${f} ${f}.gz
-              sudo ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
+          for f in $(find . -name "*FAILED*"); do
+              mv ${f} ${f}.gz
+              ln -sf ${f#*/000_FAILED_}.gz ${f}.gz
           done
 
           # append .txt to all kolla config file
-          find logs/kolla_configs -type f -exec mv '{}' '{}'.txt \;
+          find kolla_configs -type f -exec mv '{}' '{}'.txt \;
 
           # Compress all text logs
-          find logs -iname '*.txt' -execdir gzip -9 {} \+
-          find logs -iname '*.json' -execdir gzip -9 {} \+
+          find . -iname '*.txt' -execdir gzip -9 {} \+
+          find . -iname '*.json' -execdir gzip -9 {} \+
         executable: /bin/bash
-        chdir: "{{ zuul.project.src_dir }}"
-
-    - synchronize:
-        src: '{{ zuul.project.src_dir }}/logs'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - "--quiet"
+        chdir: "{{ kolla_logs_dir }}"

tests/playbooks/pre.yml

@@ -1,13 +1,17 @@
 ---
 - hosts: all
-  become: true
   tasks:
-    - name: Install dbus for debian system
-      apt: name=dbus
-      when: ansible_os_family == 'Debian'
+    - name: Create dir for kolla logs
+      file:
+        path: "{{ kolla_logs_dir }}"
+        state: directory
 
-    - name: Install ansible
-      pip:
-        name: ansible
-      when:
-        - inventory_hostname == "primary"
+    - name: Dump host info to logs
+      command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+      args:
+        chdir: "{{ kolla_logs_dir }}"
+
+    - name: Create dir for kolla build logs
+      file:
+        path: "{{ kolla_build_logs_dir }}"
+        state: directory

tests/playbooks/run.yml

@@ -1,7 +1,6 @@
 ---
 - hosts: all
   vars:
-    kolla_inventory_path: "{{ zuul.project.src_dir }}/inventory"
     nodepool_rdo_proxy: "{{ zuul_site_mirror_fqdn }}:8080/rdo"
     nodepool_pypi_mirror: "http://{{ zuul_site_mirror_fqdn }}:8080/pypi/simple"
     nodepool_mirror_host: "{{ zuul_site_mirror_fqdn }}"
@@ -12,23 +11,6 @@
     nodepool_oraclelinux_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/oraclelinux/"
     nodepool_percona_proxy: "http://{{ zuul_site_mirror_fqdn }}:8080/percona"
   tasks:
-    - name: generate kolla inventory file
-      copy:
-        dest: "{{ kolla_inventory_path }}"
-        content: |
-          {% for host in hostvars %}
-          {{ host }} ansible_host={{ hostvars[host]['ansible_host'] }} ansible_user={{ hostvars[host]['ansible_user'] }}
-          {% endfor %}
-
-    - name: setup logging
-      shell:
-        cmd: |
-          mkdir logs
-          ln -s $(pwd)/logs /tmp/logs
-          mkdir -p /tmp/logs/{ansible,build,kolla,kolla_configs}
-        executable: /bin/bash
-        chdir: "{{ zuul.project.src_dir }}"
-
     - name: copy setup script
       copy:
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tools/setup_{{ ansible_os_family }}.sh"
@@ -57,21 +39,12 @@
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2"
         dest: /etc/kolla/kolla-build.conf
 
-    - name: Template override
+    - name: Template template_overrides.j2
       template:
         src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/template_overrides.j2"
         dest: /etc/kolla/template_overrides.j2
 
-    - shell:
-        cmd: |
-          set -e
-          set -x
-
-          export ACTION={{ action }}
-          export BASE_DISTRO={{ base_distro }}
-          export INSTALL_TYPE={{ install_type }}
-          export IN_PROJECT_JOBS=1
-          tools/gate_run.sh
-        executable: /bin/bash
+    - name: Run tox
+      command: tox -e {{ action }}-{{ base_distro }}-{{ install_type }}
+      args:
         chdir: "{{ zuul.project.src_dir }}"
-      environment: '{{ zuul | zuul_legacy_vars }}'

tests/setup_gate.sh

@@ -5,5 +5,5 @@ namespace = kolla
 tag = {{ zuul.tag if zuul.pipeline == "release" else zuul.branch | basename }}
 {% endif %}
 push = false
-logs_dir = /tmp/logs/build
+logs_dir = {{ kolla_build_logs_dir }}
 template_override = /etc/kolla/template_overrides.j2

tests/templates/kolla-build.conf.j2

@@ -23,4 +23,4 @@ function dump_node_info {
     set -o errexit
 }
 
-(dump_node_info 2>&1) > /tmp/logs/node_info_$(date +%s)
+(dump_node_info 2>&1) > node_info_$(date +%s)

tools/dump_info.sh

@@ -75,11 +75,6 @@ commands =
   doc8 doc
   sphinx-build -W -b html doc/source doc/build/html
 
-[testenv:setupenv]
-commands =
-  {toxinidir}/tools/setup_gate.sh
-  {toxinidir}/tools/dump_info.sh
-
 [testenv:build-centos-binary]
 whitelist_externals = find
                       bash
@@ -89,7 +84,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestCentosBinary
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-centos-source]
 whitelist_externals = find
@@ -100,7 +94,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestCentosSource
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-ubuntu-binary]
 whitelist_externals = find
@@ -111,7 +104,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestUbuntuBinary
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-ubuntu-source]
 whitelist_externals = find
@@ -122,7 +114,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestUbuntuSource
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-debian-binary]
 whitelist_externals = find
@@ -133,7 +124,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestDebianBinary
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-debian-source]
 whitelist_externals = find
@@ -144,7 +134,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestDebianSource
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-oraclelinux-binary]
 whitelist_externals = find
@@ -155,7 +144,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestOracleLinuxBinary
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-oraclelinux-source]
 whitelist_externals = find
@@ -166,7 +154,6 @@ commands =
   find . -type f -name "*.py[c|o]" -delete -o -type l -name "*.py[c|o]" -delete
   bash -c "if [ ! -d .stestr ]; then stestr init; fi"
   stestr run test_build.BuildTestOracleLinuxSource
-  {toxinidir}/tools/dump_info.sh
 
 [testenv:genconfig]
 whitelist_externals = which