Add local version of upload-logs-s3

Author: mnasiadka

roles/upload-logs-s3/README.rst

@@ -0,0 +1,67 @@
+Upload logs to S3
+
+Before using this role, create at least one bucket and set up
+appropriate access controls or lifecycle events.  This role will not
+automatically create buckets.
+
+This role requires the ``boto3`` Python package to be
+installed in the Ansible environment on the Zuul executor.
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_site_upload_logs
+   :default: true
+
+   Controls when logs are uploaded. true, the default, means always
+   upload logs. false means never upload logs. 'failure' means to only
+   upload logs when the job has failed.
+
+   .. note:: Intended to be set by admins via site-variables.
+
+.. zuul:rolevar:: zuul_log_partition
+   :default: false
+
+   If set to true, then the first component of the log path will be
+   removed from the object name and added to the bucket name, so that
+   logs for different changes are distributed across a large number of
+   buckets.
+
+.. zuul:rolevar:: zuul_log_bucket
+
+   This role *will not* create buckets which do not already exist.  If
+   partitioning is not enabled, this is the name of the bucket which
+   will be used.  If partitioning is enabled, then this will be used
+   as the prefix for the bucket name which will be separated from the
+   partition name by an underscore.  For example, "logs_42" would be
+   the bucket name for partition 42.
+
+   Note that you will want to set this to a value that uniquely
+   identifies your Zuul installation.
+
+.. zuul:rolevar:: zuul_log_bucket_public
+   :default: true
+
+   Set to false to make logs private.
+
+.. zuul:rolevar:: zuul_log_path
+   :default: Generated by the role `set-zuul-log-path-fact`
+
+   Prepend this path to the object names when uploading.
+
+.. zuul:rolevar:: zuul_log_create_indexes
+   :default: true
+
+   Whether to create `index.html` files with directory indexes.
+
+.. zuul:rolevar:: zuul_log_aws_access_key
+
+   AWS access key to use.
+
+.. zuul:rolevar:: zuul_log_aws_secret_key
+
+   AWS secret key for the AWS access key.
+
+.. zuul:rolevar:: upload_logs_s3_endpoint
+
+   The endpoint to use when uploading logs to an s3 compatible service.
+   By default this will be automatically constructed by boto but should be set when working with non-aws hosted s3 service.

roles/upload-logs-s3/defaults/main.yaml

@@ -0,0 +1,3 @@
+zuul_log_partition: false
+zuul_log_create_indexes: true
+zuul_log_bucket_public: true

roles/upload-logs-s3/meta/main.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - role: upload-logs-base

roles/upload-logs-s3/tasks/main.yaml

@@ -0,0 +1,45 @@
+- name: Set zuul-log-path fact
+  include_role:
+    name: set-zuul-log-path-fact
+  when: zuul_log_path is not defined
+
+# Always upload (true), never upload (false) or only on failure ('failure')
+- name: Upload logs
+  when: zuul_site_upload_logs | default(true) | bool or
+        (zuul_site_upload_logs == 'failure' and not zuul_success | bool)
+  block:
+    # Use chmod instead of file because ansible 2.5 file with recurse and
+    # follow can't really handle symlinks to .
+    - name: Debug log_root
+      debug:
+        msg: "log_root {{ zuul.executor.log_root }}"
+
+    - name: Ensure logs are readable before uploading
+      delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}"
+      command: "chmod -R u=rwX,g=rX,o=rX {{ zuul.executor.log_root }}/"
+      # ANSIBLE0007 chmod used in place of argument mode to file
+      tags:
+        - skip_ansible_lint
+
+    - name: Upload logs to S3
+      delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}"
+      zuul_s3_upload:
+        endpoint: "{{ upload_logs_s3_endpoint | default(omit) }}"
+        partition: "{{ zuul_log_partition }}"
+        bucket: "{{ zuul_log_bucket }}"
+        public: "{{ zuul_log_bucket_public }}"
+        prefix: "{{ zuul_log_path }}"
+        indexes: "{{ zuul_log_create_indexes }}"
+        aws_access_key: "{{ zuul_log_aws_access_key }}"
+        aws_secret_key: "{{ zuul_log_aws_secret_key }}"
+        files:
+          - "{{ zuul.executor.log_root }}/"
+      register: upload_results
+
+- name: Return log URL to Zuul
+  delegate_to: localhost
+  zuul_return:
+    data:
+      zuul:
+        log_url: "{{ upload_results.url }}/"
+  when: upload_results is defined