CDRIVER-3935 lower evg batchtimes and add evg config generator (#774)

Author: benjirewis

.evergreen/config.yml

@@ -7,7 +7,6 @@
 # DO NOT EDIT THIS FILE
 #
 ####################################
-
 stepback: true
 command_type: system
 exec_timeout_secs: 2400
@@ -264,9 +263,9 @@ functions:
       remote_file: ${project}/${build_variant}/${revision}/${version_id}/${build_id}/abi-compliance/compat_report.html
       bucket: mciuploads
       permissions: public-read
-      display_name: 'ABI Report:'
-      content_type: text/html
       local_files_include_filter: mongoc/abi-compliance/compat_reports/**/*.html
+      content_type: text/html
+      display_name: 'ABI Report:'
   upload scan artifacts:
   - command: shell.exec
     type: test
@@ -2038,9 +2037,9 @@ tasks:
   commands:
   - command: s3.get
     params:
-      aws_key: '${toolchain_aws_key}'
-      aws_secret: '${toolchain_aws_secret}'
-      remote_file: 'mongo-c-toolchain/${distro_id}/mongo-c-toolchain.tar.gz'
+      aws_key: ${toolchain_aws_key}
+      aws_secret: ${toolchain_aws_secret}
+      remote_file: mongo-c-toolchain/${distro_id}/mongo-c-toolchain.tar.gz
       bucket: mongo-c-toolchain
       local_file: mongo-c-toolchain.tar.gz
   - command: shell.exec
@@ -23043,7 +23042,7 @@ buildvariants:
   - .debug-compile !.sspi .nossl !.sasl
   - .debug-compile .special .valgrind
   - .test-valgrind !.3.0 !.3.2 !.3.4 !.3.6
-  batchtime: 10080
+  batchtime: 1440
 - name: valgrind-ubuntu-1404
   display_name: Valgrind Tests - MongoDB (pre 4.0) (Ubuntu 14.04)
   expansions:
@@ -23057,7 +23056,7 @@ buildvariants:
   - .test-valgrind .3.2
   - .test-valgrind .3.4
   - .test-valgrind .3.6
-  batchtime: 10080
+  batchtime: 1440
 - name: asan-ubuntu
   display_name: ASAN Tests (Ubuntu 16.04)
   expansions:
@@ -23145,7 +23144,7 @@ buildvariants:
   - name: .ocsp-openssl-1.0.1
     distros:
     - ubuntu1804-test
-  batchtime: 20160
+  batchtime: 10080
 - name: packaging
   display_name: Linux Distro Packaging
   run_on: ubuntu1604-test
@@ -23162,4 +23161,4 @@ buildvariants:
   run_on: ubuntu1604-small
   tasks:
   - .tsan
-  batchtime: 10080
+  batchtime: 1440

build/evergreen_config_generator/__init__.py

@@ -0,0 +1,94 @@
+# Copyright 2018-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sys
+from collections import OrderedDict as OD
+
+try:
+    import yaml
+    import yamlordereddictloader
+except ImportError:
+    sys.stderr.write(
+        "try 'pip install -r evergreen_config_generator/requirements.txt'\n")
+    raise
+
+
+class ConfigObject(object):
+    def __init__(self, *args, **kwargs):
+        super(ConfigObject, self).__init__()
+
+    @property
+    def name(self):
+        return 'UNSET'
+
+    def to_dict(self):
+        return OD([('name', self.name)])
+
+
+# We want legible YAML tasks:
+#
+#     - name: debug-compile
+#       tags: [zlib, snappy, compression, openssl]
+#       commands:
+#       - command: shell.exec
+#         params:
+#           script: |-
+#             set -o errexit
+#             set -o xtrace
+#             ...
+#
+# Write values compactly except multiline strings, which use "|" style. Write
+# tag sets as lists.
+
+class _Dumper(yamlordereddictloader.Dumper):
+    def __init__(self, *args, **kwargs):
+        super(_Dumper, self).__init__(*args, **kwargs)
+        self.add_representer(set, type(self).represent_set)
+        # Use "multi_representer" to represent all subclasses of ConfigObject.
+        self.add_multi_representer(ConfigObject,
+                                   type(self).represent_config_object)
+
+    def represent_scalar(self, tag, value, style=None):
+        if isinstance(value, (str)) and '\n' in value:
+            style = '|'
+        return super(_Dumper, self).represent_scalar(tag, value, style)
+
+    def represent_set(self, data):
+        return super(_Dumper, self).represent_list(sorted(data))
+
+    def represent_config_object(self, obj):
+        return super(_Dumper, self).represent_data(obj.to_dict())
+
+
+def yaml_dump(obj):
+    return yaml.dump(obj, Dumper=_Dumper)
+
+
+def generate(config, path):
+    """Dump config to a file as YAML.
+    config is a dict, preferably an OrderedDict. path is a file path.
+    """
+    f = open(path, 'w+')
+    f.write('''####################################
+# Evergreen configuration
+#
+# Generated with evergreen_config_generator from
+# github.com/mongodb-labs/drivers-evergreen-tools
+#
+# DO NOT EDIT THIS FILE
+#
+####################################
+''')
+    f.write(yaml_dump(config))

build/evergreen_config_generator/functions.py

@@ -0,0 +1,116 @@
+# Copyright 2018-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict as OD
+from textwrap import dedent
+
+from evergreen_config_generator import ConfigObject
+
+
+def func(func_name, **kwargs):
+    od = OD([('func', func_name)])
+    if kwargs:
+        od['vars'] = OD(sorted(kwargs.items()))
+
+    return od
+
+
+def bootstrap(VERSION='latest', TOPOLOGY=None, **kwargs):
+    if TOPOLOGY:
+        return func('bootstrap mongo-orchestration',
+                    VERSION=VERSION,
+                    TOPOLOGY=TOPOLOGY,
+                    **kwargs)
+
+    return func('bootstrap mongo-orchestration',
+                VERSION=VERSION,
+                **kwargs)
+
+
+def run_tests(URI=None, **kwargs):
+    if URI:
+        return func('run tests', URI=URI, **kwargs)
+
+    return func('run tests', **kwargs)
+
+
+def s3_put(remote_file, project_path=True, **kwargs):
+    if project_path:
+        remote_file = '${project}/' + remote_file
+
+    od = OD([
+        ('command', 's3.put'),
+        ('params', OD([
+            ('aws_key', '${aws_key}'),
+            ('aws_secret', '${aws_secret}'),
+            ('remote_file', remote_file),
+            ('bucket', 'mciuploads'),
+            ('permissions', 'public-read')]))])
+
+    od['params'].update(kwargs)
+    return od
+
+
+def strip_lines(s):
+    return '\n'.join(line for line in s.split('\n') if line.strip())
+
+
+def shell_exec(script, test=True, errexit=True, xtrace=True, silent=False,
+               continue_on_err=False, working_dir=None, background=False):
+    dedented = ''
+    if errexit:
+        dedented += 'set -o errexit\n'
+
+    if xtrace:
+        dedented += 'set -o xtrace\n'
+
+    dedented += dedent(strip_lines(script))
+    command = OD([('command', 'shell.exec')])
+    if test:
+        command['type'] = 'test'
+
+    command['params'] = OD()
+    if silent:
+        command['params']['silent'] = True
+
+    if working_dir is not None:
+        command['params']['working_dir'] = working_dir
+
+    if continue_on_err:
+        command['params']['continue_on_err'] = True
+
+    if background:
+        command['params']['background'] = True
+
+    command['params']['shell'] = 'bash'
+    command['params']['script'] = dedented
+    return command
+
+
+def targz_pack(target, source_dir, *include):
+    return OD([
+        ('command', 'archive.targz_pack'),
+        ('params', OD([
+            ('target', target),
+            ('source_dir', source_dir),
+            ('include', list(include))]))])
+
+
+class Function(ConfigObject):
+    def __init__(self, *commands):
+        super(Function, self).__init__()
+        self.commands = commands
+
+    def to_dict(self):
+        return list(self.commands)