[dev/tooling] Use custom mkwheelhouse script to build/upload sdist packages (#847)

* Use custom mkwheelhouse script

* Also test building a wheel

* ensure 'wheel' is installed

* add missing arg

* key.name instead of key

Author: brettlangdon

.circleci/config.yml

@@ -62,11 +62,14 @@ jobs:
       # Install required dependencies
       # DEV: `pyopenssl` needed until the following PR is released
       #      https://github.com/pypa/twine/pull/447
-      - run: pip install twine readme_renderer[md] pyopenssl
+      # DEV: `wheel` is needed to run `bdist_wheel`
+      - run: pip install twine readme_renderer[md] pyopenssl wheel
       # Ensure we didn't cache from previous runs
       - run: rm -rf dist/
-      # Ensure package will build
+      # Ensure source package will build
       - run: python setup.py sdist
+      # Ensure wheel will build
+      - run: python setup.py bdist_wheel
       # Ensure package long description is valid and will render
       # https://github.com/pypa/twine/tree/6c4d5ecf2596c72b89b969ccc37b82c160645df8#twine-check
       - run: twine check dist/*

Rakefile

@@ -1,90 +1,3 @@
-desc "Starts all backing services and run all tests"
-task :test do
-  sh "docker-compose up -d | cat"
-  begin
-    sh "tox"
-  ensure
-    sh "docker-compose kill"
-  end
-  sh "python -m tests.benchmark"
-end
-
-desc 'CI dependent task; tests in parallel'
-task:test_parallel do
-
-  begin
-    test_cassandra = sh "git diff-tree --no-commit-id --name-only -r HEAD | grep ddtrace/contrib/cassandra"
-  rescue StandardError => e
-    test_cassandra = false
-  end
-
-  sh "docker-compose up -d | cat"
-
-  # If cassandra hasn't been changed ignore cassandra tests
-  if not test_cassandra
-    n_total_envs = `tox -l | grep -v cassandra | wc -l`.to_i
-    envs = 'tox -l | grep -v cassandra | tr \'\n\' \',\''
-  else
-    n_total_envs = `tox -l | wc -l`.to_i
-    envs = 'tox -l | tr \'\n\' \',\''
-  end
-
-  circle_node_tot = ENV['CIRCLE_NODE_TOTAL'].to_i
-  n_envs_chunk = n_total_envs / circle_node_tot
-  env_list_start = 1
-  env_list_end = n_envs_chunk
-  begin
-    for node_index in 0..circle_node_tot
-      if ENV['CIRCLE_NODE_INDEX'].to_i == node_index then
-        # Node 0 already does as second task wait test, the others will require it to ensure db connections
-        if node_index >= 1 then
-          sh "tox -e wait"
-        end
-        sh "#{envs} | cut -d, -f#{env_list_start}-#{env_list_end} | xargs tox -e"
-      end
-      env_list_start = env_list_end + 1
-      env_list_end = env_list_end + n_envs_chunk
-    end
-  ensure
-    sh "docker-compose kill"
-  end
-
-  sh "python -m tests.benchmark"
-end
-
-desc "Run tests with envs matching the given pattern."
-task :"test:envs", [:grep] do |t, args|
-  pattern = args[:grep]
-  if !pattern
-    puts 'specify a pattern like rake test:envs["py27.*mongo"]'
-  else
-    sh "tox -l | grep '#{pattern}' | xargs tox -e"
-  end
-end
-
-namespace :docker do
-  task :up do
-    sh "docker-compose up -d | cat"
-  end
-
-  task :down do
-    sh "docker-compose down"
-  end
-end
-
-
-desc "install the library in dev mode"
-task :dev do
-  sh "pip uninstall -y ddtrace"
-  sh "pip install -e ."
-end
-
-desc "remove artifacts"
-task :clean do
-  sh 'python setup.py clean'
-  sh 'rm -rf build *egg* *.whl dist'
-end
-
 desc "build the docs"
 task :docs do
     sh "pip install sphinx"
@@ -94,7 +7,6 @@ task :docs do
 end
 
 # Deploy tasks
-S3_BUCKET = 'pypi.datadoghq.com'
 S3_DIR = ENV['S3_DIR']
 
 desc "release the a new wheel"
@@ -105,7 +17,8 @@ task :'release:wheel' do
   #  - aws s3 cp dist/*.whl s3://pypi.datadoghq.com/#{s3_dir}/
   fail "Missing environment variable S3_DIR" if !S3_DIR or S3_DIR.empty?
 
-  sh "mkwheelhouse s3://#{S3_BUCKET}/#{S3_DIR}/ ."
+  # Use custom mkwheelhouse script to build and upload an sdist to S3 bucket
+  sh "scripts/mkwheelhouse"
 end
 
 desc "release the docs website"
@@ -169,60 +82,3 @@ namespace :pypi do
     sh "twine upload #{build}"
   end
 end
-
-namespace :version do
-
-  def get_version()
-    return `python setup.py --version`.strip()
-  end
-
-  def set_version(old, new)
-    branch = `git name-rev --name-only HEAD`.strip()
-    if  branch != "master"
-      puts "you should only tag the master branch"
-      return
-    end
-    msg = "bumping version #{old} => #{new}"
-    path = "ddtrace/__init__.py"
-    sh "sed -i 's/#{old}/#{new}/' #{path}"
-    sh "git commit -m '#{msg}' #{path}"
-    sh "git tag v#{new}"
-    puts "Verify everything looks good, then `git push && git push --tags`"
-  end
-
-  def inc_version_num(version, type)
-    split = version.split(".").map{|v| v.to_i}
-    if type == 'bugfix'
-      split[2] += 1
-    elsif type == 'minor'
-       split[1] += 1
-       split[2] = 0
-    elsif type == 'major'
-       split[0] += 1
-       split[1] = 0
-       split[2] = 0
-    end
-    return split.join(".")
-  end
-
-  def inc_version(type)
-    old = get_version()
-    new = inc_version_num(old, type)
-    set_version(old, new)
-  end
-
-  desc "Cut a new bugfix release"
-  task :bugfix do
-    inc_version("bugfix")
-  end
-
-  desc "Cut a new minor release"
-  task :minor do
-    inc_version("minor")
-  end
-
-  task :major do
-    inc_version("major")
-  end
-
-end

scripts/mkwheelhouse

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+import os
+import shutil
+import tempfile
+
+import mkwheelhouse
+
+S3_BUCKET = 'pypi.datadoghq.com'
+S3_DIR = os.environ['S3_DIR']
+
+
+# DEV: This is the same `mkwheelhouse.build_wheels` except we are running `python setup.py sdist` instead
+def build_sdist():
+    build_dir = tempfile.mkdtemp(prefix='mkwheelhouse-')
+    args = [
+        'python', 'setup.py', 'sdist',
+        '--dist-dir', build_dir,
+    ]
+    mkwheelhouse.spawn(args)
+    return build_dir
+
+
+# DEV: This is the same as `mkwheelhouse.Bucket.make_index`, except we include `*.whl` and `*.tar.gz` files
+def make_index(bucket):
+    doc, tag, text = mkwheelhouse.yattag.Doc().tagtext()
+    with tag('html'):
+        for key in bucket.list():
+            # Skip over any non-wheel or non-source dist
+            if not key.name.endswith('.whl') and not key.name.endswith('.tar.gz'):
+                continue
+
+            with tag('a', href=bucket.generate_url(key)):
+                text(key.name)
+            doc.stag('br')
+
+    return doc.getvalue()
+
+
+# DEV: This is the same as `mkwheelhouse.run` except we hard code some values and use our custom functions instead
+def run():
+    s3_url = 's3://{0}/{1}'.format(S3_BUCKET, S3_DIR)
+    acl = 'private'
+    bucket = mkwheelhouse.Bucket(s3_url)
+
+    if not bucket.has_key('index.html'):  # noqa
+        bucket.put('<!DOCTYPE html><html></html>', 'index.html', acl=acl)
+
+    index_url = bucket.generate_url('index.html')
+    build_dir = build_sdist()
+    bucket.sync(build_dir, acl=acl)
+    bucket.put(make_index(bucket), key='index.html', acl=acl)
+    shutil.rmtree(build_dir)
+    print('mkwheelhouse: index written to', index_url)
+
+
+if __name__ == '__main__':
+    run()