Merge pull request #6218 from y-p/ironcache

BLD: Cut Travis-CI build times by 30-50%

Author: y-p

.travis.yml

@@ -1,7 +1,13 @@
 language: python
 
-python:
-  - 2.6
+env:
+  global:
+  # scatterci API key
+  - secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
+  # ironcache API key
+  - secure: "TWGKPL4FGtWjCQ427QrSffaQBSCPy1QzPdhtYKAW9AlDo/agdp9RyZRQr8WTlyZ5AOG18X8MDdi0EcpFnyfDNd4thCLyZOntxwltlVV2yNNvnird3XRKUV1F2DD42L64wna04M2KYxpCKhEQ84gEnCH1DGD4g1NnR6fYuEYNSTU="
+  - secure: "NvgIWe14pv4SJ5BQhw6J0zIpCTFH4fVpzr9pRzM2tD8yf/3cspX3uyXTt4owiqtjoQ3aQGNnhWtVlmSwlgJrwu7KUaE9IPtlsENYDniAj2oJgejjx02d367pHtMB/9e3+4b2fWUsFNJgWw0ordiIT0p1lzHRdQ9ut4l/Yn/lkJs="
+  - secure: "D3ASNRu32pV79lv/Nl0dBm2ldZiTgbb6boOrN0SzIKsQU3yBwedpqX6EI6KjpVg17lGhhhFXGzL2Gz1qjU3/+m6aMvekxHgpfuc0AlEFCEqenWPxIdDDrUkdfJoCvfQQPd5oxChfHgqaEDLjuqHy1ZEgnJ2/L/6dwZ4fUt62hMk="
 
 matrix:
     include:
@@ -10,39 +16,33 @@ matrix:
       - NOSE_ARGS="not slow and not network"
       - CLIPBOARD=xclip
       - LOCALE_OVERRIDE="it_IT.UTF-8"
-      - secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
       - JOB_NAME: "26_nslow_nnet" # ScatterCI Build name, 20 chars max
     - python: 2.7
       env:
       - NOSE_ARGS="slow and not network"
       - LOCALE_OVERRIDE="zh_CN.GB18030"
       - FULL_DEPS=true
       - JOB_TAG=_LOCALE
-      - secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
       - JOB_NAME: "27_slow_nnet_LOCALE" # ScatterCI Build name, 20 chars max
     - python: 2.7
       env:
       - NOSE_ARGS="not slow"
       - FULL_DEPS=true
       - CLIPBOARD_GUI=gtk2
-      - secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
       - JOB_NAME: "27_nslow" # ScatterCI Build name, 20 chars max
     - python: 3.2
       env:
       - NOSE_ARGS="not slow"
       - FULL_DEPS=true
       - CLIPBOARD_GUI=qt4
-      - secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
       - JOB_NAME: "32_nslow"  # ScatterCI Build name, 20 chars max
     - python: 3.3
       env:
       - NOSE_ARGS="not slow"
       - FULL_DEPS=true
       - CLIPBOARD=xsel
-      - secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
       - JOB_NAME: "33_nslow"  # ScatterCI Build name, 20 chars max
-    exclude:
-    - python: 2.6
+
 
 # allow importing from site-packages,
 # so apt-get python-x works for system pythons
@@ -55,25 +55,28 @@ before_install:
   - echo $VIRTUAL_ENV
   - df -h
   - date
+  - pwd
   - uname -a
-  - ci/before_install.sh
   - python -V
+  - ci/before_install.sh
   # Xvfb stuff for clipboard functionality; see the travis-ci documentation
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
 
 install:
   - echo "install"
+  - ci/prep_ccache.sh
   - ci/install.sh
 
 before_script:
   - mysql -e 'create database pandas_nosetest;'
 
 script:
   - echo "script"
-  - ci/script.sh
+#  - ci/script.sh
 
 after_script:
   - ci/print_versions.py
   - ci/print_skipped.py /tmp/nosetests.xml
-  - ci/after_script.sh
+  - ci/after_script.sh
+  - ci/submit_ccache.sh

ci/install.sh

@@ -93,6 +93,12 @@ fi
 # build and install pandas
 time python setup.py sdist
 pip uninstall cython -y
+
+cat pandas/version.py
+
+export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH
+which gcc
+ccache -z # reset stats
 time pip install $(find dist | grep gz | head -n 1)
 
 true

ci/ironcache/get.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+import re
+import os
+import time
+import json
+import base64
+
+from iron_cache import *
+
+key='KEY'+os.environ.get('JOB_NAME','')+"."
+
+b = b''
+cache = IronCache()
+for i in range(20):
+    print("getting %s" % key+str(i))
+    try:
+        item = cache.get(cache="travis", key=key+str(i))
+        b += bytes(base64.b64decode(item.value))
+    except:
+        break
+
+with open(os.path.join(os.environ.get('HOME',''),"ccache.7z"),'wb') as f:
+    f.write(b)

ci/ironcache/put.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+import re
+import os
+import time
+import json
+import base64
+
+from iron_cache import *
+
+key='KEY'+os.environ.get('JOB_NAME','')+"."
+os.chdir(os.environ.get('HOME'))
+
+cache = IronCache()
+
+i=0
+
+for i, fname in enumerate(sorted([x for x in os.listdir('.') if re.match("ccache.\d+$",x)])):
+    print("Putting %s" % key+str(i))
+    with open(fname,"rb") as f:
+        s= f.read()
+        value=base64.b64encode(s)
+        if isinstance(value, bytes):
+            value = value.decode('ascii')
+    item = cache.put(cache="travis", key=key+str(i), value=value,options=dict(expires_in=24*60*60))
+
+# print("foo")
+for i in range(i+1,20):
+
+    try:
+        item = cache.delete(key+str(i),cache='travis')
+        print("Deleted %s" % key+str(i))
+    except:
+        break
+        pass

ci/prep_ccache.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+if [ "$IRON_TOKEN" ]; then
+    sudo apt-get $APT_ARGS install ccache p7zip-full
+    # iron_cache, pending py3 fixes upstream
+    pip install -I --allow-external --allow-insecure git+https://github.com/y-p/iron_cache_python.git
+
+
+    python ci/ironcache/get.py
+    ccache -C
+    if [ -f ~/ccache.7z ]; then
+        echo "Cache retrieved"
+        cd $HOME
+        7za e $HOME/ccache.7z
+        # ls -l $HOME
+        cd /
+        tar xvf $HOME/ccache
+        rm -rf $HOME/ccache.7z
+        rm -rf $HOME/ccache
+
+    fi
+fi
+
+exit 0

ci/submit_ccache.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+ccache -s
+
+if [ "$IRON_TOKEN" ]; then
+    rm -rf ~/ccache.7z
+
+    tar cf - $HOME/.ccache \
+        |  7za a -si ~/ccache.7z
+
+    # Caching the cython files could be be made to work.
+    # There's a timestamp issue to be handled (http://stackoverflow.com/questions/1964470)
+    # but you could get the build to reuse them.
+    # However, there's a race condition between travis cythonizing
+    # and a new commit being made on a GH server somewhere.
+    # and there's convenient causal link between the two we can
+    # take advantage of.
+    # Let's just wait another 60 seconds.
+    # "$TRAVIS_BUILD_DIR"/pandas/{index,algos,lib,tslib,parser,hashtable}.c \
+    # "$TRAVIS_BUILD_DIR"/pandas/src/{sparse,testing}.c \
+    # "$TRAVIS_BUILD_DIR"/pandas/msgpack.cpp  \
+    # |  7za a -si ~/ccache.7z
+
+    split -b 500000 -d ~/ccache.7z ~/ccache.
+
+    python ci/ironcache/put.py
+fi;
+
+exit 0