New test data

run.py

@@ -7,6 +7,11 @@
 import subprocess
 import shutil
 import re
+import json
+import getpass
+from restkit import Resource, BasicAuth
+
+owner = "avsm" # github user name of the official trunk repo
 
 cpu_regex = re.compile("^cpu(\d+)$")
 index_regex = re.compile("^index(\d+)$")
@@ -237,4 +242,112 @@ def parse_list(s):
 subprocess.check_call(["/bin/tar", "cvfz", out_file, tempdir])
 
 print "Test succeeded. Output written as", out_file
-print "Please email that file to cl-ipc-bench@lists.cam.ac.uk"
+
+path = out_file
+file_ = path[5:]
+
+git = raw_input( "Do you have a github account? Y/N" )
+
+if (git=="Y" or git=="y") is False:
+	print "Output written as", out_file
+	print "Please email that file to cl-ipc-bench@lists.cam.ac.uk"
+	sys.exit()
+
+#------------------------
+#Adding the output file to the repo and making pull request
+#------------------------
+shutil.copyfile(path, os.getcwd() + "/results/" + file_)
+
+
+user = raw_input( "Github user:" ) 
+password = getpass.getpass()
+auth = BasicAuth(user, password)
+
+
+#Getting authentication token 
+
+authreqdata = { "scopes": [ "public_repo" ], "note": "admin script" }
+
+resource = Resource('https://api.github.com/authorizations', filters=[auth])
+response = resource.post(headers={ "Content-Type": "application/json" }, payload = json.dumps(authreqdata))
+token = json.loads(response.body_string())['token']
+
+#Fork the repo
+
+resource = Resource('https://api.github.com/repos/%s/ipc-bench/forks' %owner)
+headers = {'Content-Type' : 'application/json' }
+headers['Authorization'] = 'token %s' % token
+response = resource.post(headers = headers)
+forks = json.loads(response.body_string())
+
+#Commit
+
+	#Get the SHA for the latest commit
+resource = Resource("https://api.github.com/repos/%s/ipc-bench/git/refs/heads/master" % user)
+headers = {'Content-Type' : 'application/json' }
+headers['Authorization'] = 'token %s' % token
+response = resource.get(headers = headers)
+sha_latest = json.loads(response.body_string())['object']['sha']
+
+
+	#Get the SHA for the tree
+resource = Resource("https://api.github.com/repos/%s/ipc-bench/git/commits/%s" % ( user, sha_latest ) )
+headers = {'Content-Type' : 'application/json' }
+headers['Authorization'] = 'token %s' % token
+response = resource.get(headers = headers)
+sha_tree = json.loads(response.body_string())['tree']['sha']
+
+
+print "results/" + file_
+	#Adding the file to the tree
+data = { "base_tree": sha_tree, "tree": [
+				    {
+				      "path": "results/" + file_,
+				      "mode": "160000",
+				      "type": "commit",
+				      "sha": sha_tree
+				    }
+				  ]
+       }
+resource = Resource('https://api.github.com/repos/%s/ipc-bench/git/trees ' % user)
+headers = {'Content-Type' : 'application/json' }
+headers['Authorization'] = 'token %s' % token
+response = resource.post(headers = headers, payload = json.dumps(data))
+sha_new = json.loads(response.body_string())['sha']
+
+	#Commit
+data = { "parents": [ sha_latest ], "tree": sha_new , "message": "New test data"}
+resource = Resource('https://api.github.com/repos/%s/ipc-bench/git/commits ' % user)
+headers = {'Content-Type' : 'application/json' }
+headers['Authorization'] = 'token %s' % token
+response = resource.post(headers = headers, payload = json.dumps(data))
+sha_commit = json.loads(response.body_string())['sha']
+
+	#Set head master
+data = { "sha": sha_commit, "force": True}
+resource = Resource('https://api.github.com/repos/%s/ipc-bench/git/refs/heads/master ' % user)
+headers = {'Content-Type' : 'application/json' }
+headers['Authorization'] = 'token %s' % token
+response = resource.post(headers = headers, payload = json.dumps(data))
+temp = json.loads(response.body_string())
+
+
+
+#Pull request
+
+
+
+data = {
+  		"title": "New test data",
+  		"body": "The file is " + file_,
+  		"head": "%s:master" %user,
+  		"base": "master"
+	}
+resource = Resource('https://api.github.com/repos/%s/ipc-bench/pulls ' %owner)
+headers = {'Content-Type' : 'application/json' }
+headers['Authorization'] = 'token %s' % token
+response = resource.post(headers = headers, payload = json.dumps(data))
+temp = json.loads(response.body_string())
+
+
+#------------------------

web/processing/cron.sh

@@ -3,26 +3,28 @@
 CUR_DIR=$(pwd)
 
 RESULTS_DIR=../../results
-OUT_DIR=/usr/groups/netos/html/ipc-bench
+OUT_DIR=/usr/groups/netos/html/projects/ipc-bench
 HTML_OUTFILE=results.html
 
 cd ${RESULTS_DIR}
 RESULTS=$(ls tmp*.tar.gz)
+#RESULTS="tmpsbkFiM.tar.gz"
 cd ${CUR_DIR}
 
 # Generate results.html
 cat ${PWD}/html_head.tmpl > ${OUT_DIR}/${HTML_OUTFILE}.new
 cat ${PWD}/html_results_head.tmpl >> ${OUT_DIR}/${HTML_OUTFILE}.new
 for r in ${RESULTS}; do
   ARCHIVE=${r}
-  echo -n "Extracting ${ARCHIVE}..."
-  tar -xzf ${RESULTS_DIR}/${ARCHIVE}
-  echo " done!"
-  NAME=$(echo ${r} | sed 's/\.tar\.gz//g')
-  DIR=tmp/${NAME}
-  # Check the extracted data directory exists
-  if [[ ! -d ${DIR} ]]; then
-    echo "FATAL: After extraction, ${DIR} did not exist!"
+  NAME=$(echo ${r} | sed 's/\.tar\.gz//g') DIR=tmp/${NAME}
+  if [[ ! -f ${OUT_DIR}/details/${NAME}.html ]]; then
+    echo -n "Extracting ${ARCHIVE}..."
+    tar -xzf ${RESULTS_DIR}/${ARCHIVE}
+    echo " done!"
+    # Check the extracted data directory exists
+    if [[ ! -d ${DIR} ]]; then
+      echo "FATAL: After extraction, ${DIR} did not exist!"
+    fi
   fi
 
   # Find the target cores used
@@ -33,18 +35,20 @@ for r in ${RESULTS}; do
     ${TARGET_CPUS} ${NAME}
 
   # Generate details page
-  mkdir -p ${OUT_DIR}/details
-  mkdir -p ${DIR}/graphs
-  cat ${PWD}/html_head.tmpl > ${OUT_DIR}/details/${NAME}.html
-  python gen-details-page.py ${DIR} ${OUT_DIR}/details ${TARGET_CPUS} ${NAME}
-  cat ${PWD}/html_foot.tmpl >> ${OUT_DIR}/details/${NAME}.html
-
-  # Move graphs
-  mkdir -p ${OUT_DIR}/graphs
-  rm -rf ${OUT_DIR}/graphs/${NAME}
-  mv ${DIR}/graphs ${OUT_DIR}/graphs/${NAME}
-  chmod -R g+rx ${OUT_DIR}/graphs/${NAME}
-  chmod -R o+rx ${OUT_DIR}/graphs/${NAME}
+  if [[ ! -f ${OUT_DIR}/details/${NAME}.html ]]; then
+    mkdir -p ${OUT_DIR}/details
+    mkdir -p ${DIR}/graphs
+    cat ${PWD}/html_head.tmpl > ${OUT_DIR}/details/${NAME}.html
+    python gen-details-page.py ${DIR} ${OUT_DIR}/details ${TARGET_CPUS} ${NAME}
+    cat ${PWD}/html_foot.tmpl >> ${OUT_DIR}/details/${NAME}.html
+
+    # Move graphs
+    mkdir -p ${OUT_DIR}/graphs
+    rm -rf ${OUT_DIR}/graphs/${NAME}
+    mv ${DIR}/graphs ${OUT_DIR}/graphs/${NAME}
+    chmod -R g+rx ${OUT_DIR}/graphs/${NAME}
+    chmod -R o+rx ${OUT_DIR}/graphs/${NAME}
+  fi
 done
 cat ${PWD}/html_results_foot.tmpl >> ${OUT_DIR}/${HTML_OUTFILE}.new
 cat ${PWD}/html_foot.tmpl >> ${OUT_DIR}/${HTML_OUTFILE}.new

web/processing/plot_lat.py

@@ -20,7 +20,8 @@ def get_data(filename):
     for j in range(0, len(data[i])):
       x_tmp.append(i)
       y_tmp.append(j)
-      v_tmp.append(data[i][j])
+      # * 1000000 to convert from seconds into microseconds
+      v_tmp.append(float(data[i][j]) * 1000000.0)
 
   retdata = [x_tmp, y_tmp, v_tmp]
   print len(retdata)
@@ -45,6 +46,8 @@ def get_data(filename):
 output_dir = sys.argv[3]
 
 raw_data = np.loadtxt(input_file)
+# convert from seconds to microseconds
+raw_data *= 1000000.0
 data = get_data(input_file)
 
 fig = plt.figure(figsize=(3,2))
@@ -60,8 +63,10 @@ def get_data(filename):
 #plt.clf()
 #plt.imshow(heatmap, extent=extent)
 
-if fix_scale != 0:
-  plt.matshow(raw_data, vmax=0.0001, vmin=0.00001, fignum=0)
+test_name = re.search("(.+)\.csv", sys.argv[2])
+if fix_scale != 0 or test_name.group(1) == 'mempipe_lat':
+  # 10 to 100 us
+  plt.matshow(raw_data, vmax=10, vmin=0, fignum=0)
 else:
   plt.matshow(raw_data, fignum=0)
 
@@ -70,10 +75,9 @@ def get_data(filename):
 plt.ylim(-0.5, int(sys.argv[4])-0.5)
 plt.xlabel('Core ID')
 plt.xlim(-0.5, int(sys.argv[4])-0.5)
-test_name = re.search("(.+)\.csv", sys.argv[2])
 plt.title(test_name.group(1))
 
-cb = plt.colorbar(shrink=1.0, format='%.3e')
+cb = plt.colorbar(shrink=1.0, format='%.2f')
 cb.set_label('Latency in microseconds')
 
 #plt.savefig("lat_" + sys.argv[1] + ".pdf", format="pdf", bbox_inches='tight')