Skip to content

Commit 82721d0

Browse files
Siddharth GangadharSiddharth Gangadhar
Siddharth Gangadhar
authored and
Siddharth Gangadhar
committed
initial commit
1 parent 4363710 commit 82721d0

File tree

14 files changed

+851
-0
lines changed

14 files changed

+851
-0
lines changed

‎bfsRunner.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
## -*- Mode:Python; -*- ##
2+
##
3+
## Copyright (c) 2018 ResiliNets, ITTC, University of Kansas
4+
##
5+
## Author: Siddharth Gangadhar
6+
##
7+
## James P.G. Sterbenz <jpgs@ittc.ku.edu>, director
8+
## ResiliNets Research Group http://wiki.ittc.ku.edu/resilinets
9+
## Information and Telecommunication Technology Center (ITTC)
10+
## and Department of Electrical Engineering and Computer Science
11+
## The University of Kansas Lawrence, KS USA.
12+
##
13+
14+
import sys
15+
import numpy
16+
from numpy import *
17+
import ConfigParser
18+
from ConfigParser import *
19+
import xscaleArray
20+
from xscaleArray import *
21+
import pdb
22+
import cmdRunner
23+
from cmdRunner import *
24+
import metricStats
25+
from metricStats import *
26+
import csvWriter
27+
from csvWriter import *
28+
29+
def bfsRunner(configFile, scenario):
30+
config = ConfigParser()
31+
config.read(configFile)
32+
xscale = config.get(scenario, scenario).split(',')
33+
numFlows = len(config.get(scenario, 'transProt').split(','));
34+
runs = config.getint(scenario, 'runs')
35+
#pdb.set_trace()
36+
numMetrics = 12;
37+
#convert xscale to a 1D array
38+
#pdb.set_trace()
39+
xArray = xscaleArray(xscale)
40+
#for prot in transProt:
41+
#zeros matrix of needed dimensions to store metrics statistics
42+
stats = zeros(shape=(numFlows,3,numMetrics))
43+
metrics = zeros(shape=(numFlows, len(xArray), numMetrics*3))
44+
for x in xscale:
45+
# calculate stats for each run and store as a matrix of size (pcapFiles, runs, numMetrics)
46+
(runStats, pcapFiles) = cmdRunner(x, numMetrics, scenario, config)
47+
# calculate avg, std and conf Int for the run stats over all runs
48+
# pdb.set_trace()
49+
for p in range(len(pcapFiles)):
50+
stats[p,:,:] = metricStats(runStats[p,:,:], numMetrics, runs)
51+
#pdb.set_trace();
52+
# rearrange values to follow avg, std, confInt for each metric for each pcap for each x value
53+
metrics[p, xscale.index(x)] = array(stats[p,:,:].reshape(1,numMetrics*3,order='F').copy())
54+
# write labelled metrics to csv file once the scenario is over
55+
for pcapFile in pcapFiles:
56+
csvWriter(metrics[pcapFiles.index(pcapFile),:], xArray, scenario, config, pcapFile)

‎calcQsz.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
## -*- Mode:Python; -*- ##
2+
##
3+
## Copyright (c) 2018 ResiliNets, ITTC, University of Kansas
4+
##
5+
## Author: Siddharth Gangadhar
6+
##
7+
## James P.G. Sterbenz <jpgs@ittc.ku.edu>, director
8+
## ResiliNets Research Group http://wiki.ittc.ku.edu/resilinets
9+
## Information and Telecommunication Technology Center (ITTC)
10+
## and Department of Electrical Engineering and Computer Science
11+
## The University of Kansas Lawrence, KS USA.
12+
##
13+
14+
import ConfigParser
15+
from ConfigParser import *
16+
17+
def calcQsz(scenario, x, config):
18+
bdpQsz = config.getfloat(scenario, 'bdpQsz')
19+
xscale = config.get(scenario, scenario).split(',')
20+
if scenario == "bottleneckDelay":
21+
bottleneckSpeed = config.get(scenario, 'bottleneckSpeed')
22+
speedUnit = bottleneckSpeed[-4:]
23+
if speedUnit[0] == 'K':
24+
bottleneckSpeed = 1000*int(bottleneckSpeed[:-4])
25+
elif speedUnit[0] == 'M':
26+
bottleneckSpeed = 1000000*int(bottleneckSpeed[:-4])
27+
elif speedUnit[0] == 'G':
28+
bottleneckSpeed = 1000000000*int(bottleneckSpeed[:-4])
29+
queue_size = str(int(bdpQsz*bottleneckSpeed*2*0.001*(float(xscale[xscale.index(x)])/8)))
30+
elif scenario == "bottleneckSpeed":
31+
bottleneckDelay = config.get(scenario, 'bottleneckDelay')
32+
bottleneckDelay = int(bottleneckDelay[:-2])
33+
speedUnit = config.get(scenario, 'speedUnit')
34+
if speedUnit[0] == 'K':
35+
x = 1000*int(x)
36+
elif speedUnit[0] == 'M':
37+
x = 1000000*int(x)
38+
elif speedUnit[0] == 'G':
39+
x = 1000000000*int(x)
40+
queue_size = str(int(bdpQsz*bottleneckDelay*2*0.001*(float(x/8))))
41+
else:
42+
bottleneckDelay = config.get(scenario, 'bottleneckDelay')
43+
bottleneckDelay = int(bottleneckDelay[:-2])
44+
bottleneckSpeed = config.get(scenario, 'bottleneckSpeed')
45+
speedUnit = bottleneckSpeed[-4:]
46+
if speedUnit[0] == 'K':
47+
bottleneckSpeed = 1000*int(bottleneckSpeed[:-4])
48+
elif speedUnit[0] == 'M':
49+
bottleneckSpeed = 1000000*int(bottleneckSpeed[:-4])
50+
elif speedUnit[0] == 'G':
51+
bottleneckSpeed = 1000000000*int(bottleneckSpeed[:-4])
52+
queue_size = str(int(bdpQsz*bottleneckSpeed*2*0.001*bottleneckDelay/8))
53+
# queue_size = "2000000"
54+
return queue_size

‎cfgPbsWriter.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
## -*- Mode:Python; -*- ##
2+
##
3+
## Copyright (c) 2018 ResiliNets, ITTC, University of Kansas
4+
##
5+
## Author: Siddharth Gangadhar
6+
##
7+
## James P.G. Sterbenz <jpgs@ittc.ku.edu>, director
8+
## ResiliNets Research Group http://wiki.ittc.ku.edu/resilinets
9+
## Information and Telecommunication Technology Center (ITTC)
10+
## and Department of Electrical Engineering and Computer Science
11+
## The University of Kansas Lawrence, KS USA.
12+
##
13+
14+
import os
15+
import pdb
16+
17+
def cfgPbsWriter(configFile, jobsDir):
18+
emailID = "siddharth@ittc.ku.edu"
19+
work_dir = "/work/siddharth/ns-3/dce-Nov15/dce/source/ns-3-dce/"
20+
#tcpVariants = ["bic"]
21+
# scenario = [configFile[:5]]
22+
scenario = configFile.split('_', 1)[0]
23+
#tcpVariants = ["bic", "cubic", "highspeed", "htcp", "hybla", "Illinois", "lp", "reno", "scalable", "westwood", "vegas", "yeah"]
24+
#scenario = ["error", "bandwidth", "bottleneckDelay", "bottleneckSpeed"]
25+
#driverFileName = "driver.sh"
26+
nodeSpecs = "#PBS -l nodes=1:ppn=2,mem=2000M,walltime=120:0:0"
27+
#jobsDir = "mul-flows-Jan15_11-18"
28+
cfgID = configFile[:-4]
29+
#print tcpVariants
30+
#############################################################################################################################################
31+
if not os.path.exists(jobsDir):
32+
os.makedirs(jobsDir)
33+
34+
# driverFileName = os.path.join(jobsDir, driverFileName) (change this behavior using *configFile)
35+
# driverFile = open(driverFileName, "wb") (change this behavior using *configFile)
36+
37+
for scen in scenario:
38+
#uniqueID = cfgName + "_" + tcp + "_" + scen
39+
runDir = "/tmp/" + "tmp_" + cfgID
40+
runCmd = "python " + work_dir + "dceRunner.py " + configFile + " " + scenario
41+
pbsFiName = cfgID + ".pbs"
42+
pbsFileName = os.path.join (jobsDir, pbsFiName)
43+
clusterJobName = "#PBS -N " + cfgID + "\n"
44+
pbsFile = open (pbsFileName, "wb")
45+
pbsFile.write (nodeSpecs + "\n")
46+
pbsFile.write ("#PBS -j oe" + "\n")
47+
pbsFile.write ("#PBS -S /bin/sh" + "\n")
48+
pbsFile.write ("#PBS -M " + emailID + "\n")
49+
pbsFile.write ("#PBS -m a" + "\n")
50+
pbsFile.write (clusterJobName + "\n")
51+
pbsFile.write ("cd $PBS_O_WORKDIR" + "\n")
52+
pbsFile.write ("SAVE_LDLP=$LD_LIBRARY_PATH" + "\n")
53+
pbsFile.write ("SAVE_DCE=$DCE_PATH" + "\n")
54+
pbsFile.write ("SAVE_DCE_ROOT=$DCE_ROOT" + "\n")
55+
pbsFile.write ("export LD_LIBRARY_PATH=$SAVE_LDLP:`pwd`/../../build/lib:`pwd`/../../build/bin:`pwd`/build/bin:`pwd`/../../build/bin_dce:`pwd`/../../build/lib:`pwd`/../../build/bin:`pwd`/build/bin:`pwd`/../../build/bin_dce:`pwd`/build/lib:`pwd`/build/lib:`pwd`/build/bin:/panfs/pfs.acf.ku.edu/work/siddharth/ns-3/dce-Nov15/dce/build/lib:/panfs/pfs.acf.ku.edu/work/siddharth/ns-3/dce-Nov15/dce/build/bin" + "\n")
56+
pbsFile.write ("export DCE_PATH=$SAVE_DCE:`pwd`/build/bin_dce:/panfs/pfs.acf.ku.edu/work/siddharth/ns-3/dce-Nov15/dce/build/sbin:/panfs/pfs.acf.ku.edu/work/siddharth/ns-3/dce-Nov15/dce/build/bin_dce:/usr/lib/gcc/x86_64-redhat-linux/4.4.6::`pwd`/../../build/lib:`pwd`/../../build/bin:`pwd`/build/bin:`pwd`/../../build/bin_dce:`pwd`/../../build/lib:`pwd`/../../build/bin:`pwd`/build/bin:`pwd`/../../build/bin_dce:`pwd`/build/lib:`pwd`/build/lib:`pwd`/build/bin:/panfs/pfs.acf.ku.edu/work/siddharth/ns-3/dce-Nov15/dce/build/lib:/panfs/pfs.acf.ku.edu/work/siddharth/ns-3/dce-Nov15/dce/build/bin" + "\n")
57+
pbsFile.write ("export DCE_ROOT=$SAVE_DCE_ROOT:`pwd`/build:/panfs/pfs.acf.ku.edu/work/siddharth/ns-3/dce-Nov15/dce/build" + "\n")
58+
pbsFile.write ("cd " + work_dir + "\n")
59+
pbsFile.write ("mkdir " + runDir + "\n")
60+
pbsFile.write ("cp " + configFile + " " + runDir + "\n")
61+
pbsFile.write ("cd " + runDir + "\n")
62+
pbsFile.write ("stdbuf -o L -e L " + runCmd + " | tee /projects/resilinets/siddharth/live-output_" + cfgID + ".$PBS_JOBID" + "\n")
63+
pbsFile.write ("mv *.csv *.mon " + work_dir +"\n")
64+
pbsFile.write ("cd " + runDir + "\n")
65+
pbsFile.write ("rm -rf " + runDir + "\n")
66+
# pbsFile.write ("cd " + work_dir + jobsDir + "\n")
67+
# pbsFile.write ("mkdir cfg csv logs mon pbs" + "\n")
68+
# pbsFile.write ("mv ../*.cfg cfg" + "\n")
69+
# pbsFile.write ("mv ../*.mon mon" + "\n")
70+
# pbsFile.write ("mv ../*.csv csv" + "\n")
71+
# pbsFile.write ("mv *.pbs pbs" + "\n")
72+
# pbsFile.write ("mv *.sh pbs" + "\n")
73+
# pbsFile.write ("mv ../*.o* logs" + "\n")
74+
pbsFile.close ()
75+
#driverFile.write ("qsub " + pbsFiName + "\n") (change this behavior using *configFile)
76+
#driverFile.close () (change this behavior using *configFile)
77+

‎cfgWrapper.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
## -*- Mode:Python; -*- ##
2+
##
3+
## Copyright (c) 2018 ResiliNets, ITTC, University of Kansas
4+
##
5+
## Author: Siddharth Gangadhar
6+
##
7+
## James P.G. Sterbenz <jpgs@ittc.ku.edu>, director
8+
## ResiliNets Research Group http://wiki.ittc.ku.edu/resilinets
9+
## Information and Telecommunication Technology Center (ITTC)
10+
## and Department of Electrical Engineering and Computer Science
11+
## The University of Kansas Lawrence, KS USA.
12+
##
13+
14+
import os
15+
import pdb
16+
import cfgPbsWriter
17+
from cfgPbsWriter import *
18+
19+
######################################################################################################
20+
# Initialize config parameters
21+
scenario = "error"
22+
#error = "error = 0,0.0000001,0.000001,0.00001,0.0001,0.001,0.01"
23+
#error = "error = 0"
24+
error = [0]
25+
script = "script = single-multiple-flow"
26+
simTime = [600]
27+
#simTime = [600,1200,1800]
28+
runs = 'runs = 1'
29+
pktSize = 'pktSize = 1500'
30+
accessSpeed = 'accessSpeed = 10Gbps'
31+
accessDelay = 'accessDelay = 0.1ms'
32+
pcapFile = 'pcapFile = /tmp/'
33+
csvName = 'csvName = '
34+
#qSize = [1]
35+
qSize = [0.2,0.4,0.6,0.8,1]
36+
outputFactor = 'outputFactor = Mb'
37+
dceProt = 'dceProt = '
38+
qSizeFileName = 'qSizeFileName = '
39+
#qSizeFileName = 'qMonitoring = '
40+
cmd = 'cmd = /work/siddharth/ns-3/dce-Nov15/dce/source/ns-3-dce/build/bin/%(script)s --error=%(x)s --seed=%(runNo)s --pktSize=%(pktSize)s --accessSpeed=%(accessSpeed)s --accessDelay=%(accessDelay)s --bottleneckSpeed=%(bottleneckSpeed)s --bottleneckDelay=%(bottleneckDelay)s --pcapFile=%(pcapFile)s --queue_size=%(queue_size)s --dceProt=%(dceProt)s --stopTime=%(stopTime)s --qMonitoring=%(qMonitoring)s --qSizeFileName=%(qSizeFileName)s'
41+
speed = [2000,4000,6000,8000,10000]
42+
#speed=[200,400,600,800]
43+
#speed = [200,400,600,800,1000]
44+
speedUnit = "Mbps"
45+
delay = [10,50,100,150,200,250]
46+
#delay=[50,100,150,200,250]
47+
#delay = [10,50,100,150,200,250]
48+
# for multiple flows
49+
#tcpVariants = ["cubic,reno"]
50+
tcpVariants = ["bic,bic","cubic,cubic","htcp,htcp","highspeed,highspeed","yeah,yeah","scalable,scalable","illinois,illinois","bic,reno","cubic,reno","htcp,reno","highspeed,reno","yeah,reno","scalable,reno","illinois,reno"]
51+
# for single flow
52+
#tcpVariants = ["cubic", "bic"]
53+
#tcpVariants = ["bic", "cubic", "highspeed", "htcp", "hybla", "Illinois", "lp", "reno", "scalable", "westwood", "vegas", "yeah", "veno"]
54+
#tcpVariants = ["bic", "cubic", "highspeed", "htcp", "Illinois", "scalable", "yeah"]
55+
#tcpVariants = ["bic,bic","cubic,cubic","htcp,htcp","highspeed,highspeed","yeah,yeah","scalable,scalable","illinois,illinois"]
56+
delayUnit = "ms"
57+
58+
########################################################################################################
59+
########################################################################################################
60+
# Create pbs scripts for *.cfg files in the created directory
61+
#os.chdir(cfgDir)
62+
jobsDir = "mul-flows-Jan15_11-18" # look into this later
63+
driverFileName = "driver.sh"
64+
driverFileName = os.path.join(jobsDir, driverFileName)
65+
66+
#for files in os.listdir("."):
67+
# if files.endswith(".cfg"):
68+
# cfgPbsWriter (files, jobsDir, tcpVariants)
69+
70+
#########################################################################################################
71+
72+
# Write config files in current directory
73+
74+
for err in error:
75+
for q in qSize:
76+
for sim in simTime:
77+
for spe in speed:
78+
for de in delay:
79+
for tcp in tcpVariants:
80+
bottleneckSpeed = "bottleneckSpeed = " + str(spe) + speedUnit
81+
bottleneckDelay = "bottleneckDelay = " + str(de) + delayUnit
82+
stopTime = "stopTime = " + str(sim)
83+
bdpQsz = "bdpQsz = " + str(q)
84+
errorValue = "error = " + str(err)
85+
tcpNoComma = tcp.replace(",", "-")
86+
cfgID = scenario + "_" + str(err)+ "_" + str(spe) + speedUnit + "_" + str(de) + delayUnit + "_" + str(sim) + "_" + str(q) + "_" + tcpNoComma
87+
cfgFileName = cfgID + ".cfg"
88+
qMonFileName = "qSizeFileName = " + cfgID + ".mon"
89+
cfgFile = open (cfgFileName, "wb")
90+
cfgFile.write ("[" + scenario + "]" + "\n")
91+
cfgFile.write (errorValue + "\n")
92+
cfgFile.write (script + "\n")
93+
cfgFile.write (runs + "\n")
94+
cfgFile.write (pktSize + "\n")
95+
cfgFile.write (accessSpeed + "\n")
96+
cfgFile.write (accessDelay + "\n")
97+
cfgFile.write (bottleneckSpeed + "\n")
98+
cfgFile.write (bottleneckDelay + "\n")
99+
cfgFile.write (pcapFile + cfgID + "\n")
100+
cfgFile.write (csvName + cfgID + "\n")
101+
cfgFile.write (bdpQsz + "\n")
102+
cfgFile.write (outputFactor + "\n")
103+
cfgFile.write (dceProt + tcp + "\n")
104+
cfgFile.write (stopTime + "\n")
105+
cfgFile.write (qMonFileName + "\n")
106+
cfgFile.write (bdpQsz + "\n")
107+
cfgFile.write (cmd + "\n")
108+
cfgFile.close()
109+
cfgPbsWriter (cfgFileName, jobsDir)
110+
111+
# create a shell script to qsub all pbs jobs
112+
driverFile = open(driverFileName, "wb")
113+
for files in os.listdir(jobsDir):
114+
if files.endswith(".pbs"):
115+
driverFile.write ("qsub " + files + "\n")
116+
driverFile.close ()

‎cmdRunner.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
## -*- Mode:Python; -*- ##
2+
##
3+
## Copyright (c) 2018 ResiliNets, ITTC, University of Kansas
4+
##
5+
## Author: Siddharth Gangadhar
6+
##
7+
## James P.G. Sterbenz <jpgs@ittc.ku.edu>, director
8+
## ResiliNets Research Group http://wiki.ittc.ku.edu/resilinets
9+
## Information and Telecommunication Technology Center (ITTC)
10+
## and Department of Electrical Engineering and Computer Science
11+
## The University of Kansas Lawrence, KS USA.
12+
##
13+
14+
import numpy
15+
from numpy import *
16+
import shlex
17+
import exe_comm
18+
from exe_comm import *
19+
import ConfigParser
20+
from ConfigParser import *
21+
import wafCmd
22+
from wafCmd import *
23+
import pdb
24+
import os, sys
25+
#pptrace_path = os.path.abspath('/raid5/sgondi/transport/santoshg')
26+
#sys.path.append(pptrace_path)
27+
import pp_trace
28+
from pp_trace import *
29+
import glob
30+
from glob import *
31+
32+
def cmdRunner(x, numMetrics, scenario, config):
33+
runs = config.getint(scenario, 'runs')
34+
pcapName = config.get(scenario, 'pcapFile')
35+
outputFactor = config.get(scenario, 'outputFactor')
36+
numFlows = len(config.get(scenario, 'transProt').split(','))
37+
asciiFileName = config.get(scenario, 'qSizeFileName')
38+
runStats = zeros(shape=(numFlows, runs, numMetrics))
39+
for run in range(runs):
40+
runNo = str(run+1)
41+
#pdb.set_trace()
42+
waf_cmd, qSize = wafCmd(runNo, x, scenario, config)
43+
print waf_cmd
44+
#execute waf command
45+
exe_com(shlex.split(waf_cmd))
46+
pcapFiles = glob(pcapName + "*.pcap")
47+
asciiFile = glob(asciiFileName)
48+
#pdb.set_trace()
49+
#execute tcptrace command
50+
for p in range(len(pcapFiles)):
51+
runStats[p,run,:] = array(pp_trace(pcapFiles[p], outputFactor, config, scenario, asciiFile[0], qSize))
52+
# pdb.set_trace()
53+
exe_com(shlex.split("rm " + pcapFiles[p]))
54+
# at the end, runStats would have a size of (len(pcapFiles), runs, numMetrics)
55+
#print runStats
56+
return (runStats, pcapFiles)
57+
58+
# Alternate way, might be useful for multi-threading
59+
# wafDir = os.cwd()
60+
# os.chdir('/tmp/')
61+
# os.chdir(wafDir)

0 commit comments

Comments
 (0)