Peform non-incremental run on child commit as well.

Author: jerhard

scripts/incremental/benchmarking/efficiency.py

@@ -113,23 +113,30 @@ def analyze_small_commits_in_repo(cwd, outdir, from_c, to_c):
             add_options = default_options + ['--disable', 'incremental.load', '--enable', 'incremental.save']
             utils.analyze_commit(analyzer_dir, gr, repo_path, build_compdb, parent.hash, outparent, conf_base, add_options, files)
 
+            #print('And now analyze', str(commit.hash), 'from scratch.')
+            outchild_non_incr = os.path.join(outtry, 'child-non-incr')
+            os.makedirs(outchild_non_incr)
+            # Do not save in this run to not pollute results
+            add_options = default_options + ['--disable', 'incremental.load', '--disable', 'incremental.save']
+            utils.analyze_commit(analyzer_dir, gr, repo_path, build_compdb, commit.hash, outchild_non_incr, conf_base, add_options, files)
+
             #print('And now analyze', str(commit.hash), 'incrementally.')
             outchild = os.path.join(outtry, 'child')
             os.makedirs(outchild)
             add_options = default_options + ['--enable', 'incremental.load', '--disable', 'incremental.save']
             utils.analyze_commit(analyzer_dir, gr, repo_path, build_compdb, commit.hash, outchild, conf_base, add_options, files)
 
             #print('And again incremental, this time with incremental postsolver')
-            outchildincrpost = os.path.join(outtry, 'child-incr-post')
-            os.makedirs(outchildincrpost)
+            outchild_incr_post = os.path.join(outtry, 'child-incr-post')
+            os.makedirs(outchild_incr_post)
             add_options = default_options + ['--enable', 'incremental.load', '--disable', 'incremental.save']
-            utils.analyze_commit(analyzer_dir, gr, repo_path, build_compdb, commit.hash, outchildincrpost, conf_incrpost, add_options, files)
+            utils.analyze_commit(analyzer_dir, gr, repo_path, build_compdb, commit.hash, outchild_incr_post, conf_incrpost, add_options, files)
 
             #print('And again incremental, this time with incremental postsolver and reluctant')
-            outchildrel = os.path.join(outtry, 'child-rel')
-            os.makedirs(outchildrel)
+            outchild_rel = os.path.join(outtry, 'child-rel')
+            os.makedirs(outchild_rel)
             add_options = default_options + ['--enable', 'incremental.load', '--disable', 'incremental.save', '--enable', 'incremental.reluctant.enabled']
-            utils.analyze_commit(analyzer_dir, gr, repo_path, build_compdb, commit.hash, outchildrel, conf_incrpost, add_options, files)
+            utils.analyze_commit(analyzer_dir, gr, repo_path, build_compdb, commit.hash, outchild_rel, conf_incrpost, add_options, files)
 
             count_analyzed+=1
             failed = False
@@ -149,18 +156,19 @@ def analyze_small_commits_in_repo(cwd, outdir, from_c, to_c):
 
 def collect_data(outdir):
     data = {"Commit": [], "Failed?": [], "Changed LOC": [], "Relevant changed LOC": [], "Changed/Added/Removed functions": [],
-      utils.runtime_header_parent: [], utils.runtime_header_incr_child: [],
+      utils.runtime_header_parent: [], utils.runtime_header_non_incr_child: [], utils.runtime_header_incr_child: [],
       utils.runtime_header_incr_posts_child: [], utils.runtime_header_incr_posts_rel_child: [],
-      utils.analysis_header_parent: [], utils.analysis_header_incr_child: [],
+      utils.analysis_header_parent: [], utils.analysis_header_non_incr_child: [], utils.analysis_header_incr_child: [],
       utils.analysis_header_incr_posts_child: [], utils.analysis_header_incr_posts_rel_child: [],
-      utils.solving_header_parent: [], utils.solving_header_incr_child: [],
+      utils.solving_header_parent: [], utils.solving_header_non_incr_child: [], utils.solving_header_incr_child: [],
       utils.solving_header_incr_posts_child: [], utils.solving_header_incr_posts_rel_child: [],
       "Change in number of race warnings": []}
     for t in os.listdir(outdir):
-        parentlog = os.path.join(outdir, t, 'parent', utils.analyzerlog)
-        childlog = os.path.join(outdir, t, 'child', utils.analyzerlog)
-        childpostslog = os.path.join(outdir, t, 'child-incr-post', utils.analyzerlog)
-        childpostsrellog = os.path.join(outdir, t, 'child-rel', utils.analyzerlog)
+        parent_log = os.path.join(outdir, t, 'parent', utils.analyzerlog)
+        child_non_incr_log = os.path.join(outdir, t, 'child-non-incr', utils.analyzerlog)
+        child_log = os.path.join(outdir, t, 'child', utils.analyzerlog)
+        child_posts_log = os.path.join(outdir, t, 'child-incr-post', utils.analyzerlog)
+        child_posts_rel_log = os.path.join(outdir, t, 'child-rel', utils.analyzerlog)
         commit_prop_log = os.path.join(outdir, t, 'commit_properties.log')
         t = int(t)
         commit_prop = json.load(open(commit_prop_log, "r"))
@@ -170,38 +178,45 @@ def collect_data(outdir):
         data["Commit"].append(commit_prop["hash"][:7])
         if commit_prop["failed"] == True:
             data[utils.runtime_header_parent].append(0)
+            data[utils.runtime_header_non_incr_child].append(0)
             data[utils.runtime_header_incr_child].append(0)
             data[utils.runtime_header_incr_posts_child].append(0)
             data[utils.runtime_header_incr_posts_rel_child].append(0)
             data[utils.analysis_header_parent].append(0)
+            data[utils.analysis_header_non_incr_child].append(0)
             data[utils.analysis_header_incr_child].append(0)
             data[utils.analysis_header_incr_posts_child].append(0)
             data[utils.analysis_header_incr_posts_rel_child].append(0)
             data[utils.solving_header_parent].append(0)
+            data[utils.solving_header_non_incr_child].append(0)
             data[utils.solving_header_incr_child].append(0)
             data[utils.solving_header_incr_posts_child].append(0)
             data[utils.solving_header_incr_posts_rel_child].append(0)
 
             data["Changed/Added/Removed functions"].append(0)
             data["Change in number of race warnings"].append(0)
             continue
-        parent_info = utils.extract_from_analyzer_log(parentlog)
-        child_info = utils.extract_from_analyzer_log(childlog)
-        child_posts_info = utils.extract_from_analyzer_log(childpostslog)
-        child_posts_rel_info = utils.extract_from_analyzer_log(childpostsrellog)
+
+        parent_info = utils.extract_from_analyzer_log(parent_log)
+        child_non_incr_info = utils.extract_from_analyzer_log(child_non_incr_log)
+        child_info = utils.extract_from_analyzer_log(child_log)
+        child_posts_info = utils.extract_from_analyzer_log(child_posts_log)
+        child_posts_rel_info = utils.extract_from_analyzer_log(child_posts_rel_log)
         data["Changed/Added/Removed functions"].append(int(child_info["changed"]) + int(child_info["added"]) + int(child_info["removed"]))
         data[utils.runtime_header_parent].append(float(parent_info["runtime"]))
+        data[utils.runtime_header_non_incr_child].append(float(child_non_incr_info["runtime"]))
         data[utils.runtime_header_incr_child].append(float(child_info["runtime"]))
         data[utils.runtime_header_incr_posts_child].append(float(child_posts_info["runtime"]))
         data[utils.runtime_header_incr_posts_rel_child].append(float(child_posts_rel_info["runtime"]))
 
-
         data[utils.analysis_header_parent].append(float(parent_info["analysis_time"]))
+        data[utils.analysis_header_non_incr_child].append(float(child_non_incr_info["analysis_time"]))
         data[utils.analysis_header_incr_child].append(float(child_info["analysis_time"]))
         data[utils.analysis_header_incr_posts_child].append(float(child_posts_info["analysis_time"]))
         data[utils.analysis_header_incr_posts_rel_child].append(float(child_posts_rel_info["analysis_time"]))
 
         data[utils.solving_header_parent].append(float(parent_info["solving_time"]))
+        data[utils.solving_header_non_incr_child].append(float(child_non_incr_info["solving_time"]))
         data[utils.solving_header_incr_child].append(float(child_info["solving_time"]))
         data[utils.solving_header_incr_posts_child].append(float(child_posts_info["solving_time"]))
         data[utils.solving_header_incr_posts_rel_child].append(float(child_posts_rel_info["solving_time"]))

scripts/incremental/benchmarking/plot.py

@@ -13,7 +13,7 @@ def cummulative_distr_compare2(results_dir, result_csv_filename, figure_dir):
     outfile_incr_vs_incrrel = "figure_cum_distr_rel.pdf"
     df = utils.get_cleaned_filtered_data(os.path.join(results_dir,result_csv_filename), filterDetectedChanges=True)
 
-    data, base = utils.create_cum_data(df, num_bins, [utils.runtime_header_parent, utils.runtime_header_incr_child, utils.runtime_header_incr_posts_rel_child])
+    data, base = utils.create_cum_data(df, num_bins, [utils.runtime_header_non_incr_child, utils.runtime_header_incr_child, utils.runtime_header_incr_posts_rel_child])
     datanonincr = {"values": data[0], "label": description_non_incr}
     dataincr = {"values": data[1], "label": description_incr}
 
@@ -30,7 +30,7 @@ def cummulative_distr_all4(results_dir, result_csv_filename, figure_dir):
     outfile_nonincr_vs_incr = "figure_cum_distr_all3.pdf"
     df = utils.get_cleaned_filtered_data(os.path.join(results_dir,result_csv_filename), filterDetectedChanges=True)
 
-    data, base = utils.create_cum_data(df, num_bins, [utils.runtime_header_parent, utils.runtime_header_incr_child, utils.runtime_header_incr_posts_child, utils.runtime_header_incr_posts_rel_child])
+    data, base = utils.create_cum_data(df, num_bins, [utils.runtime_header_non_incr_child, utils.runtime_header_incr_child, utils.runtime_header_incr_posts_child, utils.runtime_header_incr_posts_rel_child])
     data_non_incr = {"values": data[0], "label": description_non_incr}
     data_incr = {"values": data[1], "label": description_incr}
     data_incr_post = {"values": data[2], "label": description_incr_post}
@@ -41,7 +41,7 @@ def distribution_absdiff_plot(title, result_csv_filename, outdir, cutoffs_incr=N
     df = utils.get_cleaned_filtered_data(os.path.join(outdir,result_csv_filename), filterDetectedChanges=True)
 
     # plot incremental vs non-incremental
-    diff = df.loc[:,utils.runtime_header_parent] - df.loc[:,utils.runtime_header_incr_child]
+    diff = df.loc[:,utils.runtime_header_non_incr_child] - df.loc[:,utils.runtime_header_incr_child]
     utils.hist_plot(diff, 20, title, 'Improvement in s (incremental compared to non-incremental)', 'Number of Commits', os.path.join(outdir, "figure_absdiff_distr_incr.pdf"), cutoffs_incr)
 
     # plot reluctant vs. basic incremental
@@ -53,7 +53,7 @@ def distribution_reldiff_plot(title, result_csv_filename, outdir, cutoffs_incr=N
 
     # plot incremental vs non-incremental
     print(df[utils.runtime_header_incr_child].astype('float'))
-    diff = 1 - df[utils.runtime_header_incr_child].astype('float') / df[utils.runtime_header_parent].astype('float')
+    diff = 1 - df[utils.runtime_header_incr_child].astype('float') / df[utils.runtime_header_non_incr_child].astype('float')
     utils.hist_plot(diff, 0.01, title, "Relative Improvement in s (incremental compared to non-incremental)", 'Number of Commits', os.path.join(outdir, "figure_reldiff_distr_incr.pdf"), cutoffs_incr)
 
     # plot reluctant vs. basic incremental
@@ -62,10 +62,10 @@ def distribution_reldiff_plot(title, result_csv_filename, outdir, cutoffs_incr=N
 
 def paper_efficiency_graphs(dir_results, csv_filename, outdir, filterRelCLOC=False, filterDetectedChanges=False):
     df = utils.get_cleaned_filtered_data(os.path.join(dir_results,csv_filename), filterRelCLOC=filterRelCLOC, filterDetectedChanges=filterDetectedChanges)
-    diff1 = 1 - df[utils.runtime_header_incr_child].astype('float') / df[utils.runtime_header_parent].astype('float')
+    diff1 = 1 - df[utils.runtime_header_incr_child].astype('float') / df[utils.runtime_header_non_incr_child].astype('float')
     diff2 = 1 - df[utils.runtime_header_incr_posts_child].astype('float') / df[utils.runtime_header_incr_child].astype('float')
     diff3 = 1 - df[utils.runtime_header_incr_posts_rel_child].astype('float') / df[utils.runtime_header_incr_posts_child].astype('float')
-    diff4 = 1 - df[utils.runtime_header_incr_posts_rel_child].astype('float') / df[utils.runtime_header_parent].astype('float')
+    diff4 = 1 - df[utils.runtime_header_incr_posts_rel_child].astype('float') / df[utils.runtime_header_non_incr_child].astype('float')
     step = 0.01
     for i, diff in enumerate([diff1,diff2,diff3,diff4]):
         # output textwidth in latex with
@@ -97,9 +97,9 @@ def paper_efficiency_graphs(dir_results, csv_filename, outdir, filterRelCLOC=Fal
         for e in diff:
             if (xlimleft and e < xlimleft) or (xlimright and e > xlimright):
                 print("excluded", e, "from efficiency figure", i)
-    diff1 = df[utils.runtime_header_incr_child].astype('float') / df[utils.runtime_header_parent].astype('float')
-    diff2 = df[utils.runtime_header_incr_posts_child].astype('float') / df[utils.runtime_header_parent].astype('float')
-    diff3 = df[utils.runtime_header_incr_posts_rel_child].astype('float') / df[utils.runtime_header_parent].astype('float')
+    diff1 = df[utils.runtime_header_incr_child].astype('float') / df[utils.runtime_header_non_incr_child].astype('float')
+    diff2 = df[utils.runtime_header_incr_posts_child].astype('float') / df[utils.runtime_header_non_incr_child].astype('float')
+    diff3 = df[utils.runtime_header_incr_posts_rel_child].astype('float') / df[utils.runtime_header_non_incr_child].astype('float')
     for n, diff in [("incr", diff1), ("+ incr postsolver", diff2), ("+ reluctant", diff3)]:
         print("80% quantile for", n, "compared to from-scratch analysis:", diff.quantile(q=0.8) * 100, "%")
         print("75% quantile for", n, "compared to from-scratch analysis:", diff.quantile(q=0.75) * 100, "%")

scripts/incremental/benchmarking/utils.py

@@ -34,21 +34,25 @@
 solving_prefix = "Solving"
 
 header_parent = " for parent commit (non-incremental)"
+header_non_incr_child = " for commit (non-incremental)"
 header_incr_child = " for commit (incremental)"
 header_incr_posts_child = " for commit (incremental + incr postsolver)"
 header_incr_posts_rel_child = " for commit (incremental + incr postsolver + reluctant)"
 
 runtime_header_parent = runtime_prefix + header_parent
+runtime_header_non_incr_child = runtime_prefix + header_non_incr_child
 runtime_header_incr_child = runtime_prefix + header_incr_child
 runtime_header_incr_posts_child = runtime_prefix + header_incr_posts_child
 runtime_header_incr_posts_rel_child = runtime_prefix + header_incr_posts_rel_child
 
 analysis_header_parent = analysis_prefix + header_parent
+analysis_header_non_incr_child = analysis_prefix + header_non_incr_child
 analysis_header_incr_child = analysis_prefix + header_incr_child
 analysis_header_incr_posts_child = analysis_prefix + header_incr_posts_child
 analysis_header_incr_posts_rel_child = analysis_prefix + header_incr_posts_rel_child
 
 solving_header_parent = solving_prefix + header_parent
+solving_header_non_incr_child = solving_prefix + header_non_incr_child
 solving_header_incr_child = solving_prefix + header_incr_child
 solving_header_incr_posts_child = solving_prefix + header_incr_posts_child
 solving_header_incr_posts_rel_child = solving_prefix + header_incr_posts_rel_child