Switch from $ to format() for string formatting, other minor stuff

Author: YNedderhoff

create_screens_english.sh

@@ -198,7 +198,9 @@ def test(args):
 
     if os.stat(arguments.in_file).st_size == 0:
         print "Input file is empty"
+
     else:
+
         if arguments.train:
             print "Running in training mode\n"
             train(arguments)
@@ -212,12 +214,5 @@ def test(args):
             print "Running in evaluation mode\n"
             evaluate(arguments)
 
-        """
-        elif arguments.tag:
-            print "Running in tag mode\n"
-            t.tag(arguments.in_file, arguments.model, arguments.output_file)
-
-        """
-
     t1 = time.time()
     print "\n\tDone. Total time: " + str(t1 - t0) + " sec.\n"

create_screens_german.sh

@@ -29,7 +29,6 @@ def evaluate(args):
         print "Error in file length, Gold: " + str(len(full_graphs_gold)) + ", Predicted: " + str(
             len(full_graphs_predicted))
 
-    out = open(args.out_file, "w")
-    print >> out, "Total: " + str(total)
-    print >> out, "Correct: " + str(correct)
-    out.close()
+    with open(args.out_file, "w") as out:
+        print >> out, "Total: " + str(total)
+        print >> out, "Correct: " + str(correct)

dependency-parser.py

@@ -13,48 +13,48 @@ def give_features(hform, hlemma, hpos, dform, dlemma, dpos, bpos):
     # 6 = hlemma
     # 7 = dlemma
 
-    yield "1:%s" % hform
-    yield "2:%s" % hpos
-    yield "3:%s" % dform,
-    yield "4:%s" % dpos
-    yield "6:%s" % hlemma
-    yield "7:%s" % dlemma
-    yield "5:%s" % bpos
+    yield "1:{0}".format(hform)
+    yield "2:{0}".format(hpos)
+    yield "3:{0}".format(dform)
+    yield "4:{0}".format(dpos)
+    yield "6:{0}".format(hlemma)
+    yield "7:{0}".format(dlemma)
+    yield "5:{0}".format(bpos)
 
-    yield "1,4:%s,%s" % (hform, dpos)
-    yield "2,3:%s,%s" % (hpos, dform)
-    yield "1,2:%s,%s" % (hform, hpos)
-    yield "3,4:%s,%s" % (dform, dpos)
-    yield "1,3:%s,%s" % (hform, dform)
-    yield "2,4:%s,%s" % (hpos, dpos)
-    yield "6,4:%s,%s" % (hlemma, dpos)
-    yield "2,7:%s,%s" % (hpos, dlemma)
-    yield "6,2:%s,%s" % (hlemma, hpos)
-    yield "7,4:%s,%s" % (dlemma, dpos)
-    yield "6,7:%s,%s" % (hlemma, dlemma)
+    yield "1,4:{0},{1}".format(hform, dpos)
+    yield "2,3:{0},{1}".format(hpos, dform)
+    yield "1,2:{0},{1}".format(hform, hpos)
+    yield "3,4:{0},{1}".format(dform, dpos)
+    yield "1,3:{0},{1}".format(hform, dform)
+    yield "2,4:{0},{1}".format(hpos, dpos)
+    yield "6,4:{0},{1}".format(hlemma, dpos)
+    yield "2,7:{0},{1}".format(hpos, dlemma)
+    yield "6,2:{0},{1}".format(hlemma, hpos)
+    yield "7,4:{0},{1}".format(dlemma, dpos)
+    yield "6,7:{0},{1}".format(hlemma, dlemma)
 
-    yield "1,2,3,4:%s,%s,%s,%s" % (hform, hpos, dform, dpos)
-    yield "2,3,4:%s,%s,%s" % (hpos, dform, dpos)
-    yield "1,3,4:%s,%s,%s" % (hform, dform, dpos)
-    yield "1,2,3:%s,%s,%s" % (hform, hpos, dform)
-    yield "1,2,4:%s,%s,%s" % (hform, hpos, dpos)
-    yield "2,5,4:%s,%s,%s" % (hpos, bpos, dpos)
-    yield "2,5,3:%s,%s,%s" % (hpos, bpos, dform)
-    yield "1,5,4:%s,%s,%s" % (hform, bpos, dpos)
-    yield "1,5,3:%s,%s,%s" % (hform, bpos, dform)
+    yield "1,2,3,4:{0},{1},{2},{3}".format(hform, hpos, dform, dpos)
+    yield "2,3,4:{0},{1},{2}".format(hpos, dform, dpos)
+    yield "1,3,4:{0},{1},{2}".format(hform, dform, dpos)
+    yield "1,2,3:{0},{1},{2}".format(hform, hpos, dform)
+    yield "1,2,4:{0},{1},{2}".format(hform, hpos, dpos)
+    yield "2,5,4:{0},{1},{2}".format(hpos, bpos, dpos)
+    yield "2,5,3:{0},{1},{2}".format(hpos, bpos, dform)
+    yield "1,5,4:{0},{1},{2}".format(hform, bpos, dpos)
+    yield "1,5,3:{0},{1},{2}".format(hform, bpos, dform)
 
-    yield "6,2,7,4:%s,%s,%s,%s" % (hlemma, hpos, dlemma, dpos)
-    yield "2,7,4:%s,%s,%s" % (hpos, dlemma, dpos)
-    yield "6,7,4:%s,%s,%s" % (hlemma, dlemma, dpos)
-    yield "6,2,7:%s,%s,%s" % (hlemma, hpos, dlemma)
-    yield "6,2,4:%s,%s,%s" % (hlemma, hpos, dpos)
-    yield "2,5,7:%s,%s,%s" % (hpos, bpos, dlemma)
-    yield "6,5,4:%s,%s,%s" % (hlemma, bpos, dpos)
-    yield "6,5,7:%s,%s,%s" % (hlemma, bpos, dlemma)
+    yield "6,2,7,4:{0},{1},{2},{3}".format(hlemma, hpos, dlemma, dpos)
+    yield "2,7,4:{0},{1},{2}".format(hpos, dlemma, dpos)
+    yield "6,7,4:{0},{1},{2}".format(hlemma, dlemma, dpos)
+    yield "6,2,7:{0},{1},{2}".format(hlemma, hpos, dlemma)
+    yield "6,2,4:{0},{1},{2}".format(hlemma, hpos, dpos)
+    yield "2,5,7:{0},{1},{2}".format(hpos, bpos, dlemma)
+    yield "6,5,4:{0},{1},{2}".format(hlemma, bpos, dpos)
+    yield "6,5,7:{0},{1},{2}".format(hlemma, bpos, dlemma)
 
 
 def fm(infile):
-    # takes a file in conll09 format, returns a feature map
+    # takes a file in conll06 format, returns a feature map
     feat_map = {}  # featmap as dictionary {feature:index}
     index = 0  # index in featmap
     for sentence in sentences(codecs.open(infile, encoding='utf-8')):

kill_screens.sh

@@ -43,8 +43,8 @@ def __init__(self, tokens, mode="sparse", feat_map=None, weight_vector=None):
         self.heads = {0: []}
 
         # In every possible mode (sparse, complete-sparse, full, complete-full) at first the arcs with ROOT head are
-        # added, then every other arc. Every arc gets a sparse feature vector based on feat_map and in the completed
-        # graphs a score (based on the features and the weight vector).
+        # added, then every other arc. Every arc, except ones in the  full graph, gets a sparse feature vector based
+        # on feat_map. The completed graphs also get a score per arc (based on the features and the weight vector).
 
         # sparse arc representation
         if mode == "sparse":
@@ -212,31 +212,29 @@ def write_graph_to_file(graph, out_file, mode="normal"):  # write a graph to fil
     if mode == "normal":
 
         rev = reverse_head_graph(graph)
-        out = codecs.open(out_file, "a", "utf-8")
-        for dependent in sorted(rev):
-            # without rel
-            print >> out, "%s\t%s\t%s\t%s\t_\t_\t%s\t_\t_\t_" % (
-                rev[dependent][0].dependent,
-                rev[dependent][0].dependent_form,
-                rev[dependent][0].dependent_lemma,
-                rev[dependent][0].dependent_pos,
-                rev[dependent][0].head
-            )
-        print >> out, ""
-        out.close()
+        with codecs.open(out_file, "a", "utf-8") as out:
+            for dependent in sorted(rev):
+                # without rel
+                print >> out, "{0}\t{1}\t{2}\t{3}\t_\t_\t{4}\t_\t_\t_".format(
+                    rev[dependent][0].dependent,
+                    rev[dependent][0].dependent_form,
+                    rev[dependent][0].dependent_lemma,
+                    rev[dependent][0].dependent_pos,
+                    rev[dependent][0].head
+                )
+            print >> out, ""
 
     elif mode == "error":
         rev = reverse_head_graph(graph)
-        out = codecs.open(out_file, "a", "utf-8")
-        for dependent in sorted(rev):
-            print >> out, "%s\t%s\t%s\t%s\t_\t_\t-1\t_\t_\t_" % (
-                rev[dependent][0].dependent,
-                rev[dependent][0].dependent_form,
-                rev[dependent][0].dependent_lemma,
-                rev[dependent][0].dependent_pos,
-            )
-        print >> out, ""
-        out.close()
+        with codecs.open(out_file, "a", "utf-8") as out:
+            for dependent in sorted(rev):
+                print >> out, "{0}\t{1}\t{2}\t{3}\t_\t_\t-1\t_\t_\t_".format(
+                    rev[dependent][0].dependent,
+                    rev[dependent][0].dependent_form,
+                    rev[dependent][0].dependent_lemma,
+                    rev[dependent][0].dependent_pos,
+                )
+            print >> out, ""
 
 
 def check_graph_sanity(predicted_graph, compare_graph):  # sanity check on graph
@@ -257,11 +255,11 @@ def check_graph_sanity(predicted_graph, compare_graph):  # sanity check on graph
         if head == 0:
             root_found = True
 
-            if len(predicted_graph[head]) < 1:
+            if not predicted_graph[head]:
                 sane = False
                 print "Root has no dependent"
 
-        elif len(predicted_graph[head]) < 1:
+        elif not predicted_graph[head]:
             sane = False
             print "A head has no dependent"
 

modules/evaluation.py

@@ -1,28 +1,24 @@
 #!/bin/bash
 
 
-#CORPORA="../dependency-parsing-files/data/english"
-CORPORA="../dependency-parsing-files/data/german"
+CORPORA="../dependency-parsing-files/data/english"
+#CORPORA="../dependency-parsing-files/data/german"
 
 MODELS="models"
 PREDICTIONS="predictions"
 EVALUATIONS="evaluations"
 
 #train (on small set)
-#python -u dependency-parser.py -train -i $CORPORA/train/wsj_train.first-5k.conll06 -m model -e 20
+python -u dependency-parser.py -train -i $CORPORA/train/wsj_train.first-5k.conll06 -m model -e 1
 
 #train
-#python -u dependency-parser.py -train -i ../dependency-parsing-files/data/english/train/wsj_train.conll06 -m $MODELS/m_e-10_da_ss -e 2 -decrease-alpha -shuffle-sentences
+#python -u dependency-parser.py -train -i .$CORPORA//train/wsj_train.conll06 -m $MODELS/m_e-10_da_ss -e 2 -decrease-alpha -shuffle-sentences
 
 #test
-#python -u dependency-parser.py -test -i $CORPORA/dev/wsj_dev_without_head.conll06 -m model -o predicted.conll06
-#python -u dependency-parser.py -test -i $CORPORA/dev/wsj_dev_without_head.conll06 -m $MODELS/english/m_e-1 -o $PREDICTIONS/english/predicted_e-1.conll06
+python -u dependency-parser.py -test -i $CORPORA/dev/wsj_dev_without_head.conll06 -m model -o predicted.conll06
 
-#python -u dependency-parser.py -ev -i predicted.conll06 -g $CORPORA/dev/wsj_dev.conll06 -o evaluation_sentence.txt
-python -u dependency-parser.py -ev -i $PREDICTIONS/german/p_e-1.conll06 -g $CORPORA/dev/tiger-2.2.dev.conll06 -o $EVALUATIONS/german/evaluation_sentence_e-1.txt
-
-
-#evwaluate
-#./eval07.pl -g $CORPORA/dev/wsj_dev.conll06 -s  predicted.conll06 >> evaluation.txt 2>&1
-./eval07.pl -g $CORPORA/dev/tiger-2.2.dev.conll06 -s  $PREDICTIONS/german/p_e-1.conll06 >> $EVALUATIONS/german/evaluation_e-1.txt 2>&1
+#evaluate sentence based
+python -u dependency-parser.py -ev -i predicted.conll06 -g $CORPORA/dev/wsj_dev.conll06 -o evaluation_sentence.txt
 
+#evaluate arc based
+./eval07.pl -g $CORPORA/dev/wsj_dev.conll06 -s  predicted.conll06 >> evaluation.txt 2>&1