Adjust feature set

Author: YNedderhoff

modules/featmap.py

@@ -2,60 +2,95 @@
 import codecs
 
 
-def give_features(hform, hlemma, hpos, dform, dlemma, dpos, bpos, direction, distance):
-
+def give_features(hform, hlemma, hpos, dform, dlemma, dpos, hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos,
+                  direction, distance):
     # generator that yields features based on the following information:
 
     # 1 = hform
     # 2 = hpos
     # 3 = dform
     # 4 = dpos
-    # 5 = bpos
-    # 6 = hlemma
-    # 7 = dlemma
-
-    yield u'1,dir,dist:{0},{1},{2}'.format(hform, direction, distance)
-    yield u'2,dir,dist:{0},{1},{2}'.format(hpos, direction, distance)
-    yield u'3,dir,dist:{0},{1},{2}'.format(dform, direction, distance)
-    yield u'4,dir,dist:{0},{1},{2}'.format(dpos, direction, distance)
-    yield u'6,dir,dist:{0},{1},{2}'.format(hlemma, direction, distance)
-    yield u'7,dir,dist:{0},{1},{2}'.format(dlemma, direction, distance)
-    yield u'5,dir,dist:{0},{1},{2}'.format(bpos, direction, distance)
-
-    yield u'1,4,dir,dist:{0},{1},{2},{3}'.format(hform, dpos, direction, distance)
-    yield u'2,3,dir,dist:{0},{1},{2},{3}'.format(hpos, dform, direction, distance)
-    yield u'1,2,dir,dist:{0},{1},{2},{3}'.format(hform, hpos, direction, distance)
-    yield u'3,4,dir,dist:{0},{1},{2},{3}'.format(dform, dpos, direction, distance)
-    yield u'1,3,dir,dist:{0},{1},{2},{3}'.format(hform, dform, direction, distance)
-    yield u'2,4,dir,dist:{0},{1},{2},{3}'.format(hpos, dpos, direction, distance)
-    yield u'6,4,dir,dist:{0},{1},{2},{3}'.format(hlemma, dpos, direction, distance)
-    yield u'2,7,dir,dist:{0},{1},{2},{3}'.format(hpos, dlemma, direction, distance)
-    yield u'6,2,dir,dist:{0},{1},{2},{3}'.format(hlemma, hpos, direction, distance)
-    yield u'7,4,dir,dist:{0},{1},{2},{3}'.format(dlemma, dpos, direction, distance)
-    yield u'6,7,dir,dist:{0},{1},{2},{3}'.format(hlemma, dlemma, direction, distance)
-
-    yield u'1,2,3,4,dir,dist:{0},{1},{2},{3},{4},{5}'.format(hform, hpos, dform, dpos, direction, distance)
-    yield u'2,3,4,dir,dist:{0},{1},{2},{3},{4}'.format(hpos, dform, dpos, direction, distance)
-    yield u'1,3,4,dir,dist:{0},{1},{2},{3},{4}'.format(hform, dform, dpos, direction, distance)
-    yield u'1,2,3,dir,dist:{0},{1},{2},{3},{4}'.format(hform, hpos, dform, direction, distance)
-    yield u'1,2,4,dir,dist:{0},{1},{2},{3},{4}'.format(hform, hpos, dpos, direction, distance)
-    yield u'2,5,4,dir,dist:{0},{1},{2},{3},{4}'.format(hpos, bpos, dpos, direction, distance)
-    yield u'2,5,3,dir,dist:{0},{1},{2},{3},{4}'.format(hpos, bpos, dform, direction, distance)
-    yield u'1,5,4,dir,dist:{0},{1},{2},{3},{4}'.format(hform, bpos, dpos, direction, distance)
-    yield u'1,5,3,dir,dist:{0},{1},{2},{3},{4}'.format(hform, bpos, dform, direction, distance)
-
-    yield u'6,2,7,4,dir,dist:{0},{1},{2},{3},{4},{5}'.format(hlemma, hpos, dlemma, dpos, direction, distance)
-    yield u'2,7,4,dir,dist:{0},{1},{2},{3},{4}'.format(hpos, dlemma, dpos, direction, distance)
-    yield u'6,7,4,dir,dist:{0},{1},{2},{3},{4}'.format(hlemma, dlemma, dpos, direction, distance)
-    yield u'6,2,7,dir,dist:{0},{1},{2},{3},{4}'.format(hlemma, hpos, dlemma, direction, distance)
-    yield u'6,2,4,dir,dist:{0},{1},{2},{3},{4}'.format(hlemma, hpos, dpos, direction, distance)
-    yield u'2,5,7,dir,dist:{0},{1},{2},{3},{4}'.format(hpos, bpos, dlemma, direction, distance)
-    yield u'6,5,4,dir,dist:{0},{1},{2},{3},{4}'.format(hlemma, bpos, dpos, direction, distance)
-    yield u'6,5,7,dir,dist:{0},{1},{2},{3},{4}'.format(hlemma, bpos, dlemma, direction, distance)
+    # 5 = hlemma
+    # 6 = dlemma
+
+    # 7 = hrform
+    # 8 = hrpos
+    # 9 = hlform
+    # 10 = hlpos
+    # 11 = drform
+    # 12 = drpos
+    # 13 = dlform
+    # 14 = dlpos
+    
+    # 15 = direction
+    # 16 = distance
+
+    yield u'1,15,16:{0},{1},{2}'.format(hform, direction, distance)
+    yield u'2,15,16:{0},{1},{2}'.format(hpos, direction, distance)
+    yield u'3,15,16:{0},{1},{2}'.format(dform, direction, distance)
+    yield u'4,15,16:{0},{1},{2}'.format(dpos, direction, distance)
+    yield u'5,15,16:{0},{1},{2}'.format(hlemma, direction, distance)
+    yield u'6,15,16:{0},{1},{2}'.format(dlemma, direction, distance)
+
+    yield u'1,4,15,16:{0},{1},{2},{3}'.format(hform, dpos, direction, distance)
+    yield u'2,3,15,16:{0},{1},{2},{3}'.format(hpos, dform, direction, distance)
+    yield u'1,2,15,16:{0},{1},{2},{3}'.format(hform, hpos, direction, distance)
+    yield u'3,4,15,16:{0},{1},{2},{3}'.format(dform, dpos, direction, distance)
+    yield u'1,3,15,16:{0},{1},{2},{3}'.format(hform, dform, direction, distance)
+    yield u'2,4,15,16:{0},{1},{2},{3}'.format(hpos, dpos, direction, distance)
+    yield u'5,4,15,16:{0},{1},{2},{3}'.format(hlemma, dpos, direction, distance)
+    yield u'2,6,15,16:{0},{1},{2},{3}'.format(hpos, dlemma, direction, distance)
+    yield u'5,2,15,16:{0},{1},{2},{3}'.format(hlemma, hpos, direction, distance)
+    yield u'6,4,15,16:{0},{1},{2},{3}'.format(dlemma, dpos, direction, distance)
+    yield u'5,6,15,16:{0},{1},{2},{3}'.format(hlemma, dlemma, direction, distance)
+
+    yield u'1,2,3,4,15,16:{0},{1},{2},{3},{4},{5}'.format(hform, hpos, dform, dpos, direction, distance)
+    yield u'2,3,4,15,16:{0},{1},{2},{3},{4}'.format(hpos, dform, dpos, direction, distance)
+    yield u'1,3,4,15,16:{0},{1},{2},{3},{4}'.format(hform, dform, dpos, direction, distance)
+    yield u'1,2,3,15,16:{0},{1},{2},{3},{4}'.format(hform, hpos, dform, direction, distance)
+    yield u'1,2,4,15,16:{0},{1},{2},{3},{4}'.format(hform, hpos, dpos, direction, distance)
+
+    yield u'5,2,6,4,15,16:{0},{1},{2},{3},{4},{5}'.format(hlemma, hpos, dlemma, dpos, direction, distance)
+    yield u'2,6,4,15,16:{0},{1},{2},{3},{4}'.format(hpos, dlemma, dpos, direction, distance)
+    yield u'5,6,4,15,16:{0},{1},{2},{3},{4}'.format(hlemma, dlemma, dpos, direction, distance)
+    yield u'5,2,6,15,16:{0},{1},{2},{3},{4}'.format(hlemma, hpos, dlemma, direction, distance)
+    yield u'5,2,4,15,16:{0},{1},{2},{3},{4}'.format(hlemma, hpos, dpos, direction, distance)
+
+    if hrform != "__NULL__":
+        yield u'7,15,16:{0},{1},{2}'.format(hrform, direction, distance)
+        yield u'1,7,15,16:{0},{1},{2},{3}'.format(hform, hrform, direction, distance)
+    if hrpos != "__NULL__":
+        yield u'8,15,16:{0},{1},{2}'.format(hrpos, direction, distance)
+        yield u'2,8,15,16:{0},{1},{2},{3}'.format(hpos, hrpos, direction, distance)
+    if hlform != "__NULL__":
+        yield u'9,15,16:{0},{1},{2}'.format(hlform, direction, distance)
+        yield u'9,1,15,16:{0},{1},{2},{3}'.format(hlform, hform, direction, distance)
+    if hlpos != "__NULL__":
+        yield u'10,15,16:{0},{1},{2}'.format(hlpos, direction, distance)
+        yield u'10,2,15,16:{0},{1},{2},{3}'.format(hlpos, hpos, direction, distance)
+    if drform != "__NULL__":
+        yield u'11,15,16:{0},{1},{2}'.format(drform, direction, distance)
+        yield u'3,11,15,16:{0},{1},{2},{3}'.format(dform, drform, direction, distance)
+    if drpos != "__NULL__":
+        yield u'12,15,16:{0},{1},{2}'.format(drpos, direction, distance)
+        yield u'4,12,15,16:{0},{1},{2},{3}'.format(dpos, drpos, direction, distance)
+    if dlform != "__NULL__":
+        yield u'13,15,16:{0},{1},{2}'.format(dlform, direction, distance)
+        yield u'13,3,15,16:{0},{1},{2},{3}'.format(dlform, dform, direction, distance)
+    if dlpos != "__NULL__":
+        yield u'14,15,16:{0},{1},{2}'.format(dlpos, direction, distance)
+        yield u'14,4,15,16:{0},{1},{2},{3}'.format(dlpos, dpos, direction, distance)
+    if hlform != "__NULL__" and hrform != "__NULL":
+        yield u'9,1,7,15,16:{0},{1},{2},{3},{4}'.format(hlform, hform, hrform, direction, distance)
+    if hlpos != "__NULL__" and hrpos != "__NULL":
+        yield u'10,2,8,15,16:{0},{1},{2},{3},{4}'.format(hlpos, hpos, hrpos, direction, distance)
+    if dlform != "__NULL__" and drform != "__NULL":
+        yield u'13,3,11,15,16:{0},{1},{2},{3},{4}'.format(dlform, dform, drform, direction, distance)
+    if dlpos != "__NULL__" and drpos != "__NULL__":
+        yield u'14,4,12,15,16:{0},{1},{2},{3},{4}'.format(dlpos, dpos, drpos, direction, distance)
 
 
 def give_distance(id1, id2, direction):
-
     if direction == "right":
         d = id1 - id2
     else:
@@ -80,7 +115,7 @@ def give_distance(id1, id2, direction):
         distance = "11-20"
     else:
         distance = ">20"
-    
+
     return distance
 
 
@@ -93,34 +128,82 @@ def give_direction(id1, id2):
     return direction
 
 
-def fm(infile):
+def give_surrounding_information(sentence, id1, id2):
+    hrform = "__NULL__"
+    hrpos = "__NULL__"
+    hlform = "__NULL__"
+    hlpos = "__NULL__"
+
+    drform = "__NULL__"
+    drpos = "__NULL__"
+    dlform = "__NULL__"
+    dlpos = "__NULL__"
+
+    if id1 not in [0, 1, len(sentence)]:
+        hrform = sentence[id1].form
+        hrpos = sentence[id1].pos
+        hlform = sentence[id1 - 2].form
+        hlpos = sentence[id1 - 2].pos
+    elif id1 == 0:
+        hrform = sentence[id1].form
+        hrpos = sentence[id1].pos
+    elif id1 == 1:
+        hrform = sentence[id1].form
+        hrpos = sentence[id1].pos
+    elif id1 == len(sentence):
+        hlform = sentence[id1 - 2].form
+        hlpos = sentence[id1 - 2].pos
+
+    if id2 not in [0, 1, len(sentence)]:
+        drform = sentence[id2].form
+        drpos = sentence[id2].pos
+        dlform = sentence[id2 - 2].form
+        dlpos = sentence[id2 - 2].pos
+    elif id2 == 0:
+        drform = sentence[id2].form
+        drpos = sentence[id2].pos
+    elif id2 == 1:
+        drform = sentence[id2].form
+        drpos = sentence[id2].pos
+    elif id2 == len(sentence):
+        dlform = sentence[id2 - 2].form
+        dlpos = sentence[id2 - 2].pos
+
+    return hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos
 
+
+def fm(infile):
     # takes a file in conll06 format, returns a feature map
     feat_map = {}  # featmap as dictionary {feature:index}
     index = 0  # index in featmap
 
     for sentence in sentences(codecs.open(infile, encoding='utf-8')):
+
         for token1 in sentence:
             direction = "left"
             distance = give_distance(0, token1.id, direction)
+            hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos = "__NULL__"
+
             # add root features
             for feature in give_features("__ROOT__", "__ROOT__", "__ROOT__", token1.form, token1.lemma, token1.pos,
-                                         token1.rel, direction, distance):
+                                         hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos, direction,
+                                         distance):
                 if feature not in feat_map:
                     feat_map[feature] = index
                     index += 1
 
             # add other features
             for token2 in (token2 for token2 in sentence if token2.id != token1.id):
 
-                # direction
                 direction = give_direction(token1.id, token2.id)
-
-                # distance
                 distance = give_distance(token1.id, token2.id, direction)
+                hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos = give_surrounding_information(sentence,
+                                                                                                          token1.id,
+                                                                                                          token2.id)
 
                 for feature in give_features(token1.form, token1.lemma, token1.pos, token2.form, token2.lemma,
-                                             token2.pos, token2.rel, direction, distance):
+                                             token2.pos, hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos,
+                                             direction, distance):
                     if feature not in feat_map:
                         feat_map[feature] = index
                         index += 1

modules/graphs.py

@@ -1,10 +1,10 @@
-from featmap import give_features, give_distance, give_direction
+from featmap import give_features, give_distance, give_direction, give_surrounding_information
 import codecs
 
 
 class Arc:
     def __init__(self, mode="sparse", head=None, dependent=None, head_form=None, dependent_form=None, head_lemma=None,
-                 dependent_lemma=None, head_pos=None, dependent_pos=None, dependent_rel=None, s=0.0):
+                 dependent_lemma=None, head_pos=None, dependent_pos=None, s=0.0):
         self.head = head
         self.dependent = dependent
         self.score = s
@@ -23,8 +23,6 @@ def __init__(self, mode="sparse", head=None, dependent=None, head_form=None, dep
             self.head_pos = head_pos
             self.dependent_pos = dependent_pos
 
-            self.rel = dependent_rel
-
 
 class ManualSparseGraph:  # heads has the same structure as in the other graph classes, but for addition of arcs
     def __init__(self):
@@ -54,12 +52,13 @@ def __init__(self, tokens, mode="sparse", feat_map=None, weight_vector=None):
                 if token1.head == 0:
                     direction = "left"
                     distance = give_distance(0, token1.id, direction)
-
+                    hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos = "__NULL__"
                     new_arc = Arc("sparse", 0, token1.id)
                     if feat_map is not None:
                         new_arc.feat_vec = [f for f in (feat_map[feature] for feature in
                                                         give_features("__ROOT__", "__ROOT__", "__ROOT__", token1.form,
-                                                                      token1.lemma, token1.pos, token1.rel, direction,
+                                                                      token1.lemma, token1.pos, hrform, hrpos, hlform,
+                                                                      hlpos, drform, drpos, dlform, dlpos, direction,
                                                                       distance) if feature in feat_map)]
                     self.heads[0].append(new_arc)
 
@@ -68,14 +67,18 @@ def __init__(self, tokens, mode="sparse", feat_map=None, weight_vector=None):
                 for token2 in (token2 for token2 in tokens if token2.head == token1.id):
                     direction = give_direction(token1.id, token2.id)
                     distance = give_distance(token1.id, token2.id, direction)
+                    hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos = give_surrounding_information(tokens,
+                                                                                                              token1.id,
+                                                                                                              token2.id)
 
                     new_arc = Arc("sparse", token1.id, token2.id)
                     if feat_map is not None:
                         new_arc.feat_vec = [f for f in (feat_map[feature] for feature in
                                                         give_features(token1.form, token1.lemma, token1.pos,
-                                                                      token2.form,
-                                                                      token2.lemma, token2.pos, token2.rel, direction,
-                                                                      distance) if feature in feat_map)]
+                                                                      token2.form, token2.lemma, token2.pos, hrform,
+                                                                      hrpos, hlform, hlpos, drform, drpos, dlform,
+                                                                      dlpos, direction, distance) if
+                                                        feature in feat_map)]
                     dependents.append(new_arc)
                 if dependents:
                     self.heads[token1.id] = dependents
@@ -88,17 +91,19 @@ def __init__(self, tokens, mode="sparse", feat_map=None, weight_vector=None):
 
                 direction = "left"
                 distance = give_distance(0, token1.id, direction)
+                hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos = "__NULL__"
 
                 if mode == "complete-sparse":
                     new_arc = Arc("sparse", 0, token1.id)
                 else:
                     new_arc = Arc("full", 0, token1.id, "__ROOT__", token1.form, "__ROOT__", token1.lemma, "__ROOT__",
-                                  token1.pos, token1.rel)
+                                  token1.pos)
 
                 new_arc.feat_vec = [f for f in (feat_map[feature] for feature in
                                                 give_features("__ROOT__", "__ROOT__", "__ROOT__", token1.form,
-                                                              token1.lemma, token1.pos, token1.rel, direction, distance)
-                                                if feature in feat_map)]
+                                                              token1.lemma, token1.pos, hrform, hrpos, hlform, hlpos,
+                                                              drform, drpos, dlform, dlpos, direction, distance) if
+                                                feature in feat_map)]
                 for feature in new_arc.feat_vec:
                     new_arc.score += weight_vector[feature]
                 self.heads[0].append(new_arc)
@@ -109,16 +114,20 @@ def __init__(self, tokens, mode="sparse", feat_map=None, weight_vector=None):
 
                     direction = give_direction(token1.id, token2.id)
                     distance = give_distance(token1.id, token2.id, direction)
+                    hrform, hrpos, hlform, hlpos, drform, drpos, dlform, dlpos = give_surrounding_information(tokens,
+                                                                                                              token1.id,
+                                                                                                              token2.id)
 
                     if mode == "complete-sparse":
                         new_arc = Arc("sparse", token1.id, token2.id)
                     else:
                         new_arc = Arc("full", token1.id, token2.id, token1.form, token2.form, token1.lemma,
-                                      token2.lemma, token1.pos, token2.pos, token2.rel)
+                                      token2.lemma, token1.pos, token2.pos)
 
                     new_arc.feat_vec = [f for f in (feat_map[feature] for feature in
                                                     give_features(token1.form, token1.lemma, token1.pos, token2.form,
-                                                                  token2.lemma, token2.pos, token2.rel, direction,
+                                                                  token2.lemma, token2.pos, hrform, hrpos, hlform,
+                                                                  hlpos, drform, drpos, dlform, dlpos, direction,
                                                                   distance) if feature in feat_map)]
                     for feature in new_arc.feat_vec:
                         new_arc.score += weight_vector[feature]

modules/token.py

@@ -14,7 +14,6 @@ def __init__(self, line):
             self.head = None
         else:
             self.head = int(entries[6])
-        self.rel = entries[7].rstrip()
 
 
 def sentences(file_stream):