feat(graphs): create paper graphics

Author: AmitMY

README.md

@@ -22,7 +22,7 @@ the more effective the evaluation metric is in capturing the nuances of sign lan
 - [Tokenized BLEU](signwriting_evaluation/metrics/bleu.py) - BLEU score for tokenized SignWriting FSW strings.
 - [chrF](signwriting_evaluation/metrics/chrf.py) - chrF score for untokenized SignWriting FSW strings.
 - [CLIPScore](signwriting_evaluation/metrics/clipscore.py) - CLIPScore between SignWriting images. (Using the original CLIP model)
-- [Similarity](signwriting_evaluation/metrics/similarity.py) - symbol distance score for SignWriting FSW strings [(README)](signwriting_evaluation/metrics/similarity).
+- [Similarity](signwriting_evaluation/metrics/similarity.py) - symbol distance score for SignWriting FSW strings [(README)](signwriting_evaluation/metrics/similarity.md).
 
 ## Qualitative Evaluation
 
@@ -62,6 +62,17 @@ For each sign and metric, either the first match is incorrect, or there is a mor
 </tbody>
 </table>
 
+## Cite
+If you use our toolkit in your research or projects, please consider citing the work.
+
+```bib
+@misc{signwriting-evaluation2024,
+    title={SignWriting Evaluation: Metrics for Evaluating SignWriting Transcription and Translation Models},
+    author={Moryossef, Amit and Zilberman, Rotem and Langer, Ohad},
+    howpublished={\url{https://github.com/sign-language-processing/signwriting-evaluation}},
+    year={2024}
+}
+```
 
 ## References
 

assets/distribution/CHRF.png

@@ -14,6 +14,13 @@
 ASSETS_DIR = CURRENT_DIR.parent.parent / "assets"
 
 
+# Set the font to Times Roman
+plt.rcParams['font.family'] = 'Times New Roman'
+plt.rcParams['font.serif'] = ['Times New Roman'] + plt.rcParams['font.serif']
+# increase font size
+plt.rcParams.update({'font.size': 14})
+
+
 def load_signs(signs_file: Path):
     with open(signs_file, 'r', encoding='utf-8') as signs_f:
         signs = signs_f.read().splitlines()
@@ -78,6 +85,8 @@ def metrics_distribution(signs: list[str], metrics: list[SignWritingMetric]):
     plt.legend(loc="upper right")
     plt.tight_layout()
     plt.savefig(distribution_dir / "all.png")
+    plt.savefig(distribution_dir / "all.pdf")
+
     plt.close()
 
 

assets/distribution/CLIPScore.png

@@ -0,0 +1,44 @@
+# pylint: disable=line-too-long, duplicate-code
+from pathlib import Path
+
+if __name__ == "__main__":
+    matches_dir = Path(__file__).parent.parent.parent / "assets" / "matches"
+
+    # LaTeX header rows
+    signs_header = ["&"]
+    metrics_header = ["Rank"]
+
+    # LaTeX body rows
+    rows = [[str(i + 1)] for i in range(10)]  # 10 rows
+
+    for sign_dir in matches_dir.iterdir():
+        # pylint: disable=invalid-name
+        colspan = 0
+        for metric_dir in sign_dir.iterdir():
+            if metric_dir.is_dir():
+                colspan += 1
+                metrics_header.append(f"\\texttt{{{metric_dir.name}}}")
+                for i in range(10):
+                    rows[i].append(
+                        f"\\includegraphics[width=0.07\\textwidth]{{assets/matches/{sign_dir.name}/{metric_dir.name}/{i}.png}}"
+                    )
+        signs_header.append(
+            f"\\multicolumn{{{colspan}}}{{c|}}{{\\includegraphics[width=0.1\\textwidth]{{assets/matches/{sign_dir.name}/ref.png}}}}"
+        )
+
+    # Create LaTeX table
+    print("\\begin{table*}[ht]")
+    print("    \\centering")
+    print("    \\begin{tabular}{c|" + "c" * len(metrics_header) + "}")
+    print("        \\toprule")
+    print(f"        {' & '.join(signs_header)} \\\\")
+    print("        \\cmidrule{2-" + str(len(metrics_header) + 1) + "}")
+    print(f"        {' & '.join(metrics_header)} \\\\")
+    print("        \\midrule")
+    for row in rows:
+        print(f"        {' & '.join(row)} \\\\")
+    print("        \\bottomrule")
+    print("    \\end{tabular}")
+    print("    \\caption{Top 10 nearest neighbors for selected signs using different evaluation metrics. The reference signs are shown at the top, and the retrieved signs are displayed in order of decreasing similarity score from left to right.}")
+    print("    \\label{tab:nearest_neighbors}")
+    print("\\end{table*}")

assets/distribution/SymbolsDistances.png

@@ -7,7 +7,6 @@
 from PIL import Image
 from signwriting.visualizer.visualize import signwriting_to_image
 from tqdm import tqdm
-from transformers import AutoModel, AutoProcessor
 
 from signwriting_evaluation.metrics.base import SignWritingMetric
 
@@ -51,9 +50,10 @@ def __init__(self,
                  device=None):
         super().__init__(name="CLIPScore")
 
-        # Init CLIP model
-        self.model = AutoModel.from_pretrained(model_id)
+        # Init CLIP model pylint: disable=import-outside-toplevel
+        from transformers import AutoModel, AutoProcessor
         self.processor = AutoProcessor.from_pretrained(model_id)
+        self.model = AutoModel.from_pretrained(model_id)
 
         # Init cache
         if cache_directory is None: