Merge pull request #1 from kuefmz/updates

update readme and remove unused files

Author: kuefmz

README.md

@@ -1,40 +1,92 @@
 # Define a small taxonomy (~200 terms) for AI/ML domain
 
+
+## Overview
+
+
 The data folder is not complete as some of the sources are too large for GitHub
 
 Collections that are considered:
 - **OpenAlex**: data accessed on 14th February 2024
 - **OpenAIRE**: data accessed on 15th February 2024
 
-## Method/Usage
 
+## Methodology
+
+The methodology used can be seen on the image bellow
+
+
+![methodology](./plots/methodology.png)
+
+
+## Usage
+
+Bellow you can find the commands that need to be run to get to execute the methodology.
+
+### Step1
 First we need to collect papers and their categories from OpenAlex and OpenAIRE and align them.
 
 For this run the script 
 
-
-# Step1
+```
 python3 src/collect_initial_data.py
+```
 
 
 Some of the collected categories are noisy, this script also cleans that up.
 
-Ones the data is collected we need to look for the best match for each category from each collection because we want to map them
+### Step2
 
-For this run the script:
+Ones the data is collected we need to look for the match for each category from each collection because we want to map them. The matches are selected based on their semantic similarity.
 
+For this run the script.
 
-# Step2
+```
 python3 src/select_the_best_match_from_category_pairs.py
+```
+
+### Step3
+
+Now that the papers with their mapped categories are ready and the category similarities, it is time to upload to a Neo4J database. This can take several days, depending on the machine that is used for the database and the amount of data collected.
+
+
+```
+python3 src/store_data_into_neo4j.py
+```
+
+### Step4
+
+Ones the data is stored into the database we run a query to select all the category pairs and the supporting papers for both categories and their similarity.
+
+```
+python3 src/run_ne04j_query.py
+```
+
+### Step 5
+
+
+Now that we have the pairs, we can run another script to transform the query results to a CSV and also calculate the missing 'Agreement' metric for the analysis.
+
+```
+python3 scr/transform_result_from_json_to_csv.py
+```
+
+### Step 6
+
+This will generate the final data. The analysis is performed based on this data. The goal of this analysis is to produce a set of mapping alignments that can be aligned in the different knowledge graphs. 
+
+### Step 7
 
+The mapping produced in the previous steps should be manually validated.
 
+Assuming that the generated mapping is stored in a CSV, the following scipt collects all the papers from the collected data that belong to both of the aligned categories.
 
-To get all the best pair combinations run:
 
+```
+python3 scr/validation.py
+```
 
-# Step3.1
+These papers should be manually validated by domain experts.
 
-python3 src/get_all_the_pair_combinations.py
 
 
-To be continued.

plots/agreement_distplot.png

@@ -1,6 +1,6 @@
-<mxfile host="app.diagrams.net" modified="2024-03-14T14:59:21.439Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" etag="_qYvj7vXzO1W5TcUiSkq" version="24.0.4" type="device">
+<mxfile host="app.diagrams.net" modified="2024-04-08T16:41:30.263Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" etag="8G2JoxbQfByx-54JaGIW" version="24.2.2" type="device">
   <diagram name="1 oldal" id="1S5QFZVV1yIsBkyI2sdZ">
-    <mxGraphModel dx="1434" dy="798" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="1654" math="0" shadow="0">
+    <mxGraphModel dx="1364" dy="913" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="1654" math="0" shadow="0">
       <root>
         <mxCell id="0" />
         <mxCell id="1" parent="0" />
@@ -13,7 +13,7 @@
         <mxCell id="KIzjvLAE8B5wqseCv-Lj-1" value="&lt;font style=&quot;font-size: 14px;&quot;&gt;1. Data collection&lt;/font&gt;&lt;div style=&quot;font-size: 14px;&quot;&gt;&lt;font style=&quot;font-size: 14px;&quot;&gt;(papers, categories)&lt;/font&gt;&lt;/div&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;fontColor=#333333;container=0;" parent="1" vertex="1">
           <mxGeometry x="50" y="220" width="130.00000000000003" height="60" as="geometry" />
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-23" value="" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0.854;entryY=0.033;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="KIzjvLAE8B5wqseCv-Lj-8" target="KIzjvLAE8B5wqseCv-Lj-9">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-23" value="" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0.854;entryY=0.033;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" source="KIzjvLAE8B5wqseCv-Lj-8" target="KIzjvLAE8B5wqseCv-Lj-9" edge="1">
           <mxGeometry width="50" height="50" relative="1" as="geometry">
             <mxPoint x="1080" y="190" as="sourcePoint" />
             <mxPoint x="780.0000000000002" y="320" as="targetPoint" />
@@ -22,9 +22,9 @@
             </Array>
           </mxGeometry>
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-27" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Calculate similariy&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;value of category paires&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" vertex="1" connectable="0" parent="2AmnZrVYBXidMmyjmQXf-23">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-27" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Calculate similariy&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;value of category paires&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" parent="2AmnZrVYBXidMmyjmQXf-23" vertex="1" connectable="0">
           <mxGeometry x="0.04" y="-2" relative="1" as="geometry">
-            <mxPoint x="-65" y="8" as="offset" />
+            <mxPoint x="-62" y="-2" as="offset" />
           </mxGeometry>
         </mxCell>
         <mxCell id="KIzjvLAE8B5wqseCv-Lj-7" value="&lt;font style=&quot;font-size: 14px;&quot;&gt;2. Align papers from different KGs&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;fontColor=#333333;container=0;" parent="1" vertex="1">
@@ -33,13 +33,13 @@
         <mxCell id="KIzjvLAE8B5wqseCv-Lj-8" value="&lt;font style=&quot;font-size: 14px;&quot;&gt;3. Preprocessing&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;container=0;" parent="1" vertex="1">
           <mxGeometry x="520" y="220" width="130.00000000000003" height="60" as="geometry" />
         </mxCell>
-        <mxCell id="KIzjvLAE8B5wqseCv-Lj-13" value="" style="endArrow=classic;html=1;rounded=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" target="KIzjvLAE8B5wqseCv-Lj-7" edge="1" source="KIzjvLAE8B5wqseCv-Lj-1">
+        <mxCell id="KIzjvLAE8B5wqseCv-Lj-13" value="" style="endArrow=classic;html=1;rounded=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="KIzjvLAE8B5wqseCv-Lj-1" target="KIzjvLAE8B5wqseCv-Lj-7" edge="1">
           <mxGeometry width="50" height="50" relative="1" as="geometry">
             <mxPoint x="227.4418604651163" y="250" as="sourcePoint" />
             <mxPoint x="299.5348837209302" y="200" as="targetPoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-13" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Finalize&amp;nbsp;&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;collected&amp;nbsp;&lt;/font&gt;&lt;/div&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;data&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" vertex="1" connectable="0" parent="KIzjvLAE8B5wqseCv-Lj-13">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-13" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Finalize&amp;nbsp;&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;collected&amp;nbsp;&lt;/font&gt;&lt;/div&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;data&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" parent="KIzjvLAE8B5wqseCv-Lj-13" vertex="1" connectable="0">
           <mxGeometry x="-0.3684" y="4" relative="1" as="geometry">
             <mxPoint x="10" y="4" as="offset" />
           </mxGeometry>
@@ -50,7 +50,7 @@
             <mxPoint x="299.5348837209302" y="260" as="targetPoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-14" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Finalize&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;paper&amp;nbsp;&lt;/font&gt;&lt;div style=&quot;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;alignments&lt;/font&gt;&lt;/div&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" vertex="1" connectable="0" parent="KIzjvLAE8B5wqseCv-Lj-14">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-14" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Finalize&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;paper&amp;nbsp;&lt;/font&gt;&lt;div style=&quot;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;alignments&lt;/font&gt;&lt;/div&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" parent="KIzjvLAE8B5wqseCv-Lj-14" vertex="1" connectable="0">
           <mxGeometry x="-0.2421" y="1" relative="1" as="geometry">
             <mxPoint x="9" as="offset" />
           </mxGeometry>
@@ -65,7 +65,7 @@
             </Array>
           </mxGeometry>
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-1" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Collect more data&lt;/font&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" vertex="1" connectable="0" parent="cRcOPCo_2h5ppIKabsQA-2">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-1" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Collect more data&lt;/font&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" parent="cRcOPCo_2h5ppIKabsQA-2" vertex="1" connectable="0">
           <mxGeometry x="0.2778" y="2" relative="1" as="geometry">
             <mxPoint x="25" y="-2" as="offset" />
           </mxGeometry>
@@ -80,7 +80,7 @@
             </Array>
           </mxGeometry>
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-2" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Align preprocessed papers&lt;/font&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" vertex="1" connectable="0" parent="cRcOPCo_2h5ppIKabsQA-3">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-2" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Align preprocessed papers&lt;/font&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" parent="cRcOPCo_2h5ppIKabsQA-3" vertex="1" connectable="0">
           <mxGeometry x="0.0889" y="-1" relative="1" as="geometry">
             <mxPoint x="8" as="offset" />
           </mxGeometry>
@@ -91,7 +91,7 @@
         <mxCell id="KIzjvLAE8B5wqseCv-Lj-9" value="&lt;font style=&quot;font-size: 14px;&quot;&gt;4. Propose candidate mapping&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;container=0;" parent="1" vertex="1">
           <mxGeometry x="720" y="380" width="130" height="60" as="geometry" />
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-22" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="KIzjvLAE8B5wqseCv-Lj-10" target="KIzjvLAE8B5wqseCv-Lj-11">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-22" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="KIzjvLAE8B5wqseCv-Lj-10" target="KIzjvLAE8B5wqseCv-Lj-11" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
         <mxCell id="KIzjvLAE8B5wqseCv-Lj-10" value="&lt;font style=&quot;font-size: 14px;&quot;&gt;5. Evaluate candidate mapping&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;container=0;" parent="1" vertex="1">
@@ -109,7 +109,7 @@
             <mxPoint x="146.55172413793105" y="420" as="targetPoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-24" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Calculate&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;metrics&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" vertex="1" connectable="0" parent="KIzjvLAE8B5wqseCv-Lj-16">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-24" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Calculate&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;metrics&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" parent="KIzjvLAE8B5wqseCv-Lj-16" vertex="1" connectable="0">
           <mxGeometry x="0.15" y="-3" relative="1" as="geometry">
             <mxPoint x="12" y="3" as="offset" />
           </mxGeometry>
@@ -120,7 +120,7 @@
             <mxPoint x="367.2413793103448" y="420" as="targetPoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-25" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Define&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;threshold&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" vertex="1" connectable="0" parent="KIzjvLAE8B5wqseCv-Lj-17">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-25" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Define&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;threshold&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" parent="KIzjvLAE8B5wqseCv-Lj-17" vertex="1" connectable="0">
           <mxGeometry x="0.1" y="1" relative="1" as="geometry">
             <mxPoint x="10" y="1" as="offset" />
           </mxGeometry>
@@ -131,7 +131,7 @@
             <mxPoint x="700" y="410" as="targetPoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="2AmnZrVYBXidMmyjmQXf-26" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Select&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;papers&lt;/font&gt;&lt;/div&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;to validate&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" vertex="1" connectable="0" parent="KIzjvLAE8B5wqseCv-Lj-18">
+        <mxCell id="2AmnZrVYBXidMmyjmQXf-26" value="&lt;font style=&quot;font-size: 13px;&quot;&gt;Select&lt;/font&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;papers&lt;/font&gt;&lt;/div&gt;&lt;div style=&quot;font-size: 13px;&quot;&gt;&lt;font style=&quot;font-size: 13px;&quot;&gt;to validate&lt;/font&gt;&lt;/div&gt;" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];container=0;" parent="KIzjvLAE8B5wqseCv-Lj-18" vertex="1" connectable="0">
           <mxGeometry x="-0.175" relative="1" as="geometry">
             <mxPoint x="-9" as="offset" />
           </mxGeometry>