added functions for counting entries

Author: anngoroshi

test2text/pages/documentation.py

@@ -1,5 +1,7 @@
 import streamlit as st
 
+from test2text.services.db import get_db_client
+
 
 def show_documentation():
     st.markdown("""
@@ -9,32 +11,50 @@ def show_documentation():
                 
                     **Test2Text** is a tool for computing requirement's coverage by tests and generating relevant reports.  
                     The application provides a convenient interface for analysis the relationships between test cases and requirements.
+                    
                 """)
     st.divider()
     st.markdown("""
                     ## HOW TO USE
                     
                     ### Upload data
-                    Click :gray-badge[:material/database_upload: Annotations] or :gray-badge[:material/database_upload: Requirements] to upload annotations and requirements from CSV files to the app's database. 
+                    Click :gray-badge[:material/database_upload: Annotations] or :gray-badge[:material/database_upload: Requirements] to upload annotations and requirements from CSV files to the app's database.  
+                    Then Annotations and Requirements are loaded and Test cases are linked to Annotations go to the next chapter.
                     
                     ### Renew data
                     Click :gray-badge[:material/cached: Controls] to transform missed and new texts into numeral vectors (embeddings).  
-                    Update distances by embeddings for intelligent matching of requirements and annotations.
+                    Update distances by embeddings for intelligent matching of Requirements and Annotations.  
+                    After distances are refreshed (all Annotations linked with Requirement by distances) go to the next chapter.
                     
                     ### Generate reports
                     Click :gray-badge[:material/publish: Requirement's Report] or :gray-badge[:material/publish: Test cases Report] to make a report.  
-                    Use filters to select desired information. Analyze selected requirements or test cases by showed and plotted distances
+                    Use filters and Smart search based on embeddings to select desired information.  
+                    Analyze selected requirements or test cases by plotted distances.  
+                    List of all requirements/test cases and their annotations are shown here.
                     
                     ### Visualize saved data 
-                    Click :gray-badge[:material/dataset: Visualize vectors] to plot distances between vector representations of all requirements and annotations.
+                    Click :gray-badge[:material/dataset: Visualize vectors] to plot distances between vector representations of all requirements and annotations in multidimensional spaces.
                 
                 """)
     st.divider()
+    with get_db_client() as db:
+        st.markdown("""## Database overview""")
+        table, row_count = st.columns(2)
+        with table:
+            st.write("Table name")
+        with row_count:
+            st.write("Number of entries")
+        for table_name, count in db.get_db_full_info.items():
+            with table:
+                st.write(table_name)
+            with row_count:
+                st.write(count)
+    st.divider()
     st.markdown("""    
                     ### Methodology
                     The application use a pre-trained transformer model from the [sentence-transformers library](https://huggingface.co/sentence-transformers), specifically [nomic-ai/nomic-embed-text-v1](https://huggingface.co/nomic-ai/nomic-embed-text-v1), a model trained to produce high-quality vector embeddings for text.  
                     The model returns, for each input text, a high-dimensional NumPy array (vector) of floating point numbers (the embedding).  
-                    This arrays give us a possibility to calculate Euclidian distances between test cases annotations and requirements to view how similar or dissimilar the two texts.  
+                    This arrays give a possibility to calculate Euclidian distances between test cases annotations and requirements to show how similar or dissimilar the two texts.  
                    """)
 
     st.markdown("""     
@@ -46,26 +66,26 @@ def show_documentation():
                     Suppose we have two vectors:
                 """)
     st.latex(r"""
-                    [ \mathbf{a} = [a_1, a_2, ..., a_n] ],
+                    \mathbf{a} = [a_1, a_2, ..., a_n] ,
             """)
     st.latex(r"""
-                    [ \mathbf{b} = [b_1, b_2, ..., b_n] ]
+                    \mathbf{b} = [b_1, b_2, ..., b_n] 
             """)
 
     st.markdown("""
                     The L2 distance between **a** and **b** is calculated as:
                 """)
 
     st.latex(r"""
-                    [ L_2(\mathbf{a}, \mathbf{b}) = \sqrt{(a_1 - b_1)^2 + (a_2 - b_2)^2 + \cdots + (a_n - b_n)^2} ]
+                    L_2(\mathbf{a}, \mathbf{b}) = \sqrt{(a_1 - b_1)^2 + (a_2 - b_2)^2 + \cdots + (a_n - b_n)^2}
     """)
 
     st.markdown("""
                     Or, more compactly:
     """)
 
     st.latex(r"""
-                    [ L_2(\mathbf{a}, \mathbf{b}) = \sqrt{\sum_{i=1}^n (a_i - b_i)^2} ]
+                    L_2(\mathbf{a}, \mathbf{b}) = \sqrt{\sum_{i=1}^n (a_i - b_i)^2}
     """)
 
     st.markdown(""" 

test2text/services/db/client.py

@@ -1,4 +1,6 @@
 import sqlite3
+from typing import Union
+
 import sqlite_vec
 import logging
 
@@ -71,4 +73,61 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
     def __enter__(self):
-        return self
+        return self
+
+    def get_table_names(self):
+        """
+        Returns a list of all user-defined tables in the database.
+
+        :return: List[str] - table names
+        """
+        cursor = self.conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';"
+        )
+        tables = [row[0] for row in cursor.fetchall()]
+        cursor.close()
+        return tables
+
+
+    @property
+    def get_db_full_info(self):
+        """
+        Returns table information:
+          - row_count: number of records in the table
+          - columns: list of dicts as in get_extended_table_info (name, type, non-NULL count, typeof distribution)
+
+        :return: dict
+        """
+        db_tables_info = {}
+        table_names = self.get_table_names()
+        for table_name in table_names:
+            row_count = self.count_all_entries_in_table(table_name)
+            db_tables_info.update({
+                table_name: row_count,
+            })
+        return db_tables_info
+
+    def count_all_entries_in_table(self, table: str) -> int:
+        count = self.conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
+        return count
+
+    def count_notnull_entries_in_table(self,column: str, table: str) -> Union[int, None]:
+        if self.has_column(column, table):
+            count = self.conn.execute(
+                f"SELECT COUNT(*) FROM {table} WHERE {column} IS NOT NULL"
+            ).fetchone()[0]
+            return count
+        return None
+
+    def has_column(self, column_name: str, table_name: str) -> bool:
+        """
+        Returns True if the table has a column, otherwise False.
+
+        :param column_name: name of the column
+        :param table_name: name of the table
+        :return: bool
+        """
+        cursor = self.conn.execute(f'PRAGMA table_info("{table_name}")')
+        columns = [row[1] for row in cursor.fetchall()]  # row[1] is the column name
+        cursor.close()
+        return column_name in columns

test2text/services/embeddings/annotation_embeddings_controls.py

@@ -5,18 +5,7 @@
 BATCH_SIZE = 30
 
 
-def count_all_annotations() -> int:
-    with get_db_client() as db:
-        count = db.conn.execute("SELECT COUNT(*) FROM Annotations").fetchone()[0]
-        return count
-
 
-def count_embedded_annotations() -> int:
-    with get_db_client() as db:
-        count = db.conn.execute(
-            "SELECT COUNT(*) FROM Annotations WHERE embedding IS NOT NULL"
-        ).fetchone()[0]
-        return count
 
 
 OnProgress = Callable[[float], None]
@@ -25,8 +14,8 @@ def count_embedded_annotations() -> int:
 def embed_annotations(*_, embed_all=False, on_progress: OnProgress = None):
     with get_db_client() as db:
         from .embed import embed_annotations_batch
-        annotations_count = count_all_annotations()
-        embedded_annotations_count = count_embedded_annotations()
+        annotations_count = db.count_all_entries_in_table("Annotations")
+        embedded_annotations_count = db.count_embedded_entries_in_table("Annotations")
         if embed_all:
             annotations_to_embed = annotations_count
         else: