Merge pull request #132 from IBM/semantic_operators

Semantic operators

Author: D3f0

.github/workflows/releases.yaml

@@ -18,6 +18,9 @@ on:
       release-method:
         description: "How the release was triggered (commit-message or tag-based)"
         value: ${{ jobs.check-release.outputs.release-method }}
+      prerelease:
+        description: "Whether this is a prerelease"
+        value: ${{ jobs.check-release.outputs.prerelease }}
 
 jobs:
   check-release:
@@ -28,6 +31,7 @@ jobs:
       current-version: ${{ steps.version.outputs.current-version }}
       next-version: ${{ steps.version.outputs.next-version }}
       release-method: ${{ steps.check-commit.outputs.release-method || steps.check-tag.outputs.release-method }}
+      prerelease: "false"
 
     steps:
       - uses: actions/checkout@v4
@@ -148,18 +152,16 @@ jobs:
           git push origin "v${NEXT_VERSION}"
 
       - name: Create GitHub Release
-        uses: actions/create-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: v${{ needs.check-release.outputs.next-version }}
-          release_name: Release v${{ needs.check-release.outputs.next-version }}
-          body: |
-            Automated release: ${{ needs.check-release.outputs.bump-type }} bump
+        run: |
+          gh release create v${{ needs.check-release.outputs.next-version }} \
+            --title "Release v${{ needs.check-release.outputs.next-version }}" \
+            --body "Automated release: ${{ needs.check-release.outputs.bump-type }} bump
 
-            Previous version: ${{ needs.check-release.outputs.current-version }}
-            New version: ${{ needs.check-release.outputs.next-version }}
+Previous version: ${{ needs.check-release.outputs.current-version }}
+New version: ${{ needs.check-release.outputs.next-version }}
 
-            Triggered via: ${{ needs.check-release.outputs.release-method }}
-          draft: false
-          prerelease: false
+Triggered via: ${{ needs.check-release.outputs.release-method }}" \
+            --draft=false \
+            --prerelease=false

README.md

@@ -11,6 +11,12 @@
   Build AI-powered pipelines as <b>typed data transformations</b>—combining Pydantic schemas, LLM-powered transduction, and async execution.
 </p>
 
+---
+
+## ✨ Why Agentics
+
+Most "agent frameworks" let untyped text flow through a pipeline. Agentics flips that: **types are the interface**.
+Workflows are expressed as transformations between structured states, with predictable schemas and composable operators.
 
 ---
 
@@ -120,7 +126,7 @@ Apache 2.0
 
 ## 👥 Authors
 
-**Project Lead**
+**Principal Investigator**
 - Alfio Massimiliano Gliozzo (IBM Research) — gliozzo@us.ibm.com
 
 **Core Contributors**

examples/discovery/agentic_db.py

@@ -0,0 +1,130 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import csv
+import json
+import os
+import sqlite3
+from pathlib import Path
+from typing import IO, Optional, Union
+
+import pandas as pd
+from pydantic import BaseModel, ConfigDict, PrivateAttr
+
+
+def read_table_smart(path: str) -> pd.DataFrame:
+    # read a small sample (first line or two)
+    with open(path, "r", encoding="utf-8") as f:
+        sample = f.readline()
+
+    # count potential delimiters
+    commas = sample.count(",")
+    tabs = sample.count("\t")
+    semicolons = sample.count(";")
+
+    # choose the most frequent one
+    if tabs > commas and tabs > semicolons:
+        sep = "\t"
+    elif semicolons > commas:
+        sep = ";"
+    else:
+        sep = ","
+
+    # print(f"Detected delimiter: {repr(sep)}")
+    return pd.read_csv(path, sep=sep, engine="python")
+
+
+class AgenticDB(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    name: Optional[str] = None
+    # _conn: sqlite3.Connection = PrivateAttr(default=None)
+    dataset_description: Optional[str] = None
+    name: Optional[str] = None
+    df: Optional[dict] = None
+    path: Optional[str] = None
+    columns: Optional[list] = None
+    metadata: Optional[dict] = None
+
+    @classmethod
+    def import_from_discovery_bench_metadata(cls, metadata_path: str):
+        output_dbs = []
+        metadata = json.load(open(metadata_path, "r"))
+
+        datadaset_csvs = [
+            metadata["datasets"][i]["name"]
+            for i in range(len(metadata["datasets"]))
+            if metadata["datasets"][i]["name"].endswith("csv")
+        ]
+        datadaset_txts = [
+            metadata["datasets"][i]["name"]
+            for i in range(len(metadata["datasets"]))
+            if metadata["datasets"][i]["name"].endswith("txt")
+        ]
+        datadaset_descriptions = [
+            metadata["datasets"][i]["description"]
+            for i in range(len(metadata["datasets"]))
+        ]
+        columns = [
+            metadata["datasets"][i]["columns"]["raw"]
+            for i in range(len(metadata["datasets"]))
+        ]
+        for dataset_csv, dataset_description, column in zip(
+            datadaset_csvs, datadaset_descriptions, columns
+        ):
+            data_path = Path(os.path.dirname(metadata_path)) / dataset_csv
+            database = AgenticDB(
+                dataset_description=dataset_description,
+                df=read_table_smart(data_path).to_dict(),
+                path=str(data_path),
+                name=dataset_csv,
+                columns=column,
+            )
+            # database.import_db_from_csv(data_path)
+            output_dbs.append(database)
+        return output_dbs
+
+    def import_db_from_csv(self, source: Union[str, IO[bytes]] = None):
+        """
+        Import a CSV file into an in-memory or file-based SQLite database.
+        Supports both:
+        - path to a CSV file (string)
+        - file-like buffer (from Streamlit uploader or similar)
+        """
+        # Handle file path or buffer
+        if isinstance(source, str):
+            self.path = source
+            self.df = pd.read_csv(source)
+        elif source is not None:
+            self.path = ":memory:"  # store in memory if no path
+            self.df = pd.read_csv(source)
+        elif self.path:
+            self.df = pd.read_csv(self.path)
+        else:
+            raise ValueError(
+                "Either a CSV file path or a file-like object must be provided."
+            )
+
+        # Connect to SQLite (memory or file)
+        # self._conn = sqlite3.connect(self.path)
+
+        # Write dataframe to DB
+        # self.df.to_sql("data", self._conn, index=False, if_exists="replace")
+
+        return self.df
+
+    def query_db(self, sql: str) -> pd.DataFrame:
+        """Execute an SQL query and return the result as a pandas DataFrame."""
+        try:
+            df = pd.read_sql_query(sql, self._conn)
+            return df
+        except Exception as e:
+            print(f"Error executing query: {e}")
+            return pd.DataFrame()
+
+    def get_description(self) -> str:
+        return f"""===============================
+Dataset Path: {self.path}
+Dataset Description: {self.dataset_description}
+Columns: {self.columns}
+"""

examples/discovery/atypes.py

@@ -0,0 +1,116 @@
+import json
+import os
+from pathlib import Path
+from typing import Optional, Union
+
+from agentic_db import AgenticDB
+from pydantic import BaseModel, Field
+
+from agentics import AG
+
+
+class IntermediateEvidence(BaseModel):
+    evidence_found: Optional[bool] = Field(
+        None,
+        description="Return True if you found any relevant evidence for the QUESTION, False otherwise",
+    )
+    original_question: Optional[str] = None
+    evidence: Optional[str] = Field(
+        None,
+        description="Identify useful information needed to answer the given QUESTION",
+    )
+    partial_answer: Optional[str] = Field(
+        None,
+        description="Provide a partial answer for the given Question bsed on the SOURCE data",
+    )
+
+
+class Answer(BaseModel):
+    short_answer: Optional[str] = Field(
+        None,
+        description="Provide a one sentence answer which specifically addresses the question",
+    )
+    full_answer: Optional[str] = Field(
+        None,
+        description="Provide a detailed answer for the given question taking into consideration the evidence sources provided",
+    )
+    selected_evidence: Optional[list[IntermediateEvidence]]
+    confidence: Optional[float] = None
+
+
+class Question(BaseModel):
+    qid: Optional[Union[int, str]] = None
+    true_hypothesis: Optional[str] = None
+    generated_hypothesis: Optional[str] = Field(
+        None,
+        description="A specific hypothesis that supports the question, derived from the analysis of the input dataset",
+    )
+    domain_knowledge: Optional[str] = None
+    question_type: Optional[str] = None
+    question: Optional[str] = None
+    dataset: Optional[str] = None
+    dbs: Optional[list[AgenticDB]] = None
+    intermediate_evidence: Optional[list[IntermediateEvidence]] = []
+    full_answer: Optional[Answer] = None
+
+    @classmethod
+    def import_questions_from_metadata_as_ag(
+        cls, metadata_path: str, import_dbs: bool = False
+    ) -> AG:
+        metadata = json.load(open(metadata_path, "r"))
+        dbs = AgenticDB.import_from_discovery_bench_metadata(metadata_path)
+        output = AG(atype=Question)
+        for question in metadata["queries"][0]:
+            question_obj = Question(**question)
+            question_obj.domain_knowledge = metadata.get("domain_knowledge")
+            if import_dbs:
+                question_obj.dbs = dbs
+            output.append(question_obj)
+        return output
+
+
+class Dataset(BaseModel):
+    metadata_path: Optional[str] = None
+    datasets_descriptions: Optional[list[str]] = None
+    dbs: Optional[list[AgenticDB]] = None
+    questions: Optional[list[Question]] = []
+
+    @classmethod
+    def import_from_discovery_bench_metadata(
+        cls,
+        dataset,
+        metadata_path="/Users/gliozzo/Code/agentics911/agentics/sandbox/discoverybench/discoverybench/real/demo",
+    ) -> str:
+        dataset_obj = Dataset()
+
+        if not dataset_obj.questions:
+            dataset_obj.questions = []
+        base_path = Path(metadata_path) / dataset
+        print("Importing dataset", end="")
+        for metadata in os.listdir(base_path):
+            if metadata.endswith(".json"):
+                if not dataset_obj.dbs:
+                    dataset_obj.dbs = AgenticDB.import_from_discovery_bench_metadata(
+                        base_path / metadata
+                    )
+                print(".", end=".")
+                dataset_obj.questions += Question.import_questions_from_metadata_as_ag(
+                    base_path / metadata, import_dbs=False
+                ).states
+        dataset_obj.get_source_descriptions()
+        return dataset_obj
+
+    def get_source_descriptions(self) -> str:
+        self.datasets_descriptions = []
+        for db in self.dbs:
+
+            self.datasets_descriptions.append(
+                f"""
+            Dataset Description: {db.dataset_description}
+            Columns: {db.columns}
+            """
+            )
+        return self.datasets_descriptions
+
+    def get_questions_as_ag(self) -> AG:
+        return AG(atype=Question, states=self.questions)