Merge pull request #3 from PESU-IBM-GRM-zOS-AI-Inferencing/main

Port llama.cpp to z/OS

Author: HarithaIBM

.gitattributes

@@ -1,5 +1,6 @@
 *	text	working-tree-encoding=ISO8859-1
-*.pem   text working-tree-encoding=UTF-8
-*.png   binary
-*.gif binary
+*.pem text working-tree-encoding=UTF-8
+*.png -text
+*.gif -text
+*.mp4 -text
 *.pax.Z binary

README.md

@@ -1,6 +1,8 @@
-# Llamacpp
+[![Automatic version updates](https://github.com/zopencommunity/llama.cppport/actions/workflows/bump.yml/badge.svg)](https://github.com/ZOSOpenTools/llama.cppport/actions/workflows/bump.yml)
 
-A C++ library for writing high-performance network applications
+# llama.cpp
+
+Enable AI inferencing on z/os
 
 # Installation and Usage
 
@@ -27,7 +29,6 @@ See the [zopen porting guide](https://zopen.community/#/Guides/Porting) for more
 
 
 # Troubleshooting
-TBD
 
 # Contributing
-Contributions are welcome! Please follow the [zopen contribution guidelines](https://github.com/zopencommunity/meta/blob/main/CONTRIBUTING.md).
+Contributions are welcome! Please follow the [zopen contribution guidelines](https://github.com/zopencommunity/meta/blob/main/CONTRIBUTING.md).

buildenv

@@ -1,21 +1,23 @@
 export ZOPEN_BUILD_LINE="DEV"
-export ZOPEN_STABLE_DEPS="zoslib make cmake"
+export ZOPEN_STABLE_DEPS="zoslib make cmake curl"
 export ZOPEN_DEV_URL="https://github.com/ggerganov/llama.cpp.git"
-export ZOPEN_DEV_DEPS="zoslib make cmake"
+export ZOPEN_DEV_DEPS="zoslib make cmake curl openssl libssh2 zlib libpsl"
 export ZOPEN_CATEGORIES="ai"
-export ZOPEN_DEV_TAG="master-9e232f0"
-export ZOPEN_BUILD_LINE="DEV"
+export ZOPEN_DEV_TAG="master"
 export ZOPEN_NAME="llamacpp-master"
 export ZOPEN_RUNTIME_DEPS="ncurses"
 
-rm -f "llama"
-ln -s "llama.cpp" "llama"
-ln -s "llama.cpp" $ZOPEN_NAME
+# rm -f "llama"
+# ln -s "llama.cpp" "llama"
+# ln -s "llama.cpp" $ZOPEN_NAME
 
 export ZOPEN_COMP="CLANG"
+# set env variables
+# export CURL_HOME="/data/zopen/usr/local/zopen/curl/curl"
+# export BLAS_HOME="/usr/lpp/cbclib"
 
 export ZOPEN_CONFIGURE="cmake"
-export ZOPEN_CONFIGURE_OPTS="-B ../build --install-prefix \"\$ZOPEN_INSTALL_DIR/\" ."
+export ZOPEN_CONFIGURE_OPTS="-B ../build --install-prefix \"\$ZOPEN_INSTALL_DIR/\" -DCURL_LIBRARY=\$CURL_HOME/lib/libcurl.a -DCURL_INCLUDE_DIR=\$CURL_HOME/include -DBUILD_SHARED_LIBS_DEFAULT=OFF -DBUILD_SHARED_LIBS=OFF -DGGML_STATIC=ON -DGGML_BACKEND_DL=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS=\$BLAS_HOME/include/openblas -DBLAS_LIBRARIES=\$BLAS_HOME/lib/libopenblas.so -DLLAMA_BUILD_TESTS=ON ."
 
 export ZOPEN_MAKE="cmake"
 export ZOPEN_MAKE_OPTS="--build ../build --parallel \$ZOPEN_NUM_JOBS --config Release"
@@ -24,25 +26,39 @@ export ZOPEN_MAKE_MINIMAL=Y
 export ZOPEN_INSTALL="cmake"
 export ZOPEN_INSTALL_OPTS="--install ../build"
 
-# Needs more testing
-if false; then
-  export ZOPEN_EXTRA_CPPFLAGS="-DGGML_USE_OPENBLAS -I /home/itodoro/projects/openblas/openblas/include/openblas -mvx -mzvector -march=z15"
-  export ZOPEN_EXTRA_LIBS="/home/itodoro/projects/openblas/openblas/libopenblas.x"
-fi
+export ZOPEN_CHECK="ctest"
+export ZOPEN_CHECK_OPTS="--test-dir ../build  --output-on-failure"
 
-export ZOPEN_CHECK="skip"
+# if false; then
+#  export ZOPEN_EXTRA_CPPFLAGS="-DGGML_USE_OPENBLAS -I /home/itodoro/projects/openblas/openblas/include/openblas -mvx -mzvector -march=z15"
+#  export ZOPEN_EXTRA_LIBS="/home/itodoro/projects/openblas/openblas/libopenblas.x"
+# fi
 
 zopen_check_results()
 {
   dir="$1"
   pfx="$2"
   chk="$1/$2_check.log"
 
+  if [[ -f "$chk" ]]; then
+    total=$(grep -cE "Test #[0-9]+" "$chk")
+    failed=$(grep -cE "Failed|Subprocess aborted" "$chk")
+    skipped=$(grep -c "Skipped" "$chk")
+    passed=$((total - failed - skipped))
+  else
+    total=0
+    passed=0
+    failed=0
+    skipped=0
+  fi
+
   # Echo the following information to gauge build health
-  echo "actualFailures:0"
-  echo "totalTests:1"
+  echo "actualFailures:$failed"
+  echo "actualPassed:$passed"
+  echo "actualSkipped:$skipped"
+  echo "totalTests:$total"
   echo "expectedFailures:0"
-  echo "expectedTotalTests:1"
+  echo "expectedTotalTests:$total"
 }
 
 zopen_append_to_env()
@@ -60,4 +76,4 @@ zopen_get_version()
   # Modify to echo the version of your tool/library
   # Rather than hardcoding the version, obtain the version by running the tool/library
   echo "1.0.0"
-}
+}

cicd-stable.groovy

@@ -0,0 +1,14 @@
+node('linux')
+{
+  stage ('Poll') {
+    checkout([
+      $class: 'GitSCM',
+      branches: [[name: '*/main']],
+      doGenerateSubmoduleConfigurations: false,
+      extensions: [],
+      userRemoteConfigs: [[url: 'https://github.com/zopencommunity/llamacppport.git']]])
+  }
+  stage('Build') {
+    build job: 'Port-Pipeline', parameters: [string(name: 'PORT_GITHUB_REPO', value: 'https://github.com/zopencommunity/llamacppport.git'), string(name: 'PORT_DESCRIPTION', value: 'Enable AI inferencing on z/os' ), string(name: 'BUILD_LINE', value: 'STABLE') ]
+  }
+}

cicd.groovy

@@ -9,6 +9,6 @@ node('linux')
       userRemoteConfigs: [[url: 'https://github.com/zopencommunity/llamacppport.git']]])
   }
   stage('Build') {
-    build job: 'Port-Pipeline', parameters: [string(name: 'PORT_GITHUB_REPO', value: 'https://github.com/zopencommunity/llamacppport.git'), string(name: 'PORT_DESCRIPTION', value: "Port of Facebook's LLaMA model in C/C++" )]
+    build job: 'Port-Pipeline', parameters: [string(name: 'PORT_GITHUB_REPO', value: 'https://github.com/zopencommunity/llamacppport.git'), string(name: 'PORT_DESCRIPTION', value: 'Enable AI inferencing on z/os' ), string(name: 'BUILD_LINE', value: 'DEV') ]
   }
-}
+}

examples/README.md

@@ -0,0 +1,99 @@
+# zOpen Chat
+
+## Description
+**zOpen Chat** is a web-based interface that enables natural language interaction with AI models (like LLaMA 3.2 and Granite 3), designed to explore and work with open-source tools in the z/OS ecosystem.
+
+This project features:
+- A **web-based interface** for natural interactions.
+- Integration with **llamacpp** backend for inferencing.
+- Built-in tools to **search GitHub repositories** relevant to z/OS and files present in the **local file system**.
+
+## Use cases covered
+1. **Chat**: A conversational Q&A system where users can ask questions and receive clear, concise answers.
+2. **Explain Code**: Request short, contextual explanations for specific code files or components.
+3. **Generate Tests**: Ask the system to generate unit tests or test cases for specific files using natural language prompts.
+
+The files can be extracted from the repositories in `zopencommunity` or the `local file system`.
+> For usage examples and UI walkthroughs, see `docs/WEBUI.md`
+<!-- > Video Demonstration in `docs/videos/Final Use Cases Demo.mp4` -->
+
+## Prerequisites
+
+Before running zOpen Chat, ensure that the following are set up:
+- **llamacpp**: from the llamacpp port of z/OS. [Repository Link](https://github.com/zopencommunity/llamacppport)
+- **Node.js (LTS)**: [Download Node.js supported by z/OS](https://www.ibm.com/products/sdk-nodejs-compiler-zos)
+- **npm**: Comes with Node.js
+
+## Workflow
+
+The **Model Context Protocol (MCP)** is implemented here to orchestrate the routing in this system. It handles user inputs, tool management, and communication between the client interface and the underlying LLM infrastructure.
+
+- The client UI allows users to interact with the system via options like:
+    - Chat
+    - Explain Code
+    - Generate Test Cases
+
+- These requests are sent to the MCP Server, which serves as the orchestrator.
+
+- The MCP Server forwards the request to a Llama Server hosting the Large Language Model (LLM).
+
+- The LLM processes the input and returns a response to the MCP Server.
+
+- The response is routed back to the client UI, completing the workflow loop.
+
+> Folder structure details are explained in `docs/STRUCTURE.md`
+
+![Workflow](docs/images/workflow.png)
+
+## Setup
+
+### 1. Clone the Repository
+
+```bash
+git clone https://github.com/zopencommunity/llamacppport.git
+cd examples
+```
+
+### 2. Run LLaMA Inference Server
+Before starting any other servers, make sure the `llama-server` is running in the background.
+```bash
+llama-server -m /data/work/ai/models/granite-3.0-1b-a400m-instruct-be.Q4_K_M.gguf \
+             --host 127.0.0.1 \
+             --port 21099 \
+             --no-mmap \
+             --threads 1 \
+             -v
+```
+This command starts the LLaMA inference engine. Let it run in the background.
+
+### 3. Python Backend
+Create a virtual environment and activate it
+```bash
+python -m venv venv
+source venv/bin/activate
+```
+
+Install the dependencies
+``` bash
+cd backend
+pip install -r requirements.txt
+```
+
+Once the dependencies are installed run the Flask app
+```bash
+python app.py
+```
+This will start the Flask server on `http://127.0.0.1:21098`. To change the port, update the relevant port configuration in the `frontend/src/config.js` code.
+
+### 4. Frontend (Website)
+Install the node modules
+```bash
+cd frontend
+npm install
+```
+
+Start the website
+```bash
+npm start
+```
+The website can be accessed by going to `http://127.0.0.1:21097` on your web browser!

examples/backend/access_github.py

@@ -0,0 +1,31 @@
+import requests
+import base64
+import dotenv
+import os
+import re
+
+dotenv.load_dotenv()
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+if not GITHUB_TOKEN:
+    raise Exception("Set your GitHub Token from https://github.com/settings/tokens")
+
+def parse_input(prompt):
+    match = re.search("explain\s+(.*?)\s+(from|of|in)\s+([^\s]+)", prompt)
+    if match:
+        file, _, repo = match.groups()
+        return file.strip(), repo.strip()
+
+    return "Malformed prompt received. Please give the correct prompt."
+
+def get_file_info(file, repo, org='zopencommunity'):
+    api_url = f"https://api.github.com/repos/{org}/{repo}/contents/{file}"
+    headers = {"Authentication": GITHUB_TOKEN}
+    try:
+        res = requests.get(api_url, headers=headers)
+        res.raise_for_status()
+        data= res.json()
+        if data.get("encoding") == "base64":
+            return base64.b64decode(data["content"]).decode('utf-8', errors='ignore')
+        return data.get("content","")
+    except Exception as e:
+        return f"Exception: Could not fetch {file} from {repo} \n{e}"