diff --git a/agentic_rag/.gitignore b/agentic_rag/.gitignore index a7b9b8c..15c5616 100644 --- a/agentic_rag/.gitignore +++ b/agentic_rag/.gitignore @@ -7,6 +7,7 @@ __pycache__/ venv/ env/ .env +kubeconfig # IDE .vscode/ @@ -19,6 +20,27 @@ env/ embeddings/ chroma_db/ docs/*.json +**/.certs +**/node_modules +k8s/kustom/demo/config.yaml +k8s/kustom/demo/wallet/ +**/generated/ + +# Terraform +**/.terraform/* +*.plan +*.tfstate +*.tfstate.* +crash.log +crash.*.log +*.tfvars +*.tfvars.json +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraformrc +terraform.rc # Distribution / packaging dist/ diff --git a/agentic_rag/DEPLOY.md b/agentic_rag/DEPLOY.md new file mode 100644 index 0000000..3ecaefa --- /dev/null +++ b/agentic_rag/DEPLOY.md @@ -0,0 +1,120 @@ +# Deploy with Terraform and Kustomize + +## TODOS + +- Multiple containers for different functions + - Gradio + - Agents / Inference + - Database Access +- Hugging face token should be a secret +- Liveness and Readiness +- Use Load balancer instead of Gradio Live +- Autoscaling + +## Deploy Infrastructure + +Install scripts dependencies + +```bash +cd scripts/ && npm install && cd .. +``` + +Set environment (answer questions) and generate Terraform `tfvars` file. + +```bash +zx scripts/setenv.mjs +``` + +> Alternative: One liner for the yellow commands (for easy copy paste) +> +> ```bash +> cd tf && terraform init && terraform apply -auto-approve +> ``` + +Come back to root folder + +```bash +cd .. +``` + +Prepare Kubeconfig and namespace: + +```bash +zx scripts/kustom.mjs +``` + +## Deploy Application + +Export kubeconfig to get access to the Kubernetes Cluster + +```bash +export KUBECONFIG="$(pwd)/tf/generated/kubeconfig" +``` + +Check everything works + +```bash +kubectl cluster-info +``` + +Deploy the production overlay + +```bash +kubectl apply -k k8s/kustom/overlays/prod +``` + +Check all pods are Ready: + +```bash +kubectl get po --namespace=agentic-rag +``` + +Get Gradio Live URL: + +```bash +kubectl logs $(kubectl get po -n agentic-rag -l app=agentic-rag -o name) -n agentic-rag | grep "Running on public URL" +``` + +Open the URL from the command before in your browser. + +Also, you could get the Load Balancer Public IP address: + +```bash +echo "http://$(kubectl get service \ + -n agentic-rag \ + -o jsonpath='{.items[?(@.spec.type=="LoadBalancer")].status.loadBalancer.ingress[0].ip}')" +``` + +To troubleshoot connect to the container + +```bash +kubectl exec -it $(kubectl get po -n agentic-rag -l app=agentic-rag -o name) -n agentic-rag -- sh +``` + +## Clean up + +Delete the production overlay + +```bash +kubectl delete -k k8s/kustom/overlays/prod +``` + +Destroy infrastructure with Terraform. + +```bash +cd tf +``` + +```bash +terraform destroy -auto-approve +``` + +```bash +cd .. +``` + +Clean up the artifacts and config files + +```bash +zx scripts/clean.mjs +``` diff --git a/agentic_rag/OraDBVectorStore.py b/agentic_rag/OraDBVectorStore.py index 14b8c3a..70ef5cd 100644 --- a/agentic_rag/OraDBVectorStore.py +++ b/agentic_rag/OraDBVectorStore.py @@ -22,13 +22,21 @@ def __init__(self, persist_directory: str = "embeddings"): username = credentials.get("ORACLE_DB_USERNAME", "ADMIN") password = credentials.get("ORACLE_DB_PASSWORD", "") dsn = credentials.get("ORACLE_DB_DSN", "") + wallet_path = credentials.get("ORACLE_DB_WALLET_LOCATION") + wallet_password = credentials.get("ORACLE_DB_WALLET_PASSWORD") if not password or not dsn: raise ValueError("Oracle DB credentials not found in config.yaml. Please set ORACLE_DB_USERNAME, ORACLE_DB_PASSWORD, and ORACLE_DB_DSN.") # Connect to the database try: - conn23c = oracledb.connect(user=username, password=password, dsn=dsn) + if not wallet_path: + print(f'Connecting (no wallet) to dsn {dsn} and user {username}') + conn23c = oracledb.connect(user=username, password=password, dsn=dsn) + else: + print(f'Connecting (with wallet) to dsn {dsn} and user {username}') + conn23c = oracledb.connect(user=username, password=password, dsn=dsn, + config_dir=wallet_path, wallet_location=wallet_path, wallet_password=wallet_password) print("Oracle DB Connection successful!") except Exception as e: print("Oracle DB Connection failed!", e) diff --git a/agentic_rag/k8s/kustom/demo/config.yaml.mustache b/agentic_rag/k8s/kustom/demo/config.yaml.mustache new file mode 100644 index 0000000..ffb5ef9 --- /dev/null +++ b/agentic_rag/k8s/kustom/demo/config.yaml.mustache @@ -0,0 +1,6 @@ +HUGGING_FACE_HUB_TOKEN: "{{{ hugging_face_token }}}" +ORACLE_DB_USERNAME: "{{{ adb_username }}}" +ORACLE_DB_PASSWORD: "{{{ adb_admin_password }}}" +ORACLE_DB_DSN: "{{{ adb_service_name }}}" +ORACLE_DB_WALLET_LOCATION: "{{{ adb_wallet_location }}}" +ORACLE_DB_WALLET_PASSWORD: "{{{ adb_admin_password }}}" diff --git a/agentic_rag/k8s/kustom/demo/deployment.yaml b/agentic_rag/k8s/kustom/demo/deployment.yaml new file mode 100644 index 0000000..9371520 --- /dev/null +++ b/agentic_rag/k8s/kustom/demo/deployment.yaml @@ -0,0 +1,144 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agentic-rag + labels: + app: agentic-rag +spec: + replicas: 1 + selector: + matchLabels: + app: agentic-rag + template: + metadata: + labels: + app: agentic-rag + spec: + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "present" + effect: "NoSchedule" + initContainers: + - name: unzip + image: busybox + command: ["unzip", "/app/walletzip/wallet.zip", "-d", "/app/wallet"] + volumeMounts: + - name: wallet-config + mountPath: /app/walletzip + - name: wallet-volume + mountPath: /app/wallet + containers: + - name: agentic-rag + image: python:3.10-slim + resources: + requests: + memory: "8Gi" + cpu: "2" + ephemeral-storage: "50Gi" # Add this + limits: + memory: "16Gi" + cpu: "4" + ephemeral-storage: "100Gi" # Add this + ports: + - containerPort: 7860 + name: gradio + - containerPort: 11434 + name: ollama-api + volumeMounts: + - name: config-volume + mountPath: /app/config.yaml + subPath: config.yaml + - name: wallet-config + mountPath: /app/walletzip + - name: wallet-volume + mountPath: /app/wallet + - name: data-volume + mountPath: /app/embeddings + - name: chroma-volume + mountPath: /app/chroma_db + - name: ollama-models + mountPath: /root/.ollama + command: ["/bin/bash", "-c"] + args: + - | + apt-get update && apt-get install -y git curl gnupg + + # Install NVIDIA drivers and CUDA + echo "Installing NVIDIA drivers and CUDA..." + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + apt-get update && apt-get install -y nvidia-container-toolkit + + # Verify GPU is available + echo "Verifying GPU availability..." + nvidia-smi || echo "WARNING: nvidia-smi command failed. GPU might not be properly configured." + + # Install Ollama + echo "Installing Ollama..." + curl -fsSL https://ollama.com/install.sh | sh + + # Configure Ollama to use GPU + echo "Configuring Ollama for GPU usage..." + mkdir -p /root/.ollama + echo '{"gpu": {"enable": true}}' > /root/.ollama/config.json + + # Start Ollama in the background with GPU support + echo "Starting Ollama service with GPU support..." + ollama serve & + + # Wait for Ollama to be ready + echo "Waiting for Ollama to be ready..." + until curl -s http://localhost:11434/api/tags >/dev/null; do + sleep 5 + done + + # Verify models are using GPU + echo "Verifying models are using GPU..." + curl -s http://localhost:11434/api/tags | grep -q "llama3" && echo "llama3 model is available" + + # Clone and set up the application + cd /app + git clone -b agentic_rag_automation https://github.com/vmleon/devrel-labs.git + cd devrel-labs/agentic_rag + pip install -r requirements.txt + + # Move config.yaml file to agentic-rag folder + echo "Copying config.yaml to /app/devrel-labs/agentic_rag/config.yaml" + cp /app/config.yaml /app/devrel-labs/agentic_rag/config.yaml + + # Start the Gradio app + echo "Starting Gradio application..." + python gradio_app.py + env: + - name: PYTHONUNBUFFERED + value: "1" + - name: OLLAMA_HOST + value: "http://localhost:11434" + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "compute,utility" + - name: TORCH_CUDA_ARCH_LIST + value: "7.0;7.5;8.0;8.6" + volumes: + - name: config-volume + configMap: + name: agentic-rag-config + - name: wallet-config + configMap: + name: wallet-zip + - name: wallet-volume + emptyDir: + sizeLimit: 50Mi + - name: data-volume + persistentVolumeClaim: + claimName: agentic-rag-data-pvc + - name: chroma-volume + persistentVolumeClaim: + claimName: agentic-rag-chroma-pvc + - name: ollama-models + persistentVolumeClaim: + claimName: ollama-models-pvc \ No newline at end of file diff --git a/agentic_rag/k8s/kustom/demo/kustomization.yaml b/agentic_rag/k8s/kustom/demo/kustomization.yaml new file mode 100644 index 0000000..b9e2c26 --- /dev/null +++ b/agentic_rag/k8s/kustom/demo/kustomization.yaml @@ -0,0 +1,12 @@ +resources: + - pvcs.yaml + - deployment.yaml + - service.yaml +configMapGenerator: + - name: agentic-rag-config + files: + - config.yaml + - name: wallet-zip + files: + - wallet/wallet.zip +namespace: agentic-rag diff --git a/agentic_rag/k8s/kustom/demo/pvcs.yaml b/agentic_rag/k8s/kustom/demo/pvcs.yaml new file mode 100644 index 0000000..eb720a0 --- /dev/null +++ b/agentic_rag/k8s/kustom/demo/pvcs.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: agentic-rag-data-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: agentic-rag-chroma-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ollama-models-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi # Larger storage for model files \ No newline at end of file diff --git a/agentic_rag/k8s/kustom/demo/service.yaml b/agentic_rag/k8s/kustom/demo/service.yaml new file mode 100644 index 0000000..9de2620 --- /dev/null +++ b/agentic_rag/k8s/kustom/demo/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: agentic-rag + labels: + app: agentic-rag +spec: + type: LoadBalancer # Use NodePort if LoadBalancer is not available + ports: + - port: 80 + targetPort: 7860 + protocol: TCP + name: http + selector: + app: agentic-rag \ No newline at end of file diff --git a/agentic_rag/k8s/kustom/overlays/prod/kustomization.yaml b/agentic_rag/k8s/kustom/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..acbd030 --- /dev/null +++ b/agentic_rag/k8s/kustom/overlays/prod/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - "../../demo" \ No newline at end of file diff --git a/agentic_rag/k8s/kustom/test_gpu.yaml b/agentic_rag/k8s/kustom/test_gpu.yaml new file mode 100644 index 0000000..5169740 --- /dev/null +++ b/agentic_rag/k8s/kustom/test_gpu.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-test +spec: + containers: + - name: cuda-container + image: nvidia/cuda:11.6.2-base-ubuntu20.04 + command: ["nvidia-smi"] + resources: + limits: + nvidia.com/gpu: 1 + restartPolicy: Never \ No newline at end of file diff --git a/agentic_rag/scripts/clean.mjs b/agentic_rag/scripts/clean.mjs new file mode 100644 index 0000000..fb4ef29 --- /dev/null +++ b/agentic_rag/scripts/clean.mjs @@ -0,0 +1,31 @@ +#!/usr/bin/env zx +import Configstore from "configstore"; +import clear from "clear"; + +$.verbose = false; + +clear(); +console.log("Clean up config files, certs, ssh keys..."); + +const projectName = "arag"; + +const config = new Configstore(projectName, { projectName }); + +const privateKeyPath = config.get("privateKeyPath"); +await $`rm -f ${privateKeyPath}`; +const publicKeyPath = config.get("publicKeyPath"); +await $`rm -f ${publicKeyPath}`; + +const filesToDelete = [ + "./tf/generated", + "./tf/terraform.tfvars", + "./.certs", + "./k8s/kustom/demo/config.yaml", +]; + +filesToDelete.forEach(async (filePath) => { + await $`rm -rf ${filePath}`; + console.log(`File ${chalk.green(filePath)} deleted`); +}); + +config.clear(); diff --git a/agentic_rag/scripts/kustom.mjs b/agentic_rag/scripts/kustom.mjs new file mode 100644 index 0000000..2a92b09 --- /dev/null +++ b/agentic_rag/scripts/kustom.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env zx +import Configstore from "configstore"; +import clear from "clear"; +import Mustache from "mustache"; +import { parse, stringify } from "yaml"; +import { readFile, writeFile } from "node:fs/promises"; +import { exitWithError } from "./lib/utils.mjs"; +import { getOutputValues } from "./lib/terraform.mjs"; + +$.verbose = false; + +clear(); +console.log("Create kustomization configuration..."); + +const projectName = "arag"; + +const config = new Configstore(projectName, { projectName }); + +const profile = config.get("profile"); +const huggingfaceToken = config.get("huggingfaceToken"); + +const { adb_name, adb_admin_password } = await getOutputValues("./tf"); + +await addProfileToKubeconfig(profile); + +await createNamespace("agentic-rag"); +await createHuggingFaceTokenConfigMapFile(huggingfaceToken); +await copyWallet(); + +async function createNamespace(namespace) { + try { + const { exitCode, stdout } = + await $`KUBECONFIG="tf/generated/kubeconfig" kubectl \ + create namespace ${namespace} \ + -o yaml --dry-run=client --save-config | \ + KUBECONFIG="tf/generated/kubeconfig" kubectl apply -f -`; + if (exitCode !== 0) { + exitWithError(`${namespace} namespace not created`); + } else { + console.log(chalk.green(stdout)); + } + } catch (error) { + exitWithError(error.stderr); + } +} + +async function createHuggingFaceTokenConfigMapFile(huggingfaceToken) { + const configMapFilePath = "k8s/kustom/demo/config.yaml"; + + const ConfigMapTemplate = await fs.readFile( + `${configMapFilePath}.mustache`, + "utf-8" + ); + + const output = Mustache.render(ConfigMapTemplate, { + hugging_face_token: huggingfaceToken, + adb_username: "ADMIN", + adb_admin_password: adb_admin_password, + adb_service_name: `${adb_name}_high`, + adb_wallet_location: "/app/wallet", + }); + + await fs.writeFile(configMapFilePath, output); + + console.log(`File ${chalk.green(configMapFilePath)} created`); +} + +async function addProfileToKubeconfig(profile = "DEFAULT") { + if (profile === "DEFAULT") return; + + const kubeconfigPath = "./tf/generated/kubeconfig"; + + let yamlContent = await readFile(kubeconfigPath, { + encoding: "utf-8", + }); + + const profileFlag = "--profile"; + const profileValue = profile; + + const kubeconfig = parse(yamlContent); + const execArgs = kubeconfig.users[0].user.exec.args; + if (!execArgs.includes("--profile")) { + kubeconfig.users[0].user.exec.args = [ + ...execArgs, + profileFlag, + profileValue, + ]; + const newKubeconfigContent = stringify(kubeconfig); + + await writeFile(kubeconfigPath, newKubeconfigContent, { + encoding: "utf-8", + }); + } +} + +async function copyWallet() { + const walletPath = "k8s/kustom/demo/wallet"; + await $`mkdir -p ${walletPath}`; + const walletSourcePath = "tf/generated/wallet.zip"; + await $`cp ${walletSourcePath} ${walletPath}/`; + console.log(`File ${chalk.green(walletSourcePath)} copied`); +} diff --git a/agentic_rag/scripts/lib/README.md b/agentic_rag/scripts/lib/README.md new file mode 100644 index 0000000..52f6eb0 --- /dev/null +++ b/agentic_rag/scripts/lib/README.md @@ -0,0 +1,286 @@ +# OCI ZX Scripts + +## Usage + +From the root folder of your project, download the scripts folder: + +```sh +wget -cO - \ + https://github.com/vmleon/oci-zx-scripts/archive/refs/heads/main.zip > scripts.zip \ + && unzip scripts.zip -d scripts \ + && rm scripts.zip \ + && mv scripts/oci-zx-scripts-main scripts/lib +``` + +## Example + +Create a `setenv.mjs` script to prepare environment: + +```sh +touch scripts/setenv.mjs +``` + +Use the following example: + +```js +#!/usr/bin/env zx + +import { getNamespace } from "./lib/oci.mjs"; +const shell = process.env.SHELL | "/bin/zsh"; +$.shell = shell; +$.verbose = false; + +console.log("Check fake dependencies..."); +const dependencies = ["git", "unzip"]; +const namespace = await getNamespace(); +console.log(namespace); +``` + +Run the script: + +```sh +npx zx scripts/setenv.mjs +``` + +## Create new functions + +```javascript +#!/usr/bin/env zx +import { exitWithError } from "./utils.mjs"; + +export async function example() { + try { + const { stdout, exitCode, stderr } = await $`pwd`; + if (exitCode !== 0) { + exitWithError(stderr); + } + return stdout.trim(); + } catch (error) { + exitWithError(error.stderr); + } +} +``` + +## Get Region example + +```javascript +import { getRegions } from "./lib/oci.mjs"; +import { setVariableFromEnvOrPrompt, printRegionNames } from "./lib/utils.mjs"; + +const regions = await getRegions(); +const regionName = await setVariableFromEnvOrPrompt( + "OCI_REGION", + "OCI Region name", + async () => printRegionNames(regions) +); +const { key } = regions.find((r) => r.name === regionName); +const url = `${key}.ocir.io`; +console.log(url); +``` + +## Release Example + +```javascript +#!/usr/bin/env zx +import { + getVersion, + getNamespace, + printRegionNames, + setVariableFromEnvOrPrompt, +} from "./lib/utils.mjs"; +import { buildImage, tagImage, pushImage } from "./lib/container.mjs"; +import { buildWeb } from "./lib/npm.mjs"; +import { + buildJarGradle, + cleanGradle, + getVersionGradle, +} from "./lib/gradle.mjs"; +import { getRegions } from "./lib/oci.mjs"; + +const shell = process.env.SHELL | "/bin/zsh"; +$.shell = shell; +$.verbose = false; + +const { a, _ } = argv; +const [action] = _; + +const project = "project_name"; +const namespace = await getNamespace(); + +const regions = await getRegions(); +const regionName = await setVariableFromEnvOrPrompt( + "OCI_REGION", + "OCI Region name", + () => printRegionNames(regions) +); +const { key } = regions.find((r) => r.name === regionName); +const ocirUrl = `${key}.ocir.io`; + +if (action === "web") { + await releaseNpm("web"); + process.exit(0); +} + +if (action === "backend") { + await releaseGradle("backend"); + process.exit(0); +} + +if (a || action === "all") { + await releaseNpm("web"); + await releaseGradle("backend"); + process.exit(0); +} + +console.log("Usage:"); +console.log("\tnpx zx scripts/release.mjs all"); +console.log("\tnpx zx scripts/release.mjs -a"); +console.log("\tnpx zx scripts/release.mjs web"); +console.log("\tnpx zx scripts/release.mjs backend"); + +async function releaseNpm(service) { + await cd(service); + const currentVersion = await getVersion(); + if (service === "web") { + await buildWeb(); + } + const image_name = `${project}/${service}`; + await buildImage(`localhost/${image_name}`, currentVersion); + const local_image = `localhost/${image_name}:${currentVersion}`; + const remote_image = `${ocirUrl}/${namespace}/${image_name}:${currentVersion}`; + await tagImage(local_image, remote_image); + await pushImage(remote_image); + console.log(`Released: ${chalk.yellow(remote_image)}`); + await cd(".."); +} +async function releaseGradle(service) { + await cd(service); + await cleanGradle(); + await buildJarGradle(); + const currentVersion = await getVersionGradle(); + const image_name = `${project}/${service}`; + await buildImage(`localhost/${image_name}`, currentVersion); + const local_image = `localhost/${image_name}:${currentVersion}`; + const remote_image = `${ocirUrl}/${namespace}/${image_name}:${currentVersion}`; + await tagImage(local_image, remote_image); + await pushImage(remote_image); + console.log(`Released: ${chalk.yellow(remote_image)}`); + await cd(".."); +} +``` + +## Create RSA + +```javascript +#!/usr/bin/env zx + +import { getUserId, uploadApiKeyFile } from "./lib/oci.mjs"; +import { createRSAKeyPair } from "./lib/tls.mjs"; + +const shell = process.env.SHELL | "/bin/zsh"; +$.shell = shell; +$.verbose = false; + +async function createUserDetails() { + console.log("Generate RSA key pair..."); + + const userId = await getUserId(); + + properties = { + ...properties, + userId, + }; + + const rsaPath = "./.rsa"; + const prevKeyExists = await fs.pathExists(path.join(rsaPath, "rsa.pem")); + if (prevKeyExists) { + console.log(`${chalk.yellow("Existing RSA key pair ")} on ${rsaPath}.`); + } else { + await createRSAKeyPair(rsaPath); + const apikeyFingerprint = await uploadApiKeyFile( + userId, + path.join(rsaPath, "rsa_public.pem") + ); + properties = { ...properties, rsaPath, apikeyFingerprint }; + } + console.log(); +} +``` + +## Use .env.json + +```javascript +#!/usr/bin/env zx + +import { + getNamespace, + writeEnvJson, + generateRandomString, + readEnvJson, +} from "./lib/utils.mjs"; +import { getTenancyId } from "./lib/oci.mjs"; + +const shell = process.env.SHELL | "/bin/zsh"; +$.shell = shell; +$.verbose = false; + +let properties = await readEnvJson(); + +const namespace = await getNamespace(); +const tenancyId = await getTenancyId(); +properties = { ...properties, namespace, tenancyId }; +await writeEnvJson(properties); + +const generatedPassword = await generateRandomString(); +properties = { ...properties, generatedPassword }; + +await writeEnvJson(properties); +``` + +```javascript +#!/usr/bin/env zx + +import { readEnvJson } from "./lib/utils.mjs"; + +const shell = process.env.SHELL | "/bin/zsh"; +$.shell = shell; +$.verbose = false; + +const { namespace } = await readEnvJson(); +``` + +```javascript +#!/usr/bin/env zx + +import { setVariableFromEnvOrPrompt } from "./lib/utils.mjs"; +import { searchCompartmentIdByName } from "./lib/oci.mjs"; + +const shell = process.env.SHELL | "/bin/zsh"; +$.shell = shell; +$.verbose = false; + +const compartmentName = await setVariableFromEnvOrPrompt( + "COMPARTMENT_NAME", + "Compartment Name (root)" +); + +const compartmentId = await searchCompartmentIdByName( + compartmentName || "root" +); +``` + +## Read Terraform Output + +```javascript +#!/usr/bin/env zx + +import { getOutputValues } from "./lib/terraform.mjs"; + +const shell = process.env.SHELL | "/bin/zsh"; +$.shell = shell; +$.verbose = false; + +const tfOutput = await getOutputValues("./terraform"); + +console.log(tfOutput); +``` diff --git a/agentic_rag/scripts/lib/container.mjs b/agentic_rag/scripts/lib/container.mjs new file mode 100644 index 0000000..4d66a54 --- /dev/null +++ b/agentic_rag/scripts/lib/container.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env zx +import { exitWithError } from "./utils.mjs"; + +export async function whichContainerEngine() { + try { + const dockerPath = await which("docker"); + return !dockerPath ? "podman" : "docker"; + } catch (err) { + return "podman"; + } +} + +const ce = await whichContainerEngine(); + +export async function checkPodmanMachineRunning() { + if (ce === "podman") { + const isMachineRunning = ( + await $`podman machine info --format {{.Host.MachineState}}` + ).stdout.trim(); + if (isMachineRunning === "Stopped") { + console.log( + `Run ${chalk.yellow("podman machine start")} before continue` + ); + exitWithError("Podman machine stopped"); + } else { + console.log(`${chalk.green("[ok]")} podman machine running`); + } + } +} + +export async function containerLogin(namespace, user, token, url) { + try { + const { stdout, stderr, exitCode } = + await $`${ce} login -u ${namespace}/${user} -p ${token} ${url}`; + if (exitCode == 0) { + console.log(`${chalk.yellow(url)}: ${chalk.green(stdout.trim())}`); + } else { + console.error(chalk.red(stderr.trim())); + } + } catch (error) { + console.error(chalk.red(error.stderr.trim())); + const yellowUserString = chalk.yellow(user); + exitWithError( + `Review the user ${yellowUserString} and token pair, and try again.` + ); + } +} + +export async function tagImage(local, remote) { + console.log(`${ce} tag ${local} ${remote}`); + try { + await $`${ce} tag ${local} ${remote}`; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function pushImage(remote) { + console.log(`${ce} push ${remote}`); + try { + await $`${ce} push ${remote}`; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function buildImage(name, version) { + console.log(`${ce} build . -t ${name}:${version}`); + try { + await $`${ce} build . -t ${name}:${version}`; + } catch (error) { + exitWithError(error.stderr); + } +} diff --git a/agentic_rag/scripts/lib/crypto.mjs b/agentic_rag/scripts/lib/crypto.mjs new file mode 100644 index 0000000..af17682 --- /dev/null +++ b/agentic_rag/scripts/lib/crypto.mjs @@ -0,0 +1,59 @@ +#!/usr/bin/env zx +import { exitWithError } from "./utils.mjs"; + +export async function createSelfSignedCert(outputPath = ".") { + await $`mkdir -p ${outputPath}`; + const keyPath = path.normalize(path.join(outputPath, "tls.key")); + const certPath = path.normalize(path.join(outputPath, "tls.crt")); + try { + await $`openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ${keyPath} -out ${certPath} -subj "/CN=nginxsvc/O=nginxsvc"`; + console.log(`Cert Key created: ${chalk.green(keyPath)}`); + console.log(`Cert created: ${chalk.green(certPath)}`); + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function generateRandomString(length = 20) { + if (length < 12) { + exitWithError("Password length too short, >= 12 required"); + } + try { + const output = (await $`openssl rand -base64 ${length + 15}`).stdout.trim(); + if (output.length) { + const cleanPassword = output + .replaceAll("/", "") + .replaceAll("\\", "") + .replaceAll("=", ""); + return cleanPassword.slice(0, length); + } else { + exitWithError("random string generation failed"); + } + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function createRSAKeyPair(outputPath = ".") { + await $`mkdir -p ${outputPath}`; + const privateKeyPath = path.normalize(path.join(outputPath, "rsa.pem")); + const publicKeyPath = path.normalize(path.join(outputPath, "rsa_public.pem")); + try { + await $`openssl genrsa -out ${privateKeyPath} 2048`; + await $`openssl rsa -in ${privateKeyPath} -outform PEM -pubout -out ${publicKeyPath}`; + return { privateKeyPath, publicKeyPath }; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function createSSHKeyPair(sshPathParam) { + const defaultSSHPath = path.join(os.homedir(), ".ssh", "id_rsa"); + const sshPath = sshPathParam || defaultSSHPath; + try { + await $`ssh-keygen -t rsa -b 4096 -f ${sshPath} -q -N "" << c.current); + if (currentContext.name === contextName) { + console.log(chalk.green(`Context ${contextName} already in use.`)); + return; + } + console.log(chalk.yellow(`fn use context ${contextName}`)); + try { + const { stdout, exitCode, stderr } = await $`fn use context ${contextName}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log(chalk.green(`Context ${contextName} in use.`)); + return stdout.trim(); + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function setFnProfileName(profileName = "DEFAULT") { + console.log(chalk.yellow(`fn update context oracle.profile ${profileName}`)); + try { + const { stdout, exitCode, stderr } = + await $`fn update context oracle.profile ${profileName}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log(chalk.green(`Profile ${profileName} set.`)); + return stdout.trim(); + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function setFnCompartment(compartmentId) { + if (!compartmentId) { + exitWithError("Compartment OCID required"); + } + console.log( + chalk.yellow(`fn update context oracle.compartment-id ${compartmentId}`) + ); + try { + const { stdout, exitCode, stderr } = + await $`fn update context oracle.compartment-id ${compartmentId}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log(chalk.green(`Compartment ${compartmentId} set.`)); + return stdout.trim(); + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function setFnApiUrl(regionKey) { + if (!regionKey) { + exitWithError("Region key required"); + } + console.log( + chalk.yellow( + `fn update context api-url https://functions.${regionKey}.oci.oraclecloud.com` + ) + ); + try { + const { stdout, exitCode, stderr } = + await $`fn update context api-url https://functions.${regionKey}.oci.oraclecloud.com`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log( + chalk.green( + `API URL https://functions.${regionKey}.oci.oraclecloud.com set.` + ) + ); + return stdout.trim(); + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function setFnRegistry(regionKey, namespace, repoNamePrefix) { + if (!regionKey) { + exitWithError("Region key required"); + } + if (!namespace) { + exitWithError("Namespace required"); + } + if (!repoNamePrefix) { + exitWithError("Repo name Prefix required"); + } + console.log( + chalk.yellow( + `fn update context registry ${regionKey}.ocir.io/${namespace}/${repoNamePrefix}` + ) + ); + try { + const { stdout, exitCode, stderr } = + await $`fn update context registry ${regionKey}.ocir.io/${namespace}/${repoNamePrefix}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log( + chalk.green( + `Registry ${regionKey}.ocir.io/${namespace}/${repoNamePrefix} set.` + ) + ); + return stdout.trim(); + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function setFnRegistryCompartment(compartmentId) { + if (!compartmentId) { + exitWithError("Compartment OCID required"); + } + console.log( + chalk.yellow( + `fn update context oracle.image-compartment-id ${compartmentId}` + ) + ); + try { + const { stdout, exitCode, stderr } = + await $`fn update context oracle.image-compartment-id ${compartmentId}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log(chalk.green(`Registry compartment ${compartmentId} set.`)); + return stdout.trim(); + } catch (error) { + exitWithError(error.stderr); + } +} diff --git a/agentic_rag/scripts/lib/gradle.mjs b/agentic_rag/scripts/lib/gradle.mjs new file mode 100644 index 0000000..40ca284 --- /dev/null +++ b/agentic_rag/scripts/lib/gradle.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env zx +import { exitWithError } from "./utils.mjs"; + +export async function bumpGradle(level = "patch") { + const oldVersion = await getVersionGradle(); + let [major, minor, patch] = oldVersion.split(".").map((n) => parseInt(n)); + let newVersion; + switch (level) { + case "patch": + newVersion = `${major}.${minor}.${patch + 1}`; + break; + case "minor": + newVersion = `${major}.${minor + 1}.${0}`; + break; + case "major": + newVersion = `${major + 1}.${0}.${0}`; + break; + default: + console.log("No version bump"); + break; + } + try { + const { exitCode, stderr } = + await $`sed -I -e '/^version/s/${oldVersion}/${newVersion}/g' build.gradle`; + if (exitCode !== 0) { + exitWithError(stderr); + } + return newVersion; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function getVersionGradle() { + try { + const { stdout, exitCode, stderr } = + await $`grep "version = " build.gradle`; + if (exitCode !== 0) { + exitWithError(stderr); + } + const version = stdout.trim().split("version = ")[1].replaceAll("'", ""); + return version; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function cleanGradle() { + try { + const { stdout, exitCode, stderr } = await $`./gradlew clean`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log(stdout.trim()); + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function buildJarGradle() { + try { + const { stdout, exitCode, stderr } = await $`./gradlew bootJar`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log(stdout.trim()); + } catch (error) { + exitWithError(error.stderr); + } +} diff --git a/agentic_rag/scripts/lib/npm.mjs b/agentic_rag/scripts/lib/npm.mjs new file mode 100644 index 0000000..c4cde29 --- /dev/null +++ b/agentic_rag/scripts/lib/npm.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env zx +import { exitWithError } from "./utils.mjs"; + +export async function getNpmVersion() { + const { version } = await fs.readJson("./package.json"); + return version; +} + +export async function bump(level = "patch") { + try { + let newVersion; + switch (level) { + case "patch": + newVersion = (await $`npm version patch`).stdout.trim(); + console.log(`New version: ${newVersion}`); + break; + case "minor": + newVersion = (await $`npm version minor`).stdout.trim(); + console.log(`New version: ${newVersion}`); + break; + case "major": + newVersion = (await $`npm version major`).stdout.trim(); + console.log(`New version: ${newVersion}`); + break; + + default: + console.log("No version bump"); + break; + } + return newVersion; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function buildWeb() { + try { + console.log(`Install dependencies`); + await $`npm install`; + console.log(`Build static content`); + const output = (await $`npm run build`).stdout.trim(); + console.log(output); + } catch (error) { + exitWithError(error.stderr); + } +} diff --git a/agentic_rag/scripts/lib/oci.mjs b/agentic_rag/scripts/lib/oci.mjs new file mode 100644 index 0000000..96bbfed --- /dev/null +++ b/agentic_rag/scripts/lib/oci.mjs @@ -0,0 +1,347 @@ +#!/usr/bin/env zx +import moment from "moment"; +import { exitWithError } from "./utils.mjs"; +import { where, max, sortBy } from "underscore"; + +export async function getRegions(profile = "DEFAULT", tenancyId) { + try { + const output = ( + await $`oci iam region-subscription list \ + --tenancy-id ${tenancyId} \ + --profile ${profile}` + ).stdout.trim(); + const { data } = JSON.parse(output); + return data + .filter((r) => r.status === "READY") + .map((r) => ({ + key: r["region-key"].toLowerCase(), + name: r["region-name"], + isHomeRegion: r["is-home-region"], + })); + } catch (error) { + exitWithError(`Error: get regions ${error.message}`); + } +} + +export async function getNamespace(profile) { + const output = (await $`oci os ns get --profile ${profile}`).stdout.trim(); + const { data } = JSON.parse(output); + return data; +} + +export async function listAdbDatabases(compartmentId) { + try { + const { stdout, exitCode, stderr } = + await $`oci db autonomous-database list --all --compartment-id ${compartmentId}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + if (!stdout.length) return []; + return JSON.parse(stdout.trim()).data; + } catch (error) { + exitWithError(`Error: download wallet ${error.stderr}`); + } +} + +export async function downloadAdbWallet(adbId, walletFilePath, walletPassword) { + try { + const { exitCode, stderr } = + await $`oci db autonomous-database generate-wallet \ + --autonomous-database-id ${adbId} \ + --file ${walletFilePath} \ + --password ${walletPassword}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + console.log(`Wallet downloaded on ${chalk.green(walletFilePath)}`); + } catch (error) { + exitWithError(`Error: download wallet ${error.stderr}`); + } +} + +export async function getAvailableShapes(options = {}) { + try { + const output = ( + await $`oci compute shape list --compartment-id ${await getTenancyId()}` + ).stdout.trim(); + const { data } = JSON.parse(output); + const { type = "*", flex, includeOld = false } = options; + return data + .filter((shape) => { + if (shape.shape.includes("Standard1.")) return false; + return true; + }) + .filter((shape) => { + if (type === "*") return true; + if (type === "bm") return shape.shape.startsWith("BM."); + if (type === "vm") return shape.shape.startsWith("VM."); + }) + .filter((shape) => { + if (flex === undefined) return true; + return flex + ? shape.shape.endsWith(".Flex") + : !shape.shape.endsWith(".Flex"); + }) + .sort((s1, s2) => { + return s1.shape.localeCompare(s2.shape); + }); + } catch (error) { + exitWithError(`Error: get available shapes ${error.message}`); + } +} + +export async function getTenancyId() { + const tenancyIdEnv = process.env.OCI_TENANCY; + const tenancyId = tenancyIdEnv + ? tenancyIdEnv + : await question("OCI tenancy: "); + return tenancyId; +} + +export async function searchCompartmentIdByName( + profile = "DEFAULT", + compartmentName +) { + if (!compartmentName) { + exitWithError("Compartment name required"); + } + if (compartmentName === "root") { + return getTenancyId(); + } + try { + const { stdout, exitCode, stderr } = await $`oci iam compartment list \ + --compartment-id-in-subtree true \ + --name ${compartmentName} \ + --profile=${profile} \ + --query "data[].id"`; + if (exitCode !== 0) { + exitWithError(stderr); + } + if (!stdout.length) { + exitWithError("Compartment name not found"); + } + const compartmentId = JSON.parse(stdout.trim())[0]; + return compartmentId; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function uploadApiKeyFile(userId, publicKeyPath) { + if (!userId) { + exitWithError("User ID required"); + } + if (!publicKeyPath) { + exitWithError("Public RSA key required"); + } + const rsaPublicKeyExists = await fs.pathExists(publicKeyPath); + if (!rsaPublicKeyExists) { + exitWithError(`RSA Public key ${publicKeyPath} does not exists`); + } + try { + const { stdout, exitCode, stderr } = + await $`oci iam user api-key upload --user-id ${userId} --key-file ${publicKeyPath}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + if (!stdout.length) { + exitWithError("Compartment name not found"); + } + const { fingerprint } = JSON.parse(stdout.trim()).data; + return fingerprint; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function getUserId() { + const userIdEnv = process.env.OCI_CS_USER_OCID; + if (userIdEnv) { + return userIdEnv; + } + const userEmail = await question("OCI User email: "); + const { stdout, exitCode, stderr } = await $`oci iam user list --all`; + if (exitCode !== 0) { + exitWithError(stderr); + } + if (!stdout.length) { + exitWithError("User name not found"); + } + const data = JSON.parse(stdout.trim()).data; + const userFound = data.find((user) => user.email === userEmail); + if (!userFound) { + exitWithError(`User ${userEmail} not found`); + } + return userFound.id; +} + +export async function createBucket(compartmentId, name) { + if (!compartmentId) { + exitWithError(`Compartment Id required to create bucket`); + } + if (!name) { + exitWithError(`Name required to create bucket`); + } + const namespace = await getNamespace(); + try { + const { stdout, exitCode, stderr } = await $`oci os bucket create \ + --namespace-name ${namespace} \ + --compartment-id ${compartmentId} \ + --name ${name}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + const { fingerprint } = JSON.parse(stdout.trim()).data; + return fingerprint; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function deleteBucket(name) { + if (!name) { + exitWithError(`Name required to create bucket`); + } + const namespace = await getNamespace(); + try { + const { exitCode, stderr } = await $`oci os bucket delete \ + --bucket-name ${name} \ + --namespace-name ${namespace} \ + --empty --force`; + if (exitCode !== 0) { + exitWithError(stderr); + } + return; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function putObject(bucketName, objectName, filePath) { + if (!bucketName) { + exitWithError(`Bucket name required to put an object`); + } + if (!objectName) { + exitWithError(`Object name required to put an object`); + } + if (!filePath) { + exitWithError(`File path required to put an object`); + } + const namespace = await getNamespace(); + try { + const { exitCode, stderr } = await $`oci os object put \ + --force --name ${objectName} \ + -bn ${bucketName} -ns ${namespace} \ + --file ${filePath}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + return; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function createPARObject(bucketName, objectName, expiration) { + if (!bucketName) { + exitWithError(`Bucket name required to create a PAR`); + } + if (!objectName) { + exitWithError(`Object name required to create a PAR`); + } + if (!expiration) { + exitWithError(`RFC 3339 expiration required to create a PAR`); + } + const namespace = await getNamespace(); + + try { + const { stdout, exitCode, stderr } = await $`oci os preauth-request create \ + --bucket-name ${bucketName} \ + --namespace-name ${namespace} \ + --name ${objectName}_par \ + --access-type ObjectRead \ + --time-expires "${expiration}" \ + --object-name ${objectName}`; + if (exitCode !== 0) { + exitWithError(stderr); + } + const fullPath = JSON.parse(stdout.trim()).data["full-path"]; + return fullPath; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function listPARs(bucketName) { + if (!bucketName) { + exitWithError(`Bucket name required to list PARs`); + } + const namespace = await getNamespace(); + try { + const { stdout, exitCode, stderr } = await $`oci os preauth-request list \ + --bucket-name ${bucketName} \ + --namespace-name ${namespace} \ + --all`; + if (exitCode !== 0) { + exitWithError(stderr); + } + if (!stdout.length) return []; + const pars = JSON.parse(stdout.trim()).data; + return pars; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function deletePAR(bucketName, id) { + if (!bucketName) { + exitWithError(`Bucket name required to delete a PAR`); + } + if (!id) { + exitWithError(`PAR id required to delete a PAR`); + } + const namespace = await getNamespace(); + try { + const { exitCode, stderr } = await $`oci os preauth-request delete \ + --bucket-name ${bucketName} \ + --namespace-name ${namespace} \ + --par-id ${id} --force`; + if (exitCode !== 0) { + exitWithError(stderr); + } + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function listBuckets(compartmentId) { + if (!compartmentId) { + exitWithError(`Compartment Id required to create bucket`); + } + const namespace = await getNamespace(); + try { + const { stdout, exitCode, stderr } = await $`oci os bucket list \ + --compartment-id ${compartmentId} \ + --namespace-name ${namespace} \ + --all`; + if (exitCode !== 0) { + exitWithError(stderr); + } + if (!stdout.length) return []; + const compartmentList = JSON.parse(stdout.trim()).data; + return compartmentList; + } catch (error) { + exitWithError(error.stderr); + } +} + +export async function getBucket(compartmentId, name) { + if (!compartmentId) { + exitWithError(`Compartment Id required to create bucket`); + } + if (!name) { + exitWithError(`Name required to create bucket`); + } + const listBucket = await listBuckets(compartmentId); + return listBucket.find((b) => b.name === name); +} diff --git a/agentic_rag/scripts/lib/terraform.mjs b/agentic_rag/scripts/lib/terraform.mjs new file mode 100644 index 0000000..5c7e6e0 --- /dev/null +++ b/agentic_rag/scripts/lib/terraform.mjs @@ -0,0 +1,23 @@ +#!/usr/bin/env zx +import { exitWithError } from "./utils.mjs"; + +export async function getOutputValues(terraformPath = "") { + const tfOutputPathExists = await fs.pathExists(terraformPath); + if (!tfOutputPathExists) exitWithError("Terraform path doesn't exist"); + + const { stdout: stdoutPwd } = await $`pwd`; + const currentPath = stdoutPwd.trim(); + await cd(path.normalize(terraformPath)); + + const { stdout } = await $`terraform output -json`; + const terraformOutput = JSON.parse(stdout); + + const values = {}; + for (const [key, content] of Object.entries(terraformOutput)) { + values[key] = content.value; + } + + await cd(currentPath); + + return values; +} diff --git a/agentic_rag/scripts/lib/utils.mjs b/agentic_rag/scripts/lib/utils.mjs new file mode 100644 index 0000000..5de87aa --- /dev/null +++ b/agentic_rag/scripts/lib/utils.mjs @@ -0,0 +1,78 @@ +#!/usr/bin/env zx + +export async function readEnvJson() { + const envFilePath = ".env.json"; + const envFileExists = await fs.pathExists(envFilePath); + if (envFileExists) { + return fs.readJson(envFilePath); + } + return writeEnvJson({}); +} + +export async function writeEnvJson(properties) { + const envFilePath = ".env.json"; + await fs.writeJson(envFilePath, properties, { spaces: 2 }); + return properties; +} + +export async function validateBumpLevel(level) { + if (!["major", "minor", "patch"].includes(level)) { + exitWithError("Error: release version must be 'major', 'minor' or 'patch'"); + } + return level; +} + +export async function printRegionNames(regions) { + console.log("printRegionNames"); + const regionsByZone = regions.reduce((acc, cur) => { + const zone = cur.name.split("-")[0]; + if (acc[zone]) { + acc[zone].push(cur.name); + } else { + acc[zone] = [cur.name]; + } + return acc; + }, {}); + Object.keys(regionsByZone).forEach((zone) => + console.log(`\t${chalk.yellow(zone)}: ${regionsByZone[zone].join(", ")}`) + ); +} + +export async function getRegionByKey(code = "fra") { + const output = (await $`oci iam region list`).stdout.trim(); + const { data } = JSON.parse(output); + return data.find((r) => code.toUpperCase() === r.key); +} + +export async function setVariableFromEnvOrPrompt( + envKey, + questionText, + printChoices +) { + const envValue = process.env[envKey]; + if (envValue) { + return envValue; + } else { + if (printChoices) { + printChoices(); + } + const answer = await question(`${questionText}: `); + return answer; + } +} + +export async function exitWithError(errorMessage = "") { + console.error(chalk.red(errorMessage.trim())); + process.exit(1); +} + +export async function checkRequiredProgramsExist(programs) { + try { + for (let program of programs) { + await which(program); + console.log(`${chalk.green("[ok]")} ${program}`); + } + } catch (error) { + exitWithError(`Error: Required command ${error.message}`); + } +} diff --git a/agentic_rag/scripts/package-lock.json b/agentic_rag/scripts/package-lock.json new file mode 100644 index 0000000..402c586 --- /dev/null +++ b/agentic_rag/scripts/package-lock.json @@ -0,0 +1,751 @@ +{ + "name": "scripts", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "scripts", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "clear": "^0.1.0", + "configstore": "^7.0.0", + "ini": "^5.0.0", + "inquirer": "^12.5.2", + "moment": "^2.30.1", + "mustache": "^4.2.0", + "underscore": "^1.13.7", + "yaml": "^2.7.1", + "zx": "^8.5.3" + } + }, + "node_modules/@inquirer/checkbox": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.1.5.tgz", + "integrity": "sha512-swPczVU+at65xa5uPfNP9u3qx/alNwiaykiI/ExpsmMSQW55trmZcwhYWzw/7fj+n6Q8z1eENvR7vFfq9oPSAQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/figures": "^1.0.11", + "@inquirer/type": "^3.0.6", + "ansi-escapes": "^4.3.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/confirm": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.9.tgz", + "integrity": "sha512-NgQCnHqFTjF7Ys2fsqK2WtnA8X1kHyInyG+nMIuHowVTIgIuS10T4AznI/PvbqSpJqjCUqNBlKGh1v3bwLFL4w==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/type": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "10.1.10", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.10.tgz", + "integrity": "sha512-roDaKeY1PYY0aCqhRmXihrHjoSW2A00pV3Ke5fTpMCkzcGF64R8e0lw3dK+eLEHwS4vB5RnW1wuQmvzoRul8Mw==", + "license": "MIT", + "dependencies": { + "@inquirer/figures": "^1.0.11", + "@inquirer/type": "^3.0.6", + "ansi-escapes": "^4.3.2", + "cli-width": "^4.1.0", + "mute-stream": "^2.0.0", + "signal-exit": "^4.1.0", + "wrap-ansi": "^6.2.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/editor": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.10.tgz", + "integrity": "sha512-5GVWJ+qeI6BzR6TIInLP9SXhWCEcvgFQYmcRG6d6RIlhFjM5TyG18paTGBgRYyEouvCmzeco47x9zX9tQEofkw==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/type": "^3.0.6", + "external-editor": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/expand": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.12.tgz", + "integrity": "sha512-jV8QoZE1fC0vPe6TnsOfig+qwu7Iza1pkXoUJ3SroRagrt2hxiL+RbM432YAihNR7m7XnU0HWl/WQ35RIGmXHw==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/type": "^3.0.6", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.11.tgz", + "integrity": "sha512-eOg92lvrn/aRUqbxRyvpEWnrvRuTYRifixHkYVpJiygTgVSBIHDqLh0SrMQXkafvULg3ck11V7xvR+zcgvpHFw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/input": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.9.tgz", + "integrity": "sha512-mshNG24Ij5KqsQtOZMgj5TwEjIf+F2HOESk6bjMwGWgcH5UBe8UoljwzNFHqdMbGYbgAf6v2wU/X9CAdKJzgOA==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/type": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/number": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.12.tgz", + "integrity": "sha512-7HRFHxbPCA4e4jMxTQglHJwP+v/kpFsCf2szzfBHy98Wlc3L08HL76UDiA87TOdX5fwj2HMOLWqRWv9Pnn+Z5Q==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/type": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/password": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.12.tgz", + "integrity": "sha512-FlOB0zvuELPEbnBYiPaOdJIaDzb2PmJ7ghi/SVwIHDDSQ2K4opGBkF+5kXOg6ucrtSUQdLhVVY5tycH0j0l+0g==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/type": "^3.0.6", + "ansi-escapes": "^4.3.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/prompts": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.4.1.tgz", + "integrity": "sha512-UlmM5FVOZF0gpoe1PT/jN4vk8JmpIWBlMvTL8M+hlvPmzN89K6z03+IFmyeu/oFCenwdwHDr2gky7nIGSEVvlA==", + "license": "MIT", + "dependencies": { + "@inquirer/checkbox": "^4.1.5", + "@inquirer/confirm": "^5.1.9", + "@inquirer/editor": "^4.2.10", + "@inquirer/expand": "^4.0.12", + "@inquirer/input": "^4.1.9", + "@inquirer/number": "^3.0.12", + "@inquirer/password": "^4.0.12", + "@inquirer/rawlist": "^4.0.12", + "@inquirer/search": "^3.0.12", + "@inquirer/select": "^4.1.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/rawlist": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.0.12.tgz", + "integrity": "sha512-wNPJZy8Oc7RyGISPxp9/MpTOqX8lr0r+lCCWm7hQra+MDtYRgINv1hxw7R+vKP71Bu/3LszabxOodfV/uTfsaA==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/type": "^3.0.6", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/search": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.12.tgz", + "integrity": "sha512-H/kDJA3kNlnNIjB8YsaXoQI0Qccgf0Na14K1h8ExWhNmUg2E941dyFPrZeugihEa9AZNW5NdsD/NcvUME83OPQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/figures": "^1.0.11", + "@inquirer/type": "^3.0.6", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/select": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.1.1.tgz", + "integrity": "sha512-IUXzzTKVdiVNMA+2yUvPxWsSgOG4kfX93jOM4Zb5FgujeInotv5SPIJVeXQ+fO4xu7tW8VowFhdG5JRmmCyQ1Q==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/figures": "^1.0.11", + "@inquirer/type": "^3.0.6", + "ansi-escapes": "^4.3.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/type": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.6.tgz", + "integrity": "sha512-/mKVCtVpyBu3IDarv0G+59KC4stsD5mDsGpYh+GKs1NZT88Jh52+cuoA1AtLk2Q0r/quNl+1cSUyLRHBFeD0XA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/atomically": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-2.0.3.tgz", + "integrity": "sha512-kU6FmrwZ3Lx7/7y3hPS5QnbJfaohcIul5fGqf7ok+4KklIEk9tJ0C2IQPdacSbVUWv6zVHXEBWoWd6NrVMT7Cw==", + "dependencies": { + "stubborn-fs": "^1.2.5", + "when-exit": "^2.1.1" + } + }, + "node_modules/chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", + "license": "MIT" + }, + "node_modules/clear": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/clear/-/clear-0.1.0.tgz", + "integrity": "sha512-qMjRnoL+JDPJHeLePZJuao6+8orzHMGP04A8CdwCNsKhRbOnKRjefxONR7bwILT3MHecxKBjHkKL/tkZ8r4Uzw==", + "engines": { + "node": "*" + } + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/configstore": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-7.0.0.tgz", + "integrity": "sha512-yk7/5PN5im4qwz0WFZW3PXnzHgPu9mX29Y8uZ3aefe2lBPC1FYttWZRcaW9fKkT0pBCJyuQ2HfbmPVaODi9jcQ==", + "license": "BSD-2-Clause", + "dependencies": { + "atomically": "^2.0.3", + "dot-prop": "^9.0.0", + "graceful-fs": "^4.2.11", + "xdg-basedir": "^5.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/yeoman/configstore?sponsor=1" + } + }, + "node_modules/dot-prop": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-9.0.0.tgz", + "integrity": "sha512-1gxPBJpI/pcjQhKgIU91II6Wkay+dLcN3M6rf2uwP8hRur3HtQXjVrdAK3sjC0piaEuxzMwjXChcETiJl47lAQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^4.18.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "license": "MIT", + "dependencies": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ini": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-5.0.0.tgz", + "integrity": "sha512-+N0ngpO3e7cRUWOJAS7qw0IZIVc6XPrW4MlFBdD066F2L4k1L6ker3hLqSq7iXxU5tgS4WGkIUElWn5vogAEnw==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/inquirer": { + "version": "12.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-12.5.2.tgz", + "integrity": "sha512-qoDk/vdSTIaXNXAoNnlg7ubexpJfUo7t8GT2vylxvE49BrLhToFuPPdMViidG2boHV7+AcP1TCkJs/+PPoF2QQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.10", + "@inquirer/prompts": "^7.4.1", + "@inquirer/type": "^3.0.6", + "ansi-escapes": "^4.3.2", + "mute-stream": "^2.0.0", + "run-async": "^3.0.0", + "rxjs": "^7.8.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/mute-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", + "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/run-async": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", + "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/stubborn-fs": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", + "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==" + }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "license": "MIT", + "dependencies": { + "os-tmpdir": "~1.0.2" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-fest": { + "version": "4.40.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.40.0.tgz", + "integrity": "sha512-ABHZ2/tS2JkvH1PEjxFDTUWC8dB5OsIGZP4IFLhR293GqT5Y5qB1WwL2kMPYhQW9DVgVD8Hd7I8gjwPIf5GFkw==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/underscore": { + "version": "1.13.7", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz", + "integrity": "sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g==", + "license": "MIT" + }, + "node_modules/when-exit": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/when-exit/-/when-exit-2.1.4.tgz", + "integrity": "sha512-4rnvd3A1t16PWzrBUcSDZqcAmsUIy4minDXT/CZ8F2mVDgd65i4Aalimgz1aQkRGU0iH5eT5+6Rx2TK8o443Pg==", + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/xdg-basedir": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-5.1.0.tgz", + "integrity": "sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/yaml": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.1.tgz", + "integrity": "sha512-10ULxpnOCQXxJvBgxsn9ptjq6uviG/htZKk9veJGhlqn3w/DxQ631zFF+nlQXLwmImeS5amR2dl2U8sg6U9jsQ==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/yoctocolors-cjs": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz", + "integrity": "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zx": { + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/zx/-/zx-8.5.3.tgz", + "integrity": "sha512-TsGLAt8Ngr4wDXLZmN9BT+6FWVLFbqdQ0qpXkV3tIfH7F+MgN/WUeSY7W4nNqAntjWunmnRaznpyxtJRPhCbUQ==", + "license": "Apache-2.0", + "bin": { + "zx": "build/cli.js" + }, + "engines": { + "node": ">= 12.17.0" + } + } + } +} diff --git a/agentic_rag/scripts/package.json b/agentic_rag/scripts/package.json new file mode 100644 index 0000000..d5ab93b --- /dev/null +++ b/agentic_rag/scripts/package.json @@ -0,0 +1,26 @@ +{ + "name": "scripts", + "version": "1.0.0", + "main": "index.js", + "directories": { + "lib": "lib" + }, + "scripts": { + "setenv": "node setenv.mjs" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "clear": "^0.1.0", + "configstore": "^7.0.0", + "ini": "^5.0.0", + "inquirer": "^12.5.2", + "moment": "^2.30.1", + "mustache": "^4.2.0", + "underscore": "^1.13.7", + "yaml": "^2.7.1", + "zx": "^8.5.3" + } +} \ No newline at end of file diff --git a/agentic_rag/scripts/setenv.mjs b/agentic_rag/scripts/setenv.mjs new file mode 100644 index 0000000..b10302e --- /dev/null +++ b/agentic_rag/scripts/setenv.mjs @@ -0,0 +1,195 @@ +#!/usr/bin/env zx +import { readFile } from "node:fs/promises"; +import { parse as iniParse } from "ini"; +import Mustache from "mustache"; +import Configstore from "configstore"; +import inquirer from "inquirer"; +import clear from "clear"; +import { + getNamespace, + getRegions, + searchCompartmentIdByName, +} from "./lib/oci.mjs"; +import { createSSHKeyPair, createSelfSignedCert } from "./lib/crypto.mjs"; + +$.verbose = false; + +clear(); +console.log("Set up environment..."); + +const projectName = "arag"; + +const config = new Configstore(projectName, { projectName }); + +await selectProfile(); +const profile = config.get("profile"); +const tenancyId = config.get("tenancyId"); + +await setNamespaceEnv(); +await setRegionEnv(); +const regionName = config.get("regionName"); +await setHuggingFaceToken(); +await setCompartmentEnv(); +const compartmentId = config.get("compartmentId"); + +await createSSHKeys(projectName); +await createCerts(); + +await generateTFVars(); + +console.log(`\nConfiguration file saved in: ${chalk.green(config.path)}`); + +async function selectProfile() { + let ociConfigFile = await readFile(`${os.homedir()}/.oci/config`, { + encoding: "utf-8", + }); + + const ociConfig = iniParse(ociConfigFile); + const profileList = Object.keys(ociConfig); + + if (profileList.length === 1) { + config.set("profile", profileList[0]); + config.set("tenancyId", ociConfig[profileList[0]].tenancy); + } else { + await inquirer + .prompt([ + { + type: "list", + name: "profile", + message: "Select the OCI Config Profile", + choices: profileList, + }, + ]) + .then((answers) => { + config.set("profile", answers.profile); + config.set("tenancyId", ociConfig[answers.profile].tenancy); + }); + } +} + +async function setNamespaceEnv() { + const namespace = await getNamespace(profile); + config.set("namespace", namespace); +} + +async function setRegionEnv() { + const listSubscribedRegions = (await getRegions(profile, tenancyId)).sort( + (r1, r2) => r1.isHomeRegion > r2.isHomeRegion + ); + + if (listSubscribedRegions.length === 1) { + config.set("regionName", listSubscribedRegions[0].name); + config.set("regionKey", listSubscribedRegions[0].key); + } else { + await inquirer + .prompt([ + { + type: "list", + name: "region", + message: "Select the region", + choices: listSubscribedRegions.map((r) => r.name), + filter(val) { + return listSubscribedRegions.find((r) => r.name === val); + }, + }, + ]) + .then((answers) => { + config.set("regionName", answers.region.name); + config.set("regionKey", answers.region.key); + }); + } +} + +async function setHuggingFaceToken() { + await inquirer + .prompt([ + { + type: "input", + name: "huggingfaceToken", + message: "Hugging Face Token", + }, + ]) + .then(async (answers) => { + const huggingfaceToken = answers.huggingfaceToken; + config.set("huggingfaceToken", huggingfaceToken); + }); +} + +async function setCompartmentEnv() { + await inquirer + .prompt([ + { + type: "input", + name: "compartmentName", + message: "Compartment Name", + default() { + return "root"; + }, + }, + ]) + .then(async (answers) => { + const compartmentName = answers.compartmentName; + const compartmentId = await searchCompartmentIdByName( + profile, + compartmentName || "root" + ); + config.set("compartmentName", compartmentName); + config.set("compartmentId", compartmentId); + }); +} + +async function createSSHKeys(name) { + const sshPathParam = path.join(os.homedir(), ".ssh", name); + const publicKeyContent = await createSSHKeyPair(sshPathParam); + config.set("privateKeyPath", sshPathParam); + config.set("publicKeyContent", publicKeyContent); + config.set("publicKeyPath", `${sshPathParam}.pub`); + console.log(`SSH key pair created: ${chalk.green(sshPathParam)}`); +} + +async function createCerts() { + const certPath = path.join(__dirname, "..", ".certs"); + await $`mkdir -p ${certPath}`; + await createSelfSignedCert(certPath); + config.set("certFullchain", path.join(certPath, "tls.crt")); + config.set("certPrivateKey", path.join(certPath, "tls.key")); +} + +async function generateTFVars() { + const publicKeyContent = config.get("publicKeyContent"); + const sshPrivateKeyPath = config.get("privateKeyPath"); + const certFullchain = config.get("certFullchain"); + const certPrivateKey = config.get("certPrivateKey"); + const compartmentName = config.get("compartmentName"); + + const tfVarsPath = "tf/terraform.tfvars"; + + const tfvarsTemplate = await fs.readFile(`${tfVarsPath}.mustache`, "utf-8"); + + const output = Mustache.render(tfvarsTemplate, { + region_name: regionName, + config_file_profile: profile, + tenancy_id: tenancyId, + compartment_id: compartmentId, + cert_fullchain: certFullchain, + cert_private_key: certPrivateKey, + ssh_public_key: publicKeyContent, + ssh_private_key_path: sshPrivateKeyPath, + }); + + console.log( + `Terraform will deploy resources in ${chalk.green( + regionName + )} in compartment ${ + compartmentName ? chalk.green(compartmentName) : chalk.green("root") + }` + ); + + await fs.writeFile(tfVarsPath, output); + + console.log(`File ${chalk.green(tfVarsPath)} created`); + + console.log(`1. ${chalk.yellow("cd tf")}`); + console.log(`2. ${chalk.yellow("terraform init")}`); + console.log(`3. ${chalk.yellow("terraform apply -auto-approve")}`); +} diff --git a/agentic_rag/src/inference/Containerfile b/agentic_rag/src/inference/Containerfile new file mode 100644 index 0000000..ea65202 --- /dev/null +++ b/agentic_rag/src/inference/Containerfile @@ -0,0 +1,14 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Use non-root user for security +RUN useradd -m app +USER app + +CMD ["python", "app.py"] \ No newline at end of file diff --git a/agentic_rag/src/ui/Containerfile b/agentic_rag/src/ui/Containerfile new file mode 100644 index 0000000..ea65202 --- /dev/null +++ b/agentic_rag/src/ui/Containerfile @@ -0,0 +1,14 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Use non-root user for security +RUN useradd -m app +USER app + +CMD ["python", "app.py"] \ No newline at end of file diff --git a/agentic_rag/src/vector/Containerfile b/agentic_rag/src/vector/Containerfile new file mode 100644 index 0000000..ea65202 --- /dev/null +++ b/agentic_rag/src/vector/Containerfile @@ -0,0 +1,14 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Use non-root user for security +RUN useradd -m app +USER app + +CMD ["python", "app.py"] \ No newline at end of file diff --git a/agentic_rag/tf/.terraform.lock.hcl b/agentic_rag/tf/.terraform.lock.hcl new file mode 100644 index 0000000..c15a805 --- /dev/null +++ b/agentic_rag/tf/.terraform.lock.hcl @@ -0,0 +1,185 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.7.0" + constraints = "~> 2.7.0" + hashes = [ + "h1:1niS9AcwxN8CrWemnJS2Xf6vM72+48Xh3xFSS3DFWQo=", + "zh:04e23bebca7f665a19a032343aeecd230028a3822e546e6f618f24c47ff87f67", + "zh:5bb38114238e25c45bf85f5c9f627a2d0c4b98fe44a0837e37d48574385f8dad", + "zh:64584bc1db4c390abd81c76de438d93acf967c8a33e9b923d68da6ed749d55bd", + "zh:697695ab9cce351adf91a1823bdd72ce6f0d219138f5124ef7645cedf8f59a1f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7edefb1d1e2fead8fd155f7b50a2cb49f2f3fed154ac3ef5f991ccaff93d6120", + "zh:807fb15b75910bf14795f2ad1a2d41b069f9ef52c242131b2964c8527312e235", + "zh:821d9148d261df1d1a8e5a4812df2a6a3ffaf0d2070dad3c785382e489069239", + "zh:a7d92251118fb723048c482154a6ac6368aad583d28d15fffc6f5dafd9507463", + "zh:b627d4cef192b3c12ddaf9cb2c4f98c10d0129883c8c2a9c0049983f9de7030d", + "zh:dfb70306fcc0ad1d512ab7c24765703783cc286062d4849de4fbe23526f5dc8e", + "zh:f21de276f857b7e51fa2593d8fef05a7faafb0a7b62db14ac58a03ce1be7d881", + ] +} + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.3.6" + constraints = ">= 2.2.0" + hashes = [ + "h1:afnqn3XPnO40laFt+SVHPPKsg1j3HXT0VAO0xBVvmrY=", + "zh:1321b5ddede56be3f9b35bf75d7cda79adcb357fad62eb8677b6595e0baaa6cd", + "zh:265d66e61b9cd16ca1182ebf094cc0a08fb3687e8193a1dbac6899b16c237151", + "zh:3875c3a20e082ac55d5ff24bcaf7133ebc90c7f999fd0fb37cf0f0003474c94c", + "zh:68ce41ccd07757c451682703840cae1ec270ed5275cd491bbf8279782dfcbb73", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8dca3bb3f85ff8ac4d1b3f93975dcb751ed788396c56ebf0c3737ce1a4c60492", + "zh:9339bdaa99939291cedf543861353c8e7171ec5231c0dfacaa9bdb3338978dab", + "zh:a8510c2256e9a78697910bb5542aeca457c81225ea88130335f6d14a36a36c74", + "zh:af7ed71b8fceb60a5e3b7fa663be171e0bd41bb0af30e0e1f06a004c7b584e4a", + "zh:bc9de0f921b69d07f5fc1ea65f8af71d8d1a7053aafb500788b30bfce64b8fbe", + "zh:bccd0a49f161a91660d7d30dd6b389e6820f29752ccf351f10a3297c96973823", + "zh:c69321caca20009abead617f888a67aca990276cb7388b738b19157b88749190", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.17.0" + constraints = ">= 2.9.0" + hashes = [ + "h1:kQMkcPVvHOguOqnxoEU2sm1ND9vCHiT8TvZ2x6v/Rsw=", + "zh:06fb4e9932f0afc1904d2279e6e99353c2ddac0d765305ce90519af410706bd4", + "zh:104eccfc781fc868da3c7fec4385ad14ed183eb985c96331a1a937ac79c2d1a7", + "zh:129345c82359837bb3f0070ce4891ec232697052f7d5ccf61d43d818912cf5f3", + "zh:3956187ec239f4045975b35e8c30741f701aa494c386aaa04ebabffe7749f81c", + "zh:66a9686d92a6b3ec43de3ca3fde60ef3d89fb76259ed3313ca4eb9bb8c13b7dd", + "zh:88644260090aa621e7e8083585c468c8dd5e09a3c01a432fb05da5c4623af940", + "zh:a248f650d174a883b32c5b94f9e725f4057e623b00f171936dcdcc840fad0b3e", + "zh:aa498c1f1ab93be5c8fbf6d48af51dc6ef0f10b2ea88d67bcb9f02d1d80d3930", + "zh:bf01e0f2ec2468c53596e027d376532a2d30feb72b0b5b810334d043109ae32f", + "zh:c46fa84cc8388e5ca87eb575a534ebcf68819c5a5724142998b487cb11246654", + "zh:d0c0f15ffc115c0965cbfe5c81f18c2e114113e7a1e6829f6bfd879ce5744fbb", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/http" { + version = "3.4.5" + constraints = ">= 3.2.1" + hashes = [ + "h1:eSVCYfvn5JyV3LC0+mrLlLtgLv4B+RWeNqz02miBcMY=", + "zh:2072006c177efc101471f3d5eb8e1d8e6c68778cbfd6db3d3f22f59cfe6ce6ae", + "zh:3ac4cc0efe11ee054300769cfcc37491433937a8824621d1f8f7a18e7401da87", + "zh:63997e5457c9ddf9cfff17bd7bf9f083cbeff3105452045662109dd6be499ef9", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:826819bb8ab7d6e3095f597083d5b1ab93d1854312b9e1b6c18288fff9664f34", + "zh:8ad74e7d8ec2e226a73d49c7c317108f61a4cb803972fb3f945d1709d5115fcd", + "zh:a609ca9e0c91d250ac80295e39d5f524e8c0872d33ba8fde3c3e41893b4b015d", + "zh:ae07d19babc452f63f6a6511b944990e819dc20687b6c8f01d1676812f5ada53", + "zh:b7c827dc32a1a5d77185a78cd391b01217894b384f58169f98a96d683730d8ce", + "zh:d045e3db9f5e39ce78860d3fd94e04604fcbe246f6fe346ee50a971f936e9ccd", + "zh:ec28f9b52c74edd47eebbb5c254a6df5706360cde5ccd65097976efca23a2977", + "zh:f24982eaa7d34fd66554c3cf94873713a0dff14da9ea4c4be0cc76f1a6146d59", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.5.2" + constraints = "~> 2.5.2" + hashes = [ + "h1:IyFbOIO6mhikFNL/2h1iZJ6kyN3U00jgkpCLUCThAfE=", + "zh:136299545178ce281c56f36965bf91c35407c11897f7082b3b983d86cb79b511", + "zh:3b4486858aa9cb8163378722b642c57c529b6c64bfbfc9461d940a84cd66ebea", + "zh:4855ee628ead847741aa4f4fc9bed50cfdbf197f2912775dd9fe7bc43fa077c0", + "zh:4b8cd2583d1edcac4011caafe8afb7a95e8110a607a1d5fb87d921178074a69b", + "zh:52084ddaff8c8cd3f9e7bcb7ce4dc1eab00602912c96da43c29b4762dc376038", + "zh:71562d330d3f92d79b2952ffdda0dad167e952e46200c767dd30c6af8d7c0ed3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:805f81ade06ff68fa8b908d31892eaed5c180ae031c77ad35f82cb7a74b97cf4", + "zh:8b6b3ebeaaa8e38dd04e56996abe80db9be6f4c1df75ac3cccc77642899bd464", + "zh:ad07750576b99248037b897de71113cc19b1a8d0bc235eb99173cc83d0de3b1b", + "zh:b9f1c3bfadb74068f5c205292badb0661e17ac05eb23bfe8bd809691e4583d0e", + "zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + constraints = ">= 3.2.1" + hashes = [ + "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.1" + constraints = ">= 3.4.3, ~> 3.7.1" + hashes = [ + "h1:t152MY0tQH4a8fLzTtEWx70ITd3azVOrFDn/pQblbto=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.13.0" + constraints = ">= 0.9.1" + hashes = [ + "h1:iwR4JouIoeVPDabb8XCqsiaZlZ28IcB3tDD9MuPeSXE=", + "zh:3776dd78ef3053562ccb2f8916d5d3f21a28f05e78859f0f1e4510525f891ecb", + "zh:541ca0b56f808c15d208b9396f149563b133223c4b66cdefbcfe2d8f1c23497e", + "zh:67ed315f3572eb20ce6778423b14fbb6faba3090f454bc20ec4146489b4738c0", + "zh:69dc375845bcfc451426480119f2941ee28b9ef01273d228bb66918180863b3a", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:93c24b7c87b5db9721f60782ac784152599aa78b30fdea2fc9c594d46d92767c", + "zh:95441cf14312041ae0b34640ff33975c09540125b01f9131358fca50e7be239d", + "zh:a294103aeed868c58987e131357a3ec259316c937c909e8a726b862d5a227b82", + "zh:adf6ded3f2e2f318e8aebf1040bc2791b448d006af7d12f7ddc3e8d40b22047a", + "zh:b2d9c16b7acd20d3813060c4d3647dc5f40598ebbdf59f642d53d189e4e3870a", + "zh:bc76a5161e9bcf74cadd76b3d4a51de508aa0c62e7f7ae536a87cd7595d81ebf", + "zh:ce6df2c1052c60b4432cb5c0ead471d7cdb4b285b807c265328a358631fc3610", + ] +} + +provider "registry.terraform.io/oracle/oci" { + version = "6.35.0" + constraints = ">= 4.67.3, >= 4.119.0, >= 6.17.0, ~> 6.35.0" + hashes = [ + "h1:8siNIZ7wAMDt3f8I9SfW+WQ5KFwose6x2CN7aLpn94w=", + "zh:00decedf7ab26f1d5e4727e6f1862a03539e0a51968248171c2c526c4abeed92", + "zh:1c02aa05cfbc6c76c46ab0a16c509452bda9293054f04cf4864d59f63a9a0f02", + "zh:73583d2f20cb2007e876377efe3bfb463b2f85a3e235e56915b42d2d44126009", + "zh:806c666229a4879e9c7e939d1c6b5418a180007436592b036eb033a294f686af", + "zh:876a9f7dd845cb11c0e558efa67391ad1737d4cf1fc24b96163e158956f23212", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a32dad3e2f716449debbe96c8bc0fb91c0c2c5ffb36b8da6bdda3a1b10a67521", + "zh:ac8e4e2508d036e9b4b6ee066af0086fd5940a9c95f10c56b217cf714600c47b", + "zh:b67060c8c933190ce29ab4b61b70bb301235bb56f721853331c1643971f68422", + "zh:b7d4e45702d79ed6aceaaca5dcba23bd585b933144f850f91f3e3c87668ab9b0", + "zh:bdd6c687e36877b77ddfb9c5db9e07eda0a249b2512a8af4dcc035c78ec4b85f", + "zh:c35e29af0280a4a5ee07edda8875e558ce88dcaa1c26e5bd47f6cfd552cc761f", + "zh:da032a625e6ed943aa254058de3a35b031e27898bc5f447849aa42fcb31b8849", + "zh:e2c30a0b2856f1ebd127f688ea865d793a5436e1b0c79a5ac22495d91d36f076", + "zh:e5b56f287224269c91896a116f0b6c13af75601cd5959a09eb1c757c88361acb", + ] +} diff --git a/agentic_rag/tf/adb.tf b/agentic_rag/tf/adb.tf new file mode 100644 index 0000000..b06fa69 --- /dev/null +++ b/agentic_rag/tf/adb.tf @@ -0,0 +1,30 @@ +locals { + db_version = data.oci_database_autonomous_db_versions.adb_versions.autonomous_db_versions[0].version +} + +resource "oci_database_autonomous_database" "adb" { + compartment_id = var.compartment_ocid + db_name = "${var.project_name}${local.deploy_id}" + + admin_password = random_password.adb_admin_password.result + cpu_core_count = var.autonomous_database_cpu_core_count + data_storage_size_in_tbs = var.autonomous_database_data_storage_size_in_tbs + db_workload = var.autonomous_database_db_workload + db_version = local.db_version + display_name = "${var.project_name}${local.deploy_id}" + is_mtls_connection_required = true + whitelisted_ips = var.autonomous_database_db_whitelisted_ips + is_auto_scaling_enabled = true + license_model = var.autonomous_database_db_license +} + +resource "oci_database_autonomous_database_wallet" "adb_wallet" { + autonomous_database_id = oci_database_autonomous_database.adb.id + password = random_password.adb_admin_password.result + base64_encode_content = "true" +} + +resource "local_file" "adb_wallet_file" { + content_base64 = oci_database_autonomous_database_wallet.adb_wallet.content + filename = "${path.module}/generated/wallet.zip" +} \ No newline at end of file diff --git a/agentic_rag/tf/data.tf b/agentic_rag/tf/data.tf new file mode 100644 index 0000000..38a2bc2 --- /dev/null +++ b/agentic_rag/tf/data.tf @@ -0,0 +1,28 @@ +data "oci_identity_tenancy" "tenant_details" { + tenancy_id = var.tenancy_ocid + + provider = oci +} + +data "oci_identity_regions" "home" { + filter { + name = "key" + values = [data.oci_identity_tenancy.tenant_details.home_region_key] + } + + provider = oci +} + +data "oci_containerengine_cluster_option" "oke" { + cluster_option_id = "all" +} + +data "oci_database_autonomous_db_versions" "adb_versions" { + compartment_id = var.compartment_ocid + db_workload = var.autonomous_database_db_workload + + filter { + name = "version" + values = ["23ai"] + } +} \ No newline at end of file diff --git a/agentic_rag/tf/locals.tf b/agentic_rag/tf/locals.tf new file mode 100644 index 0000000..f3b574b --- /dev/null +++ b/agentic_rag/tf/locals.tf @@ -0,0 +1,6 @@ +locals { + project_name = var.project_name + deploy_id = random_string.deploy_id.result + anywhere = "0.0.0.0/0" + tcp = "6" +} \ No newline at end of file diff --git a/agentic_rag/tf/main.tf b/agentic_rag/tf/main.tf new file mode 100644 index 0000000..3d1df08 --- /dev/null +++ b/agentic_rag/tf/main.tf @@ -0,0 +1,5 @@ +resource "random_string" "deploy_id" { + length = 2 + special = false + upper = false +} \ No newline at end of file diff --git a/agentic_rag/tf/oke.tf b/agentic_rag/tf/oke.tf new file mode 100644 index 0000000..fa50d2e --- /dev/null +++ b/agentic_rag/tf/oke.tf @@ -0,0 +1,124 @@ +locals { + cluster_k8s_latest_version = reverse(sort(data.oci_containerengine_cluster_option.oke.kubernetes_versions))[0] + lb_subnet_cidr = "10.22.128.0/27" + workers_subnet_cidr = "10.22.144.0/20" + cp_subnet_cidr = "10.22.0.8/29" + vcn_cidr = "10.22.0.0/16" +} + +module "oke" { + source = "oracle-terraform-modules/oke/oci" + version = "5.2.4" + + providers = { + oci = oci + oci.home = oci.home + } + + tenancy_id = var.tenancy_ocid + compartment_id = var.compartment_ocid + region = var.region + home_region = var.tenancy_ocid + + kubernetes_version = local.cluster_k8s_latest_version + cluster_name = "${local.project_name}-${local.deploy_id}-oke" + vcn_name = "${local.project_name}-${local.deploy_id}-vcn" + + assign_public_ip_to_control_plane = true + control_plane_is_public = true + control_plane_allowed_cidrs = ["0.0.0.0/0"] + + create_bastion = false + create_operator = false + + ssh_private_key_path = var.ssh_private_key_path + ssh_public_key = var.ssh_public_key + + # IAM - Policies + create_iam_autoscaler_policy = "never" + create_iam_kms_policy = "never" + create_iam_operator_policy = "never" + create_iam_worker_policy = "never" + + # Network module - VCN + subnets = { + bastion = { + create = "never" + } + operator = { + create = "never" + } + cp = { + create = "always", + cidr = local.cp_subnet_cidr + } + pub_lb = { + create = "always", + cidr = local.lb_subnet_cidr + } + workers = { + create = "always", + cidr = local.workers_subnet_cidr + } + int_lb = { + create = "never" + } + pods = { + create = "never" + } + } + nsgs = { + bastion = { create = "never" } + operator = { create = "never" } + cp = { create = "always" } + int_lb = { create = "never" } + pub_lb = { create = "always" } // never + workers = { create = "always" } + pods = { create = "always" } // never + } + + assign_dns = true + create_vcn = true + vcn_cidrs = [local.vcn_cidr] + vcn_dns_label = "oke" + + create_cluster = true + cni_type = "npn" + cluster_type = "enhanced" + + pods_cidr = "10.244.0.0/16" + services_cidr = "10.96.0.0/16" + use_signed_images = false + use_defined_tags = false + + # Workers + worker_pool_mode = "node-pool" + worker_pool_size = 2 + + worker_pools = { + cpu_pool_1 = { + size = 1, + shape = "VM.Standard.E5.Flex", + ocpus = 4, + memory = 64, + boot_volume_size = 120, + create = true + } + gpu_pool_1 = { + size = 1, + shape = "VM.GPU.A10.1", + placement_ads = [1] + create = true + } + } +} + +data "oci_containerengine_cluster_kube_config" "kube_config" { + cluster_id = module.oke.cluster_id +} + + +resource "local_file" "kubeconfig" { + content = data.oci_containerengine_cluster_kube_config.kube_config.content + filename = "${path.module}/generated/kubeconfig" +} \ No newline at end of file diff --git a/agentic_rag/tf/output.tf b/agentic_rag/tf/output.tf new file mode 100644 index 0000000..4c1395e --- /dev/null +++ b/agentic_rag/tf/output.tf @@ -0,0 +1,20 @@ +output "deployment" { + value = "${local.project_name}${local.deploy_id}" +} + +output "adb_name" { + value = oci_database_autonomous_database.adb.db_name +} + +output "adb_display_name" { + value = oci_database_autonomous_database.adb.display_name +} + +output "adb_admin_password" { + value = random_password.adb_admin_password.result + sensitive = true +} + +output "oke_cluster_id" { + value = module.oke.cluster_id +} \ No newline at end of file diff --git a/agentic_rag/tf/provider.tf b/agentic_rag/tf/provider.tf new file mode 100644 index 0000000..7273136 --- /dev/null +++ b/agentic_rag/tf/provider.tf @@ -0,0 +1,12 @@ +provider "oci" { + tenancy_ocid = var.tenancy_ocid + region = var.region + config_file_profile = var.config_file_profile +} + +provider "oci" { + alias = "home" + tenancy_ocid = var.tenancy_ocid + region = lookup(data.oci_identity_regions.home.regions[0], "name") + config_file_profile = var.config_file_profile +} diff --git a/agentic_rag/tf/random.tf b/agentic_rag/tf/random.tf new file mode 100644 index 0000000..5df3827 --- /dev/null +++ b/agentic_rag/tf/random.tf @@ -0,0 +1,9 @@ +resource "random_password" "adb_admin_password" { + length = 16 + special = true + min_numeric = 3 + min_special = 3 + min_lower = 3 + min_upper = 3 + override_special = "()-_[]{}?" +} \ No newline at end of file diff --git a/agentic_rag/tf/terraform.tfvars.mustache b/agentic_rag/tf/terraform.tfvars.mustache new file mode 100644 index 0000000..bff026b --- /dev/null +++ b/agentic_rag/tf/terraform.tfvars.mustache @@ -0,0 +1,10 @@ +region = "{{ region_name }}" +config_file_profile = "{{ config_file_profile }}" +tenancy_ocid = "{{ tenancy_id }}" +compartment_ocid = "{{ compartment_id }}" + +cert_fullchain = "{{{ cert_fullchain }}}" +cert_private_key = "{{{ cert_private_key }}}" + +ssh_public_key = "{{{ ssh_public_key}}}" +ssh_private_key_path = "{{{ ssh_private_key_path}}}" \ No newline at end of file diff --git a/agentic_rag/tf/variables.tf b/agentic_rag/tf/variables.tf new file mode 100644 index 0000000..a822e87 --- /dev/null +++ b/agentic_rag/tf/variables.tf @@ -0,0 +1,63 @@ +variable "tenancy_ocid" { + type = string +} + +variable "region" { + type = string +} + +variable "config_file_profile" { + type = string +} + +variable "compartment_ocid" { + type = string +} + +variable "ssh_private_key_path" { + type = string +} + +variable "ssh_public_key" { + type = string +} + +variable "project_name" { + type = string + default = "arag" +} + +variable "cert_fullchain" { + type = string +} + +variable "cert_private_key" { + type = string +} + + +variable "autonomous_database_db_workload" { + type = string + default = "OLTP" +} + +variable "autonomous_database_db_license" { + type = string + description = "BRING_YOUR_OWN_LICENSE, LICENSE_INCLUDED" + default = "BRING_YOUR_OWN_LICENSE" +} + +variable "autonomous_database_db_whitelisted_ips" { + type = list(string) + default = ["0.0.0.0/0"] # Don't do this in prod +} + +variable "autonomous_database_cpu_core_count" { + type = number + default = 1 +} + +variable "autonomous_database_data_storage_size_in_tbs" { + type = number + default = 1 +} \ No newline at end of file diff --git a/agentic_rag/tf/versions.tf b/agentic_rag/tf/versions.tf new file mode 100644 index 0000000..5d3507f --- /dev/null +++ b/agentic_rag/tf/versions.tf @@ -0,0 +1,24 @@ +terraform { + required_providers { + oci = { + source = "oracle/oci" + version = "~> 6.35.0" + configuration_aliases = [oci.home] + } + local = { + source = "hashicorp/local" + version = "~> 2.5.2" + # https://registry.terraform.io/providers/hashicorp/local/ + } + random = { + source = "hashicorp/random" + version = "~> 3.7.1" + # https://registry.terraform.io/providers/hashicorp/random/ + } + archive = { + source = "hashicorp/archive" + version = "~> 2.7.0" + # https://registry.terraform.io/providers/hashicorp/archive/ + } + } +}