Introduce build index on cloud (#35375)

* add entities._indexes

* cleanup and refactor

* cleanup and fix gates

* move ignorePath to cspell in root

* serverless support

* depend on azureml-rag

* remove langchain

* remove extra constants

* cspell

* address comments

* pylint

* remove _createComponent

* mypy and pylint

* pylint errors

* pylint

* docstrings

* remove notebook

* remove dependency on azureml.rag

* remove langchain retriever

* mypy, pylint and expose configs

* extra constants and pylint

* run black

* address comments

* address more comments

* mypy & black

* pylint

* import Input

* one last mypy

Author: jingyizhu99

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_common.py

@@ -950,6 +950,24 @@ class AzureDevopsArtifactsType:
     ARTIFACT = "artifact"
 
 
+class DataIndexTypes:
+    """DataIndexTypes is an enumeration of values for the types out indexes which can be written to by DataIndex."""
+
+    ACS = "acs"
+    """Azure Cognitive Search index type."""
+    PINECONE = "pinecone"
+    """Pinecone index type."""
+    FAISS = "faiss"
+    """Faiss index type."""
+
+
+class IndexInputType:
+    """An enumeration of values for the types of input data for an index."""
+
+    GIT = "git"
+    LOCAL = "local"
+
+
 class ScheduleType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     JOB = "job"
     MONITOR = "monitor"

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_component.py

@@ -1,6 +1,7 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+# pylint: disable=line-too-long
 
 from typing import Dict
 
@@ -62,6 +63,24 @@ class DataTransferBuiltinComponentUri(object):
     EXPORT_DATABASE = "azureml://registries/azureml/components/export_data_database/versions/0.0.1"
 
 
+class LLMRAGComponentUri:
+    LLM_RAG_CRACK_AND_CHUNK = "azureml://registries/azureml/components/llm_rag_crack_and_chunk/labels/default"
+    LLM_RAG_GENERATE_EMBEDDINGS = "azureml://registries/azureml/components/llm_rag_generate_embeddings/labels/default"
+    LLM_RAG_CRACK_AND_CHUNK_AND_EMBED = (
+        "azureml://registries/azureml/components/llm_rag_crack_and_chunk_and_embed/labels/default"
+    )
+    LLM_RAG_UPDATE_ACS_INDEX = "azureml://registries/azureml/components/llm_rag_update_acs_index/labels/default"
+    LLM_RAG_UPDATE_PINECONE_INDEX = (
+        "azureml://registries/azureml/components/llm_rag_update_pinecone_index/labels/default"
+    )
+    LLM_RAG_CREATE_FAISS_INDEX = "azureml://registries/azureml/components/llm_rag_create_faiss_index/labels/default"
+    LLM_RAG_REGISTER_MLINDEX_ASSET = (
+        "azureml://registries/azureml/components/llm_rag_register_mlindex_asset/labels/default"
+    )
+    LLM_RAG_VALIDATE_DEPLOYMENTS = "azureml://registries/azureml/components/llm_rag_validate_deployments/labels/default"
+    LLM_RAG_CREATE_PROMPTFLOW = "azureml://registries/azureml/components/llm_rag_create_promptflow/labels/default"
+
+
 class ComponentSource:
     """Indicate where the component is constructed."""
 

sdk/ml/azure-ai-ml/azure/ai/ml/entities/__init__.py

@@ -173,6 +173,13 @@
 from ._feature_store_entity.data_column import DataColumn
 from ._feature_store_entity.data_column_type import DataColumnType
 from ._feature_store_entity.feature_store_entity import FeatureStoreEntity
+from ._indexes import (
+    AzureAISearchConfig,
+    IndexDataSource,
+    GitSource,
+    LocalSource,
+)
+from ._indexes import ModelConfiguration as IndexModelConfiguration
 from ._job.command_job import CommandJob
 from ._job.compute_configuration import ComputeConfiguration
 from ._job.input_port import InputPort
@@ -548,6 +555,11 @@
     "AadCredentialConfiguration",
     "Index",
     "AzureOpenAIDeployment",
+    "AzureAISearchConfig",
+    "IndexDataSource",
+    "GitSource",
+    "LocalSource",
+    "IndexModelConfiguration",
 ]
 
 # Allow importing these types for backwards compatibility

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_indexes/__init__.py

@@ -0,0 +1,16 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""AzureML Retrieval Augmented Generation (RAG) utilities."""
+
+from .input._ai_search_config import AzureAISearchConfig
+from .input._index_data_source import IndexDataSource, GitSource, LocalSource
+from .model_config import ModelConfiguration
+
+__all__ = [
+    "ModelConfiguration",
+    "AzureAISearchConfig",
+    "IndexDataSource",
+    "GitSource",
+    "LocalSource",
+]