Merge remote-tracking branch 'upstream/master'

Author: dchourasia

ui/.gitignore

@@ -1,2 +1,3 @@
 src/protos.d.ts
 src/protos.js
+feature_repo/data/*.parquet

ui/.npmrc

@@ -1 +1 @@
-//registry.npmjs.org/:_authToken=${NPM_TOKEN}
+# //registry.npmjs.org/:_authToken=${NPM_TOKEN}

ui/feature_repo/apply_permissions.py

@@ -0,0 +1,25 @@
+from feast import FeatureStore
+from features import (
+    zipcode_features_permission,
+    zipcode_source_permission,
+    model_v1_permission,
+    risky_features_permission,
+    document_embeddings_permission,
+    document_metadata_permission,
+    rag_model_permission,
+)
+
+store = FeatureStore(repo_path=".")
+
+store.apply([
+    zipcode_features_permission,
+    zipcode_source_permission,
+    model_v1_permission,
+    risky_features_permission,
+    document_embeddings_permission,
+    document_metadata_permission,
+    rag_model_permission,
+])
+
+print("Permissions applied successfully!")
+print("Current permissions:", store.list_permissions())

ui/feature_repo/apply_rag_data.py

@@ -0,0 +1,32 @@
+import pandas as pd
+import numpy as np
+from datetime import datetime, timedelta
+
+now = datetime.now()
+embeddings = []
+for i in range(10):
+    embeddings.append({
+        'document_id': f'doc_{i}',
+        'embedding': np.random.rand(768).astype(np.float32),
+        'event_timestamp': now - timedelta(days=i),
+        'created_timestamp': now - timedelta(days=i, hours=1)
+    })
+df_embeddings = pd.DataFrame(embeddings)
+df_embeddings.to_parquet('data/document_embeddings.parquet', index=False)
+
+metadata = []
+for i in range(10):
+    metadata.append({
+        'document_id': f'doc_{i}',
+        'title': f'Document {i}',
+        'content': f'This is the content of document {i}',
+        'source': 'web',
+        'author': f'author_{i}',
+        'publish_date': (now - timedelta(days=i*30)).strftime('%Y-%m-%d'),
+        'event_timestamp': now - timedelta(days=i),
+        'created_timestamp': now - timedelta(days=i, hours=1)
+    })
+df_metadata = pd.DataFrame(metadata)
+df_metadata.to_parquet('data/document_metadata.parquet', index=False)
+
+print('Created RAG data files successfully!')

ui/feature_repo/data/document_embeddings.parquet

@@ -5,3 +5,4 @@ online_store:
   type: sqlite
 offline_store:
   type: file
+entity_key_serialization_version: 2

ui/feature_repo/data/document_metadata.parquet

@@ -1,11 +1,15 @@
 from datetime import timedelta
 
 import pandas as pd
+import numpy as np
 
 from feast import Entity, FeatureService, FeatureView, Field, FileSource
 from feast.data_source import RequestSource
 from feast.on_demand_feature_view import on_demand_feature_view
-from feast.types import Bool, Int64, String
+from feast.permissions.action import AuthzedAction, READ
+from feast.permissions.permission import Permission
+from feast.permissions.policy import RoleBasedPolicy
+from feast.types import Bool, Int64, String, Float32, Array
 
 zipcode = Entity(
     name="zipcode",
@@ -199,3 +203,153 @@ def transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame:
     tags={"owner": "[email protected]", "stage": "dev"},
     description="Location model",
 )
+
+zipcode_features_permission = Permission(
+    name="zipcode-features-reader",
+    types=[FeatureView],
+    name_patterns=["zipcode_features"],
+    policy=RoleBasedPolicy(roles=["analyst", "data_scientist"]),
+    actions=[AuthzedAction.DESCRIBE, *READ],
+)
+
+zipcode_source_permission = Permission(
+    name="zipcode-source-writer",
+    types=[FileSource],
+    name_patterns=["zipcode"],
+    policy=RoleBasedPolicy(roles=["admin", "data_engineer"]),
+    actions=[AuthzedAction.CREATE, AuthzedAction.UPDATE, AuthzedAction.WRITE_OFFLINE],
+)
+
+model_v1_permission = Permission(
+    name="credit-score-v1-reader",
+    types=[FeatureService],
+    name_patterns=["credit_score_v1"],
+    policy=RoleBasedPolicy(roles=["model_user", "data_scientist"]),
+    actions=[AuthzedAction.DESCRIBE, AuthzedAction.READ_ONLINE],
+)
+
+risky_features_permission = Permission(
+    name="risky-features-reader",
+    types=[FeatureView, FeatureService],
+    required_tags={"stage": "prod"},
+    policy=RoleBasedPolicy(roles=["trusted_analyst"]),
+    actions=[AuthzedAction.READ_OFFLINE],
+)
+
+document = Entity(
+    name="document_id",
+    description="Document identifier for RAG system",
+    tags={
+        "owner": "[email protected]",
+        "team": "rag",
+    },
+)
+
+document_source = FileSource(
+    name="document_embeddings",
+    path="data/document_embeddings.parquet",
+    timestamp_field="event_timestamp",
+    created_timestamp_column="created_timestamp",
+)
+
+document_metadata_source = FileSource(
+    name="document_metadata",
+    path="data/document_metadata.parquet",
+    timestamp_field="event_timestamp",
+    created_timestamp_column="created_timestamp",
+)
+
+document_embeddings_view = FeatureView(
+    name="document_embeddings",
+    entities=[document],
+    ttl=timedelta(days=365),
+    schema=[
+        Field(name="embedding", dtype=Array(Float32, 768)),
+        Field(name="document_id", dtype=String),
+    ],
+    source=document_source,
+    tags={
+        "date_added": "2025-05-04",
+        "model": "sentence-transformer",
+        "access_group": "[email protected]",
+        "stage": "prod",
+    },
+    online=True,
+)
+
+document_metadata_view = FeatureView(
+    name="document_metadata",
+    entities=[document],
+    ttl=timedelta(days=365),
+    schema=[
+        Field(name="title", dtype=String),
+        Field(name="content", dtype=String),
+        Field(name="source", dtype=String),
+        Field(name="author", dtype=String),
+        Field(name="publish_date", dtype=String),
+        Field(name="document_id", dtype=String),
+    ],
+    source=document_metadata_source,
+    tags={
+        "date_added": "2025-05-04",
+        "access_group": "[email protected]",
+        "stage": "prod",
+    },
+    online=True,
+)
+
+# Define a request data source for query embeddings
+query_request = RequestSource(
+    name="query",
+    schema=[
+        Field(name="query_embedding", dtype=Array(Float32, 768)),
+    ],
+)
+
+# Define an on-demand feature view for similarity calculation
+@on_demand_feature_view(
+    sources=[document_embeddings_view, query_request],
+    schema=[
+        Field(name="similarity_score", dtype=Float32),
+    ],
+)
+def document_similarity(inputs: pd.DataFrame) -> pd.DataFrame:
+    """Calculate cosine similarity between query and document embeddings."""
+    df = pd.DataFrame()
+    df["similarity_score"] = 0.95  # Placeholder value
+    return df
+
+rag_model = FeatureService(
+    name="rag_retriever",
+    features=[
+        document_embeddings_view,
+        document_metadata_view,
+        document_similarity,
+    ],
+    tags={"owner": "[email protected]", "stage": "prod"},
+    description="Retrieval Augmented Generation model",
+)
+
+document_embeddings_permission = Permission(
+    name="document-embeddings-reader",
+    types=[FeatureView],
+    name_patterns=["document_embeddings"],
+    policy=RoleBasedPolicy(roles=["ml_engineer", "data_scientist"]),
+    actions=[AuthzedAction.DESCRIBE, *READ],
+)
+
+document_metadata_permission = Permission(
+    name="document-metadata-reader",
+    types=[FeatureView],
+    name_patterns=["document_metadata"],
+    policy=RoleBasedPolicy(roles=["ml_engineer", "content_manager"]),
+    actions=[AuthzedAction.DESCRIBE, *READ],
+)
+
+rag_model_permission = Permission(
+    name="rag-model-user",
+    types=[FeatureService],
+    name_patterns=["rag_retriever"],
+    policy=RoleBasedPolicy(roles=["ml_engineer", "app_developer"]),
+    actions=[AuthzedAction.DESCRIBE, AuthzedAction.READ_ONLINE],
+)

ui/feature_repo/feature_store.yaml

@@ -25,6 +25,7 @@
   "dependencies": {
     "@elastic/datemath": "^5.0.3",
     "@elastic/eui": "^95.12.0",
+    "@elastic/eui-theme-borealis": "1.0.0",
     "@emotion/css": "^11.13.0",
     "@emotion/react": "^11.13.3",
     "@types/dagre": "^0.7.52",

ui/feature_repo/features.py

@@ -1,6 +1,5 @@
 import React from "react";
 
-import "@elastic/eui/dist/eui_theme_light.css";
 import "./index.css";
 
 import { Routes, Route } from "react-router-dom";
@@ -23,6 +22,8 @@ import FeatureServiceInstance from "./pages/feature-services/FeatureServiceInsta
 import DataSourceInstance from "./pages/data-sources/DataSourceInstance";
 import RootProjectSelectionPage from "./pages/RootProjectSelectionPage";
 import DatasetInstance from "./pages/saved-data-sets/DatasetInstance";
+import PermissionsIndex from "./pages/permissions/Index";
+import LineageIndex from "./pages/lineage/Index";
 import NoProjectGuard from "./components/NoProjectGuard";
 
 import TabsRegistryContext, {
@@ -144,6 +145,8 @@ const FeastUISansProvidersInner = ({
                       path="data-set/:datasetName/*"
                       element={<DatasetInstance />}
                     />
+                    <Route path="permissions/" element={<PermissionsIndex />} />
+                    <Route path="lineage/" element={<LineageIndex />} />
                   </Route>
                 </Route>
                 <Route path="*" element={<NoMatch />} />