Load app templates from yml files

Author: ajhai

apps/apis.py

@@ -42,9 +42,10 @@
 from apps.integration_configs import WebIntegrationConfig
 from emails.sender import EmailSender
 from emails.templates.factory import EmailTemplateFactory
-from processors.models import ApiBackend
+from processors.models import ApiBackend, ApiProvider
 from processors.models import Endpoint
 from base.models import Profile
+from apps.yaml_loader import get_app_template_by_slug, get_app_templates_from_contrib
 
 logger = logging.getLogger(__name__)
 
@@ -205,15 +206,36 @@ def getCloneableApps(self, request, uid=None):
     def getTemplates(self, request, slug=None):
         json_data = None
         if slug:
-            object = get_object_or_404(AppTemplate, slug=slug)
-            serializer = AppTemplateSerializer(
-                instance=object, context={'hide_details': False},
-            )
-            json_data = serializer.data
+            object = get_app_template_by_slug(slug)
+            if object:
+                object_dict = object.dict()
+                # For backward compatibility with old app templates
+                for page in object_dict['pages']:
+                    page['schema'] = page['input_schema']
+                    page['ui_schema'] = page['input_ui_schema']
+                json_data = object_dict
+            else:
+                object = get_object_or_404(AppTemplate, slug=slug)
+                serializer = AppTemplateSerializer(
+                    instance=object, context={'hide_details': False},
+                )
+                json_data = serializer.data
         else:
             queryset = AppTemplate.objects.all().order_by('order')
             serializer = AppTemplateSerializer(queryset, many=True)
             json_data = serializer.data
+
+            # Add app templates from contrib
+            app_template_slugs = list(map(lambda x: x['slug'], json_data))
+            app_templates_from_yaml = get_app_templates_from_contrib()
+            for app_template in app_templates_from_yaml:
+                if app_template.slug not in app_template_slugs:
+                    # When listing, do not show pages and app
+                    app_template_dict = app_template.dict()
+                    app_template_dict.pop('pages')
+                    app_template_dict.pop('app')
+                    json_data.append(app_template_dict)
+
         return DRFResponse(json_data)
 
     def publish(self, request, uid):
@@ -302,12 +324,12 @@ def delete(self, request, uid):
         # Cleanup app run_graph
         run_graph_entries = app.run_graph.all()
         endpoint_entries = list(filter(lambda x: x != None, set(list(map(lambda x: x.entry_endpoint, run_graph_entries)) +
-                                                           list(map(lambda x: x.exit_endpoint, run_graph_entries)))))
+                                                                list(map(lambda x: x.exit_endpoint, run_graph_entries)))))
 
-        # Cleanup rungraph 
+        # Cleanup rungraph
         # Delete all the run_graph entries
         run_graph_entries.delete()
-            
+
         # Delete all the endpoint entries
         for entry in endpoint_entries:
             EndpointViewSet.delete(self, request, id=str(
@@ -418,7 +440,8 @@ def patch(self, request, uid):
 
     def post(self, request):
         owner = request.user
-        app_type = get_object_or_404(AppType, id=request.data['app_type'])
+        app_type = get_object_or_404(AppType, id=request.data['app_type']) if 'app_type' in request.data else get_object_or_404(
+            AppType, slug=request.data['app_type_slug'])
         app_name = request.data['name']
         app_description = request.data['description'] if 'description' in request.data else ''
         app_config = request.data['config'] if 'config' in request.data else {}
@@ -435,9 +458,18 @@ def post(self, request):
         # Iterate over processors and create an endpoint for each using api_backend
         # and then use that to create AppRunGraphEntry to create the graph
         for processor in processors:
-            api_backend_obj = get_object_or_404(
-                ApiBackend, id=processor['api_backend'],
-            )
+            api_backend_obj = None
+            if 'processor_slug' in processor and 'provider_slug' in processor:
+                provider = get_object_or_404(
+                    ApiProvider, slug=processor['provider_slug'],
+                )
+                api_backend_obj = get_object_or_404(
+                    ApiBackend, slug=processor['processor_slug'], api_provider=provider,
+                )
+            else:
+                api_backend_obj = get_object_or_404(
+                    ApiBackend, id=processor['api_backend'],
+                )
             endpoint = Endpoint.objects.create(
                 name=f'{app_name}:{api_backend_obj.name}',
                 owner=owner,
@@ -485,9 +517,10 @@ def post(self, request):
             edge.save()
             graph_edges.append(edge)
 
+        template_slug = request.data['template_slug'] if 'template_slug' in request.data else None
         template = AppTemplate.objects.filter(
-            slug=request.data['template_slug'],
-        ).first() if 'template_slug' in request.data else None
+            slug=template_slug,
+        ).first() if template_slug else None
 
         app = App.objects.create(
             name=app_name,
@@ -500,6 +533,7 @@ def post(self, request):
             output_template=app_output_template,
             data_transformer=app_data_transformer,
             template=template,
+            template_slug=template_slug,
         )
         app.run_graph.set(graph_edges)
         app.save()

apps/schemas.py

@@ -0,0 +1,104 @@
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel, Field
+
+"""
+This file contains YAML schemas for the apps and app templates.
+"""
+
+
+class InputField(BaseModel):
+    advanced_parameter: Optional[bool]
+    default: Optional[Any]
+    description: Optional[str]
+    enum: Optional[List[Any]]
+    enumNames: Optional[List[str]]
+    format: Optional[str]
+    items: Optional[Dict[str, Any]]
+    maximum: Optional[int]
+    minimum: Optional[int]
+    multipleOf: Optional[int]
+    name: str
+    path: Optional[str]
+    pattern: Optional[str]
+    required: Optional[bool]
+    title: str
+    type: str = "string"
+    widget: Optional[str]
+
+
+class OutputTemplate(BaseModel):
+    """
+    OutputTemplate schema
+    """
+    markdown: str = Field(None, description='Markdown template for output')
+
+
+class Provider(BaseModel):
+    """
+    Provider schema
+    """
+    name: str = Field(None, description='Name of the provider')
+    slug: str = Field(
+        None, description='Unique slug for the provider. Must be @github username for individual contributions')
+    description: str = Field(None, description='Description of the provider')
+
+
+class Processor(BaseModel):
+    """
+    Processor schema
+    """
+    id: str = Field(
+        None, description='Unique identifier for the processor in the app run graph')
+    provider_slug: str = Field(
+        None, description='Slug of the processor provider')
+    processor_slug: str = Field(None, description='Slug of the processor')
+    # TODO: Validate input and config against backing processor's schemas
+    input: dict = Field(None, description='Input for the processor')
+    config: dict = Field(None, description='Configuration for the processor')
+
+
+class App(BaseModel):
+    """
+    App schema
+    """
+    name: str = Field(None, description='Name of the app')
+    slug: str = Field(None, description='Unique slug for the app')
+    type_slug: str = Field(None, description='Slug of the app type')
+    description: str = Field(None, description='Description of the app')
+    input_fields: Optional[List[InputField]] = Field(
+        None, description='Input fields for the app')
+    input_schema: Optional[dict] = Field(
+        None, description='Input schema for the app')
+    input_ui_schema: Optional[dict] = Field(
+        None, description='Input UI schema for the app')
+    output_template: OutputTemplate = Field(
+        None, description='Output template for the app')
+    processors: List[Processor] = Field(
+        None, description='Processors for the app')
+
+
+class AppTemplatePage(BaseModel):
+    """
+    AppTemplatePage schema
+    """
+    title: str = Field(None, description='Title of the page')
+    description: str = Field(None, description='Description of the page')
+    input_fields: Optional[List[InputField]] = Field(
+        None, description='Input fields for the page')
+    input_schema: Optional[dict] = Field(
+        None, description='Schema for the page')
+    input_ui_schema: Optional[dict] = Field(
+        None, description='UI schema for the page')
+
+
+class AppTemplate(BaseModel):
+    """
+    AppTemplate schema
+    """
+    name: str = Field(None, description='Name of the app template')
+    slug: str = Field(None, description='Unique slug for the app template')
+    description: str = Field(
+        None, description='Description of the app template')
+    pages: List[AppTemplatePage] = Field(
+        None, description='Pages of the app template')
+    app: App = Field(None, description='App of the app template')

apps/serializers.py

@@ -1,5 +1,7 @@
 from rest_framework import serializers
 
+from apps.yaml_loader import get_app_template_by_slug
+
 from .models import App
 from .models import AppHub
 from .models import AppRunGraphEntry
@@ -191,7 +193,13 @@ def get_last_modified_by_email(self, obj):
         return obj.last_modified_by.email if (obj.last_modified_by and obj.has_write_permission(self._request_user)) else None
 
     def get_template(self, obj):
-        return AppTemplateSerializer(instance=obj.template).data if obj.template else None
+        if obj.template:
+            return AppTemplateSerializer(instance=obj.template).data
+        elif obj.template_slug is not None:
+            app_template = get_app_template_by_slug(obj.template_slug)
+            if app_template:
+                return app_template.dict()
+        return None
 
     def get_web_config(self, obj):
         return obj.web_config if obj.has_write_permission(self._request_user) else None

apps/yaml_loader.py

@@ -0,0 +1,97 @@
+"""
+Utils to convert yaml to App and AppTemplate schema models and vice versa
+"""
+import yaml
+import os
+from django.conf import settings
+from django.core.cache import cache
+from typing import List, Type
+from pydantic import BaseModel, Field, create_model
+from apps.schemas import AppTemplate
+
+from common.blocks.base.schema import get_ui_schema_from_json_schema
+
+
+def get_input_model_from_fields(name: str, input_fields: list) -> Type['BaseModel']:
+    """
+    Dynamically creates a Pydantic model from a list of input fields.
+
+    Args:
+        name (str): The name of the model to be created.
+        input_fields (list): A list of dictionaries representing the input fields of the model.
+
+    Returns:
+        Type['BaseModel']: The dynamically created Pydantic model.
+    """
+    return create_model(
+        f'{name}', **{field['name']: (int if field['type'] == 'int' else str, Field(**{k: field[k] for k in field})) for field in input_fields})
+
+
+def get_app_template_from_yaml(yaml_file: str) -> dict:
+    """
+    Reads a YAML file and returns a dictionary containing the app template.
+
+    Args:
+        yaml_file (str): The path to the YAML file.
+
+    Returns:
+        dict: A dictionary containing the app template.
+    """
+    with open(yaml_file, 'r') as f:
+        yaml_dict = yaml.safe_load(f)
+
+        # Construct dynamic models for app template page input and app input
+        pages = yaml_dict.get('pages', [])
+        for page in pages:
+            input_fields = page.get('input_fields', [])
+            input_model = get_input_model_from_fields(
+                page["title"], input_fields)
+            page['input_schema'] = input_model.schema()
+            page['input_ui_schema'] = get_ui_schema_from_json_schema(
+                input_model.schema())
+            page.pop('input_fields')
+
+        app = yaml_dict.get('app', {})
+        input_fields = app.get('input_fields', [])
+        input_model = get_input_model_from_fields(
+            app["name"], input_fields)
+        app['input_schema'] = input_model.schema()
+        app['input_schema'].pop('title')
+        app['input_ui_schema'] = get_ui_schema_from_json_schema(
+            input_model.schema())
+        app.pop('input_fields')
+
+        return AppTemplate(**yaml_dict)
+
+
+def get_app_templates_from_contrib() -> List[AppTemplate]:
+    """
+    Loads app templates from yaml files in settings.APP_TEMPLATES_DIR and caches them in memory.
+    """
+    app_templates = cache.get('app_templates')
+    if app_templates:
+        return app_templates
+
+    app_templates = []
+    if not hasattr(settings, 'APP_TEMPLATES_DIR'):
+        return app_templates
+
+    for file in os.listdir(settings.APP_TEMPLATES_DIR):
+        if file.endswith('.yml'):
+            app_template = get_app_template_from_yaml(
+                os.path.join(settings.APP_TEMPLATES_DIR, file))
+            if app_template:
+                app_templates.append(app_template)
+    cache.set('app_templates', app_templates)
+
+    return app_templates
+
+
+def get_app_template_by_slug(slug: str) -> dict:
+    """
+    Returns an app template by slug.
+    """
+    for app_template in get_app_templates_from_contrib():
+        if app_template.slug == slug:
+            return app_template
+    return None

client/src/components/apps/AppTemplatesList.jsx

@@ -56,10 +56,13 @@ export function AppTemplatesList() {
       ...(appTemplate.app || {}),
       name: appName || appTemplate?.name || "Untitled",
       app_type: appTemplate.app?.type,
+      app_type_slug: appTemplate.app?.type_slug,
       template_slug: appTemplate.slug,
     };
     payload.processors = payload.processors.map((processor) => ({
-      api_backend: processor.api_backend.id,
+      api_backend: processor.api_backend?.id,
+      provider_slug: processor.provider_slug,
+      processor_slug: processor.processor_slug,
       config: processor.config,
       input: processor.input,
     }));