Merge pull request #455 from nerdalert/file-conversion-service

Add a docling service file conversion feature

Author: vishnoianil

.env.native.example

@@ -10,3 +10,5 @@ IL_ENABLE_DEV_MODE=true #Enable this option if you want to enable UI features th
 NEXT_PUBLIC_TAXONOMY_ROOT_DIR=
 
 NEXT_PUBLIC_EXPERIMENTAL_FEATURES=false
+
+# IL_FILE_CONVERSION_SERVICE=http://localhost:8000 # Uncomment and fill in the http://host:port if the docling conversion service is running.

src/app/api/native/convert/route.ts

@@ -0,0 +1,68 @@
+// src/app/api/native/convert/route.ts
+import { NextResponse } from 'next/server';
+
+`use server`;
+
+interface ConvertRequestBody {
+  options?: {
+    output_markdown?: boolean;
+    include_images?: boolean;
+  };
+  file_source: {
+    base64_string: string;
+    filename: string;
+  };
+}
+
+// This route calls the external REST service to convert any doc => markdown
+export async function POST(request: Request) {
+  try {
+    // 1. Parse JSON body from client
+    const body: ConvertRequestBody = await request.json();
+
+    // 2. Read the IL_FILE_CONVERSION_SERVICE from .env (fallback to localhost if not set)
+    const baseUrl = process.env.IL_FILE_CONVERSION_SERVICE || 'http://localhost:8000';
+
+    // 3. Check the health of the conversion service before proceeding
+    const healthRes = await fetch(`${baseUrl}/health`);
+    if (!healthRes.ok) {
+      console.error('The file conversion service is offline or returned non-OK status:', healthRes.status, healthRes.statusText);
+      return NextResponse.json({ error: 'Conversion service is offline, only markdown files accepted.' }, { status: 503 });
+    }
+
+    // Parse the health response body in case we need to verify its "status":"ok"
+    const healthData = await healthRes.json();
+    if (!healthData.status || healthData.status !== 'ok') {
+      console.error('Doc->md conversion service health check response not "ok":', healthData);
+      return NextResponse.json({ error: 'Conversion service is offline, only markdown files accepted.' }, { status: 503 });
+    }
+
+    // 4. Service is healthy, proceed with md conversion
+    const res = await fetch(`${baseUrl}/convert/markdown`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify(body)
+    });
+
+    if (!res.ok) {
+      console.error('Conversion service responded with error', res.status, res.statusText);
+      return NextResponse.json({ error: `Conversion service call failed. ${res.statusText}` }, { status: 500 });
+    }
+
+    // 5. Wait for the docling service to return the user submitted file converted to markdown
+    const data = await res.text();
+
+    // Return the markdown wrapped in JSON so the client side can parse it
+    return NextResponse.json({ content: data }, { status: 200 });
+  } catch (error: unknown) {
+    if (error instanceof Error) {
+      console.error('Error during doc->md conversion route call:', error);
+      return NextResponse.json({ error: 'md conversion failed.', message: error.message }, { status: 500 });
+    } else {
+      console.error('Unknown error during conversion route call:', error);
+      return NextResponse.json({ error: 'conversion failed due to an unknown error.' }, { status: 500 });
+    }
+  }
+}

src/components/Contribute/Knowledge/Native/DocumentInformation/DocumentInformation.tsx

@@ -1,24 +1,17 @@
 // src/components/Contribute/Knowledge/Native/DocumentInformation/DocumentInformation.tsx
 import React, { useEffect, useState } from 'react';
+import { FormFieldGroupHeader, FormGroup, FormHelperText } from '@patternfly/react-core/dist/dynamic/components/Form';
+import { Button } from '@patternfly/react-core/dist/dynamic/components/Button';
+import { TextInput } from '@patternfly/react-core/dist/dynamic/components/TextInput';
+import { Alert, AlertActionLink, AlertActionCloseButton } from '@patternfly/react-core/dist/dynamic/components/Alert';
+import { HelperText } from '@patternfly/react-core/dist/dynamic/components/HelperText';
+import { HelperTextItem } from '@patternfly/react-core/dist/dynamic/components/HelperText';
+import ExclamationCircleIcon from '@patternfly/react-icons/dist/dynamic/icons/exclamation-circle-icon';
+import { ValidatedOptions } from '@patternfly/react-core/dist/esm/helpers/constants';
+import { Modal, ModalVariant } from '@patternfly/react-core/dist/esm/deprecated/components/Modal/Modal';
 import { UploadFile } from '@/components/Contribute/Knowledge/UploadFile';
 import { checkKnowledgeFormCompletion } from '@/components/Contribute/Knowledge/validation';
 import { KnowledgeFormData } from '@/types';
-import {
-  ValidatedOptions,
-  FormFieldGroupHeader,
-  FormGroup,
-  Button,
-  Modal,
-  ModalVariant,
-  TextInput,
-  FormHelperText,
-  HelperText,
-  HelperTextItem,
-  Alert,
-  AlertActionCloseButton,
-  AlertActionLink
-} from '@patternfly/react-core';
-import { ExclamationCircleIcon } from '@patternfly/react-icons';
 
 interface Props {
   reset: boolean;
@@ -252,24 +245,29 @@ const DocumentInformation: React.FC<Props> = ({
           </Button>
         </div>
       </FormGroup>
-      <Modal variant={ModalVariant.medium} title="Data Loss Warning" isOpen={isModalOpen} onClose={() => setIsModalOpen(false)}>
-        <p>{modalText}</p>
-        <div style={{ display: 'flex', justifyContent: 'flex-end', gap: '10px', marginTop: '20px' }}>
-          <Button variant="secondary" onClick={handleModalContinue}>
+      <Modal
+        variant={ModalVariant.medium}
+        title="Data Loss Warning"
+        titleIconVariant="warning"
+        isOpen={isModalOpen}
+        onClose={() => setIsModalOpen(false)}
+        actions={[
+          <Button key="Continue" variant="secondary" onClick={handleModalContinue}>
             Continue
-          </Button>
-          <Button variant="secondary" onClick={() => setIsModalOpen(false)}>
+          </Button>,
+          <Button key="cancel" variant="secondary" onClick={() => setIsModalOpen(false)}>
             Cancel
           </Button>
-        </div>
+        ]}
+      >
+        <p>{modalText}</p>
       </Modal>
       {!useFileUpload ? (
         <>
           <FormGroup isRequired key={'doc-info-details-id'} label="Repo URL or Server Side File Path">
             <TextInput
               isRequired
-              // TODO: once all of the different potential filepaths/url/types are determined, add back stricter validation
-              type="text"
+              type="url"
               aria-label="repo"
               validated={validRepo}
               placeholder="Enter repo URL where document exists"
@@ -328,8 +326,6 @@ const DocumentInformation: React.FC<Props> = ({
           </Button>
         </>
       )}
-
-      {/* Informational Alert */}
       {alertInfo && (
         <Alert variant={alertInfo.type} title={alertInfo.title} actionClose={<AlertActionCloseButton onClose={() => setAlertInfo(undefined)} />}>
           {alertInfo.message}
@@ -340,8 +336,6 @@ const DocumentInformation: React.FC<Props> = ({
           )}
         </Alert>
       )}
-
-      {/* Success Alert */}
       {successAlertTitle && successAlertMessage && (
         <Alert
           variant="success"
@@ -358,8 +352,6 @@ const DocumentInformation: React.FC<Props> = ({
           {successAlertMessage}
         </Alert>
       )}
-
-      {/* Failure Alert */}
       {failureAlertTitle && failureAlertMessage && (
         <Alert variant="danger" title={failureAlertTitle} actionClose={<AlertActionCloseButton onClose={onCloseFailureAlert} />}>
           {failureAlertMessage}

src/components/Contribute/Knowledge/Native/KnowledgeQuestionAnswerPairsNative/KnowledgeQuestionAnswerPairs.tsx

@@ -60,6 +60,8 @@ const KnowledgeQuestionAnswerPairsNative: React.FC<Props> = ({
   const [expandedFiles, setExpandedFiles] = useState<Record<string, boolean>>({});
   const [selectedWordCount, setSelectedWordCount] = useState<number>(0);
   const [showAllCommits, setShowAllCommits] = useState<boolean>(false);
+  const [contextWordCount, setContextWordCount] = useState(0);
+  const MAX_WORDS = 500;
 
   // Ref for the <pre> elements to track selections TODO: figure out how to make text expansions taller in PF without a custom-pre
   const preRefs = useRef<Record<string, HTMLPreElement | null>>({});
@@ -217,6 +219,27 @@ const KnowledgeQuestionAnswerPairsNative: React.FC<Props> = ({
     []
   );
 
+  // TODO: replace with a tokenizer library
+  const countWords = (text: string) => {
+    return text.trim().split(/\s+/).filter(Boolean).length;
+  };
+
+  // Update word count whenever context changes
+  useEffect(() => {
+    setContextWordCount(countWords(seedExample.context));
+  }, [seedExample.context]);
+
+  // Handle context input change with word count validation
+  const onContextChange = (_event: React.FormEvent<HTMLTextAreaElement>, contextValue: string) => {
+    const wordCount = countWords(contextValue);
+    if (wordCount <= MAX_WORDS) {
+      handleContextInputChange(seedExampleIndex, contextValue);
+    } else {
+      // allow the overage and show validation error
+      handleContextInputChange(seedExampleIndex, contextValue);
+    }
+  };
+
   return (
     <FormGroup style={{ padding: '20px' }}>
       <Tooltip content={<div>Select context from your knowledge files</div>} position="top">
@@ -232,11 +255,18 @@ const KnowledgeQuestionAnswerPairsNative: React.FC<Props> = ({
         placeholder="Enter the context from which the Q&A pairs are derived. (500 words max)"
         value={seedExample.context}
         validated={seedExample.isContextValid}
-        maxLength={500}
+        onChange={onContextChange}
         style={{ marginBottom: '20px' }}
-        onChange={(_event, contextValue: string) => handleContextInputChange(seedExampleIndex, contextValue)}
         onBlur={() => handleContextBlur(seedExampleIndex)}
       />
+      {/* Display word count */}
+      <FormHelperText>
+        <HelperText>
+          <HelperTextItem>
+            {contextWordCount} / {MAX_WORDS} words
+          </HelperTextItem>
+        </HelperText>
+      </FormHelperText>
       {seedExample.isContextValid === ValidatedOptions.error && (
         <FormHelperText>
           <HelperText>
@@ -252,7 +282,6 @@ const KnowledgeQuestionAnswerPairsNative: React.FC<Props> = ({
           <div
             style={{
               padding: '10px',
-              // backgroundColor: '#f2f2f2',
               borderRadius: '5px',
               marginBottom: '10px',
               fontSize: '14px',

src/components/Contribute/Knowledge/Native/index.tsx

@@ -490,14 +490,16 @@ export const KnowledgeFormNative: React.FunctionComponent<KnowledgeFormProps> =
       immutable: true,
       isExpanded: false,
       context: yamlSeedExample.context,
-      // TODO: Hardcoding yaml QnA uploads seed_examples to success until working with Validate.tsx - Bug #465
-      isContextValid: ValidatedOptions.success,
+      isContextValid: ValidatedOptions.default,
+      validationError: '',
       questionAndAnswers: yamlSeedExample.questions_and_answers.map((qa) => ({
         immutable: true,
         question: qa.question,
         answer: qa.answer,
-        isQuestionValid: ValidatedOptions.success,
-        isAnswerValid: ValidatedOptions.success
+        isQuestionValid: ValidatedOptions.default,
+        questionValidationError: '',
+        isAnswerValid: ValidatedOptions.default,
+        answerValidationError: ''
       }))
     }));