store rag file metadata in db

Author: Veikkosuhonen

src/client/components/Rag/ProgressReporter.tsx

@@ -0,0 +1,73 @@
+import { DataTypes } from 'sequelize'
+
+import { Migration } from '../connection'
+
+export const up: Migration = ({ context: queryInterface }) =>
+  queryInterface.createTable('rag_files', {
+    id: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      primaryKey: true,
+      autoIncrement: true,
+    },
+    rag_index_id: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      references: {
+        model: 'rag_indices',
+        key: 'id',
+      },
+      onUpdate: 'CASCADE',
+      onDelete: 'CASCADE',
+    },
+    pipeline_stage: {
+      type: DataTypes.STRING,
+      allowNull: false,
+      defaultValue: 'pending',
+      comment: 'Stage of the ingestion pipeline',
+    },
+    error: {
+      type: DataTypes.STRING,
+      allowNull: true,
+      comment: 'Error message if any error occurred during the ingestion pipeline',
+    },
+    filename: {
+      type: DataTypes.STRING,
+      allowNull: false,
+      comment: 'Name of the uploaded file',
+    },
+    file_type: {
+      type: DataTypes.STRING,
+      allowNull: false,
+      comment: 'Type of the uploaded file (e.g., pdf, md, text)',
+    },
+    file_size: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      comment: 'Number of characters in the file after text extraction',
+    },
+    num_chunks: {
+      type: DataTypes.INTEGER,
+      allowNull: true,
+      comment: 'Number of chunks created from the file',
+    },
+    user_id: {
+      type: DataTypes.STRING,
+      allowNull: false,
+      comment: 'ID of the user who uploaded the file',
+    },
+    metadata: {
+      type: DataTypes.JSONB,
+      allowNull: true,
+    },
+    created_at: {
+      type: DataTypes.DATE,
+      allowNull: false,
+    },
+    updated_at: {
+      type: DataTypes.DATE,
+      allowNull: false,
+    },
+  })
+
+export const down: Migration = ({ context: queryInterface }) => queryInterface.dropTable('rag_files')

src/server/db/migrations/20250521_00_add_files_to_rag_indexes.ts

@@ -6,6 +6,7 @@ import Enrolment from './enrolment'
 import Responsibility from './responsibilities'
 import Discussion from './discussion'
 import RagIndex from './ragIndex'
+import RagFile from './ragFile'
 
 User.belongsToMany(ChatInstance, {
   through: UserChatInstanceUsage,
@@ -47,4 +48,10 @@ User.hasMany(RagIndex, { as: 'ragIndices' })
 
 RagIndex.belongsTo(User, { as: 'user' })
 
-export { User, ChatInstance, UserChatInstanceUsage, Prompt, Enrolment, Responsibility, Discussion, RagIndex }
+RagFile.belongsTo(RagIndex, { as: 'ragIndex' })
+
+RagIndex.hasMany(RagFile, { as: 'ragFiles' })
+
+RagFile.belongsTo(User, { as: 'user' })
+
+export { User, ChatInstance, UserChatInstanceUsage, Prompt, Enrolment, Responsibility, Discussion, RagIndex, RagFile }

src/server/db/migrations/20250521_00_init_rag_files.ts

@@ -0,0 +1,89 @@
+import { Model, InferAttributes, InferCreationAttributes, CreationOptional, DataTypes } from 'sequelize'
+
+import { sequelize } from '../connection'
+import { IngestionPipelineStageKey } from '../../../shared/constants'
+
+class RagFile extends Model<InferAttributes<RagFile>, InferCreationAttributes<RagFile>> {
+  declare id: CreationOptional<number>
+
+  declare ragIndexId: number
+
+  declare pipelineStage: CreationOptional<IngestionPipelineStageKey>
+
+  declare error: CreationOptional<string | null>
+
+  declare filename: string
+
+  declare fileType: string
+
+  declare fileSize: number
+
+  declare numChunks: CreationOptional<number | null>
+
+  declare userId: string
+
+  declare metadata: Record<string, unknown> | null
+}
+
+RagFile.init(
+  {
+    id: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      primaryKey: true,
+      autoIncrement: true,
+    },
+    ragIndexId: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      references: {
+        model: 'rag_indices',
+        key: 'id',
+      },
+      onUpdate: 'CASCADE',
+      onDelete: 'CASCADE',
+    },
+    pipelineStage: {
+      type: DataTypes.STRING,
+      allowNull: false,
+      defaultValue: 'pending',
+    },
+    error: {
+      type: DataTypes.STRING,
+      allowNull: true,
+    },
+    filename: {
+      type: DataTypes.STRING,
+      allowNull: false,
+    },
+    fileType: {
+      type: DataTypes.STRING,
+      allowNull: false,
+    },
+    fileSize: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+    },
+    numChunks: {
+      type: DataTypes.INTEGER,
+      allowNull: true,
+    },
+    userId: {
+      type: DataTypes.STRING,
+      allowNull: false,
+    },
+    metadata: {
+      type: DataTypes.JSONB,
+      allowNull: true,
+    },
+  },
+  {
+    underscored: true,
+    timestamps: true,
+    sequelize,
+  },
+)
+
+export default RagFile
+
+export type RagFileAttributes = InferAttributes<RagFile>