diff --git a/.adminjs/bundle.js b/.adminjs/bundle.js deleted file mode 100644 index 0bbb8e4..0000000 --- a/.adminjs/bundle.js +++ /dev/null @@ -1,6 +0,0 @@ -(function () { - 'use strict'; - - AdminJS.UserComponents = {}; -})(); -//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYnVuZGxlLmpzIiwic291cmNlcyI6WyJlbnRyeS5qcyJdLCJzb3VyY2VzQ29udGVudCI6WyJBZG1pbkpTLlVzZXJDb21wb25lbnRzID0ge31cbiJdLCJuYW1lcyI6WyJBZG1pbkpTIiwiVXNlckNvbXBvbmVudHMiXSwibWFwcGluZ3MiOiI7OztDQUFBQSxPQUFPLENBQUNDLGNBQWMsR0FBRyxFQUFFOzs7Ozs7In0= diff --git a/.adminjs/entry.js b/.adminjs/entry.js deleted file mode 100644 index cbe129d..0000000 --- a/.adminjs/entry.js +++ /dev/null @@ -1 +0,0 @@ -AdminJS.UserComponents = {}; diff --git a/src/controllers/ToolsCatalogController.ts b/src/controllers/ToolsCatalogController.ts deleted file mode 100644 index 5929de3..0000000 --- a/src/controllers/ToolsCatalogController.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { validate } from 'class-validator'; -import { Request, Response } from 'express'; -import { AppDataSource } from '../db/data-source.js'; -import { ResponseUtil } from '../utils/Response.js'; -import { CreateSLCToolsDTO } from '../dtos/SLCToolsDTO.js'; // Define this DTO as per your requirements -import { slc_tools_catalog } from '../db/entities/SLCToolsCatalog.js'; - -export class ToolsCatalogController { - async getToolsCatalog(req: Request, res: Response): Promise { - const toolsCatalogData = await AppDataSource.getRepository(slc_tools_catalog).find(); - return ResponseUtil.sendResponse(res, toolsCatalogData, 200); - } - - async createToolsCatalogItem(req: Request, res: Response): Promise { - try { - const toolsCatalogData = req.body; - - // Assign default values, might serve from a utility file soon - toolsCatalogData.tool_description = toolsCatalogData.tool_description || 'No tool description'; - toolsCatalogData.keywords = toolsCatalogData.keywords || 'no, Keywords'; - toolsCatalogData.contact_email = toolsCatalogData.contact_email || 'default@email.com'; - - const dto = new CreateSLCToolsDTO(); - Object.assign(dto, toolsCatalogData); - - console.time('DTO Validation'); - const errors = await validate(dto); - console.timeEnd('DTO Validation'); - - if (errors.length > 0) { - return ResponseUtil.sendError(res, 'Invalid data', 400, errors); - } - - const repo = AppDataSource.getRepository(slc_tools_catalog); - const toolsCatalogItem = repo.create(toolsCatalogData); - await repo.save(toolsCatalogItem); - return ResponseUtil.sendResponse(res, toolsCatalogItem, 201); - } catch (error) { - console.error('Error in saving tool:', error); - return ResponseUtil.sendError(res, 'Server Error', 500, error); - } - } -} diff --git a/src/controllers/ViewController.ts b/src/controllers/ViewController.ts index 852d4bd..264fbce 100644 --- a/src/controllers/ViewController.ts +++ b/src/controllers/ViewController.ts @@ -3,20 +3,12 @@ import { AppDataSource } from '../db/data-source.js'; import { slc_item_catalog } from '../db/entities/SLCItemCatalog.js'; import logger from '../utils/logger.js'; import fs from 'fs'; -import { CreateSLCItemDTO } from '../dtos/SLCItemDTO.js'; -import { validate } from 'class-validator'; -import { ResponseUtil } from '../utils/Response.js'; import { slc_tools_catalog } from '../db/entities/SLCToolsCatalog.js'; import { dataset_catalog } from '../db/entities/DatasetCatalog.js'; -import { CreateDatasetCatalogDTO } from '../dtos/DatasetCatalogDTO.js'; import { ReviewController } from './ReviewController.js'; -import { ValidationManager } from '../services/ValidationManager.js'; import { ValidationResults } from '../db/entities/ValidationResults.js'; const reviewController = new ReviewController(); -const validationResultsRepository = AppDataSource.getRepository(ValidationResults); -const catalogRepository = AppDataSource.getRepository(slc_item_catalog); -const validationManager = new ValidationManager(validationResultsRepository, catalogRepository); import { SearchController } from './SearchController.js'; @@ -96,83 +88,6 @@ export class ViewController { res.status(500).json({ error: 'Internal Server Error' }); } } - - async approveAll(req: Request, res: Response) { - const data = req.body.data; - logger.info('Approve All called with data:', data); - const jsonArray = JSON.parse(data); - logger.info('Parsed JSON data:', jsonArray); - - let processedCount = 0; - const itemsToClassify: CreateSLCItemDTO[] = []; - - for (const item of jsonArray) { - let dto, repo; - - // Check if the item has an exercise_name - if (!item.exercise_name || item.exercise_name.trim() === '') { - logger.info('Skipping item without exercise_name:', item); - continue; // Skip this item if it doesn't have an exercise_name - } - - switch (item.catalog_type) { - case 'SLCItemCatalog': { - logger.info('SLCItemCatalog called with data, logger'); - dto = new CreateSLCItemDTO(); - Object.assign(dto, item); - repo = AppDataSource.getRepository(slc_item_catalog); - itemsToClassify.push(dto); // Add to classification list - break; - } - - case 'DatasetCatalog': { - const dto = new CreateDatasetCatalogDTO(); - Object.assign(dto, item); - const validationErrors = await validate(dto); - if (validationErrors.length > 0) { - logger.error(`Validation errors for ${item.catalog_type}:`, validationErrors); - continue; // Skip this item if validation fails - } - const repo = AppDataSource.getRepository(dataset_catalog); - const catalogItem = repo.create(dto); - await repo.save(catalogItem); - processedCount++; - break; - } - - default: - logger.warn(`Unrecognized catalog type: ${item.catalog_type}`); - continue; // Skip this item - } - - // Common validation and saving logic for the catalog types - if (dto && repo) { - const validationErrors = await validate(dto); - if (validationErrors.length > 0) { - logger.error(`Validation errors for ${item.catalog_type}:`, validationErrors); - } else { - const catalogItem = repo.create(dto); - await repo.save(catalogItem); - processedCount++; - } - } - } - - // Proceed to store and classify items after processing all entries - if (itemsToClassify.length > 0) { - logger.info('calling storeAndClassifyItems'); - const categoryReport = await validationManager.generateCategoryReport(itemsToClassify); - await validationManager.storeAndClassifyItems(categoryReport); - } - - return ResponseUtil.sendResponse(res, `${processedCount} entries processed successfully`, 201); - const catalog_data = await AppDataSource.getRepository(slc_item_catalog).find(); - res.render('pages/index', { - catalog: catalog_data, - title: 'SPLICE Catalog', - }); - } - profileView(req: Request, res: Response) { // res.render('pages/profile', { title: 'Profile', user: JSON.stringify(req.oidc.user, null, 2) }); res.render('pages/profile', { title: 'Profile' }); diff --git a/src/dtos/DatasetCatalogDTO.ts b/src/dtos/DatasetCatalogDTO.ts deleted file mode 100644 index 19bd01a..0000000 --- a/src/dtos/DatasetCatalogDTO.ts +++ /dev/null @@ -1,174 +0,0 @@ -// src/dtos/DatasetCatalogDTO.ts -import { IsOptional, IsString, IsUrl, IsInt, IsArray, ValidateNested, IsNumber, IsDateString } from 'class-validator'; -import { Type } from 'class-transformer'; - -class ContributorDTO { - @IsOptional() - @IsString() - displayName?: string; - - @IsOptional() - @IsString() - givenName?: string; - - @IsOptional() - @IsString() - familyName?: string; - - @IsOptional() - @IsString() - identifier?: string; - - @IsOptional() - @IsString() - affiliation?: string; -} - -export class CreateDatasetCatalogDTO { - // Text fields - @IsOptional() - @IsString() - title?: string; - - @IsOptional() - @IsString() - platform?: string; - - @IsOptional() - @IsString() - datasetName?: string; - - @IsOptional() - @IsString() - description?: string; - - // Arrays of strings - @IsOptional() - @IsArray() - @IsString({ each: true }) - dataFormats?: string[]; - - @IsOptional() - @IsArray() - @IsString({ each: true }) - dataType?: string[]; - - @IsOptional() - @IsArray() - @IsString({ each: true }) - keywords?: string[]; - - // JSON array of objects - @IsOptional() - @IsArray() - @ValidateNested({ each: true }) - @Type(() => ContributorDTO) - contributors?: ContributorDTO[]; - - // Optional metadata - @IsOptional() - @IsString() - language?: string; - - @IsOptional() - @IsInt() - publicationYear?: number; - - @IsOptional() - @IsString() - publisher?: string; - - @IsOptional() - @IsUrl() - resourceUrl?: string; - - @IsOptional() - @IsString() - resourceUrlType?: string; - - @IsOptional() - @IsString() - bibtexSource?: string; - - @IsOptional() - @IsString() - rights?: string; - - @IsOptional() - @IsString() - availability?: string; - - @IsOptional() - @IsNumber() - fairnessScore?: number; - - @IsOptional() - @IsArray() - @IsString({ each: true }) - programmingLanguages?: string[]; - - // Dates - @IsOptional() - @IsDateString() - dataCollectionStartDate?: string; - - @IsOptional() - @IsDateString() - dataCollectionEndDate?: string; - - // Numeric counts - @IsOptional() - @IsInt() - taskNumber?: number; - - @IsOptional() - @IsInt() - sampleSize?: number; - - // Free-text long fields - @IsOptional() - @IsString() - sampleDemographics?: string; - - // Country list - @IsOptional() - @IsArray() - @IsString({ each: true }) - country?: string[]; - - @IsOptional() - @IsString() - educationalInstitution?: string; - - @IsOptional() - @IsString() - dataStandard?: string; - - @IsOptional() - @IsString() - learningEnvironment?: string; - - @IsOptional() - @IsString() - aggregation?: string; - - @IsOptional() - @IsString() - aggregationLevel?: string; - - @IsOptional() - @IsUrl() - relatedPublicationUrl?: string; - - @IsOptional() - @IsString() - relatedPublication?: string; - - @IsOptional() - @IsString() - researchQuestion?: string; - - @IsOptional() - @IsString() - futureWork?: string; -} diff --git a/src/dtos/ItemClassificationDTO.ts b/src/dtos/ItemClassificationDTO.ts deleted file mode 100644 index 4611117..0000000 --- a/src/dtos/ItemClassificationDTO.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { IsNotEmpty, IsInt } from 'class-validator'; - -export class CreateItemClassificationDTO { - @IsNotEmpty() - @IsInt() - item_id!: number; - - @IsNotEmpty() - @IsInt() - class_id!: number; -} diff --git a/src/dtos/OntologyAliasDTO.ts b/src/dtos/OntologyAliasDTO.ts deleted file mode 100644 index 62def05..0000000 --- a/src/dtos/OntologyAliasDTO.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { IsNotEmpty, IsString, IsBoolean, IsInt, IsOptional } from 'class-validator'; - -export class CreateOntologyAliasDTO { - @IsNotEmpty() - @IsInt() - class_id!: number; - - @IsNotEmpty() - @IsString() - alias!: string; - - @IsOptional() - @IsBoolean() - is_active?: boolean = true; -} diff --git a/src/dtos/OntologyClassDTO.ts b/src/dtos/OntologyClassDTO.ts deleted file mode 100644 index 4ad93ea..0000000 --- a/src/dtos/OntologyClassDTO.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { IsNotEmpty, IsString, IsBoolean, IsOptional } from 'class-validator'; - -export class CreateOntologyClassDTO { - @IsNotEmpty() - @IsString() - class_uri!: string; - - @IsNotEmpty() - @IsString() - label!: string; - - @IsOptional() - @IsString() - comment?: string; - - @IsOptional() - @IsString() - description?: string; - - @IsOptional() - @IsBoolean() - is_active?: boolean = true; -} diff --git a/src/dtos/OntologyRelationDTO.ts b/src/dtos/OntologyRelationDTO.ts deleted file mode 100644 index e58018d..0000000 --- a/src/dtos/OntologyRelationDTO.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { IsNotEmpty, IsString, IsInt } from 'class-validator'; - -export class CreateOntologyRelationDTO { - @IsNotEmpty() - @IsInt() - parent_class_id!: number; - - @IsNotEmpty() - @IsInt() - child_class_id!: number; - - @IsNotEmpty() - @IsString() - relationship_type!: string; -} diff --git a/src/dtos/SLCToolsDTO.ts b/src/dtos/SLCToolsDTO.ts deleted file mode 100644 index 4458a59..0000000 --- a/src/dtos/SLCToolsDTO.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { IsNotEmpty, IsString, IsEmail, IsOptional, IsUrl } from 'class-validator'; - -export class CreateSLCToolsDTO { - @IsNotEmpty() - @IsString() - platform_name!: string; - - @IsNotEmpty() - @IsUrl() - @IsString() - url!: string; - - @IsString() - @IsOptional() - tool_description!: string; - - @IsString() - @IsOptional() - license!: string; - - @IsString() - @IsOptional() - interface!: string; - - @IsString() - @IsOptional() - keywords!: string; - - @IsEmail() - @IsOptional() - contact_email!: string; -} diff --git a/src/dtos/ValidationResultsDTO.ts b/src/dtos/ValidationResultsDTO.ts deleted file mode 100644 index 392b540..0000000 --- a/src/dtos/ValidationResultsDTO.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { IsNotEmpty, IsString, IsOptional, IsBoolean, IsInt } from 'class-validator'; - -export class CreateValidationResultsDTO { - @IsNotEmpty() - @IsString() - user!: string; - - @IsOptional() - @IsString() - metadataIssues?: string; - - @IsNotEmpty() - @IsInt() - itemId!: number; - - @IsOptional() - @IsBoolean() - isUrlValid?: boolean; - - @IsOptional() - @IsString() - categorizationResults?: string; - - @IsOptional() - @IsString() - validationStatus?: string; -} diff --git a/src/routes/review.ts b/src/routes/review.ts index 5cfa2f3..e8707e3 100644 --- a/src/routes/review.ts +++ b/src/routes/review.ts @@ -7,7 +7,6 @@ const viewController = new ViewController(); router.get('/catalog', viewController.catalogView); router.post('/upload', viewController.uploadPost); -router.post('/approve', viewController.approveAll); router.post('/validate', async (req: Request) => { console.log('Received JSON data:', req.body); // proceed with validation logic diff --git a/src/services/Categorizer.ts b/src/services/Categorizer.ts deleted file mode 100644 index 207e070..0000000 --- a/src/services/Categorizer.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { AppDataSource } from '../db/data-source.js'; -import { ItemClassification } from '../db/entities/ItemClassification.js'; -import { OntologyClasses } from '../db/entities/OntologyClass.js'; -import { slc_item_catalog } from '../db/entities/SLCItemCatalog.js'; -import { SLCItem, MatchedItem } from '../types/ItemTypes.js'; -import logger from '../utils/logger.js'; - -export class Categorizer { - private itemRepo = AppDataSource.getRepository(slc_item_catalog); - private classificationRepo = AppDataSource.getRepository(ItemClassification); - private ontologyRepo = AppDataSource.getRepository(OntologyClasses); - - /** - * Stores items in the catalog and classifies them based on the ontology classes. - * @param items Items to be stored and classified. - * @param matchedItems Matched items with their corresponding classes. - */ - public async storeItemsAndClassify(items: SLCItem[], matchedItems: MatchedItem[]) { - for (const item of items) { - try { - //Find or insert the item in slc_item_catalog - let itemRecord = await this.itemRepo.findOne({ where: { title: item.title } }); - - if (!itemRecord) { - logger.info(`Inserting new item into slc_item_catalog: ${item.title}`); - itemRecord = await this.itemRepo.save(item); - logger.info(`Inserted item with ID: ${itemRecord?.id} for exercise: ${item.title}`); - } else { - logger.info(`Found existing item with ID: ${itemRecord.id} for exercise: ${item.title}`); - } - - if (!itemRecord || !itemRecord.id) { - logger.error(`Item ${item.title} could not be saved or retrieved from slc_item_catalog.`); - continue; // Skip if itemRecord is null - } - - //Find or assign class - const matchedItem = matchedItems.find((m) => m.item.title === item.title); - let classEntity: OntologyClasses | null = null; - - if (matchedItem && matchedItem.matchedClass) { - classEntity = await this.ontologyRepo.findOne({ where: { label: matchedItem.matchedClass } }); - } - - if (!classEntity) { - logger.info(`Assigning 'Unclassified' class to item ${item.title}`); - classEntity = await this.ontologyRepo.findOne({ where: { label: 'Unclassified' } }); - if (!classEntity) { - logger.error(`"Unclassified" class not found in ontology.`); - continue; - } - } - - // Create classification instance by assigning entities directly - const classification = this.classificationRepo.create({ - item: itemRecord, - ontologyClass: classEntity, - }); - - // Log the classification object to ensure values are set correctly - logger.debug(`Prepared classification: ${JSON.stringify(classification, null, 2)}`); - logger.info(`Saving classification for item_id: ${itemRecord.id}, class_id: ${classEntity.id}`); - - await this.classificationRepo.save(classification); - - logger.info(`Successfully classified item ${item.title} under class ${classEntity.label}`); - } catch (error) { - logger.error(`Failed to classify item ${item.title}:`, error); - } - } - } -} diff --git a/src/services/CategoryReport.ts b/src/services/CategoryReport.ts deleted file mode 100644 index e1cbb9c..0000000 --- a/src/services/CategoryReport.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { KeywordMatch } from './KeywordMatch.js'; -import { OntologyClasses } from '../db/entities/OntologyClass.js'; -import logger from '../utils/logger.js'; -import { AppDataSource } from '../db/data-source.js'; -import { SLCItem, MatchedItem } from '../types/ItemTypes.js'; - -export class CategoryReport { - private matcher = new KeywordMatch(); - private ontologyRepo = AppDataSource.getRepository(OntologyClasses); - - public async generateReport(items: SLCItem[]): Promise<{ - matched: MatchedItem[]; - unclassified: MatchedItem[]; - unmatched: MatchedItem[]; - }> { - const matched: MatchedItem[] = []; - const unclassified: MatchedItem[] = []; - const unmatched: MatchedItem[] = []; - - for (const item of items) { - logger.info(`Processing item: ${item.title}`); - const matchedClasses = await this.matcher.matchKeywordsToOntology({ - keywords: item.keywords || [], - }); - - if (matchedClasses && matchedClasses.length > 0 && matchedClasses[0]) { - matched.push({ item, matchedClass: matchedClasses[0].label }); - } else { - const unclassifiedClass = await this.ontologyRepo.findOne({ where: { label: 'Unclassified' } }); - if (unclassifiedClass) { - unclassified.push({ item, matchedClass: 'Unclassified' }); - } else { - unmatched.push({ item, matchedClass: 'None' }); - } - } - } - - return { - matched, - unclassified, - unmatched, - }; - } -} diff --git a/src/services/IframeValidatorService.ts b/src/services/IframeValidatorService.ts deleted file mode 100644 index 289863f..0000000 --- a/src/services/IframeValidatorService.ts +++ /dev/null @@ -1,55 +0,0 @@ -import puppeteer from 'puppeteer'; - -export async function runIframeValidation(iframeUrl: string): Promise<{ passed: boolean; message?: string }> { - const validatorPageUrl = `http://localhost:3000/iframeValidator.html?iframe=${encodeURIComponent(iframeUrl)}`; - - const browser = await puppeteer.connect({ browserWSEndpoint: 'ws://chrome:3000' }); - const page = await browser.newPage(); - - let result: { passed: boolean; message?: string } = { passed: false }; - - try { - await page.goto(validatorPageUrl, { waitUntil: 'networkidle2', timeout: 15000 }); - - const checkResult = await page.evaluate(() => { - return new Promise<{ passed: boolean; message?: string }>((resolve) => { - const statusDiv = document.getElementById('status'); - - if (!statusDiv) return resolve({ passed: false, message: 'No status element found' }); - - const maxWaitTime = 8000; - const pollInterval = 500; - let elapsed = 0; - - const poll = setInterval(() => { - const statusText = statusDiv.textContent || ''; - if (statusText.includes('Passed')) { - clearInterval(poll); - resolve({ passed: true, message: statusText }); - } else if (statusText.includes('Failed')) { - clearInterval(poll); - resolve({ passed: false, message: statusText }); - } - - elapsed += pollInterval; - if (elapsed >= maxWaitTime) { - clearInterval(poll); - resolve({ passed: false, message: 'Timeout waiting for validation result' }); - } - }, pollInterval); - }); - }); - - result = checkResult; - } catch (err: unknown) { - let message = 'Unknown error'; - if (err instanceof Error) { - message = err.message; - } - result = { passed: false, message: `Error during Puppeteer run: ${message}` }; - } finally { - await browser.close(); - } - - return result; -} diff --git a/src/services/KeywordMatch.ts b/src/services/KeywordMatch.ts deleted file mode 100644 index 1a01c7a..0000000 --- a/src/services/KeywordMatch.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { AppDataSource } from '../db/data-source.js'; -import { OntologyClasses } from '../db/entities/OntologyClass.js'; -import { OntologyAliases } from '../db/entities/OntologyAlias.js'; -import logger from '../utils/logger.js'; -import { Metadata } from '../types/ItemTypes.js'; - -export class KeywordMatch { - private ontologyRepo = AppDataSource.getRepository(OntologyClasses); - private aliasRepo = AppDataSource.getRepository(OntologyAliases); - - public async matchKeywordsToOntology(metadata: Metadata): Promise { - try { - if (!metadata?.keywords || !Array.isArray(metadata.keywords)) { - logger.warn('Invalid or missing keywords in metadata'); - return null; - } - - const matchedClasses: OntologyClasses[] = []; - logger.info(`Starting keyword matching for: ${JSON.stringify(metadata.keywords)}`); - - for (const keyword of metadata.keywords) { - try { - if (!keyword || typeof keyword !== 'string') { - logger.warn('Skipping invalid keyword:', keyword); - continue; - } - - logger.debug(`Processing keyword: ${keyword}`); - - // Try direct class match first - const classMatch = await this.findClassMatch(keyword); - if (classMatch) { - matchedClasses.push(classMatch); - continue; - } - - // Try alias match if no direct class match found - const aliasMatch = await this.findAliasMatch(keyword); - if (aliasMatch) { - matchedClasses.push(aliasMatch); - } - } catch (error) { - logger.error(`Error processing keyword "${keyword}":`, error); - continue; - } - } - - logger.info( - 'Keyword matching completed. Matched classes:', - matchedClasses.map((mc) => mc.label), - ); - - return matchedClasses.length > 0 ? matchedClasses : null; - } catch (error) { - logger.error('Error in matchKeywordsToOntology:', error); - throw error; - } - } - - private async findClassMatch(keyword: string): Promise { - try { - const classMatch = await this.ontologyRepo - .createQueryBuilder('oc') - .where('LOWER(oc.label) LIKE LOWER(:keyword)', { keyword: `%${keyword}%` }) - .getOne(); - - if (classMatch) { - logger.debug(`Found direct class match: ${classMatch.label} for keyword: ${keyword}`); - } - return classMatch; - } catch (error) { - logger.error(`Error in findClassMatch for keyword "${keyword}":`, error); - return null; - } - } - - private async findAliasMatch(keyword: string): Promise { - try { - const aliasMatch = await this.aliasRepo - .createQueryBuilder('oa') - .leftJoinAndSelect('oa.class', 'class') - .where('LOWER(oa.alias) LIKE LOWER(:keyword)', { keyword: `%${keyword}%` }) - .getOne(); - - if (aliasMatch?.class) { - logger.debug(`Found alias match: ${aliasMatch.class.label} for keyword: ${keyword}`); - return aliasMatch.class; - } - return null; - } catch (error) { - logger.error(`Error in findAliasMatch for keyword "${keyword}":`, error); - return null; - } - } -} diff --git a/src/services/MetadataExtractor.ts b/src/services/MetadataExtractor.ts deleted file mode 100644 index c948c89..0000000 --- a/src/services/MetadataExtractor.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { CreateSLCItemDTO } from '../dtos/SLCItemDTO.js'; - -export class MetadataExtractor { - public static extractMetadata(item: CreateSLCItemDTO) { - return { - keywords: item.keywords, - title: item.title, - author: item.author, - }; - } -} diff --git a/src/services/MetadataValidator.ts b/src/services/MetadataValidator.ts deleted file mode 100644 index adbad1d..0000000 --- a/src/services/MetadataValidator.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { validate, ValidationError } from 'class-validator'; -import { CreateSLCItemDTO } from '../dtos/SLCItemDTO.js'; -import { ValidationIssue } from '../types/ValidationTypes.js'; - -export class MetadataValidator { - async validate(jsonArray: CreateSLCItemDTO[]): Promise<{ - issues: ValidationIssue[]; - validItems: CreateSLCItemDTO[]; - totalSubmissions: number; - successfulVerifications: number; - }> { - const issues: ValidationIssue[] = []; - const validItems: CreateSLCItemDTO[] = []; - let successfulVerifications = 0; - - for (const rawItem of jsonArray) { - const dto = new CreateSLCItemDTO(); - Object.assign(dto, rawItem); - - const validationErrors: ValidationError[] = await validate(dto); - if (validationErrors.length > 0) { - issues.push({ item: dto, validationErrors }); - } else { - validItems.push(dto); - successfulVerifications++; - } - } - - return { issues, validItems, totalSubmissions: jsonArray.length, successfulVerifications }; - } -} diff --git a/src/services/URLValidator.ts b/src/services/URLValidator.ts deleted file mode 100644 index 3a22c96..0000000 --- a/src/services/URLValidator.ts +++ /dev/null @@ -1,71 +0,0 @@ -import axios from 'axios'; -import * as https from 'https'; -import { URLValidationItem, URLValidationIssue } from '../types/ValidationTypes.js'; - -export class URLValidator { - async validate(validItems: URLValidationItem[]): Promise<{ - urlsChecked: number; - successfulUrls: number; - unsuccessfulUrls: number; - issues: URLValidationIssue[]; - }> { - let urlsChecked = 0; - let successfulUrls = 0; - let unsuccessfulUrls = 0; - const issues: URLValidationIssue[] = []; - - const urlPromises = validItems.map(async (item) => { - try { - // Increment urlsChecked for every URL attempted - urlsChecked++; - // const response = await axios.get(item.url); - // Disable SSL certificate validation for axios - const response = await axios.get(item.iframe_url, { - httpsAgent: new https.Agent({ rejectUnauthorized: false }), - timeout: 4000, - }); - - if (response.status === 200) { - successfulUrls++; - console.log(`success ${item.iframe_url} is reachable, returned status: ${response.status}`); - } else { - unsuccessfulUrls++; - const msg = `[ERROR] ${item.iframe_url} returned status ${response.status}`; - console.error(msg); - issues.push({ item, error: `URL returned status ${response.status}` }); - } - } catch (error) { - unsuccessfulUrls++; - - let errorMessage = 'error message'; - - if (axios.isAxiosError(error)) { - if (error.response) { - errorMessage = `Server responded with status code ${error.response.status}`; - } else if (error.request) { - errorMessage = `No response received from the server, code: ${error.code}, message: ${error.message}`; - } else { - errorMessage = `Axios Error: ${error.message}`; - } - } else { - errorMessage = 'Unknown error occurred.'; - } - - console.error(`error ${item.iframe_url} message: ${errorMessage}`); - issues.push({ item, error: errorMessage }); - } - }); - - await Promise.all(urlPromises); - console.log( - `successful url's : ${successfulUrls} , unsuccessful url's: ${unsuccessfulUrls}, issues: ${issues.length}`, - ); - - return { urlsChecked: urlsChecked, successfulUrls, unsuccessfulUrls, issues }; - } -} -/* -1 add method to review controller to catch the results that we have, map them to the database, catch specific for each item -2 add functionality into validation manager such that any errors caught can be sent -for getting specifc details will be applied such that each of the services sends individual data back to review controller. -*/ diff --git a/src/services/ValidationManager.ts b/src/services/ValidationManager.ts deleted file mode 100644 index facd254..0000000 --- a/src/services/ValidationManager.ts +++ /dev/null @@ -1,368 +0,0 @@ -import logger from '../utils/logger.js'; -import { MetadataValidator } from './MetadataValidator.js'; -import { URLValidator } from './URLValidator.js'; -import { CategoryReport } from './CategoryReport.js'; -import { Categorizer } from './Categorizer.js'; -import { MetadataIssue, URLValidationResult } from '../types/ValidationTypes.js'; -import { SLCItem } from '../types/ItemTypes.js'; -import { CreateSLCItemDTO } from '../dtos/SLCItemDTO.js'; -import { CategorizationReport } from '../types/CategorizationTypes.js'; -import { Repository } from 'typeorm'; -import { ValidationResults } from '../db/entities/ValidationResults.js'; -import { slc_item_catalog } from '../db/entities/SLCItemCatalog.js'; -import { validateLTI } from './ValidatorLTI.js'; -import { runIframeValidation } from './IframeValidatorService.js'; - -export class ValidationManager { - private metadataValidator: MetadataValidator; - private urlValidator: URLValidator; - private categoryReport: CategoryReport; - private categorizer: Categorizer; - private validationResultsRepository: Repository; - private catalogRepository: Repository; - - constructor( - validationResultsRepository: Repository, - catalogRepository: Repository, - ) { - this.metadataValidator = new MetadataValidator(); - this.urlValidator = new URLValidator(); - this.categoryReport = new CategoryReport(); - this.categorizer = new Categorizer(); - this.validationResultsRepository = validationResultsRepository; - this.catalogRepository = catalogRepository; - } - private async getOrCreateValidationResultByUrl(url: string, slcItem?: SLCItem): Promise { - let catalogItem = await this.catalogRepository.findOne({ - where: { iframe_url: slcItem?.iframe_url }, - relations: ['validationResults'], - }); - - const default_key = 'empty'; - - if (!catalogItem) { - logger.warn(`Catalog item not found for URL: ${url}`); - - // rework this code such that if they are not in the database it puts it in their - // also create validated slc item - const newItem = { - catalog_type: slcItem?.catalog_type ?? default_key, - persistentID: slcItem?.persistentID ?? default_key, - platform_name: slcItem?.platform_name ?? default_key, - iframe_url: slcItem?.iframe_url ?? default_key, - license: slcItem?.license ?? default_key, - description: slcItem?.description ?? default_key, - author: Array.isArray(slcItem?.author) ? slcItem.author : [default_key], - institution: Array.isArray(slcItem?.institution) ? slcItem.institution : [default_key], - keywords: Array.isArray(slcItem?.keywords) ? slcItem.keywords : [default_key], - features: Array.isArray(slcItem?.features) ? slcItem.features : [default_key], - title: slcItem?.title ?? default_key, - programming_language: Array.isArray(slcItem?.programming_language) - ? slcItem.programming_language - : [default_key], - natural_language: Array.isArray(slcItem?.natural_language) ? slcItem.natural_language : [default_key], - protocol: Array.isArray(slcItem?.protocol) ? slcItem.protocol : [default_key], - protocol_url: Array.isArray(slcItem?.protocol_url) ? slcItem.protocol_url : [default_key], - }; - - catalogItem = this.catalogRepository.create(newItem); - await this.catalogRepository.save(catalogItem); - logger.info(`Creating new catalog item with data:`, newItem); - } - - let validationResult = await this.validationResultsRepository.findOne({ - where: { item: { id: catalogItem.id } }, - order: { dateLastUpdated: 'DESC' }, - }); - - if (!validationResult) { - validationResult = this.validationResultsRepository.create({ item: catalogItem, user: 'user' }); - await this.validationResultsRepository.save(validationResult); - } - - return validationResult; - } - /** - * Validates metadata items and returns the result. - * @param jsonArray - Array of SLCItems to validate - * @returns Metadata validation result - */ - async validateMetadata(jsonArray: SLCItem[]): Promise<{ - issues: MetadataIssue[]; - validItems: SLCItem[]; - totalSubmissions: number; - successfulVerifications: number; - }> { - try { - logger.info(`DENIS IS STARTING metadata validation for ${jsonArray.length} items`); - - // 1. Transform SLCItem[] -> CreateSLCItemDTO[] - const createDtoArray: CreateSLCItemDTO[] = jsonArray; - - // 2. Validate the CreateSLCItemDTO[] array - const result = await this.metadataValidator.validate(createDtoArray); - - // Save validation results for each item - for (const item of jsonArray) { - const validationErrors = - result.issues.find((issue) => issue.item.persistentID === item.persistentID)?.validationErrors || null; - - const catalogItem = await this.catalogRepository.findOne({ where: { persistentID: item.persistentID } }); - - if (!catalogItem) { - logger.warn(`No catalog item found for persistentID: ${item.persistentID}`); - continue; - } - - const validationResult = await this.getOrCreateValidationResultByUrl(item.iframe_url, item); - if (!validationResult) continue; - - // Build detailed metadata error string - const formattedErrors = validationErrors?.length - ? validationErrors - .map((err) => { - const constraints = err.constraints ? Object.values(err.constraints).join(', ') : 'Unknown issue'; - return `Field "${err.property}": ${constraints}`; - }) - .join('; ') - : 'no issues'; - - validationResult.metadataIssues = formattedErrors; - validationResult.user = item.title; - validationResult.dateLastUpdated = new Date(); - await this.validationResultsRepository.save(validationResult); - } - logger.info(`Metadata validation completed: ${result.validItems.length} valid items`); - //logger.info(); print the validation results - const validSLCItems: SLCItem[] = result.validItems.map((dto) => ({ - ...dto, - - keywords: dto.keywords ?? [], - })); - - return { - issues: result.issues, - validItems: validSLCItems, - totalSubmissions: result.totalSubmissions, - successfulVerifications: result.successfulVerifications, - }; - } catch (error) { - logger.error('Error in metadata validation:', error); - throw error; - } - } - - /** - * Validates URLs for valid metadata items. - * @param validItems - Array of valid SLCItems - * @returns URL validation result - */ - async validateUrls(validItems: SLCItem[]): Promise { - try { - logger.info(`Starting URL validation for ${validItems.length} items`); - const result = await this.urlValidator.validate(validItems); - // Save individual URL validation results into the database - for (const [index, item] of validItems.entries()) { - const isValid = result.successfulUrls > index; - - const validationResult = await this.getOrCreateValidationResultByUrl(item.iframe_url, item); - if (!validationResult) continue; - - validationResult.isUrlValid = isValid; - try { - const iframeResult = await runIframeValidation(item.iframe_url); - validationResult.iframeValidationError = iframeResult.passed - ? 'Passed: SPLICE message received' - : iframeResult.message || 'Unknown error'; - } catch (iframeError) { - logger.error(`Iframe validation failed for ${item.iframe_url}:`, iframeError); - validationResult.iframeValidationError = 'Iframe validation failed'; - } - - function getLTIURL(item: { protocol?: string[]; protocol_url?: string[] }): string | undefined { - const protocolIndex = item.protocol?.findIndex((protocol) => protocol === 'LTI'); - if ( - protocolIndex !== undefined && - protocolIndex >= 0 && - item.protocol_url && - item.protocol_url[protocolIndex] - ) { - return item.protocol_url[protocolIndex]; - } - return undefined; - } - - const lti_url = getLTIURL(item); - - //lti validation - if (lti_url) { - try { - const ltiPayload = { - launch_url: 'https://codeworkout.cs.vt.edu/lti/launch', - key: 'canvas_key', // replace with real values when available - secret: 'canvas_secret', - }; - - const ltiResult = await validateLTI(ltiPayload); - - validationResult.ltiValidationStatus = ltiResult.launchable - ? 'Launchable' - : `Not Launchable (status ${ltiResult.status_code || 'unknown'})`; - } catch (error: unknown) { - let message = 'Unknown error'; - - if (error instanceof Error) { - message = error.message; - } - - logger.error(`Error validating LTI URL for ${lti_url}:`, error); - validationResult.ltiValidationStatus = `Validation Failed: ${message}`; - } - } else { - validationResult.ltiValidationStatus = 'No LTI URL Provided'; - } - //end of lti validation - - validationResult.dateLastUpdated = new Date(); - - await this.validationResultsRepository.save(validationResult); - } - logger.info(`URL validation completed: ${result.successfulUrls} successful URLs`); - return result; - } catch (error) { - logger.error('Error in URL validation:', error); - throw error; - } - } - - /** - * Generates a category report for given items. - * @param items - Array of items to classify - * @returns Categorization report with matched/unclassified/unmatched - */ - async generateCategoryReport(items: SLCItem[]): Promise { - try { - logger.info(`Generating category report for ${items.length} items`); - const report = await this.categoryReport.generateReport(items); - - logger.info('Category report generated successfully:', { - matched: report.matched.length, - unclassified: report.unclassified.length, - unmatched: report.unmatched.length, - }); - - return report; - } catch (error) { - logger.error('Error generating category report:', error); - throw error; - } - } - - /** - * Stores items in the catalog and classifies them. - * @param report - Categorization report containing matched, unclassified, and unmatched items - */ - async storeAndClassifyItems(report: CategorizationReport): Promise { - try { - logger.info('Starting items storage and classification'); - const items = [...report.matched, ...report.unclassified, ...report.unmatched]; - - if (!items.length) { - const error = new Error('No items to process'); - logger.error(error.message); - throw error; - } - - // The categorizer expects an array of SLCItems and matched items - const itemsToProcess: SLCItem[] = items.map((obj) => obj.item); - await this.categorizer.storeItemsAndClassify(itemsToProcess, report.matched); - //new stuff - for (const item of itemsToProcess) { - try { - await this.categorizer.storeItemsAndClassify([item], report.matched); - const validationResult = await this.getOrCreateValidationResultByUrl(item.iframe_url, item); - if (validationResult) { - validationResult.categorizationResults = 'Success'; - validationResult.dateLastUpdated = new Date(); - await this.validationResultsRepository.save(validationResult); - } - } catch (error) { - logger.error(`Error reprocessing item for error capture: ${item.iframe_url}`, error); - const validationResult = await this.getOrCreateValidationResultByUrl(item.iframe_url); - if (!validationResult) return; - validationResult.categorizationResults = `Categorization failed: No matching ontology class for keywords [${item.keywords?.join(', ') || 'none'}]`; - validationResult.dateLastUpdated = new Date(); - await this.validationResultsRepository.save(validationResult); - throw error; - } - } - logger.info(`Successfully stored and classified ${items.length} items`); - } catch (error) { - logger.error('Failed to store and classify items:', error); - throw error; - } - } - - /** - * Executes the full validation workflow: metadata validation, URL validation, categorization, and storing results. - * @param items - Array of SLCItems to process - * @returns Summary of the validation process - */ - async fullValidationWorkflow(items: SLCItem[]): Promise<{ - metadataValidated: number; - urlsChecked: number; - urlsValidated: number; - reportSummary: { - matched: number; - unclassified: number; - unmatched: number; - }; - }> { - try { - if (!items?.length) { - throw new Error('No items provided for validation'); - } - - logger.info(`Starting full validation workflow for ${items.length} items`); - - // Step 1: Metadata Validation - const metadataResult = await this.validateMetadata(items); - const validatedMetadataItems = metadataResult.validItems; - - if (!validatedMetadataItems.length) { - throw new Error('No items passed metadata validation'); - } - - // Step 2: URL Validation - const urlValidationResult = await this.validateUrls(validatedMetadataItems); - const successfulUrlItems = validatedMetadataItems.filter( - (_, index) => urlValidationResult.successfulUrls > index, - ); - - if (!successfulUrlItems.length) { - throw new Error('No items passed URL validation'); - } - - // Step 3: Category Report Generation - const report = await this.generateCategoryReport(successfulUrlItems); - - // Step 4: Store and Classify - await this.storeAndClassifyItems(report); - - // Return summary - return { - metadataValidated: validatedMetadataItems.length, - urlsChecked: urlValidationResult.urlsChecked, - urlsValidated: successfulUrlItems.length, - reportSummary: { - matched: report.matched.length, - unclassified: report.unclassified.length, - unmatched: report.unmatched.length, - }, - }; - } catch (error) { - logger.error('Error in validation workflow:', error); - throw error; - } - } -} diff --git a/src/services/ValidatorLTI.ts b/src/services/ValidatorLTI.ts deleted file mode 100644 index d9ee1c6..0000000 --- a/src/services/ValidatorLTI.ts +++ /dev/null @@ -1,29 +0,0 @@ -import axios from 'axios'; - -export interface LTIValidationResponse { - launchable: boolean; - status_code?: number; - launch_url?: string; - error?: string; -} - -export async function validateLTI(payload: object): Promise { - try { - const response = await axios.post('http://lti-validator:4000/validate', payload); - return response.data; - } catch (error: unknown) { - let message = 'Unknown error'; - if (axios.isAxiosError(error)) { - message = error.response?.data?.error || error.message; - } else if (error instanceof Error) { - message = error.message; - } - - console.error('LTI validation failed:', message); - - return { - launchable: false, - error: message, - }; - } -} diff --git a/src/types/CategorizationTypes.ts b/src/types/CategorizationTypes.ts deleted file mode 100644 index c7dcda7..0000000 --- a/src/types/CategorizationTypes.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { CreateSLCItemDTO } from '../dtos/SLCItemDTO.js'; -import { MatchedItem } from './ItemTypes.js'; - -export interface CategorizedItem { - item: CreateSLCItemDTO; - matchedClass: string; -} - -export interface CategorizationReport { - matched: MatchedItem[]; - unclassified: MatchedItem[]; - unmatched: MatchedItem[]; -} diff --git a/src/views/pages/review-dashboard.ejs b/src/views/pages/review-dashboard.ejs deleted file mode 100644 index 5a0fb36..0000000 --- a/src/views/pages/review-dashboard.ejs +++ /dev/null @@ -1,230 +0,0 @@ - - -<%- include('../partials/head') -%> -<%- include('../partials/header') -%> - -
-

Review Dashboard

-
-

Total Submissions: <%= totalSubmissions %>

-

Successful Verifications: <%= successfulVerifications %>

-

Submissions with Issues: <%= issues.length %>

-

Total Categorized Items: <%= categorizationResults.length %>

-

URLs Checked: <%= urlsChecked %>

-

Successful URLs: <%= successfulUrls %>

-

Unsuccessful URLs: <%= unsuccessfulUrls %>

-
    - <% const issueCounts = { missingFields: 0, invalidUrls: 0, formatInconsistencies: 0 }; %> - <% issues.forEach(issue => { %> - <% issue.validationErrors.forEach(error => { %> - <% if (error.constraints) { %> - <% Object.keys(error.constraints).forEach(constraint => { %> - <% if (constraint === 'isNotEmpty') issueCounts.missingFields++; %> - <% if (constraint === 'isUrl') issueCounts.invalidUrls++; %> - <% if (constraint === 'isString') issueCounts.formatInconsistencies++; %> - <% }); %> - <% } %> - <% }); %> - <% }); %> -
  • Missing Fields: <%= issueCounts.missingFields %>
    • -
  • Invalid URLs: <%= issueCounts.invalidUrls %>
    • -
  • Format Inconsistencies: <%= issueCounts.formatInconsistencies %>
    • -
-
- - - - <% if (categorizationResults.length > 0) { %> -
Automatic Categorization Results
- -
- <% categorizationResults.forEach((result, index) => { %> -
-
-

Item: <%= result.item %>

-

Matched Class: <%= result.matchedClass %>

-

Status: <%= result.status %>

-
-
- <% }) %> -
- <% } %> - - <% if (issues.length > 0) { %> -
Entries with Issues
- -
- <% issues.forEach((issue, index) => { %> -
-
-

<%= issue.item.exercise_name %>

-
<%= issue.summary %>
- -
-
-

Catalog Type: <%= issue.item.catalog_type %>

-

Platform Name: <%= issue.item.platform_name %>

-

URL: <%= issue.item.url %>

-

Keywords: <%= issue.item.keywords.join(', ') %>

-

LTI Instructions URL: <%= issue.item.lti_instructions_url %>

-

Exercise Type: <%= issue.item.exercise_type %>

-

Author: <%= issue.item.author %>

-

Institution: <%= issue.item.institution %>

-

Exercise Name: <%= issue.item.exercise_name %>

-

Language: <%= issue.item.language %>

-

iFrame URL: <%= issue.item.iframe_url %>

-

LTI URL: <%= issue.item.lti_url %>

-
-
-

Validation Errors:

-
    - <% issue.validationErrors.forEach(error => { %> -
  • <%= error.property %>: <%= error.constraints ? Object.values(error.constraints).join(', ') : 'Unknown error' %>
    • - <% }) %> -
-
- <% if (issue.error || issue.validationErrors.length > 0) { %> -
-

Recommendations:

-
    - <% if (issue.error) { %> -
  • Fix the URL or ensure the server is reachable: <%= issue.error %>
    • - <% } %> - <% issue.validationErrors.forEach(error => { %> -
  • Correct <%= error.property %>: <%= error.constraints ? Object.values(error.constraints).join(', ') : 'Refer to the guidelines' %>
    • - <% }) %> -
-
- <% } %> -
-
-
- <% }) %> -
-
-

Resources:

- -
- - -
- - -
- <% } else { %> -

No issues found!

- <% } %> - <% if (validationResults && validationResults.length > 0) { %> -
Individual Results
-
- - Download Validation Results JSON - -
-
- - - - - - - - - - - - - - <% validationResults.forEach(result => { %> - - - - - - <% const iframeMessage = result.iframeValidation?.message; %> - - - - - <% }) %> - -
Exercise NameURLMetadata IssuesURL ValidiFrame URLCategorization (keywords)LTI Launchable
<%= result.user %><%= result.item.url %> - <% if (result.metadataIssues === 'no issues') { %> -

Success

- <% } else if (result.metadataIssues) { %> -

failure

<%= result.metadataIssues %> - <% } else { %> - Pending - <% } %> - 		- <% if (result.isUrlValid === true) { %> -

Success

- <% } else if (result.isUrlValid === false) { %> -

failure

- <% } else { %> - N/A - <% } %> - 		- <% if (iframeMessage === 'Iframe validation failed' || iframeMessage == null) { %> -

Failure

- <% } else { %> -

Success

- <% } %> - 		- <% if (result.categorizationResults?.startsWith('Success')) { %> -

Success

- <% } else if (result.categorizationResults) { %> -

failure

-

Categorization failed: No matching ontology class for keywords

- <% } else { %> -

failure

-

Categorization failed: No matching ontology class for keywords

- <% } %> - 		- <% if (result.ltiValidationStatus === 'Launchable') { %> - Launchable - <% } else if (result.ltiValidationStatus === 'Not Launchable' || result.ltiValidationStatus === 'Validation Failed') { %> - <%= result.ltiValidationStatus %> - <% } else { %> - N/A - <% } %> -
-
- <% } else { %> -

No validation results available yet.

- <% } %> - - - - - - <%- include('../partials/footer') -%> - - -