From bf4b19ee66aefcd749622c8c39be9941d1c66078 Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Fri, 15 Nov 2024 13:43:50 +0100 Subject: [PATCH 1/4] feat: Add audio-text-to-text task. --- packages/tasks/src/pipelines.ts | 6 ++++++ packages/tasks/src/tasks/index.ts | 2 ++ 2 files changed, 8 insertions(+) diff --git a/packages/tasks/src/pipelines.ts b/packages/tasks/src/pipelines.ts index 237469b89b..e272d2eeff 100644 --- a/packages/tasks/src/pipelines.ts +++ b/packages/tasks/src/pipelines.ts @@ -355,6 +355,12 @@ export const PIPELINE_DATA = { modality: "audio", color: "green", }, + "audio-text-to-text": { + name: "Audio-Text-to-Text", + modality: "multimodal", + color: "red", + hideInDatasets: true, + }, "voice-activity-detection": { name: "Voice Activity Detection", modality: "audio", diff --git a/packages/tasks/src/tasks/index.ts b/packages/tasks/src/tasks/index.ts index 7392cfa912..5e02cd57d9 100644 --- a/packages/tasks/src/tasks/index.ts +++ b/packages/tasks/src/tasks/index.ts @@ -116,6 +116,7 @@ export const TASKS_MODEL_LIBRARIES: Record = { "audio-classification": ["speechbrain", "transformers", "transformers.js"], "audio-to-audio": ["asteroid", "fairseq", "speechbrain"], "automatic-speech-recognition": ["espnet", "nemo", "speechbrain", "transformers", "transformers.js"], + "audio-text-to-text": [], "depth-estimation": ["transformers", "transformers.js"], "document-question-answering": ["transformers", "transformers.js"], "feature-extraction": ["sentence-transformers", "transformers", "transformers.js"], @@ -197,6 +198,7 @@ export const TASKS_DATA: Record = { "any-to-any": getData("any-to-any", placeholder), "audio-classification": getData("audio-classification", audioClassification), "audio-to-audio": getData("audio-to-audio", audioToAudio), + "audio-text-to-text": undefined, "automatic-speech-recognition": getData("automatic-speech-recognition", automaticSpeechRecognition), "depth-estimation": getData("depth-estimation", depthEstimation), "document-question-answering": getData("document-question-answering", documentQuestionAnswering), From 77a0ea59400a13d91fe518f1b659a146ec49b46f Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Fri, 15 Nov 2024 16:09:01 +0100 Subject: [PATCH 2/4] code review. --- packages/tasks/src/tasks/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/tasks/index.ts b/packages/tasks/src/tasks/index.ts index 5e02cd57d9..1d4d3c8119 100644 --- a/packages/tasks/src/tasks/index.ts +++ b/packages/tasks/src/tasks/index.ts @@ -198,7 +198,7 @@ export const TASKS_DATA: Record = { "any-to-any": getData("any-to-any", placeholder), "audio-classification": getData("audio-classification", audioClassification), "audio-to-audio": getData("audio-to-audio", audioToAudio), - "audio-text-to-text": undefined, + "audio-text-to-text": getData("audio-text-to-text", placeholder), "automatic-speech-recognition": getData("automatic-speech-recognition", automaticSpeechRecognition), "depth-estimation": getData("depth-estimation", depthEstimation), "document-question-answering": getData("document-question-answering", documentQuestionAnswering), From 0379bb4cff43d619371b42b85776e022bff16a70 Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Fri, 15 Nov 2024 16:52:53 +0100 Subject: [PATCH 3/4] add icon. --- .../widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte index 1429e1cb4c..092b478280 100644 --- a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte +++ b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte @@ -4,6 +4,7 @@ import IconAudioClassification from "../Icons/IconAudioClassification.svelte"; import IconAudioToAudio from "../Icons/IconAudioToAudio.svelte"; import IconAutomaticSpeechRecognition from "../Icons/IconAutomaticSpeechRecognition.svelte"; + import IconAudioTextToText from "../Icons/IconAudioToAudio.svelte"; import IconConversational from "../Icons/IconConversational.svelte"; import IconFeatureExtraction from "../Icons/IconFeatureExtraction.svelte"; import IconFillMask from "../Icons/IconFillMask.svelte"; @@ -73,6 +74,7 @@ "automatic-speech-recognition": IconAutomaticSpeechRecognition, "audio-to-audio": IconAudioToAudio, "audio-classification": IconAudioClassification, + "audio-text-to-text": IconAudioTextToText, "voice-activity-detection": IconVoiceActivityDetection, "depth-estimation": IconDepthEstimation, "image-classification": IconImageClassification, From 50b60e68e245cbb3072208e8e5ae3e9d037097e2 Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Fri, 15 Nov 2024 20:34:14 +0100 Subject: [PATCH 4/4] up. --- .../src/lib/components/PipelineIcon/PipelineIcon.svelte | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte index 092b478280..0d85bf76ac 100644 --- a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte +++ b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte @@ -4,7 +4,6 @@ import IconAudioClassification from "../Icons/IconAudioClassification.svelte"; import IconAudioToAudio from "../Icons/IconAudioToAudio.svelte"; import IconAutomaticSpeechRecognition from "../Icons/IconAutomaticSpeechRecognition.svelte"; - import IconAudioTextToText from "../Icons/IconAudioToAudio.svelte"; import IconConversational from "../Icons/IconConversational.svelte"; import IconFeatureExtraction from "../Icons/IconFeatureExtraction.svelte"; import IconFillMask from "../Icons/IconFillMask.svelte"; @@ -74,7 +73,7 @@ "automatic-speech-recognition": IconAutomaticSpeechRecognition, "audio-to-audio": IconAudioToAudio, "audio-classification": IconAudioClassification, - "audio-text-to-text": IconAudioTextToText, + "audio-text-to-text": IconAudioToAudio, "voice-activity-detection": IconVoiceActivityDetection, "depth-estimation": IconDepthEstimation, "image-classification": IconImageClassification,