Skip to content

Embedder rework #79

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andhreljaKern merged 1 commit into from
Jul 30, 2025
Merged

Embedder rework #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 4 additions & 16 deletions src/components/projects/projectId/settings/embeddings/AddNewEmbeddingModal.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ export default function AddNewEmbeddingModal() {
function changePlatformOrGranularity() {
prepareSuggestions();
const savePlatform = platform.platform;
if (savePlatform == PlatformType.COHERE || savePlatform == PlatformType.OPEN_AI || savePlatform == PlatformType.AZURE) {
if (savePlatform == PlatformType.OPEN_AI || savePlatform == PlatformType.AZURE) {
setGranularity(GRANULARITY_TYPES_ARRAY.find((g) => g.value == EmbeddingType.ON_ATTRIBUTE));
if (savePlatform == PlatformType.AZURE) {
const azureUrls = localStorage.getItem('azureUrls');
Expand Down Expand Up @@ -137,7 +137,7 @@ export default function AddNewEmbeddingModal() {

function checkIfPlatformHasToken() {
if (!platform) return;
if (platform.name == platformNamesDict[PlatformType.COHERE] || platform.name == platformNamesDict[PlatformType.OPEN_AI] || platform.name == platformNamesDict[PlatformType.AZURE]) {
if (platform.name == platformNamesDict[PlatformType.OPEN_AI] || platform.name == platformNamesDict[PlatformType.AZURE]) {
setGranularityArray(GRANULARITY_TYPES_ARRAY.filter((g) => g.value != EmbeddingType.ON_TOKEN));
} else {
setGranularityArray(GRANULARITY_TYPES_ARRAY);
Expand Down Expand Up @@ -179,13 +179,11 @@ export default function AddNewEmbeddingModal() {
filterAttributes: filteredAttributes
}

if (platform.name == platformNamesDict[PlatformType.HUGGING_FACE] || platform.name == platformNamesDict[PlatformType.PYTHON]) {
if (platform.name == platformNamesDict[PlatformType.HUGGING_FACE]) {
config.model = model;
} else if (platform.name == platformNamesDict[PlatformType.OPEN_AI]) {
config.model = model;
config.apiToken = apiToken;
} else if (platform.name == platformNamesDict[PlatformType.COHERE]) {
config.apiToken = apiToken;
} else if (platform.name == platformNamesDict[PlatformType.AZURE]) {
config.model = engine; //note that is handled internally as model so we use the model field for the request
config.apiToken = apiToken;
Expand Down Expand Up @@ -251,15 +249,6 @@ export default function AddNewEmbeddingModal() {
<input placeholder="Enter your API token" onChange={(e) => setApiToken(e.target.value)} value={apiToken}
className="h-9 w-full text-sm border-gray-300 rounded-md placeholder-italic border text-gray-900 pl-4 placeholder:text-gray-400 focus:outline-none focus:ring-2 focus:ring-gray-300 focus:ring-offset-2 focus:ring-offset-gray-100" />
</>}
{platform && platform.name == platformNamesDict[PlatformType.COHERE] && <>
<Tooltip content={TOOLTIPS_DICT.PROJECT_SETTINGS.EMBEDDINGS.API_TOKEN} placement="right" color="invert">
<span className="card-title mb-0 label-text flex"><span className="cursor-help underline filtersUnderline">API token</span></span>
</Tooltip>
<input placeholder="Enter your API token" onChange={(e) => setApiToken(e.target.value)} value={apiToken}
className="h-9 w-full text-sm border-gray-300 rounded-md placeholder-italic border text-gray-900 pl-4 placeholder:text-gray-400 focus:outline-none focus:ring-2 focus:ring-gray-300 focus:ring-offset-2 focus:ring-offset-gray-100" />
</>}
{platform && platform.name == platformNamesDict[PlatformType.PYTHON] && <SuggestionsModel options={embeddingHandles[targetAttribute]} selectedOption={(option: string) => setModel(option)} />}

{platform && platform.name == platformNamesDict[PlatformType.AZURE] && <>
<Tooltip content={TOOLTIPS_DICT.PROJECT_SETTINGS.EMBEDDINGS.API_TOKEN} placement="right" color="invert">
<span className="card-title mb-0 label-text flex"><span className="cursor-help underline filtersUnderline">API token</span></span>
Expand Down Expand Up @@ -302,11 +291,10 @@ export default function AddNewEmbeddingModal() {
</>}
</>}
</div>
{platform && (platform.name == platformNamesDict[PlatformType.COHERE] || platform.name == platformNamesDict[PlatformType.OPEN_AI] || platform.name == platformNamesDict[PlatformType.AZURE]) && <div className="text-center mt-3">
{platform && (platform.name == platformNamesDict[PlatformType.OPEN_AI] || platform.name == platformNamesDict[PlatformType.AZURE]) && <div className="text-center mt-3">
<div className="border border-gray-300 text-xs text-gray-500 p-2.5 rounded-lg text-justify">
<label ref={gdprText} className="text-gray-700">
{selectedPlatform.splitTerms[0]}
{platform.name == platformNamesDict[PlatformType.COHERE] && <a href={selectedPlatform.link} target="_blank" className="underline">cohere terms of service.</a>}
{platform.name == platformNamesDict[PlatformType.OPEN_AI] && <a href={selectedPlatform.link} target="_blank" className="underline">openai terms of service.</a>}
{platform.name == platformNamesDict[PlatformType.AZURE] && <a href={selectedPlatform.link} target="_blank" className="underline">azure terms of service.</a>}
<div>{selectedPlatform.splitTerms[1]}</div>
Expand Down
2 changes: 1 addition & 1 deletion src/components/shared/upload/helper-components/UploadWrapper.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export default function UploadWrapper(props: UploadWrapperProps) {
const uploadFileType = useSelector(selectUploadData).uploadFileType;
const importOptions = useSelector(selectUploadData).importOptions;
const embeddings = useSelector(selectEmbeddings);
const recalculationCosts = embeddings.some((e: Embedding) => e.platform == PlatformType.COHERE || e.platform == PlatformType.OPEN_AI || e.platform == PlatformType.AZURE);
const recalculationCosts = embeddings.some((e: Embedding) => e.platform == PlatformType.OPEN_AI || e.platform == PlatformType.AZURE);

const [selectedFile, setSelectedFile] = useState<File | null>(null);
const [fileEndsWithZip, setFileEndsWithZip] = useState<boolean>(false);
Expand Down
2 changes: 0 additions & 2 deletions src/types/components/projects/projectId/settings/embeddings.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ export type EmbeddingPlatform = {
export enum PlatformType {
HUGGING_FACE = "huggingface",
OPEN_AI = "openai",
COHERE = "cohere",
PYTHON = "python",
AZURE = "azure"
}

Expand Down
10 changes: 3 additions & 7 deletions src/util/components/projects/projectId/settings/embeddings-helper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ export function postProcessingEmbeddingPlatforms(platforms: EmbeddingPlatform[],
export const platformNamesDict = {
[PlatformType.HUGGING_FACE]: "Hugging Face",
[PlatformType.OPEN_AI]: "OpenAI",
[PlatformType.COHERE]: "Cohere",
[PlatformType.PYTHON]: "Python",
[PlatformType.AZURE]: "Azure"
}

Expand All @@ -73,9 +71,9 @@ function buildExpectedEmbeddingName(data: any): string {
let toReturn = data.targetAttribute.name;
toReturn += "-" + (data.granularity.value == EmbeddingType.ON_ATTRIBUTE ? 'classification' : 'extraction');
const platform = data.platform;
if (platform == PlatformType.HUGGING_FACE || platform == PlatformType.PYTHON) {
if (platform == PlatformType.HUGGING_FACE) {
toReturn += "-" + platform + "-" + data.model;
} else if (platform == PlatformType.OPEN_AI || platform == PlatformType.COHERE || platform == PlatformType.AZURE) {
} else if (platform == PlatformType.OPEN_AI || platform == PlatformType.AZURE) {
toReturn += buildEmbeddingNameWithApiToken(data);
}
return toReturn;
Expand Down Expand Up @@ -106,12 +104,10 @@ export function checkIfCreateEmbeddingIsDisabled(props: EmbeddingCreationEnabled
const engine = props.engine;
const version = props.version;
const url = props.url;
if (platform.name == platformNamesDict[PlatformType.HUGGING_FACE] || platform.name == platformNamesDict[PlatformType.PYTHON]) {
if (platform.name == platformNamesDict[PlatformType.HUGGING_FACE]) {
checkFormFields = model == null || model == "";
} else if (platform.name == platformNamesDict[PlatformType.OPEN_AI]) {
checkFormFields = model == null || apiToken == null || apiToken == "" || !termsAccepted;
} else if (platform.name == platformNamesDict[PlatformType.COHERE]) {
checkFormFields = apiToken == null || apiToken == "" || !termsAccepted;
} else if (platform.name == platformNamesDict[PlatformType.AZURE]) {
checkFormFields = apiToken == null || apiToken == "" || url == null || url == "" || version == null || version == "" || !termsAccepted || !engine;
}
Expand Down
2 changes: 1 addition & 1 deletion src/util/components/projects/projectId/settings/project-export-helper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export function postProcessingFormGroups(projectSize: any, embeddings: Embedding
projectSize.forEach((element: any) => {
let hasGdpr = false;
if (element.table == ProjectExportGroup.EMBEDDING_TENSORS) {
hasGdpr = embeddings.some((e: any) => e.name.split("-")[2] == PlatformType.COHERE || e.name.split("-")[2] == PlatformType.OPEN_AI || e.name.split("-")[2] == PlatformType.AZURE);
hasGdpr = embeddings.some((e: any) => e.name.split("-")[2] == PlatformType.OPEN_AI || e.name.split("-")[2] == PlatformType.AZURE);
}
let group = {
export: element.default,
Expand Down