Skip to content

Commit e02e7f2

Browse files
wukkowukko
committed
web: very early proof-of-concept of on-device image background removal
1 parent 2b95af1 commit e02e7f2

File tree

8 files changed

+667
-5
lines changed

8 files changed

+667
-5
lines changed

‎pnpm-lock.yaml‎

Lines changed: 452 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎web/i18n/en/settings.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,9 @@
113113

114114
"advanced.data": "data management",
115115

116-
"advanced.duck": "local processing",
117-
"advanced.duck.title": "process everything on device",
118-
"advanced.duck.description": "very wip, may cause critical issues or not work at all. this toggle will probably be gone by release.",
116+
"advanced.duck": "duck",
117+
"advanced.duck.title": "enable new on-device features",
118+
"advanced.duck.description": "very wip, WILL cause critical issues or not work at all. this toggle will be gone by release.",
119119

120120
"processing.community": "community instances",
121121
"processing.enable_custom.title": "use a custom processing server",

‎web/i18n/en/tabs.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,6 @@
44
"updates": "updates",
55
"donate": "donate",
66
"about": "about",
7-
"remux": "remux"
7+
"remux": "remux",
8+
"cutout": "cut out"
89
}

‎web/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
"@fontsource-variable/noto-sans-mono": "^5.0.20",
2929
"@fontsource/ibm-plex-mono": "^5.0.13",
3030
"@fontsource/redaction-10": "^5.0.2",
31+
"@huggingface/transformers": "^3.2.4",
3132
"@imput/libav.js-remux-cli": "^5.5.6",
3233
"@imput/version-info": "workspace:^",
3334
"@sveltejs/adapter-static": "^3.0.6",

‎web/src/components/sidebar/Sidebar.svelte‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
<script lang="ts">
2+
import settings from "$lib/state/settings";
3+
4+
import { device } from "$lib/device";
25
import { t } from "$lib/i18n/translations";
36
import { defaultNavPage } from "$lib/subnav";
47
@@ -14,6 +17,8 @@
1417
import IconHeart from "@tabler/icons-svelte/IconHeart.svelte";
1518
import IconInfoCircle from "@tabler/icons-svelte/IconInfoCircle.svelte";
1619
20+
import IconCut from "@tabler/icons-svelte/IconCut.svelte";
21+
1722
let screenWidth: number;
1823
let settingsLink = defaultNavPage("settings");
1924
let aboutLink = defaultNavPage("about");
@@ -30,6 +35,9 @@
3035
<div id="sidebar-tabs" role="tablist">
3136
<div id="sidebar-actions" class="sidebar-inner-container">
3237
<SidebarTab name="save" path="/" icon={IconDownload} />
38+
{#if $settings.advanced.duck && !device.is.mobile && $settings.advanced.debug}
39+
<SidebarTab name="cutout" path="/cutout" icon={IconCut} beta />
40+
{/if}
3341
<SidebarTab name="remux" path="/remux" icon={IconRepeat} beta />
3442
</div>
3543
<div id="sidebar-info" class="sidebar-inner-container">

‎web/src/components/sidebar/SidebarTab.svelte‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
1010
export let beta = false;
1111
12-
const firstTabPage = ["save", "remux", "settings"];
12+
const firstTabPage = ["save", "cutout", "remux", "settings"];
1313
1414
let tab: HTMLElement;
1515

‎web/src/lib/workers/removebg.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import { AutoModel, AutoProcessor, RawImage } from "@huggingface/transformers";
2+
3+
const models = {
4+
light: {
5+
id: "briaai/RMBG-1.4",
6+
input: "input",
7+
modelConfig: {
8+
device: "wasm",
9+
dtype: "fp32",
10+
},
11+
processorConfig: {},
12+
},
13+
heavy: {
14+
id: "onnx-community/BiRefNet_lite",
15+
input: "input_image",
16+
modelConfig: {
17+
device: "webgpu",
18+
dtype: "fp16",
19+
},
20+
processorConfig: {},
21+
}
22+
}
23+
24+
export const removeImageBackground = async (file: File) => {
25+
const image = await RawImage.fromBlob(new Blob([file]));
26+
27+
const model_type = "light";
28+
const model = await AutoModel.from_pretrained(models[model_type].id, models[model_type].modelConfig);
29+
30+
const processor = await AutoProcessor.from_pretrained(models[model_type].id, models[model_type].processorConfig);
31+
32+
if (model && processor) {
33+
const { pixel_values } = await processor(image);
34+
35+
const { output } = await model({ [models[model_type].input]: pixel_values });
36+
37+
const mask = await RawImage.fromTensor(output[0].mul(255).to('uint8')).resize(image.width, image.height);
38+
39+
const canvas = document.createElement('canvas');
40+
canvas.width = image.width;
41+
canvas.height = image.height;
42+
const ctx = canvas.getContext('2d');
43+
44+
if (!ctx) return;
45+
46+
ctx.drawImage(image.toCanvas(), 0, 0);
47+
48+
const pixelData = ctx.getImageData(0, 0, image.width, image.height);
49+
for (let i = 0; i < mask.data.length; ++i) {
50+
pixelData.data[4 * i + 3] = mask.data[i];
51+
}
52+
ctx.putImageData(pixelData, 0, 0);
53+
54+
return canvas;
55+
}
56+
}

‎web/src/routes/cutout/+page.svelte‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
<script lang="ts">
2+
import settings from "$lib/state/settings";
3+
4+
import { onMount } from "svelte";
5+
import { goto } from "$app/navigation";
6+
7+
import { downloadFile } from "$lib/download";
8+
import { removeImageBackground } from "$lib/workers/removebg";
9+
10+
import DropReceiver from "$components/misc/DropReceiver.svelte";
11+
import FileReceiver from "$components/misc/FileReceiver.svelte";
12+
import Skeleton from "$components/misc/Skeleton.svelte";
13+
14+
let imageContainer: HTMLElement;
15+
16+
let draggedOver = false;
17+
let file: File | undefined;
18+
19+
let thinking = false;
20+
let done = false;
21+
22+
let result: HTMLCanvasElement;
23+
24+
const processImage = async () => {
25+
if (file) {
26+
thinking = true;
27+
const removedBackground = await removeImageBackground(file);
28+
if (removedBackground) {
29+
thinking = false;
30+
done = true;
31+
result = removedBackground;
32+
imageContainer.append(removedBackground);
33+
}
34+
}
35+
};
36+
37+
const exportImage = async () => {
38+
result.toBlob(async (blob) => {
39+
if (!blob || !file) return;
40+
return await downloadFile({
41+
file: new File([blob], `${file.name} (cutout).png`, {
42+
type: "image/png",
43+
}),
44+
});
45+
}, "image/png");
46+
};
47+
48+
onMount(() => {
49+
if (!($settings.advanced.duck && $settings.advanced.debug)) {
50+
goto("/", { replaceState: true });
51+
}
52+
});
53+
</script>
54+
55+
<DropReceiver bind:file bind:draggedOver id="cutout-container">
56+
{#if !thinking && !done}
57+
<FileReceiver
58+
bind:draggedOver
59+
bind:file
60+
acceptTypes={["image/*"]}
61+
acceptExtensions={["jpg", "png", "webp"]}
62+
/>
63+
<div class="subtext early-note">
64+
this is a very early & basic proof-of-concept, nothing about this feature is final or complete. please don't share or talk about it.
65+
</div>
66+
{#if file}
67+
<button on:click={processImage}>process imported stuff</button>
68+
{/if}
69+
{/if}
70+
71+
{#if thinking}
72+
<div>thinking very hard rn...</div>
73+
{/if}
74+
75+
{#if done}
76+
<div>thought a lot, here's what i got:</div>
77+
{/if}
78+
79+
{#if thinking || done}
80+
<div id="image-preview" bind:this={imageContainer}>
81+
{#if !done}
82+
<Skeleton width="100%" height="100%" class="big" />
83+
{/if}
84+
</div>
85+
{/if}
86+
87+
{#if done}
88+
<div id="finished-actions">
89+
<button
90+
on:click={() => {
91+
done = false;
92+
file = undefined;
93+
}}
94+
>
95+
start over
96+
</button>
97+
<button on:click={exportImage}>save the result</button>
98+
</div>
99+
{/if}
100+
</DropReceiver>
101+
102+
<style>
103+
:global(#cutout-container) {
104+
display: flex;
105+
justify-content: center;
106+
align-items: center;
107+
flex-direction: column;
108+
width: 100%;
109+
gap: 18px;
110+
}
111+
112+
.early-note {
113+
max-width: 400px;
114+
text-align: center;
115+
}
116+
117+
#image-preview {
118+
background: var(--button);
119+
display: flex;
120+
flex-direction: column;
121+
justify-content: center;
122+
align-items: center;
123+
width: 720px;
124+
height: 480px;
125+
overflow: hidden;
126+
border-radius: var(--border-radius);
127+
box-shadow: var(--button-box-shadow);
128+
}
129+
130+
#finished-actions {
131+
display: flex;
132+
flex-direction: row;
133+
gap: 6px;
134+
}
135+
136+
button {
137+
padding: 12px 24px;
138+
border-radius: 200px;
139+
}
140+
141+
:global(canvas) {
142+
height: 100%;
143+
}
144+
</style>

0 commit comments

Comments
 (0)