diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index 97b6282a9b8..53e8ff01b5c 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -2517,6 +2517,13 @@ "fitModeContain": "Contain", "fitModeCover": "Cover", "fitModeFill": "Fill", + "smoothing": "Smoothing", + "smoothingDesc": "Apply a high-quality backend resample when committing transforms.", + "smoothingMode": "Resample Mode", + "smoothingModeBilinear": "Bilinear", + "smoothingModeBicubic": "Bicubic", + "smoothingModeHamming": "Hamming", + "smoothingModeLanczos": "Lanczos", "reset": "Reset", "apply": "Apply", "cancel": "Cancel" diff --git a/invokeai/frontend/web/src/features/controlLayers/components/Transform/Transform.tsx b/invokeai/frontend/web/src/features/controlLayers/components/Transform/Transform.tsx index ed0bc6c4d4b..b8459eed180 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/Transform/Transform.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/Transform/Transform.tsx @@ -3,6 +3,7 @@ import { useStore } from '@nanostores/react'; import { useFocusRegion, useIsRegionFocused } from 'common/hooks/focus'; import { CanvasOperationIsolatedLayerPreviewSwitch } from 'features/controlLayers/components/CanvasOperationIsolatedLayerPreviewSwitch'; import { TransformFitToBboxButtons } from 'features/controlLayers/components/Transform/TransformFitToBboxButtons'; +import { TransformSmoothingControls } from 'features/controlLayers/components/Transform/TransformSmoothingControls'; import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate'; import type { CanvasEntityAdapter } from 'features/controlLayers/konva/CanvasEntity/types'; import { useRegisteredHotkeys } from 'features/system/components/HotkeysModal/useHotkeyData'; @@ -59,6 +60,8 @@ const TransformContent = memo(({ adapter }: { adapter: CanvasEntityAdapter }) => + + diff --git a/invokeai/frontend/web/src/features/controlLayers/components/Transform/TransformSmoothingControls.tsx b/invokeai/frontend/web/src/features/controlLayers/components/Transform/TransformSmoothingControls.tsx new file mode 100644 index 00000000000..65a9f538395 --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/components/Transform/TransformSmoothingControls.tsx @@ -0,0 +1,52 @@ +import { Flex, FormControl, FormLabel, Select, Switch, Tooltip } from '@invoke-ai/ui-library'; +import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; +import { + selectTransformSmoothingEnabled, + selectTransformSmoothingMode, + settingsTransformSmoothingEnabledToggled, + settingsTransformSmoothingModeChanged, + type TransformSmoothingMode, +} from 'features/controlLayers/store/canvasSettingsSlice'; +import type { ChangeEvent } from 'react'; +import { memo, useCallback } from 'react'; +import { useTranslation } from 'react-i18next'; + +export const TransformSmoothingControls = memo(() => { + const { t } = useTranslation(); + const dispatch = useAppDispatch(); + const smoothingEnabled = useAppSelector(selectTransformSmoothingEnabled); + const smoothingMode = useAppSelector(selectTransformSmoothingMode); + + const onToggle = useCallback(() => { + dispatch(settingsTransformSmoothingEnabledToggled()); + }, [dispatch]); + + const onModeChange = useCallback( + (e: ChangeEvent) => { + dispatch(settingsTransformSmoothingModeChanged(e.target.value as TransformSmoothingMode)); + }, + [dispatch] + ); + + return ( + + + + {t('controlLayers.transform.smoothing')} + + + + + {t('controlLayers.transform.smoothingMode')} + + + + ); +}); + +TransformSmoothingControls.displayName = 'TransformSmoothingControls'; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer.ts index 57aea0630cd..23fc4b8498a 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer.ts @@ -556,37 +556,67 @@ export class CanvasEntityObjectRenderer extends CanvasModuleBase { this.rasterCacheKeys.clear(); }; - cloneObjectGroup = (arg: { attrs?: GroupConfig } = {}): Konva.Group => { - const { attrs } = arg; + cloneObjectGroup = ( + arg: { attrs?: GroupConfig; cache?: { pixelRatio?: number; imageSmoothingEnabled?: boolean } } = {} + ): Konva.Group => { + const { attrs, cache } = arg; const clone = this.konva.objectGroup.clone(); if (attrs) { clone.setAttrs(attrs); } if (clone.hasChildren()) { - clone.cache({ pixelRatio: 1, imageSmoothingEnabled: false }); + const { pixelRatio = 1, imageSmoothingEnabled = false } = cache ?? {}; + clone.cache({ pixelRatio, imageSmoothingEnabled }); } return clone; }; - getCanvas = (arg: { rect?: Rect; attrs?: GroupConfig; bg?: string } = {}): HTMLCanvasElement => { - const { rect, attrs, bg } = arg; - const clone = this.cloneObjectGroup({ attrs }); - const canvas = konvaNodeToCanvas({ node: clone, rect, bg }); + getCanvas = ( + arg: { + rect?: Rect; + attrs?: GroupConfig; + bg?: string; + imageSmoothingEnabled?: boolean; + pixelRatio?: number; + cache?: { pixelRatio?: number; imageSmoothingEnabled?: boolean }; + } = {} + ): HTMLCanvasElement => { + const { rect, attrs, bg, imageSmoothingEnabled, pixelRatio, cache } = arg; + const clone = this.cloneObjectGroup({ attrs, cache }); + const canvas = konvaNodeToCanvas({ node: clone, rect, bg, imageSmoothingEnabled, pixelRatio }); clone.destroy(); return canvas; }; - getBlob = async (arg: { rect?: Rect; attrs?: GroupConfig; bg?: string } = {}): Promise => { - const { rect, attrs, bg } = arg; - const clone = this.cloneObjectGroup({ attrs }); - const blob = await konvaNodeToBlob({ node: clone, rect, bg }); + getBlob = async ( + arg: { + rect?: Rect; + attrs?: GroupConfig; + bg?: string; + imageSmoothingEnabled?: boolean; + pixelRatio?: number; + cache?: { pixelRatio?: number; imageSmoothingEnabled?: boolean }; + } = {} + ): Promise => { + const { rect, attrs, bg, imageSmoothingEnabled, pixelRatio, cache } = arg; + const clone = this.cloneObjectGroup({ attrs, cache }); + const blob = await konvaNodeToBlob({ node: clone, rect, bg, imageSmoothingEnabled, pixelRatio }); return blob; }; - getImageData = (arg: { rect?: Rect; attrs?: GroupConfig; bg?: string } = {}): ImageData => { - const { rect, attrs, bg } = arg; - const clone = this.cloneObjectGroup({ attrs }); - const imageData = konvaNodeToImageData({ node: clone, rect, bg }); + getImageData = ( + arg: { + rect?: Rect; + attrs?: GroupConfig; + bg?: string; + imageSmoothingEnabled?: boolean; + pixelRatio?: number; + cache?: { pixelRatio?: number; imageSmoothingEnabled?: boolean }; + } = {} + ): ImageData => { + const { rect, attrs, bg, imageSmoothingEnabled, pixelRatio, cache } = arg; + const clone = this.cloneObjectGroup({ attrs, cache }); + const imageData = konvaNodeToImageData({ node: clone, rect, bg, imageSmoothingEnabled, pixelRatio }); clone.destroy(); return imageData; }; diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityTransformer.ts b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityTransformer.ts index 9107988f14d..5b204003173 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityTransformer.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityTransformer.ts @@ -14,14 +14,19 @@ import { offsetCoord, roundRect, } from 'features/controlLayers/konva/util'; +import type { TransformSmoothingMode } from 'features/controlLayers/store/canvasSettingsSlice'; import { selectSelectedEntityIdentifier } from 'features/controlLayers/store/selectors'; import type { Coordinate, LifecycleCallback, Rect, RectWithRotation } from 'features/controlLayers/store/types'; +import { imageDTOToImageObject } from 'features/controlLayers/store/util'; +import { Graph } from 'features/nodes/util/graph/generation/Graph'; import { toast } from 'features/toast/toast'; import Konva from 'konva'; import type { GroupConfig } from 'konva/lib/Group'; import { atom } from 'nanostores'; import type { Logger } from 'roarr'; import { serializeError } from 'serialize-error'; +import { uploadImage } from 'services/api/endpoints/images'; +import type { ImageDTO } from 'services/api/types'; import { assert } from 'tsafe'; type CanvasEntityTransformerConfig = { @@ -90,6 +95,8 @@ const DEFAULT_CONFIG: CanvasEntityTransformerConfig = { ROTATE_ANCHOR_SIZE: 12, }; +const TRANSFORM_SMOOTHING_PIXEL_RATIO = 2; + export class CanvasEntityTransformer extends CanvasModuleBase { readonly type = 'entity_transformer'; readonly id: string; @@ -807,23 +814,105 @@ export class CanvasEntityTransformer extends CanvasModuleBase { this.log.debug('Applying transform'); this.$isProcessing.set(true); this._setInteractionMode('off'); - const rect = this.getRelativeRect(); - const rasterizeResult = await withResultAsync(() => - this.parent.renderer.rasterize({ - rect: roundRect(rect), - replaceObjects: true, - ignoreCache: true, + const rect = roundRect(this.getRelativeRect()); + const { transformSmoothingEnabled, transformSmoothingMode } = this.manager.stateApi.getSettings(); + if (!transformSmoothingEnabled) { + const rasterizeResult = await withResultAsync(() => + this.parent.renderer.rasterize({ + rect, + replaceObjects: true, + ignoreCache: true, + attrs: { opacity: 1, filters: [] }, + }) + ); + if (rasterizeResult.isErr()) { + toast({ status: 'error', title: 'Failed to apply transform' }); + this.log.error({ error: serializeError(rasterizeResult.error) }, 'Failed to rasterize entity'); + } + this.requestRectCalculation(); + this.stopTransform(); + return; + } + + const rasterizeResult = await withResultAsync(async () => { + const blob = await this.parent.renderer.getBlob({ + rect, attrs: { opacity: 1, filters: [] }, - }) - ); + imageSmoothingEnabled: true, + pixelRatio: TRANSFORM_SMOOTHING_PIXEL_RATIO, + cache: { + imageSmoothingEnabled: true, + pixelRatio: TRANSFORM_SMOOTHING_PIXEL_RATIO, + }, + }); + + return await uploadImage({ + file: new File([blob], `${this.parent.id}_transform.png`, { type: 'image/png' }), + image_category: 'other', + is_intermediate: true, + silent: true, + }); + }); + if (rasterizeResult.isErr()) { toast({ status: 'error', title: 'Failed to apply transform' }); this.log.error({ error: serializeError(rasterizeResult.error) }, 'Failed to rasterize entity'); + this.requestRectCalculation(); + this.stopTransform(); + return; } + + const resizeResult = await withResultAsync(() => + this.manager.stateApi.runGraphAndReturnImageOutput({ + ...CanvasEntityTransformer.buildTransformSmoothingGraph(rasterizeResult.value, rect, transformSmoothingMode), + options: { + prepend: true, + }, + }) + ); + + if (resizeResult.isErr()) { + toast({ status: 'error', title: 'Failed to apply transform' }); + this.log.error({ error: serializeError(resizeResult.error) }, 'Failed to smooth transformed entity'); + this.requestRectCalculation(); + this.stopTransform(); + return; + } + + const imageObject = imageDTOToImageObject(resizeResult.value); + await this.parent.bufferRenderer.setBuffer(imageObject); + this.parent.bufferRenderer.commitBuffer({ pushToState: false }); + this.manager.stateApi.rasterizeEntity({ + entityIdentifier: this.parent.entityIdentifier, + imageObject, + position: { + x: rect.x, + y: rect.y, + }, + replaceObjects: true, + }); this.requestRectCalculation(); this.stopTransform(); }; + private static buildTransformSmoothingGraph = ( + imageDTO: ImageDTO, + rect: Rect, + resampleMode: TransformSmoothingMode + ): { graph: Graph; outputNodeId: string } => { + const graph = new Graph(getPrefixedId('transform_smoothing')); + const outputNodeId = getPrefixedId('transform_smoothing_resize'); + graph.addNode({ + id: outputNodeId, + type: 'img_resize', + image: { image_name: imageDTO.image_name }, + width: rect.width, + height: rect.height, + resample_mode: resampleMode, + }); + return { graph, outputNodeId }; + }; + resetTransform = () => { this.resetScale(); this.updatePosition(); diff --git a/invokeai/frontend/web/src/features/controlLayers/konva/util.ts b/invokeai/frontend/web/src/features/controlLayers/konva/util.ts index 6189b3eef78..4b8e2ad97f7 100644 --- a/invokeai/frontend/web/src/features/controlLayers/konva/util.ts +++ b/invokeai/frontend/web/src/features/controlLayers/konva/util.ts @@ -368,9 +368,15 @@ export const dataURLToImageData = (dataURL: string, width: number, height: numbe }); }; -export const konvaNodeToCanvas = (arg: { node: Konva.Node; rect?: Rect; bg?: string }): HTMLCanvasElement => { - const { node, rect, bg } = arg; - const canvas = node.toCanvas({ ...(rect ?? {}), imageSmoothingEnabled: false, pixelRatio: 1 }); +export const konvaNodeToCanvas = (arg: { + node: Konva.Node; + rect?: Rect; + bg?: string; + imageSmoothingEnabled?: boolean; + pixelRatio?: number; +}): HTMLCanvasElement => { + const { node, rect, bg, imageSmoothingEnabled = false, pixelRatio = 1 } = arg; + const canvas = node.toCanvas({ ...(rect ?? {}), imageSmoothingEnabled, pixelRatio }); if (!bg) { return canvas; @@ -382,7 +388,7 @@ export const konvaNodeToCanvas = (arg: { node: Konva.Node; rect?: Rect; bg?: str bgCanvas.height = canvas.height; const bgCtx = bgCanvas.getContext('2d'); assert(bgCtx !== null, 'bgCtx is null'); - bgCtx.imageSmoothingEnabled = false; + bgCtx.imageSmoothingEnabled = imageSmoothingEnabled; bgCtx.fillStyle = bg; bgCtx.fillRect(0, 0, bgCanvas.width, bgCanvas.height); bgCtx.drawImage(canvas, 0, 0); @@ -419,9 +425,15 @@ export const canvasToImageData = (canvas: HTMLCanvasElement): ImageData => { * @param rect - The bounding box to crop to * @returns A Promise that resolves with ImageData object of the node cropped to the bounding box */ -export const konvaNodeToImageData = (arg: { node: Konva.Node; rect?: Rect; bg?: string }): ImageData => { - const { node, rect, bg } = arg; - const canvas = konvaNodeToCanvas({ node, rect, bg }); +export const konvaNodeToImageData = (arg: { + node: Konva.Node; + rect?: Rect; + bg?: string; + imageSmoothingEnabled?: boolean; + pixelRatio?: number; +}): ImageData => { + const { node, rect, bg, imageSmoothingEnabled, pixelRatio } = arg; + const canvas = konvaNodeToCanvas({ node, rect, bg, imageSmoothingEnabled, pixelRatio }); return canvasToImageData(canvas); }; @@ -431,9 +443,15 @@ export const konvaNodeToImageData = (arg: { node: Konva.Node; rect?: Rect; bg?: * @param rect - The bounding box to crop to * @returns A Promise that resolves to the Blob or null, */ -export const konvaNodeToBlob = (arg: { node: Konva.Node; rect?: Rect; bg?: string }): Promise => { - const { node, rect, bg } = arg; - const canvas = konvaNodeToCanvas({ node, rect, bg }); +export const konvaNodeToBlob = (arg: { + node: Konva.Node; + rect?: Rect; + bg?: string; + imageSmoothingEnabled?: boolean; + pixelRatio?: number; +}): Promise => { + const { node, rect, bg, imageSmoothingEnabled, pixelRatio } = arg; + const canvas = konvaNodeToCanvas({ node, rect, bg, imageSmoothingEnabled, pixelRatio }); return canvasToBlob(canvas); }; diff --git a/invokeai/frontend/web/src/features/controlLayers/store/canvasSettingsSlice.ts b/invokeai/frontend/web/src/features/controlLayers/store/canvasSettingsSlice.ts index 0307f958079..f76654d4c80 100644 --- a/invokeai/frontend/web/src/features/controlLayers/store/canvasSettingsSlice.ts +++ b/invokeai/frontend/web/src/features/controlLayers/store/canvasSettingsSlice.ts @@ -9,6 +9,9 @@ import { z } from 'zod'; const zAutoSwitchMode = z.enum(['off', 'switch_on_start', 'switch_on_finish']); export type AutoSwitchMode = z.infer; +const zTransformSmoothingMode = z.enum(['bilinear', 'bicubic', 'hamming', 'lanczos']); +export type TransformSmoothingMode = z.infer; + const zCanvasSettingsState = z.object({ /** * Whether to show HUD (Heads-Up Display) on the canvas. @@ -85,6 +88,14 @@ const zCanvasSettingsState = z.object({ * Whether to show the rule of thirds composition guide overlay on the canvas. */ ruleOfThirds: z.boolean(), + /** + * Whether to apply smoothing when rasterizing transformed layers. + */ + transformSmoothingEnabled: z.boolean().default(false), + /** + * The resampling mode to use when smoothing transformed layers. + */ + transformSmoothingMode: zTransformSmoothingMode.default('bicubic'), /** * Whether to save all staging images to the gallery instead of keeping them as intermediate images. */ @@ -123,6 +134,8 @@ const getInitialState = (): CanvasSettingsState => ({ saveAllImagesToGallery: false, stagingAreaAutoSwitch: 'switch_on_start', fillColorPickerPinned: false, + transformSmoothingEnabled: false, + transformSmoothingMode: 'bicubic', }); const slice = createSlice({ @@ -196,6 +209,15 @@ const slice = createSlice({ settingsSaveAllImagesToGalleryToggled: (state) => { state.saveAllImagesToGallery = !state.saveAllImagesToGallery; }, + settingsTransformSmoothingEnabledToggled: (state) => { + state.transformSmoothingEnabled = !state.transformSmoothingEnabled; + }, + settingsTransformSmoothingModeChanged: ( + state, + action: PayloadAction + ) => { + state.transformSmoothingMode = action.payload; + }, settingsStagingAreaAutoSwitchChanged: ( state, action: PayloadAction @@ -230,6 +252,8 @@ export const { settingsPressureSensitivityToggled, settingsRuleOfThirdsToggled, settingsSaveAllImagesToGalleryToggled, + settingsTransformSmoothingEnabledToggled, + settingsTransformSmoothingModeChanged, settingsStagingAreaAutoSwitchChanged, settingsFillColorPickerPinnedSet, } = slice.actions; @@ -267,3 +291,7 @@ export const selectPressureSensitivity = createCanvasSettingsSelector((settings) export const selectRuleOfThirds = createCanvasSettingsSelector((settings) => settings.ruleOfThirds); export const selectSaveAllImagesToGallery = createCanvasSettingsSelector((settings) => settings.saveAllImagesToGallery); export const selectStagingAreaAutoSwitch = createCanvasSettingsSelector((settings) => settings.stagingAreaAutoSwitch); +export const selectTransformSmoothingEnabled = createCanvasSettingsSelector( + (settings) => settings.transformSmoothingEnabled +); +export const selectTransformSmoothingMode = createCanvasSettingsSelector((settings) => settings.transformSmoothingMode);