Feat(UI): Canvas high level transform smoothing

invokeai/frontend/web/public/locales/en.json

@@ -2517,6 +2517,13 @@
             "fitModeContain": "Contain",
             "fitModeCover": "Cover",
             "fitModeFill": "Fill",
+            "smoothing": "Smoothing",
+            "smoothingDesc": "Apply a high-quality backend resample when committing transforms.",
+            "smoothingMode": "Resample Mode",
+            "smoothingModeBilinear": "Bilinear",
+            "smoothingModeBicubic": "Bicubic",
+            "smoothingModeHamming": "Hamming",
+            "smoothingModeLanczos": "Lanczos",
             "reset": "Reset",
             "apply": "Apply",
             "cancel": "Cancel"

invokeai/frontend/web/src/features/controlLayers/components/Transform/Transform.tsx

@@ -3,6 +3,7 @@ import { useStore } from '@nanostores/react';
 import { useFocusRegion, useIsRegionFocused } from 'common/hooks/focus';
 import { CanvasOperationIsolatedLayerPreviewSwitch } from 'features/controlLayers/components/CanvasOperationIsolatedLayerPreviewSwitch';
 import { TransformFitToBboxButtons } from 'features/controlLayers/components/Transform/TransformFitToBboxButtons';
+import { TransformSmoothingControls } from 'features/controlLayers/components/Transform/TransformSmoothingControls';
 import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
 import type { CanvasEntityAdapter } from 'features/controlLayers/konva/CanvasEntity/types';
 import { useRegisteredHotkeys } from 'features/system/components/HotkeysModal/useHotkeyData';
@@ -59,6 +60,8 @@ const TransformContent = memo(({ adapter }: { adapter: CanvasEntityAdapter }) =>
         <CanvasOperationIsolatedLayerPreviewSwitch />
       </Flex>
 
+      <TransformSmoothingControls />
+
       <TransformFitToBboxButtons adapter={adapter} />
 
       <ButtonGroup isAttached={false} size="sm" w="full" alignItems="center">

invokeai/frontend/web/src/features/controlLayers/components/Transform/TransformSmoothingControls.tsx

@@ -0,0 +1,52 @@
+import { Flex, FormControl, FormLabel, Select, Switch, Tooltip } from '@invoke-ai/ui-library';
+import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
+import {
+  selectTransformSmoothingEnabled,
+  selectTransformSmoothingMode,
+  settingsTransformSmoothingEnabledToggled,
+  settingsTransformSmoothingModeChanged,
+  type TransformSmoothingMode,
+} from 'features/controlLayers/store/canvasSettingsSlice';
+import type { ChangeEvent } from 'react';
+import { memo, useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+
+export const TransformSmoothingControls = memo(() => {
+  const { t } = useTranslation();
+  const dispatch = useAppDispatch();
+  const smoothingEnabled = useAppSelector(selectTransformSmoothingEnabled);
+  const smoothingMode = useAppSelector(selectTransformSmoothingMode);
+
+  const onToggle = useCallback(() => {
+    dispatch(settingsTransformSmoothingEnabledToggled());
+  }, [dispatch]);
+
+  const onModeChange = useCallback(
+    (e: ChangeEvent<HTMLSelectElement>) => {
+      dispatch(settingsTransformSmoothingModeChanged(e.target.value as TransformSmoothingMode));
+    },
+    [dispatch]
+  );
+
+  return (
+    <Flex w="full" gap={4} alignItems="center" flexWrap="wrap">
+      <Tooltip label={t('controlLayers.transform.smoothingDesc')}>
+        <FormControl w="min-content">
+          <FormLabel m={0}>{t('controlLayers.transform.smoothing')}</FormLabel>
+          <Switch size="sm" isChecked={smoothingEnabled} onChange={onToggle} />
+        </FormControl>
+      </Tooltip>
+      <FormControl flex={1} minW={200} maxW={280}>
+        <FormLabel m={0}>{t('controlLayers.transform.smoothingMode')}</FormLabel>
+        <Select size="sm" value={smoothingMode} onChange={onModeChange} isDisabled={!smoothingEnabled}>
+          <option value="bilinear">{t('controlLayers.transform.smoothingModeBilinear')}</option>
+          <option value="bicubic">{t('controlLayers.transform.smoothingModeBicubic')}</option>
+          <option value="hamming">{t('controlLayers.transform.smoothingModeHamming')}</option>
+          <option value="lanczos">{t('controlLayers.transform.smoothingModeLanczos')}</option>
+        </Select>
+      </FormControl>
+    </Flex>
+  );
+});
+
+TransformSmoothingControls.displayName = 'TransformSmoothingControls';

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer.ts

@@ -556,37 +556,67 @@ export class CanvasEntityObjectRenderer extends CanvasModuleBase {
     this.rasterCacheKeys.clear();
   };
 
-  cloneObjectGroup = (arg: { attrs?: GroupConfig } = {}): Konva.Group => {
-    const { attrs } = arg;
+  cloneObjectGroup = (
+    arg: { attrs?: GroupConfig; cache?: { pixelRatio?: number; imageSmoothingEnabled?: boolean } } = {}
+  ): Konva.Group => {
+    const { attrs, cache } = arg;
     const clone = this.konva.objectGroup.clone();
     if (attrs) {
       clone.setAttrs(attrs);
     }
     if (clone.hasChildren()) {
-      clone.cache({ pixelRatio: 1, imageSmoothingEnabled: false });
+      const { pixelRatio = 1, imageSmoothingEnabled = false } = cache ?? {};
+      clone.cache({ pixelRatio, imageSmoothingEnabled });
     }
     return clone;
   };
 
-  getCanvas = (arg: { rect?: Rect; attrs?: GroupConfig; bg?: string } = {}): HTMLCanvasElement => {
-    const { rect, attrs, bg } = arg;
-    const clone = this.cloneObjectGroup({ attrs });
-    const canvas = konvaNodeToCanvas({ node: clone, rect, bg });
+  getCanvas = (
+    arg: {
+      rect?: Rect;
+      attrs?: GroupConfig;
+      bg?: string;
+      imageSmoothingEnabled?: boolean;
+      pixelRatio?: number;
+      cache?: { pixelRatio?: number; imageSmoothingEnabled?: boolean };
+    } = {}
+  ): HTMLCanvasElement => {
+    const { rect, attrs, bg, imageSmoothingEnabled, pixelRatio, cache } = arg;
+    const clone = this.cloneObjectGroup({ attrs, cache });
+    const canvas = konvaNodeToCanvas({ node: clone, rect, bg, imageSmoothingEnabled, pixelRatio });
     clone.destroy();
     return canvas;
   };
 
-  getBlob = async (arg: { rect?: Rect; attrs?: GroupConfig; bg?: string } = {}): Promise<Blob> => {
-    const { rect, attrs, bg } = arg;
-    const clone = this.cloneObjectGroup({ attrs });
-    const blob = await konvaNodeToBlob({ node: clone, rect, bg });
+  getBlob = async (
+    arg: {
+      rect?: Rect;
+      attrs?: GroupConfig;
+      bg?: string;
+      imageSmoothingEnabled?: boolean;
+      pixelRatio?: number;
+      cache?: { pixelRatio?: number; imageSmoothingEnabled?: boolean };
+    } = {}
+  ): Promise<Blob> => {
+    const { rect, attrs, bg, imageSmoothingEnabled, pixelRatio, cache } = arg;
+    const clone = this.cloneObjectGroup({ attrs, cache });
+    const blob = await konvaNodeToBlob({ node: clone, rect, bg, imageSmoothingEnabled, pixelRatio });
     return blob;
   };
 
-  getImageData = (arg: { rect?: Rect; attrs?: GroupConfig; bg?: string } = {}): ImageData => {
-    const { rect, attrs, bg } = arg;
-    const clone = this.cloneObjectGroup({ attrs });
-    const imageData = konvaNodeToImageData({ node: clone, rect, bg });
+  getImageData = (
+    arg: {
+      rect?: Rect;
+      attrs?: GroupConfig;
+      bg?: string;
+      imageSmoothingEnabled?: boolean;
+      pixelRatio?: number;
+      cache?: { pixelRatio?: number; imageSmoothingEnabled?: boolean };
+    } = {}
+  ): ImageData => {
+    const { rect, attrs, bg, imageSmoothingEnabled, pixelRatio, cache } = arg;
+    const clone = this.cloneObjectGroup({ attrs, cache });
+    const imageData = konvaNodeToImageData({ node: clone, rect, bg, imageSmoothingEnabled, pixelRatio });
     clone.destroy();
     return imageData;
   };

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityTransformer.ts

@@ -14,14 +14,19 @@ import {
   offsetCoord,
   roundRect,
 } from 'features/controlLayers/konva/util';
+import type { TransformSmoothingMode } from 'features/controlLayers/store/canvasSettingsSlice';
 import { selectSelectedEntityIdentifier } from 'features/controlLayers/store/selectors';
 import type { Coordinate, LifecycleCallback, Rect, RectWithRotation } from 'features/controlLayers/store/types';
+import { imageDTOToImageObject } from 'features/controlLayers/store/util';
+import { Graph } from 'features/nodes/util/graph/generation/Graph';
 import { toast } from 'features/toast/toast';
 import Konva from 'konva';
 import type { GroupConfig } from 'konva/lib/Group';
 import { atom } from 'nanostores';
 import type { Logger } from 'roarr';
 import { serializeError } from 'serialize-error';
+import { uploadImage } from 'services/api/endpoints/images';
+import type { ImageDTO } from 'services/api/types';
 import { assert } from 'tsafe';
 
 type CanvasEntityTransformerConfig = {
@@ -90,6 +95,8 @@ const DEFAULT_CONFIG: CanvasEntityTransformerConfig = {
   ROTATE_ANCHOR_SIZE: 12,
 };
 
+const TRANSFORM_SMOOTHING_PIXEL_RATIO = 2;
+
 export class CanvasEntityTransformer extends CanvasModuleBase {
   readonly type = 'entity_transformer';
   readonly id: string;
@@ -807,23 +814,105 @@ export class CanvasEntityTransformer extends CanvasModuleBase {
     this.log.debug('Applying transform');
     this.$isProcessing.set(true);
     this._setInteractionMode('off');
-    const rect = this.getRelativeRect();
-    const rasterizeResult = await withResultAsync(() =>
-      this.parent.renderer.rasterize({
-        rect: roundRect(rect),
-        replaceObjects: true,
-        ignoreCache: true,
+    const rect = roundRect(this.getRelativeRect());
+    const { transformSmoothingEnabled, transformSmoothingMode } = this.manager.stateApi.getSettings();
+    if (!transformSmoothingEnabled) {
+      const rasterizeResult = await withResultAsync(() =>
+        this.parent.renderer.rasterize({
+          rect,
+          replaceObjects: true,
+          ignoreCache: true,
+          attrs: { opacity: 1, filters: [] },
+        })
+      );
+      if (rasterizeResult.isErr()) {
+        toast({ status: 'error', title: 'Failed to apply transform' });
+        this.log.error({ error: serializeError(rasterizeResult.error) }, 'Failed to rasterize entity');
+      }
+      this.requestRectCalculation();
+      this.stopTransform();
+      return;
+    }
+
+    const rasterizeResult = await withResultAsync(async () => {
+      const blob = await this.parent.renderer.getBlob({
+        rect,
         attrs: { opacity: 1, filters: [] },
-      })
-    );
+        imageSmoothingEnabled: true,
+        pixelRatio: TRANSFORM_SMOOTHING_PIXEL_RATIO,
+        cache: {
+          imageSmoothingEnabled: true,
+          pixelRatio: TRANSFORM_SMOOTHING_PIXEL_RATIO,
+        },
+      });
+
+      return await uploadImage({
+        file: new File([blob], `${this.parent.id}_transform.png`, { type: 'image/png' }),
+        image_category: 'other',
+        is_intermediate: true,
+        silent: true,
+      });
+    });
+
     if (rasterizeResult.isErr()) {
       toast({ status: 'error', title: 'Failed to apply transform' });
       this.log.error({ error: serializeError(rasterizeResult.error) }, 'Failed to rasterize entity');
+      this.requestRectCalculation();
+      this.stopTransform();
+      return;
     }
+
+    const resizeResult = await withResultAsync(() =>
+      this.manager.stateApi.runGraphAndReturnImageOutput({
+        ...CanvasEntityTransformer.buildTransformSmoothingGraph(rasterizeResult.value, rect, transformSmoothingMode),
+        options: {
+          prepend: true,
+        },
+      })
+    );
+
+    if (resizeResult.isErr()) {
+      toast({ status: 'error', title: 'Failed to apply transform' });
+      this.log.error({ error: serializeError(resizeResult.error) }, 'Failed to smooth transformed entity');
+      this.requestRectCalculation();
+      this.stopTransform();
+      return;
+    }
+
+    const imageObject = imageDTOToImageObject(resizeResult.value);
+    await this.parent.bufferRenderer.setBuffer(imageObject);
+    this.parent.bufferRenderer.commitBuffer({ pushToState: false });
+    this.manager.stateApi.rasterizeEntity({
+      entityIdentifier: this.parent.entityIdentifier,
+      imageObject,
+      position: {
+        x: rect.x,
+        y: rect.y,
+      },
+      replaceObjects: true,
+    });
     this.requestRectCalculation();
     this.stopTransform();
   };
 
+  private static buildTransformSmoothingGraph = (
+    imageDTO: ImageDTO,
+    rect: Rect,
+    resampleMode: TransformSmoothingMode
+  ): { graph: Graph; outputNodeId: string } => {
+    const graph = new Graph(getPrefixedId('transform_smoothing'));
+    const outputNodeId = getPrefixedId('transform_smoothing_resize');
+    graph.addNode({
+      id: outputNodeId,
+      type: 'img_resize',
+      image: { image_name: imageDTO.image_name },
+      width: rect.width,
+      height: rect.height,
+      resample_mode: resampleMode,
+    });
+    return { graph, outputNodeId };
+  };
+
   resetTransform = () => {
     this.resetScale();
     this.updatePosition();

invokeai/frontend/web/src/features/controlLayers/konva/util.ts

@@ -368,9 +368,15 @@ export const dataURLToImageData = (dataURL: string, width: number, height: numbe
   });
 };
 
-export const konvaNodeToCanvas = (arg: { node: Konva.Node; rect?: Rect; bg?: string }): HTMLCanvasElement => {
-  const { node, rect, bg } = arg;
-  const canvas = node.toCanvas({ ...(rect ?? {}), imageSmoothingEnabled: false, pixelRatio: 1 });
+export const konvaNodeToCanvas = (arg: {
+  node: Konva.Node;
+  rect?: Rect;
+  bg?: string;
+  imageSmoothingEnabled?: boolean;
+  pixelRatio?: number;
+}): HTMLCanvasElement => {
+  const { node, rect, bg, imageSmoothingEnabled = false, pixelRatio = 1 } = arg;
+  const canvas = node.toCanvas({ ...(rect ?? {}), imageSmoothingEnabled, pixelRatio });
 
   if (!bg) {
     return canvas;
@@ -382,7 +388,7 @@ export const konvaNodeToCanvas = (arg: { node: Konva.Node; rect?: Rect; bg?: str
   bgCanvas.height = canvas.height;
   const bgCtx = bgCanvas.getContext('2d');
   assert(bgCtx !== null, 'bgCtx is null');
-  bgCtx.imageSmoothingEnabled = false;
+  bgCtx.imageSmoothingEnabled = imageSmoothingEnabled;
   bgCtx.fillStyle = bg;
   bgCtx.fillRect(0, 0, bgCanvas.width, bgCanvas.height);
   bgCtx.drawImage(canvas, 0, 0);
@@ -419,9 +425,15 @@ export const canvasToImageData = (canvas: HTMLCanvasElement): ImageData => {
  * @param rect - The bounding box to crop to
  * @returns A Promise that resolves with ImageData object of the node cropped to the bounding box
  */
-export const konvaNodeToImageData = (arg: { node: Konva.Node; rect?: Rect; bg?: string }): ImageData => {
-  const { node, rect, bg } = arg;
-  const canvas = konvaNodeToCanvas({ node, rect, bg });
+export const konvaNodeToImageData = (arg: {
+  node: Konva.Node;
+  rect?: Rect;
+  bg?: string;
+  imageSmoothingEnabled?: boolean;
+  pixelRatio?: number;
+}): ImageData => {
+  const { node, rect, bg, imageSmoothingEnabled, pixelRatio } = arg;
+  const canvas = konvaNodeToCanvas({ node, rect, bg, imageSmoothingEnabled, pixelRatio });
   return canvasToImageData(canvas);
 };
 
@@ -431,9 +443,15 @@ export const konvaNodeToImageData = (arg: { node: Konva.Node; rect?: Rect; bg?:
  * @param rect - The bounding box to crop to
  * @returns A Promise that resolves to the Blob or null,
  */
-export const konvaNodeToBlob = (arg: { node: Konva.Node; rect?: Rect; bg?: string }): Promise<Blob> => {
-  const { node, rect, bg } = arg;
-  const canvas = konvaNodeToCanvas({ node, rect, bg });
+export const konvaNodeToBlob = (arg: {
+  node: Konva.Node;
+  rect?: Rect;
+  bg?: string;
+  imageSmoothingEnabled?: boolean;
+  pixelRatio?: number;
+}): Promise<Blob> => {
+  const { node, rect, bg, imageSmoothingEnabled, pixelRatio } = arg;
+  const canvas = konvaNodeToCanvas({ node, rect, bg, imageSmoothingEnabled, pixelRatio });
   return canvasToBlob(canvas);
 };