diff --git a/src/hammer.ts b/src/hammer.ts
index 681a2536..5277f684 100644
--- a/src/hammer.ts
+++ b/src/hammer.ts
@@ -95,7 +95,11 @@ function handlePan(chart: Chart, state: State, e: HammerInput) {
   const delta = state.delta
   if (delta) {
     state.panning = true
-    pan(chart, { x: e.deltaX - delta.x, y: e.deltaY - delta.y }, state.panScales)
+    pan(
+      chart,
+      { x: e.deltaX - delta.x, y: e.deltaY - delta.y },
+      state.panScales && state.panScales.map((i) => chart.scales[i]).filter(Boolean)
+    )
     state.delta = { x: e.deltaX, y: e.deltaY }
   }
 }
@@ -115,7 +119,7 @@ function startPan(chart: Chart, state: State, event: HammerInput) {
     return onPanRejected?.({ chart, event })
   }
 
-  state.panScales = getEnabledScalesByPoint(state.options.pan, point, chart)
+  state.panScales = getEnabledScalesByPoint(state.options.pan, point, chart).map((i) => i.id)
   state.delta = { x: 0, y: 0 }
   handlePan(chart, state, event)
 }
diff --git a/src/state.ts b/src/state.ts
index 94ea535d..fd72bf2f 100644
--- a/src/state.ts
+++ b/src/state.ts
@@ -1,4 +1,4 @@
-import { Chart, Scale, type Point } from 'chart.js'
+import { Chart, type Point } from 'chart.js'
 import type { ZoomPluginOptions } from './options'
 
 export type ScaleRange = { min: number; max: number }
@@ -36,7 +36,7 @@ export interface State {
   filterNextClick?: boolean
   scale?: number | null
   delta?: Point | null
-  panScales?: Scale[]
+  panScales?: string[]
 }
 
 const chartStates = new WeakMap<Chart, State>()