diff --git a/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp b/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp
index f89e6cc3030..b6c90522d0a 100644
--- a/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp
+++ b/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp
@@ -244,12 +244,13 @@ void deformable_im2col(
 }
 
 int get_greatest_divisor_below_bound(int n, int bound) {
-  for (int k = bound; k > 1; --k) {
-    if (n % k == 0) {
-      return k;
+    int limit = std::min(n/2, bound);
+    for (int k = limit; k > 1; --k) {
+        if (n % k == 0) {
+          return k;
+        }
     }
-  }
-  return 1;
+    return 1;
 }
 
 template <typename scalar_t>
diff --git a/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu b/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu
index ae496b37d48..24bf57724f3 100644
--- a/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu
+++ b/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu
@@ -305,7 +305,8 @@ void deformable_im2col(
 }
 
 int get_greatest_divisor_below_bound(int n, int bound) {
-  for (int k = bound; k > 1; --k) {
+  int limit = std::min(n/2, bound);
+  for (int k = limit; k > 1; --k) {
     if (n % k == 0) {
       return k;
     }