diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
new file mode 100644
index 00000000..394a021d
--- /dev/null
+++ b/.buildkite/pipeline.yml
@@ -0,0 +1,18 @@
+steps:
+  - label: "CUDA"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1"
+    agents:
+      queue: "juliagpu"
+      cuda: "*"
+    command: |
+      julia --project=. -e "using Pkg; Pkg.instantiate()"
+      julia --project=. -e "using Pkg; Pkg.test()"
+    env:
+      GROUP: 'CUDA'
+      JULIA_PKG_SERVER: "" # it often struggles with our large artifacts
+      # SECRET_CODECOV_TOKEN: "..."
+    timeout_in_minutes: 180
+    # Don't run Buildkite if the commit message includes the text [skip tests]
+    if: build.message !~ /\[skip tests\]/
diff --git a/Project.toml b/Project.toml
index e43900af..161b93f7 100644
--- a/Project.toml
+++ b/Project.toml
@@ -40,9 +40,10 @@ Calculus = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9"
 DiffTests = "de460e47-3fe3-5279-bb4a-814414816d5d"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 IrrationalConstants = "92d709cd-6900-40b7-9082-c6be49f344b6"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Calculus", "DiffTests", "IrrationalConstants", "SparseArrays", "StaticArrays", "Test", "InteractiveUtils"]
+test = ["Calculus", "DiffTests", "IrrationalConstants", "Pkg", "SparseArrays", "StaticArrays", "Test", "InteractiveUtils"]
diff --git a/test/gpu/GPUGradientTest.jl b/test/gpu/GPUGradientTest.jl
new file mode 100644
index 00000000..f4c81e42
--- /dev/null
+++ b/test/gpu/GPUGradientTest.jl
@@ -0,0 +1,20 @@
+using ForwardDiff, CUDA, Test
+
+fn(x) = sum(x .^ 2 ./ 2)
+
+x = [1.0, 2.0, 3.0]
+x_jl = CuArray(x)
+
+grad = ForwardDiff.gradient(fn, x)
+grad_jl = ForwardDiff.gradient(fn, x_jl)
+
+@test grad_jl isa CuArray
+@test Array(grad_jl) ≈ grad
+
+cfg = ForwardDiff.GradientConfig(
+    fn, x_jl, ForwardDiff.Chunk{2}(), ForwardDiff.Tag(fn, eltype(x))
+)
+grad_jl = ForwardDiff.gradient(fn, x_jl, cfg)
+
+@test grad_jl isa CuArray
+@test Array(grad_jl) ≈ grad
\ No newline at end of file
diff --git a/test/gpu/GPUJacobianTest.jl b/test/gpu/GPUJacobianTest.jl
new file mode 100644
index 00000000..16fd3439
--- /dev/null
+++ b/test/gpu/GPUJacobianTest.jl
@@ -0,0 +1,12 @@
+using ForwardDiff, CUDA, Test
+
+f(x) = x .^ 2 ./ 2
+
+x = [1.0, 2.0, 3.0]
+x_jl = CuArray(x)
+
+jac = ForwardDiff.jacobian(f, x)
+jac_jl = ForwardDiff.jacobian(f, x_jl)
+
+@test jac_jl isa CuArray
+@test Array(jac_jl) ≈ jac
\ No newline at end of file
diff --git a/test/gpu/Project.toml b/test/gpu/Project.toml
new file mode 100644
index 00000000..29894da7
--- /dev/null
+++ b/test/gpu/Project.toml
@@ -0,0 +1,5 @@
+[deps]
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+
+[compat]
+CUDA = "5"
diff --git a/test/runtests.jl b/test/runtests.jl
index e440af65..de072faf 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,55 +1,80 @@
 using ForwardDiff, Test, Random
+using Pkg
+
+const GROUP = get(ENV, "GROUP", "All")
+
+function activate_gpu_env()
+    Pkg.activate("gpu")
+    Pkg.develop(PackageSpec(path = dirname(@__DIR__)))
+    Pkg.instantiate()
+end
 
 SEED = trunc(Int, time())
 println("##### Random.seed!($SEED), on VERSION == $VERSION")
 Random.seed!(SEED)
 
-@testset "ForwardDiff.jl" begin
-    t0 = time()
-    @testset "Partials" begin
-        println("##### Testing Partials...")
-        t = @elapsed include("PartialsTest.jl")
-        println("##### done (took $t seconds).")
-    end
-    @testset "Dual" begin
-        println("##### Testing Dual...")
-        t = @elapsed include("DualTest.jl")
-        println("##### done (took $t seconds).")
-    end
-    @testset "Derivatives" begin
-        println("##### Testing derivative functionality...")
-        t = @elapsed include("DerivativeTest.jl")
-        println("##### done (took $t seconds).")
-    end
-    @testset "Gradients" begin
-        println("##### Testing gradient functionality...")
-        t = @elapsed include("GradientTest.jl")
-        println("##### done (took $t seconds).")
+if GROUP == "All" || GROUP == "Core"
+    @testset "ForwardDiff.jl" begin
+        t0 = time()
+        @testset "Partials" begin
+            println("##### Testing Partials...")
+            t = @elapsed include("PartialsTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "Dual" begin
+            println("##### Testing Dual...")
+            t = @elapsed include("DualTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "Derivatives" begin
+            println("##### Testing derivative functionality...")
+            t = @elapsed include("DerivativeTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "Gradients" begin
+            println("##### Testing gradient functionality...")
+            t = @elapsed include("GradientTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "Jacobians" begin
+            println("##### Testing jacobian functionality...")
+            t = @elapsed include("JacobianTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "Hessians" begin
+            println("##### Testing hessian functionality...")
+            t = @elapsed include("HessianTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "Perturbation Confusion" begin
+            println("##### Testing perturbation confusion functionality...")
+            t = @elapsed include("ConfusionTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "Miscellaneous" begin
+            println("##### Testing miscellaneous functionality...")
+            t = @elapsed include("MiscTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "Allocations" begin
+            println("##### Testing allocations...")
+            t = @elapsed include("AllocationsTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        println("##### Running all ForwardDiff tests took $(time() - t0) seconds.")
     end
-    @testset "Jacobians" begin
-        println("##### Testing jacobian functionality...")
-        t = @elapsed include("JacobianTest.jl")
-        println("##### done (took $t seconds).")
+elseif GROUP == "CUDA"
+    @testset "ForwardDiff.jl CUDA" begin
+        activate_gpu_env()
+        @testset "CUDA Gradients" begin
+            println("##### Testing CUDA gradients...")
+            t = @elapsed include("gpu/CUDAGradientTest.jl")
+            println("##### done (took $t seconds).")
+        end
+        @testset "CUDA Jacobians" begin
+            println("##### Testing CUDA Jacobians...")
+            t = @elapsed include("gpu/CUDAJacobianTest.jl")
+            println("##### done (took $t seconds).")
+        end
     end
-    @testset "Hessians" begin
-        println("##### Testing hessian functionality...")
-        t = @elapsed include("HessianTest.jl")
-        println("##### done (took $t seconds).")
-    end
-    @testset "Perturbation Confusion" begin
-        println("##### Testing perturbation confusion functionality...")
-        t = @elapsed include("ConfusionTest.jl")
-        println("##### done (took $t seconds).")
-    end
-    @testset "Miscellaneous" begin
-        println("##### Testing miscellaneous functionality...")
-        t = @elapsed include("MiscTest.jl")
-        println("##### done (took $t seconds).")
-    end
-    @testset "Allocations" begin
-        println("##### Testing allocations...")
-        t = @elapsed include("AllocationsTest.jl")
-        println("##### done (took $t seconds).")
-    end
-    println("##### Running all ForwardDiff tests took $(time() - t0) seconds.")
-end
+end
\ No newline at end of file