Skip to content

Commit 205b67c

Browse files
alexottalexott
authored
Use token-based pagination for Jobs List operations (#2810)
As we couldn't easily switch `databricks_job` resource to use Go SDK, we were still using offset-based pagination that has a significant load onto the control plane. This PR changes list operation to use recommended token-based approach until we switch to Go SDK. This fixes #2807
1 parent cb4a6f0 commit 205b67c

File tree

5 files changed

+25
-21
lines changed

5 files changed

+25
-21
lines changed

‎exporter/exporter_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) {
473473
},
474474
{
475475
Method: "GET",
476-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
476+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
477477
Response: jobs.JobListResponse{},
478478
},
479479
{
@@ -588,7 +588,7 @@ func TestImportingNoResourcesError(t *testing.T) {
588588
},
589589
{
590590
Method: "GET",
591-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
591+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
592592
Response: jobs.JobListResponse{},
593593
},
594594
{
@@ -632,7 +632,7 @@ func TestImportingClusters(t *testing.T) {
632632
},
633633
{
634634
Method: "GET",
635-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
635+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
636636
Response: jobs.JobListResponse{},
637637
},
638638
{
@@ -816,7 +816,7 @@ func TestImportingJobs_JobList(t *testing.T) {
816816
emptyRepos,
817817
{
818818
Method: "GET",
819-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
819+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
820820
Response: jobs.JobListResponse{
821821
Jobs: []jobs.Job{
822822
{
@@ -1035,7 +1035,7 @@ func TestImportingJobs_JobListMultiTask(t *testing.T) {
10351035
emptyRepos,
10361036
{
10371037
Method: "GET",
1038-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
1038+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
10391039
Response: jobs.JobListResponse{
10401040
Jobs: []jobs.Job{
10411041
{
@@ -1290,7 +1290,7 @@ func TestImportingSecrets(t *testing.T) {
12901290
},
12911291
{
12921292
Method: "GET",
1293-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
1293+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
12941294
Response: jobs.JobListResponse{},
12951295
},
12961296
{

‎jobs/data_job_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ import (
99
)
1010

1111
func commonFixtures(name string) []qa.HTTPFixture {
12-
resource := "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0"
12+
resource := "/api/2.1/jobs/list?expand_tasks=false&limit=25"
1313
if name != "" {
14-
resource = fmt.Sprintf("/api/2.1/jobs/list?expand_tasks=true&limit=25&name=%s&offset=0", name)
14+
resource = fmt.Sprintf("/api/2.1/jobs/list?expand_tasks=true&limit=25&name=%s", name)
1515
}
1616
return []qa.HTTPFixture{
1717
{
@@ -195,7 +195,7 @@ func TestDataSourceQueryableJobNoMatchName(t *testing.T) {
195195
Fixtures: []qa.HTTPFixture{
196196
{
197197
Method: "GET",
198-
Resource: "/api/2.1/jobs/list?expand_tasks=true&limit=25&name=Third&offset=0",
198+
Resource: "/api/2.1/jobs/list?expand_tasks=true&limit=25&name=Third",
199199
Response: JobListResponse{
200200
Jobs: []Job{},
201201
},

‎jobs/data_jobs_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ func TestJobsData(t *testing.T) {
1111
Fixtures: []qa.HTTPFixture{
1212
{
1313
Method: "GET",
14-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
14+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
1515
Response: JobListResponse{
1616
Jobs: []Job{
1717
{

‎jobs/resource_job.go‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -311,8 +311,10 @@ func (js *JobSettings) sortWebhooksByID() {
311311

312312
// JobListResponse returns a list of all jobs
313313
type JobListResponse struct {
314-
Jobs []Job `json:"jobs"`
315-
HasMore bool `json:"has_more,omitempty"`
314+
Jobs []Job `json:"jobs"`
315+
HasMore bool `json:"has_more,omitempty"`
316+
NextPageToken string `json:"next_page_token,omitempty"`
317+
PrevPageToken string `json:"prev_page_token,omitempty"`
316318
}
317319

318320
// Job contains the information when using a GET request from the Databricks Jobs api
@@ -417,12 +419,14 @@ func (a JobsAPI) ListByName(name string, expandTasks bool) ([]Job, error) {
417419
if name != "" {
418420
params["name"] = name
419421
}
420-
offset := 0
421422

423+
nextPageToken := ""
422424
ctx := context.WithValue(a.context, common.Api, common.API_2_1)
423425
for {
424426
var resp JobListResponse
425-
params["offset"] = offset
427+
if nextPageToken != "" {
428+
params["page_token"] = nextPageToken
429+
}
426430
err := a.client.Get(ctx, "/jobs/list", params, &resp)
427431
if err != nil {
428432
return nil, err
@@ -431,7 +435,7 @@ func (a JobsAPI) ListByName(name string, expandTasks bool) ([]Job, error) {
431435
if !resp.HasMore {
432436
break
433437
}
434-
offset += len(resp.Jobs)
438+
nextPageToken = resp.NextPageToken
435439
}
436440
return jobs, nil
437441
}

‎jobs/resource_job_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2196,7 +2196,7 @@ func TestJobsAPIList(t *testing.T) {
21962196
qa.HTTPFixturesApply(t, []qa.HTTPFixture{
21972197
{
21982198
Method: "GET",
2199-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
2199+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
22002200
Response: JobListResponse{
22012201
Jobs: []Job{
22022202
{
@@ -2217,26 +2217,26 @@ func TestJobsAPIListMultiplePages(t *testing.T) {
22172217
qa.HTTPFixturesApply(t, []qa.HTTPFixture{
22182218
{
22192219
Method: "GET",
2220-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=0",
2220+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25",
22212221
Response: JobListResponse{
22222222
Jobs: []Job{
22232223
{
22242224
JobID: 1,
22252225
},
22262226
},
2227-
HasMore: true,
2227+
HasMore: true,
2228+
NextPageToken: "aaaa",
22282229
},
22292230
},
22302231
{
22312232
Method: "GET",
2232-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&offset=1",
2233+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&page_token=aaaa",
22332234
Response: JobListResponse{
22342235
Jobs: []Job{
22352236
{
22362237
JobID: 2,
22372238
},
22382239
},
2239-
HasMore: false,
22402240
},
22412241
},
22422242
}, func(ctx context.Context, client *common.DatabricksClient) {
@@ -2251,7 +2251,7 @@ func TestJobsAPIListByName(t *testing.T) {
22512251
qa.HTTPFixturesApply(t, []qa.HTTPFixture{
22522252
{
22532253
Method: "GET",
2254-
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&name=test&offset=0",
2254+
Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25&name=test",
22552255
Response: JobListResponse{
22562256
Jobs: []Job{
22572257
{

0 commit comments

Comments
 (0)