diff --git a/application/application.go b/application/application.go index 7c5082e..4c8fdbf 100644 --- a/application/application.go +++ b/application/application.go @@ -23,6 +23,7 @@ type JobService interface { GetJob(ctx context.Context, jobNumber int) (*domain.Job, error) ClaimJob(ctx context.Context) (*domain.Job, error) UpdateJobState(ctx context.Context, jobNumber int, newState domain.State, userID string) error + UpdateJobCollectionID(ctx context.Context, jobNumber int, collectionID string) error GetJobs(ctx context.Context, field sort.SortParameterField, direction sort.SortParameterDirection, states []domain.State, limit, offset int) ([]*domain.Job, int, error) GetJobTasks(ctx context.Context, states []domain.State, jobNumber int, limit, offset int) ([]*domain.Task, int, error) CreateTask(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) @@ -112,6 +113,27 @@ func (js *jobService) GetJob(ctx context.Context, jobNumber int) (*domain.Job, e return js.store.GetJob(ctx, jobNumber) } +// UpdateJobCollectionID updates the collection ID of a migration job. +func (js *jobService) UpdateJobCollectionID(ctx context.Context, jobNumber int, collectionID string) error { + job, err := js.store.GetJob(ctx, jobNumber) + if err != nil { + return err + } + + job.Config.CollectionID = collectionID + + now := time.Now().UTC() + + job.LastUpdated = now + + err = js.store.UpdateJob(ctx, job) + if err != nil { + return fmt.Errorf("failed to update job collection ID: %w", err) + } + + return nil +} + // UpdateJobState updates the state of a migration job and logs // an event with the requesting user's ID. func (js *jobService) UpdateJobState(ctx context.Context, jobNumber int, newState domain.State, userID string) error { diff --git a/application/application_test.go b/application/application_test.go index ac45305..cebc37f 100644 --- a/application/application_test.go +++ b/application/application_test.go @@ -1295,6 +1295,120 @@ func TestUpdateTaskState(t *testing.T) { }) } +func TestUpdateJobCollectionID(t *testing.T) { + Convey("Given a job service and store with an existing job", t, func() { + oldTime := time.Date(2024, time.January, 10, 12, 0, 0, 0, time.UTC) + fakeJob := &domain.Job{ + ID: "test-job-id", + JobNumber: testJobNumber, + Config: &domain.JobConfig{ + CollectionID: "old-collection-id", + }, + LastUpdated: oldTime, + } + + mockMongo := &storeMocks.MongoDBMock{ + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return fakeJob, nil + }, + UpdateJobFunc: func(ctx context.Context, job *domain.Job) error { + return nil + }, + } + + mockStore := store.Datastore{ + Backend: mockMongo, + } + + mockClients := clients.ClientList{} + cfg := &config.Config{EnableEventLogging: false} + jobService := Setup(&mockStore, &mockClients, cfg) + + ctx := context.Background() + newCollectionID := "new-collection-id" + + Convey("When UpdateJobCollectionID is called", func() { + err := jobService.UpdateJobCollectionID(ctx, fakeJob.JobNumber, newCollectionID) + + Convey("Then the store should be called to update the job", func() { + So(len(mockMongo.UpdateJobCalls()), ShouldEqual, 1) + So(mockMongo.UpdateJobCalls()[0].Job.Config.CollectionID, ShouldEqual, newCollectionID) + So(mockMongo.UpdateJobCalls()[0].Job.LastUpdated.After(oldTime), ShouldBeTrue) + + Convey("And no error should be returned", func() { + So(err, ShouldBeNil) + }) + }) + }) + }) + + Convey("Given a job service and store that returns an error when getting a job", t, func() { + mockMongo := &storeMocks.MongoDBMock{ + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return nil, appErrors.ErrJobNotFound + }, + } + + mockStore := store.Datastore{ + Backend: mockMongo, + } + + mockClients := clients.ClientList{} + cfg := &config.Config{EnableEventLogging: false} + jobService := Setup(&mockStore, &mockClients, cfg) + + ctx := context.Background() + + Convey("When UpdateJobCollectionID is called", func() { + err := jobService.UpdateJobCollectionID(ctx, testJobNumber, "new-collection-id") + + Convey("Then the error should be returned and no update performed", func() { + So(err, ShouldEqual, appErrors.ErrJobNotFound) + So(len(mockMongo.UpdateJobCalls()), ShouldEqual, 0) + }) + }) + }) + + Convey("Given a job service and store that returns an error when updating a job", t, func() { + fakeJob := &domain.Job{ + ID: "test-job-id", + JobNumber: testJobNumber, + Config: &domain.JobConfig{ + CollectionID: "old-collection-id", + }, + } + + mockMongo := &storeMocks.MongoDBMock{ + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return fakeJob, nil + }, + UpdateJobFunc: func(ctx context.Context, job *domain.Job) error { + return fmt.Errorf("fake error for testing") + }, + } + + mockStore := store.Datastore{ + Backend: mockMongo, + } + + mockClients := clients.ClientList{} + cfg := &config.Config{EnableEventLogging: false} + jobService := Setup(&mockStore, &mockClients, cfg) + + ctx := context.Background() + + Convey("When UpdateJobCollectionID is called", func() { + err := jobService.UpdateJobCollectionID(ctx, testJobNumber, "new-collection-id") + + Convey("Then an error should be returned", func() { + So(len(mockMongo.UpdateJobCalls()), ShouldEqual, 1) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldContainSubstring, "failed to update job collection ID") + }) + }) + }) +} + func TestGetJobTasks(t *testing.T) { Convey("Given a job service and store that has stored tasks for a job", t, func() { mockMongo := &storeMocks.MongoDBMock{ diff --git a/application/mock/jobservice.go b/application/mock/jobservice.go index e540337..c60bfaf 100644 --- a/application/mock/jobservice.go +++ b/application/mock/jobservice.go @@ -57,6 +57,9 @@ var _ application.JobService = &JobServiceMock{} // GetNextJobNumberFunc: func(ctx context.Context) (*domain.Counter, error) { // panic("mock out the GetNextJobNumber method") // }, +// UpdateJobCollectionIDFunc: func(ctx context.Context, jobNumber int, collectionID string) error { +// panic("mock out the UpdateJobCollectionID method") +// }, // UpdateJobStateFunc: func(ctx context.Context, jobNumber int, newState domain.State, userID string) error { // panic("mock out the UpdateJobState method") // }, @@ -109,6 +112,9 @@ type JobServiceMock struct { // GetNextJobNumberFunc mocks the GetNextJobNumber method. GetNextJobNumberFunc func(ctx context.Context) (*domain.Counter, error) + // UpdateJobCollectionIDFunc mocks the UpdateJobCollectionID method. + UpdateJobCollectionIDFunc func(ctx context.Context, jobNumber int, collectionID string) error + // UpdateJobStateFunc mocks the UpdateJobState method. UpdateJobStateFunc func(ctx context.Context, jobNumber int, newState domain.State, userID string) error @@ -224,6 +230,15 @@ type JobServiceMock struct { // Ctx is the ctx argument value. Ctx context.Context } + // UpdateJobCollectionID holds details about calls to the UpdateJobCollectionID method. + UpdateJobCollectionID []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // JobNumber is the jobNumber argument value. + JobNumber int + // CollectionID is the collectionID argument value. + CollectionID string + } // UpdateJobState holds details about calls to the UpdateJobState method. UpdateJobState []struct { // Ctx is the ctx argument value. @@ -264,6 +279,7 @@ type JobServiceMock struct { lockGetJobTasks sync.RWMutex lockGetJobs sync.RWMutex lockGetNextJobNumber sync.RWMutex + lockUpdateJobCollectionID sync.RWMutex lockUpdateJobState sync.RWMutex lockUpdateTask sync.RWMutex lockUpdateTaskState sync.RWMutex @@ -741,6 +757,46 @@ func (mock *JobServiceMock) GetNextJobNumberCalls() []struct { return calls } +// UpdateJobCollectionID calls UpdateJobCollectionIDFunc. +func (mock *JobServiceMock) UpdateJobCollectionID(ctx context.Context, jobNumber int, collectionID string) error { + if mock.UpdateJobCollectionIDFunc == nil { + panic("JobServiceMock.UpdateJobCollectionIDFunc: method is nil but JobService.UpdateJobCollectionID was just called") + } + callInfo := struct { + Ctx context.Context + JobNumber int + CollectionID string + }{ + Ctx: ctx, + JobNumber: jobNumber, + CollectionID: collectionID, + } + mock.lockUpdateJobCollectionID.Lock() + mock.calls.UpdateJobCollectionID = append(mock.calls.UpdateJobCollectionID, callInfo) + mock.lockUpdateJobCollectionID.Unlock() + return mock.UpdateJobCollectionIDFunc(ctx, jobNumber, collectionID) +} + +// UpdateJobCollectionIDCalls gets all the calls that were made to UpdateJobCollectionID. +// Check the length with: +// +// len(mockedJobService.UpdateJobCollectionIDCalls()) +func (mock *JobServiceMock) UpdateJobCollectionIDCalls() []struct { + Ctx context.Context + JobNumber int + CollectionID string +} { + var calls []struct { + Ctx context.Context + JobNumber int + CollectionID string + } + mock.lockUpdateJobCollectionID.RLock() + calls = mock.calls.UpdateJobCollectionID + mock.lockUpdateJobCollectionID.RUnlock() + return calls +} + // UpdateJobState calls UpdateJobStateFunc. func (mock *JobServiceMock) UpdateJobState(ctx context.Context, jobNumber int, newState domain.State, userID string) error { if mock.UpdateJobStateFunc == nil { diff --git a/clients/interfaces.go b/clients/interfaces.go index 53c12a5..4ac8bd6 100644 --- a/clients/interfaces.go +++ b/clients/interfaces.go @@ -23,9 +23,11 @@ type RedirectAPIClient interface { // ZebedeeClient is an interface defining the methods for the Zebedee // (github.com/ONSdigital/zebedee) client. type ZebedeeClient interface { + CreateCollection(ctx context.Context, userAuthToken string, collection zebedee.Collection) (zebedee.Collection, error) GetDataset(ctx context.Context, userAccessToken, collectionID, lang, path string) (d zebedee.Dataset, err error) GetDatasetLandingPage(ctx context.Context, userAccessToken, collectionID, lang, path string) (d zebedee.DatasetLandingPage, err error) GetFileSize(ctx context.Context, userAccessToken, collectionID, lang, uri string) (f zebedee.FileSize, err error) GetPageData(ctx context.Context, userAuthToken, collectionID, lang, path string) (m zebedee.PageData, err error) GetResourceStream(ctx context.Context, userAuthToken, collectionID, lang, path string) (s io.ReadCloser, err error) + SaveContentToCollection(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error } diff --git a/clients/mock/zebedee.go b/clients/mock/zebedee.go index 73b2900..845870f 100644 --- a/clients/mock/zebedee.go +++ b/clients/mock/zebedee.go @@ -21,6 +21,12 @@ var _ clients.ZebedeeClient = &ZebedeeClientMock{} // // // make and configure a mocked clients.ZebedeeClient // mockedZebedeeClient := &ZebedeeClientMock{ +// ApproveCollectionFunc: func(ctx context.Context, userAuthToken string, collectionID string) error { +// panic("mock out the ApproveCollection method") +// }, +// CreateCollectionFunc: func(ctx context.Context, userAuthToken string, collection zebedee.Collection) (zebedee.Collection, error) { +// panic("mock out the CreateCollection method") +// }, // GetDatasetFunc: func(ctx context.Context, userAccessToken string, collectionID string, lang string, path string) (zebedee.Dataset, error) { // panic("mock out the GetDataset method") // }, @@ -36,6 +42,9 @@ var _ clients.ZebedeeClient = &ZebedeeClientMock{} // GetResourceStreamFunc: func(ctx context.Context, userAuthToken string, collectionID string, lang string, path string) (io.ReadCloser, error) { // panic("mock out the GetResourceStream method") // }, +// SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken string, collectionID string, path string, content interface{}) error { +// panic("mock out the SaveContentToCollection method") +// }, // } // // // use mockedZebedeeClient in code that requires clients.ZebedeeClient @@ -43,6 +52,12 @@ var _ clients.ZebedeeClient = &ZebedeeClientMock{} // // } type ZebedeeClientMock struct { + // ApproveCollectionFunc mocks the ApproveCollection method. + ApproveCollectionFunc func(ctx context.Context, userAuthToken string, collectionID string) error + + // CreateCollectionFunc mocks the CreateCollection method. + CreateCollectionFunc func(ctx context.Context, userAuthToken string, collection zebedee.Collection) (zebedee.Collection, error) + // GetDatasetFunc mocks the GetDataset method. GetDatasetFunc func(ctx context.Context, userAccessToken string, collectionID string, lang string, path string) (zebedee.Dataset, error) @@ -58,8 +73,29 @@ type ZebedeeClientMock struct { // GetResourceStreamFunc mocks the GetResourceStream method. GetResourceStreamFunc func(ctx context.Context, userAuthToken string, collectionID string, lang string, path string) (io.ReadCloser, error) + // SaveContentToCollectionFunc mocks the SaveContentToCollection method. + SaveContentToCollectionFunc func(ctx context.Context, userAuthToken string, collectionID string, path string, content interface{}) error + // calls tracks calls to the methods. calls struct { + // ApproveCollection holds details about calls to the ApproveCollection method. + ApproveCollection []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // UserAuthToken is the userAuthToken argument value. + UserAuthToken string + // CollectionID is the collectionID argument value. + CollectionID string + } + // CreateCollection holds details about calls to the CreateCollection method. + CreateCollection []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // UserAuthToken is the userAuthToken argument value. + UserAuthToken string + // Collection is the collection argument value. + Collection zebedee.Collection + } // GetDataset holds details about calls to the GetDataset method. GetDataset []struct { // Ctx is the ctx argument value. @@ -125,12 +161,108 @@ type ZebedeeClientMock struct { // Path is the path argument value. Path string } + // SaveContentToCollection holds details about calls to the SaveContentToCollection method. + SaveContentToCollection []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // UserAuthToken is the userAuthToken argument value. + UserAuthToken string + // CollectionID is the collectionID argument value. + CollectionID string + // Path is the path argument value. + Path string + // Content is the content argument value. + Content interface{} + } } - lockGetDataset sync.RWMutex - lockGetDatasetLandingPage sync.RWMutex - lockGetFileSize sync.RWMutex - lockGetPageData sync.RWMutex - lockGetResourceStream sync.RWMutex + lockApproveCollection sync.RWMutex + lockCreateCollection sync.RWMutex + lockGetDataset sync.RWMutex + lockGetDatasetLandingPage sync.RWMutex + lockGetFileSize sync.RWMutex + lockGetPageData sync.RWMutex + lockGetResourceStream sync.RWMutex + lockSaveContentToCollection sync.RWMutex +} + +// ApproveCollection calls ApproveCollectionFunc. +func (mock *ZebedeeClientMock) ApproveCollection(ctx context.Context, userAuthToken string, collectionID string) error { + if mock.ApproveCollectionFunc == nil { + panic("ZebedeeClientMock.ApproveCollectionFunc: method is nil but ZebedeeClient.ApproveCollection was just called") + } + callInfo := struct { + Ctx context.Context + UserAuthToken string + CollectionID string + }{ + Ctx: ctx, + UserAuthToken: userAuthToken, + CollectionID: collectionID, + } + mock.lockApproveCollection.Lock() + mock.calls.ApproveCollection = append(mock.calls.ApproveCollection, callInfo) + mock.lockApproveCollection.Unlock() + return mock.ApproveCollectionFunc(ctx, userAuthToken, collectionID) +} + +// ApproveCollectionCalls gets all the calls that were made to ApproveCollection. +// Check the length with: +// +// len(mockedZebedeeClient.ApproveCollectionCalls()) +func (mock *ZebedeeClientMock) ApproveCollectionCalls() []struct { + Ctx context.Context + UserAuthToken string + CollectionID string +} { + var calls []struct { + Ctx context.Context + UserAuthToken string + CollectionID string + } + mock.lockApproveCollection.RLock() + calls = mock.calls.ApproveCollection + mock.lockApproveCollection.RUnlock() + return calls +} + +// CreateCollection calls CreateCollectionFunc. +func (mock *ZebedeeClientMock) CreateCollection(ctx context.Context, userAuthToken string, collection zebedee.Collection) (zebedee.Collection, error) { + if mock.CreateCollectionFunc == nil { + panic("ZebedeeClientMock.CreateCollectionFunc: method is nil but ZebedeeClient.CreateCollection was just called") + } + callInfo := struct { + Ctx context.Context + UserAuthToken string + Collection zebedee.Collection + }{ + Ctx: ctx, + UserAuthToken: userAuthToken, + Collection: collection, + } + mock.lockCreateCollection.Lock() + mock.calls.CreateCollection = append(mock.calls.CreateCollection, callInfo) + mock.lockCreateCollection.Unlock() + return mock.CreateCollectionFunc(ctx, userAuthToken, collection) +} + +// CreateCollectionCalls gets all the calls that were made to CreateCollection. +// Check the length with: +// +// len(mockedZebedeeClient.CreateCollectionCalls()) +func (mock *ZebedeeClientMock) CreateCollectionCalls() []struct { + Ctx context.Context + UserAuthToken string + Collection zebedee.Collection +} { + var calls []struct { + Ctx context.Context + UserAuthToken string + Collection zebedee.Collection + } + mock.lockCreateCollection.RLock() + calls = mock.calls.CreateCollection + mock.lockCreateCollection.RUnlock() + return calls } // GetDataset calls GetDatasetFunc. @@ -372,3 +504,51 @@ func (mock *ZebedeeClientMock) GetResourceStreamCalls() []struct { mock.lockGetResourceStream.RUnlock() return calls } + +// SaveContentToCollection calls SaveContentToCollectionFunc. +func (mock *ZebedeeClientMock) SaveContentToCollection(ctx context.Context, userAuthToken string, collectionID string, path string, content interface{}) error { + if mock.SaveContentToCollectionFunc == nil { + panic("ZebedeeClientMock.SaveContentToCollectionFunc: method is nil but ZebedeeClient.SaveContentToCollection was just called") + } + callInfo := struct { + Ctx context.Context + UserAuthToken string + CollectionID string + Path string + Content interface{} + }{ + Ctx: ctx, + UserAuthToken: userAuthToken, + CollectionID: collectionID, + Path: path, + Content: content, + } + mock.lockSaveContentToCollection.Lock() + mock.calls.SaveContentToCollection = append(mock.calls.SaveContentToCollection, callInfo) + mock.lockSaveContentToCollection.Unlock() + return mock.SaveContentToCollectionFunc(ctx, userAuthToken, collectionID, path, content) +} + +// SaveContentToCollectionCalls gets all the calls that were made to SaveContentToCollection. +// Check the length with: +// +// len(mockedZebedeeClient.SaveContentToCollectionCalls()) +func (mock *ZebedeeClientMock) SaveContentToCollectionCalls() []struct { + Ctx context.Context + UserAuthToken string + CollectionID string + Path string + Content interface{} +} { + var calls []struct { + Ctx context.Context + UserAuthToken string + CollectionID string + Path string + Content interface{} + } + mock.lockSaveContentToCollection.RLock() + calls = mock.calls.SaveContentToCollection + mock.lockSaveContentToCollection.RUnlock() + return calls +} diff --git a/config/config.go b/config/config.go index a7edd4a..d4835d1 100644 --- a/config/config.go +++ b/config/config.go @@ -118,6 +118,7 @@ func Get() (*Config, error) { }, AuthConfig: authorisation.NewDefaultConfig(), SlackConfig: &slack.Config{}, + ServiceAuthToken: "migrationservicetestauthtoken", RedirectAPIURL: "http://localhost:29900", TopicAPIURL: "http://localhost:25300", TopicCacheUpdateInterval: 10 * time.Minute, diff --git a/config/config_test.go b/config/config_test.go index c758360..c124440 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -26,6 +26,7 @@ func TestConfig(t *testing.T) { configuration, err = Get() // This Get() is only called once, when inside this function So(err, ShouldBeNil) So(configuration, ShouldResemble, &Config{ + AuthConfig: authorisation.NewDefaultConfig(), BindAddr: "localhost:30100", DatasetAPIURL: "http://localhost:22000", DefaultLimit: 10, @@ -33,6 +34,7 @@ func TestConfig(t *testing.T) { DefaultMaxLimit: 100, EnableEventLogging: false, EnableMockClients: false, + EnableTopicCache: false, FilesAPIURL: "http://localhost:26900", GracefulShutdownTimeout: 5 * time.Second, HealthCheckInterval: 30 * time.Second, @@ -60,12 +62,11 @@ func TestConfig(t *testing.T) { }, }, }, - AuthConfig: authorisation.NewDefaultConfig(), - SlackConfig: &slack.Config{}, RedirectAPIURL: "http://localhost:29900", + ServiceAuthToken: "migrationservicetestauthtoken", + SlackConfig: &slack.Config{}, TopicAPIURL: "http://localhost:25300", TopicCacheUpdateInterval: 10 * time.Minute, - EnableTopicCache: false, UploadServiceURL: "http://localhost:25100", ZebedeeURL: "http://localhost:8082", }) diff --git a/domain/collection.go b/domain/collection.go new file mode 100644 index 0000000..c3a80d8 --- /dev/null +++ b/domain/collection.go @@ -0,0 +1,22 @@ +package domain + +import ( + "strconv" + + "github.com/ONSdigital/dp-api-clients-go/v2/zebedee" +) + +const ( + // CollectionNamePrefix is the prefix given to + // collections created in zebedee, i.e. "Migration Collection for Job 1" + CollectionNamePrefix = "Migration Collection for Job" +) + +// NewMigrationCollection creates a new zebedee.Collection +// for a given job number. +func NewMigrationCollection(jobNumber int) zebedee.Collection { + return zebedee.Collection{ + Name: CollectionNamePrefix + " " + strconv.Itoa(jobNumber), + Type: zebedee.CollectionTypeManual, + } +} diff --git a/domain/collection_test.go b/domain/collection_test.go new file mode 100644 index 0000000..2446cb5 --- /dev/null +++ b/domain/collection_test.go @@ -0,0 +1,23 @@ +package domain + +import ( + "testing" + + "github.com/ONSdigital/dp-api-clients-go/v2/zebedee" + . "github.com/smartystreets/goconvey/convey" +) + +func TestNewMigrationCollection(t *testing.T) { + Convey("Given a job number", t, func() { + jobNumber := 123 + + Convey("When NewMigrationCollection is called", func() { + collection := NewMigrationCollection(jobNumber) + + Convey("Then it returns a collection with the expected name and type", func() { + So(collection.Name, ShouldEqual, CollectionNamePrefix+" 123") + So(collection.Type, ShouldEqual, zebedee.CollectionTypeManual) + }) + }) + }) +} diff --git a/domain/job_config.go b/domain/job_config.go index 204ac48..18e33c6 100644 --- a/domain/job_config.go +++ b/domain/job_config.go @@ -9,10 +9,11 @@ import ( // JobConfig represents the configuration for a migration job type JobConfig struct { - SourceID string `json:"source_id" bson:"source_id"` - TargetID string `json:"target_id" bson:"target_id"` - Type JobType `json:"type" bson:"type"` - Validator JobValidator `json:"-" bson:"-"` + CollectionID string `json:"collection_id,omitempty" bson:"collection_id,omitempty"` + SourceID string `json:"source_id" bson:"source_id"` + TargetID string `json:"target_id" bson:"target_id"` + Type JobType `json:"type" bson:"type"` + Validator JobValidator `json:"-" bson:"-"` } // ValidateInternal performs internal validation of the JobConfig fields diff --git a/executor/job_static_dataset.go b/executor/job_static_dataset.go index c5efcc3..01270b5 100644 --- a/executor/job_static_dataset.go +++ b/executor/job_static_dataset.go @@ -11,21 +11,23 @@ import ( // StaticDatasetJobExecutor executes migration jobs for static datasets. type StaticDatasetJobExecutor struct { - clientList *clients.ClientList - jobService application.JobService + clientList *clients.ClientList + jobService application.JobService + serviceAuthToken string } // NewStaticDatasetJobExecutor creates a new StaticDatasetJobExecutor -func NewStaticDatasetJobExecutor(jobService application.JobService, clientList *clients.ClientList) *StaticDatasetJobExecutor { +func NewStaticDatasetJobExecutor(jobService application.JobService, clientList *clients.ClientList, serviceAuthToken string) *StaticDatasetJobExecutor { return &StaticDatasetJobExecutor{ - jobService: jobService, - clientList: clientList, + jobService: jobService, + clientList: clientList, + serviceAuthToken: serviceAuthToken, } } // Migrate handles the migration operations for a static dataset job. func (e *StaticDatasetJobExecutor) Migrate(ctx context.Context, job *domain.Job) error { - logData := log.Data{"job_id": job.ID} + logData := log.Data{"job_number": job.JobNumber} log.Info(ctx, "starting migration for job", logData) datasetSeriesTask := domain.NewTask(job.JobNumber) @@ -39,7 +41,27 @@ func (e *StaticDatasetJobExecutor) Migrate(ctx context.Context, job *domain.Job) ID: job.Config.TargetID, } - _, err := e.jobService.CreateTask(ctx, job.JobNumber, &datasetSeriesTask) + collection := domain.NewMigrationCollection(job.JobNumber) + + logData["collection_name"] = collection.Name + log.Info(ctx, "creating collection for migration job", logData) + + createdCollection, err := e.clientList.Zebedee.CreateCollection(ctx, e.serviceAuthToken, collection) + if err != nil { + log.Error(ctx, "failed to create collection for migration job", err, logData) + return err + } + + logData["collection_id"] = createdCollection.ID + log.Info(ctx, "updating job with collection id", logData) + + err = e.jobService.UpdateJobCollectionID(ctx, job.JobNumber, createdCollection.ID) + if err != nil { + log.Error(ctx, "failed to update job collection ID", err, logData) + return err + } + + _, err = e.jobService.CreateTask(ctx, job.JobNumber, &datasetSeriesTask) if err != nil { logData["task_source_id"] = datasetSeriesTask.Source.ID log.Error(ctx, "failed to create migration task", err, logData) diff --git a/executor/job_static_dataset_test.go b/executor/job_static_dataset_test.go index a980125..8842534 100644 --- a/executor/job_static_dataset_test.go +++ b/executor/job_static_dataset_test.go @@ -7,12 +7,19 @@ import ( applicationMocks "github.com/ONSdigital/dis-migration-service/application/mock" "github.com/ONSdigital/dis-migration-service/clients" + clientMocks "github.com/ONSdigital/dis-migration-service/clients/mock" "github.com/ONSdigital/dis-migration-service/domain" + "github.com/ONSdigital/dp-api-clients-go/v2/zebedee" . "github.com/smartystreets/goconvey/convey" ) const ( - testJobNumber = 1 + testJobNumber = 1 + testCollectionID = "migration-collection-1" +) + +var ( + errTest = errors.New("test error") ) func TestJobStaticDataset(t *testing.T) { @@ -21,12 +28,23 @@ func TestJobStaticDataset(t *testing.T) { CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { return &domain.Task{}, nil }, + UpdateJobCollectionIDFunc: func(ctx context.Context, jobNumber int, collectionID string) error { + return nil + }, + } + mockZebedeeClient := &clientMocks.ZebedeeClientMock{ + CreateCollectionFunc: func(ctx context.Context, userAuthToken string, collection zebedee.Collection) (zebedee.Collection, error) { + collection.ID = testCollectionID + return collection, nil + }, + } + mockClientList := &clients.ClientList{ + Zebedee: mockZebedeeClient, } - mockClientList := &clients.ClientList{} ctx := context.Background() - executor := NewStaticDatasetJobExecutor(mockJobService, mockClientList) + executor := NewStaticDatasetJobExecutor(mockJobService, mockClientList, "faketoken") Convey("When migrate is called for a job", func() { job := &domain.Job{ @@ -43,34 +61,92 @@ func TestJobStaticDataset(t *testing.T) { Convey("Then no error is returned", func() { So(err, ShouldBeNil) - Convey("And a dataset series migration task is created for the dataset", func() { - So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 1) - So(mockJobService.CreateTaskCalls()[0].JobNumber, ShouldEqual, 1) - So(mockJobService.CreateTaskCalls()[0].Task.Type, ShouldEqual, domain.TaskTypeDatasetSeries) - So(mockJobService.CreateTaskCalls()[0].Task.Source.ID, ShouldEqual, "source-dataset-id") - So(mockJobService.CreateTaskCalls()[0].Task.Target.ID, ShouldEqual, "target-dataset-id") + Convey("And a collection is created for the migration job", func() { + So(len(mockZebedeeClient.CreateCollectionCalls()), ShouldEqual, 1) + So(mockZebedeeClient.CreateCollectionCalls()[0].Collection.Name, ShouldEqual, "Migration Collection for Job 1") + So(mockZebedeeClient.CreateCollectionCalls()[0].Collection.Type, ShouldEqual, "manual") + So(mockJobService.UpdateJobCollectionIDCalls()[0].CollectionID, ShouldEqual, testCollectionID) + + Convey("And a dataset series migration task is created for the dataset", func() { + So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 1) + So(mockJobService.CreateTaskCalls()[0].JobNumber, ShouldEqual, testJobNumber) + So(mockJobService.CreateTaskCalls()[0].Task.Type, ShouldEqual, domain.TaskTypeDatasetSeries) + So(mockJobService.CreateTaskCalls()[0].Task.Source.ID, ShouldEqual, "source-dataset-id") + So(mockJobService.CreateTaskCalls()[0].Task.Target.ID, ShouldEqual, "target-dataset-id") + }) }) }) }) }) + Convey("Given a static dataset job executor and a job service that errors when creating a task", t, func() { mockJobService := &applicationMocks.JobServiceMock{ CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { - return nil, errors.New("create task error") + return nil, errTest }, UpdateJobStateFunc: func(ctx context.Context, jobNumber int, state domain.State, userID string) error { return nil }, + UpdateJobCollectionIDFunc: func(ctx context.Context, jobNumber int, collectionID string) error { + return nil + }, + } + mockZebedeeClient := &clientMocks.ZebedeeClientMock{ + CreateCollectionFunc: func(ctx context.Context, userAuthToken string, collection zebedee.Collection) (zebedee.Collection, error) { + return collection, nil + }, + } + mockClientList := &clients.ClientList{ + Zebedee: mockZebedeeClient, + } + + ctx := context.Background() + + executor := NewStaticDatasetJobExecutor(mockJobService, mockClientList, "faketoken") + + Convey("When migrate is called for a job", func() { + job := &domain.Job{ + JobNumber: testJobNumber, + Config: &domain.JobConfig{ + SourceID: "source-dataset-id", + TargetID: "target-dataset-id", + }, + State: domain.StateMigrating, + } + + err := executor.Migrate(ctx, job) + + Convey("Then an error is returned", func() { + So(err, ShouldEqual, errTest) + }) + }) + }) + + Convey("Given a static dataset job executor and a zebedee client that errors when creating a collection", t, func() { + mockJobService := &applicationMocks.JobServiceMock{ + CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { + return &domain.Task{}, nil + }, + UpdateJobStateFunc: func(ctx context.Context, jobNumber int, state domain.State, userID string) error { + return nil + }, + } + mockZebedeeClient := &clientMocks.ZebedeeClientMock{ + CreateCollectionFunc: func(ctx context.Context, userAuthToken string, collection zebedee.Collection) (zebedee.Collection, error) { + return zebedee.Collection{}, errTest + }, + } + mockClientList := &clients.ClientList{ + Zebedee: mockZebedeeClient, } - mockClientList := &clients.ClientList{} ctx := context.Background() - executor := NewStaticDatasetJobExecutor(mockJobService, mockClientList) + executor := NewStaticDatasetJobExecutor(mockJobService, mockClientList, "faketoken") Convey("When migrate is called for a job", func() { job := &domain.Job{ - ID: "job-1", + JobNumber: testJobNumber, Config: &domain.JobConfig{ SourceID: "source-dataset-id", TargetID: "target-dataset-id", @@ -81,7 +157,8 @@ func TestJobStaticDataset(t *testing.T) { err := executor.Migrate(ctx, job) Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) + So(err, ShouldEqual, errTest) + So(mockJobService.UpdateJobCollectionIDCalls(), ShouldHaveLength, 0) }) }) }) diff --git a/executor/task_dataset_series.go b/executor/task_dataset_series.go index 75603bd..5357662 100644 --- a/executor/task_dataset_series.go +++ b/executor/task_dataset_series.go @@ -67,6 +67,25 @@ func (e *DatasetSeriesTaskExecutor) Migrate(ctx context.Context, task *domain.Ta return err } + job, err := e.jobService.GetJob(ctx, task.JobNumber) + if err != nil { + log.Error(ctx, "failed to get job for dataset series task", err, logData) + return err + } + + sourceData.Description.MigrationLink = mapper.CreateDatasetLink(targetData) + err = e.clientList.Zebedee.SaveContentToCollection( + ctx, + e.serviceAuthToken, + job.Config.CollectionID, + task.Source.ID, + sourceData, + ) + if err != nil { + log.Error(ctx, "failed to save updated dataset landing page with migration link to zebedee collection", err, logData) + return err + } + for _, edition := range sourceData.Datasets { editionTask := domain.NewTask(task.JobNumber) diff --git a/executor/task_dataset_series_test.go b/executor/task_dataset_series_test.go index f9b23c0..737958c 100644 --- a/executor/task_dataset_series_test.go +++ b/executor/task_dataset_series_test.go @@ -47,6 +47,13 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { return &domain.Task{}, nil }, + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil + }, UpdateTaskStateFunc: func(ctx context.Context, taskID string, state domain.State) error { return nil }, } mockDatasetClient := &datasetSDKMock.ClienterMock{ @@ -54,25 +61,29 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { return models.DatasetUpdate{}, nil }, } + mockZebedeeClient := &clientMocks.ZebedeeClientMock{ + GetDatasetLandingPageFunc: func(ctx context.Context, collectionID, edition, lang, datasetID string) (zebedee.DatasetLandingPage, error) { + return zebedee.DatasetLandingPage{ + Type: zebedee.PageTypeDatasetLandingPage, + URI: "/economy/datasets/test-dataset", + Datasets: []zebedee.Link{ + { + URI: getEditionURI(testDatasetSeriesURI, "2021"), + }, + { + URI: getEditionURI(testDatasetSeriesURI, "2022"), + }, + }, + }, nil + }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil + }, + } mockClientList := &clients.ClientList{ DatasetAPI: mockDatasetClient, - Zebedee: &clientMocks.ZebedeeClientMock{ - GetDatasetLandingPageFunc: func(ctx context.Context, collectionID, edition, lang, datasetID string) (zebedee.DatasetLandingPage, error) { - return zebedee.DatasetLandingPage{ - Type: zebedee.PageTypeDatasetLandingPage, - URI: "/economy/datasets/test-dataset", - Datasets: []zebedee.Link{ - { - URI: getEditionURI(testDatasetSeriesURI, "2021"), - }, - { - URI: getEditionURI(testDatasetSeriesURI, "2022"), - }, - }, - }, nil - }, - }, + Zebedee: mockZebedeeClient, } ctx := context.Background() @@ -91,6 +102,12 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { So(mockDatasetClient.CreateDatasetCalls()[0].Dataset.ID, ShouldEqual, testDatasetSeriesID) So(mockDatasetClient.CreateDatasetCalls()[0].Headers.AccessToken, ShouldEqual, testServiceAuthToken) + Convey("And the collection is updated with the migrationLink", func() { + So(len(mockZebedeeClient.SaveContentToCollectionCalls()), ShouldEqual, 1) + So(mockZebedeeClient.SaveContentToCollectionCalls()[0].CollectionID, ShouldEqual, testCollectionID) + So(mockZebedeeClient.SaveContentToCollectionCalls()[0].Path, ShouldEqual, testSeriesTask.Source.ID) + }) + Convey("And an edition task is created for each dataset", func() { So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 2) So(mockJobService.CreateTaskCalls()[0].Task.Type, ShouldEqual, domain.TaskTypeDatasetEdition) @@ -181,12 +198,67 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { }) }) + Convey("Given a dataset series task executor and a zebedee API client that fails to update a collection", t, func() { + mockJobService := &applicationMocks.JobServiceMock{ + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil + }, + } + + mockZebedeeClient := &clientMocks.ZebedeeClientMock{ + GetDatasetLandingPageFunc: func(ctx context.Context, collectionID, edition, lang, datasetID string) (zebedee.DatasetLandingPage, error) { + return zebedee.DatasetLandingPage{ + Type: "dataset_landing_page", + Datasets: []zebedee.Link{ + { + URI: "/datasets/test-dataset/editions/2021/versions/1", + }, + }, + URI: "/economy/datasets/test-dataset", + }, nil + }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return errTest + }, + } + + mockClientList := &clients.ClientList{ + DatasetAPI: &datasetSDKMock.ClienterMock{ + CreateDatasetFunc: func(ctx context.Context, headers sdk.Headers, dataset models.Dataset) (models.DatasetUpdate, error) { + return models.DatasetUpdate{}, nil + }, + }, + Zebedee: mockZebedeeClient, + } + + ctx := context.Background() + + topicCache, _ := cache.NewPopulatedTopicCacheForTest(ctx) + executor := NewDatasetSeriesTaskExecutor(mockJobService, mockClientList, testServiceAuthToken, topicCache) + + Convey("When migrate is called for a task", func() { + err := executor.Migrate(ctx, testSeriesTask) + + Convey("Then an error is returned", func() { + So(err, ShouldEqual, errTest) + + Convey("And no edition tasks are created for the dataset", func() { + So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 0) + }) + }) + }) + }) + Convey("Given a dataset series task executor and a dataset API client that fails to create a dataset", t, func() { mockJobService := &applicationMocks.JobServiceMock{} mockClientList := &clients.ClientList{ DatasetAPI: &datasetSDKMock.ClienterMock{ CreateDatasetFunc: func(ctx context.Context, headers sdk.Headers, dataset models.Dataset) (models.DatasetUpdate, error) { - return models.DatasetUpdate{}, errors.New("failed to create dataset") + return models.DatasetUpdate{}, errTest }, }, Zebedee: &clientMocks.ZebedeeClientMock{ @@ -198,6 +270,7 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { URI: "/datasets/test-dataset/editions/2021/versions/1", }, }, + URI: "/economy/datasets/test-dataset", }, nil }, }, @@ -212,7 +285,7 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { err := executor.Migrate(ctx, testSeriesTask) Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) + So(err, ShouldEqual, errTest) Convey("And no edition tasks are created for the dataset", func() { So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 0) @@ -227,7 +300,14 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { return &domain.Task{}, nil }, UpdateTaskStateFunc: func(ctx context.Context, taskID string, state domain.State) error { - return errors.New("failed to update task") + return errTest + }, + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil }, } mockClientList := &clients.ClientList{ @@ -245,8 +325,12 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { URI: getEditionURI(testDatasetSeriesURI, "2021"), }, }, + URI: "/economy/datasets/test-dataset", }, nil }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil + }, }, } @@ -259,7 +343,7 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { err := executor.Migrate(ctx, testSeriesTask) Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) + So(err, ShouldEqual, errTest) }) }) }) @@ -267,7 +351,14 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { Convey("Given a dataset series task executor and a jobService that fails to create an edition task", t, func() { mockJobService := &applicationMocks.JobServiceMock{ CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { - return nil, errors.New("failed to create task") + return nil, errTest + }, + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil }, UpdateTaskStateFunc: func(ctx context.Context, taskID string, state domain.State) error { return nil }, } @@ -292,6 +383,9 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { }, }, nil }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil + }, }, } @@ -304,7 +398,7 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { err := executor.Migrate(ctx, testSeriesTask) Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) + So(err, ShouldEqual, errTest) Convey("And no further edition tasks are created for the dataset", func() { So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 1) @@ -324,6 +418,13 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { return &domain.Task{}, nil }, + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil + }, UpdateTaskStateFunc: func(ctx context.Context, taskID string, state domain.State) error { return nil }, } mockDatasetClient := &datasetSDKMock.ClienterMock{ @@ -334,23 +435,28 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { }, } - mockClientList := &clients.ClientList{ - DatasetAPI: mockDatasetClient, - Zebedee: &clientMocks.ZebedeeClientMock{ - GetDatasetLandingPageFunc: func(ctx context.Context, collectionID, edition, lang, datasetID string) (zebedee.DatasetLandingPage, error) { - return zebedee.DatasetLandingPage{ - Type: zebedee.PageTypeDatasetLandingPage, - URI: "/economy/inflationandpriceindices/datasets/cpih01", - Datasets: []zebedee.Link{ - { - URI: getEditionURI(testDatasetSeriesURI, "2021"), - }, + mockZebedeeClient := &clientMocks.ZebedeeClientMock{ + GetDatasetLandingPageFunc: func(ctx context.Context, collectionID, edition, lang, datasetID string) (zebedee.DatasetLandingPage, error) { + return zebedee.DatasetLandingPage{ + Type: zebedee.PageTypeDatasetLandingPage, + URI: "/economy/inflationandpriceindices/datasets/cpih01", + Datasets: []zebedee.Link{ + { + URI: getEditionURI(testDatasetSeriesURI, "2021"), }, - }, nil - }, + }, + }, nil + }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil }, } + mockClientList := &clients.ClientList{ + DatasetAPI: mockDatasetClient, + Zebedee: mockZebedeeClient, + } + executor := NewDatasetSeriesTaskExecutor(mockJobService, mockClientList, testServiceAuthToken, topicCache) Convey("When migrate is called for a task with topic cache", func() { @@ -384,6 +490,13 @@ func TestDatasetSeriesTaskExecutor(t *testing.T) { CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { return &domain.Task{}, nil }, + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil + }, UpdateTaskStateFunc: func(ctx context.Context, taskID string, state domain.State) error { return nil }, } mockDatasetClient := &datasetSDKMock.ClienterMock{ diff --git a/executor/task_dataset_version.go b/executor/task_dataset_version.go index 3f74e7a..34bfc8d 100644 --- a/executor/task_dataset_version.go +++ b/executor/task_dataset_version.go @@ -81,6 +81,28 @@ func (e *DatasetVersionTaskExecutor) Migrate(ctx context.Context, task *domain.T return err } + job, err := e.jobService.GetJob(ctx, task.JobNumber) + if err != nil { + log.Error(ctx, "failed to get job for dataset version task", err, logData) + return err + } + + logData["collection_id"] = job.Config.CollectionID + log.Info(ctx, "Sa", logData) + + sourceData.Description.MigrationLink = mapper.CreateDatasetVersionLink(datasetVersion) + err = e.clientList.Zebedee.SaveContentToCollection( + ctx, + e.serviceAuthToken, + job.Config.CollectionID, + task.Source.ID, + sourceData, + ) + if err != nil { + log.Error(ctx, "failed to save updated dataset landing page with migration link to zebedee collection", err, logData) + return err + } + headers := sdk.Headers{ AccessToken: e.serviceAuthToken, } diff --git a/executor/task_dataset_version_test.go b/executor/task_dataset_version_test.go index f4b13b0..fb96c9d 100644 --- a/executor/task_dataset_version_test.go +++ b/executor/task_dataset_version_test.go @@ -2,7 +2,6 @@ package executor import ( "context" - "errors" "strconv" "testing" @@ -43,6 +42,13 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { return &domain.Task{}, nil }, + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil + }, UpdateTaskStateFunc: func(ctx context.Context, taskID string, state domain.State) error { return nil }, UpdateTaskFunc: func(ctx context.Context, task *domain.Task) error { return nil }, } @@ -69,6 +75,9 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { Type: zebedee.PageTypeDatasetLandingPage, }, nil }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil + }, }, } @@ -125,6 +134,9 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { Type: zebedee.PageTypeDatasetLandingPage, }, nil }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil + }, }, } @@ -173,7 +185,7 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { DatasetAPI: mockDatasetClient, Zebedee: &clientMocks.ZebedeeClientMock{ GetDatasetFunc: func(ctx context.Context, collectionID, edition, lang, datasetID string) (zebedee.Dataset, error) { - return zebedee.Dataset{}, errors.New("unknown error") + return zebedee.Dataset{}, errTest }, }, } @@ -188,7 +200,7 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { err := executor.Migrate(ctx, task) Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) + So(err, ShouldEqual, errTest) Convey("And the datasetAPI should not be called to create a dataset", func() { So(len(mockDatasetClient.PostVersionCalls()), ShouldEqual, 0) @@ -230,7 +242,7 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { err := executor.Migrate(ctx, task) Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "invalid page type for dataset version page") Convey("And the datasetAPI should not be called to create a dataset", func() { So(len(mockDatasetClient.PostVersionCalls()), ShouldEqual, 0) @@ -242,8 +254,76 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { }) }) + Convey("Given a dataset version task executor and a zebedee API client that fails to update a collection", t, func() { + mockJobService := &applicationMocks.JobServiceMock{ + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil + }, + UpdateTaskFunc: func(ctx context.Context, task *domain.Task) error { + return nil + }, + } + + mockZebedeeClient := &clientMocks.ZebedeeClientMock{ + GetDatasetLandingPageFunc: func(ctx context.Context, collectionID, edition, lang, datasetID string) (zebedee.DatasetLandingPage, error) { + return zebedee.DatasetLandingPage{ + Type: "dataset_landing_page", + Datasets: []zebedee.Link{ + { + URI: "/datasets/test-dataset/editions/2021/versions/1", + }, + }, + }, nil + }, + GetDatasetFunc: func(ctx context.Context, userAccessToken, collectionID, lang, path string) (zebedee.Dataset, error) { + return zebedee.Dataset{ + Type: zebedee.PageTypeDataset, + }, nil + }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return errTest + }, + } + + mockClientList := &clients.ClientList{ + DatasetAPI: &datasetSDKMock.ClienterMock{ + CreateDatasetFunc: func(ctx context.Context, headers sdk.Headers, dataset models.Dataset) (models.DatasetUpdate, error) { + return models.DatasetUpdate{}, nil + }, + }, + Zebedee: mockZebedeeClient, + } + + ctx := context.Background() + + executor := NewDatasetVersionTaskExecutor(mockJobService, mockClientList, testServiceAuthToken) + + Convey("When migrate is called for a task", func() { + err := executor.Migrate(ctx, testVersionTask) + + Convey("Then an error is returned", func() { + So(err, ShouldEqual, errTest) + + Convey("And no download tasks are created for the dataset", func() { + So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 0) + }) + }) + }) + }) + Convey("Given a dataset version task executor and a dataset API client that fails to create a version", t, func() { mockJobService := &applicationMocks.JobServiceMock{ + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil + }, UpdateTaskFunc: func(ctx context.Context, task *domain.Task) error { return nil }, @@ -251,7 +331,7 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { mockClientList := &clients.ClientList{ DatasetAPI: &datasetSDKMock.ClienterMock{ PostVersionFunc: func(ctx context.Context, headers sdk.Headers, datasetID, editionID, versionID string, version models.Version) (*models.Version, error) { - return &models.Version{}, errors.New("failed to create version") + return &models.Version{}, errTest }, }, Zebedee: &clientMocks.ZebedeeClientMock{ @@ -270,6 +350,9 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { Type: zebedee.PageTypeDataset, }, nil }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil + }, }, } @@ -287,7 +370,7 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { So(mockJobService.UpdateTaskCalls()[0].Task.Target.ID, ShouldEqual, "1") Convey("And an error is returned", func() { - So(err, ShouldNotBeNil) + So(err, ShouldEqual, errTest) Convey("And no download tasks are created for the dataset", func() { So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 0) @@ -302,9 +385,16 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { return &domain.Task{}, nil }, + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil + }, UpdateTaskFunc: func(ctx context.Context, task *domain.Task) error { return nil }, UpdateTaskStateFunc: func(ctx context.Context, taskID string, state domain.State) error { - return errors.New("failed to update task") + return errTest }, } mockClientList := &clients.ClientList{ @@ -329,6 +419,9 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { Type: zebedee.PageTypeDataset, }, nil }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil + }, }, } @@ -342,7 +435,7 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { err := executor.Migrate(ctx, task) Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) + So(err, ShouldEqual, errTest) }) }) }) @@ -350,7 +443,14 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { Convey("Given a dataset series task executor and a jobService that fails to create a download task", t, func() { mockJobService := &applicationMocks.JobServiceMock{ CreateTaskFunc: func(ctx context.Context, jobNumber int, task *domain.Task) (*domain.Task, error) { - return nil, errors.New("failed to create task") + return nil, errTest + }, + GetJobFunc: func(ctx context.Context, jobNumber int) (*domain.Job, error) { + return &domain.Job{ + Config: &domain.JobConfig{ + CollectionID: testCollectionID, + }, + }, nil }, UpdateTaskFunc: func(ctx context.Context, task *domain.Task) error { return nil }, UpdateTaskStateFunc: func(ctx context.Context, taskID string, state domain.State) error { return nil }, @@ -364,7 +464,7 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { Zebedee: &clientMocks.ZebedeeClientMock{ GetDatasetLandingPageFunc: func(ctx context.Context, collectionID, edition, lang, datasetID string) (zebedee.DatasetLandingPage, error) { return zebedee.DatasetLandingPage{ - Type: zebedee.PageTypeDatasetLandingPage, + Type: zebedee.PageTypeDataset, }, nil }, GetDatasetFunc: func(ctx context.Context, userAccessToken, collectionID, lang, path string) (zebedee.Dataset, error) { @@ -377,6 +477,9 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { }, }, nil }, + SaveContentToCollectionFunc: func(ctx context.Context, userAuthToken, collectionID, path string, content interface{}) error { + return nil + }, }, } @@ -390,7 +493,7 @@ func TestDatasetVersionTaskExecutor(t *testing.T) { err := executor.Migrate(ctx, task) Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) + So(err, ShouldEqual, errTest) Convey("And no further tasks are created for the version", func() { So(len(mockJobService.CreateTaskCalls()), ShouldEqual, 1) diff --git a/features/migrate_static_dataset.feature b/features/migrate_static_dataset.feature index a5eec79..d972ccb 100644 --- a/features/migrate_static_dataset.feature +++ b/features/migrate_static_dataset.feature @@ -48,6 +48,7 @@ Feature: Migrate a static dataset job "state": "in_review", "last_updated": "{{DYNAMIC_TIMESTAMP}}", "config": { + "collection_id": "migration-job-test-collection", "source_id": "/test-static-dataset-job", "target_id": "test-target-id", "type": "static_dataset" @@ -97,6 +98,7 @@ Feature: Migrate a static dataset job "state": "failed_migration", "last_updated": "{{DYNAMIC_TIMESTAMP}}", "config": { + "collection_id": "migration-job-test-collection", "source_id": "/test-static-dataset-job-failed", "target_id": "test-target-id", "type": "static_dataset" diff --git a/features/steps/fakeapi.go b/features/steps/fakeapi.go index 8d96a4e..b68ca29 100644 --- a/features/steps/fakeapi.go +++ b/features/steps/fakeapi.go @@ -6,25 +6,45 @@ import ( "net/http" "strings" + "github.com/ONSdigital/dp-api-clients-go/v2/zebedee" datasetModels "github.com/ONSdigital/dp-dataset-api/models" datasetError "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/maxcnunes/httpfake" ) +const ( + testCollectionID = "migration-job-test-collection" +) + // FakeAPI contains all the information for a fake component API type FakeAPI struct { - fakeHTTP *httpfake.HTTPFake - datasetCreateHandler *httpfake.Request + fakeHTTP *httpfake.HTTPFake + datasetCreateHandler *httpfake.Request + collectionCreateHandler *httpfake.Request + collectionUpdateHandler *httpfake.Request } // NewFakeAPI creates a new fake component API func NewFakeAPI() *FakeAPI { fakeAPI := httpfake.New() + // This is setting success criteria for collection interactions with zebedee. + // To control this from component tests you will need to implement steps to update + // these responses. + collectionCreateHandler := fakeAPI.NewHandler().Post("/collection") + collectionCreateHandler.Reply(200).BodyStruct(zebedee.Collection{ + ID: testCollectionID, + }) + + collectionUpdateHandler := fakeAPI.NewHandler().Post(fmt.Sprintf("/content/%s", testCollectionID)) + collectionUpdateHandler.Reply(200) + return &FakeAPI{ - fakeHTTP: fakeAPI, - datasetCreateHandler: fakeAPI.NewHandler().Post("/datasets"), + fakeHTTP: fakeAPI, + datasetCreateHandler: fakeAPI.NewHandler().Post("/datasets"), + collectionCreateHandler: collectionCreateHandler, + collectionUpdateHandler: collectionUpdateHandler, } } diff --git a/go.mod b/go.mod index 96ca7d1..8540999 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.24.12 require ( github.com/ONSdigital/dis-redirect-api v0.3.8 - github.com/ONSdigital/dp-api-clients-go/v2 v2.273.1 + github.com/ONSdigital/dp-api-clients-go/v2 v2.274.0 github.com/ONSdigital/dp-authorisation/v2 v2.33.1 github.com/ONSdigital/dp-cache v0.6.1 github.com/ONSdigital/dp-component-test v1.2.4-alpha @@ -35,16 +35,9 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ONSdigital/dp-api-clients-go v1.43.0 // indirect - github.com/ONSdigital/dp-graph/v2 v2.18.0 // indirect github.com/ONSdigital/dp-kafka/v3 v3.11.0 // indirect - github.com/ONSdigital/dp-kafka/v4 v4.2.0 // indirect - github.com/ONSdigital/dp-mongodb-in-memory v1.8.1 // indirect github.com/ONSdigital/dp-net/v2 v2.22.0 // indirect github.com/ONSdigital/dp-s3/v3 v3.3.0 // indirect - github.com/ONSdigital/golang-neo4j-bolt-driver v0.0.0-20241121114036-9f4b82bb9d37 // indirect - github.com/ONSdigital/graphson v0.3.0 // indirect - github.com/ONSdigital/gremgo-neptune v1.1.0 // indirect github.com/Shopify/sarama v1.38.1 // indirect github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect @@ -72,7 +65,6 @@ require ( github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d // indirect github.com/chromedp/chromedp v0.14.2 // indirect github.com/chromedp/sysutil v1.1.0 // indirect - github.com/cloudflare/cloudflare-go/v6 v6.4.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -105,7 +97,6 @@ require ( github.com/gobwas/ws v1.4.0 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/golang/glog v1.2.5 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/gorilla/schema v1.4.1 // indirect @@ -123,7 +114,6 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/jinzhu/copier v0.4.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.18.3 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -152,17 +142,12 @@ require ( github.com/shirou/gopsutil/v4 v4.25.12 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/smarty/assertions v1.16.0 // indirect - github.com/spf13/afero v1.15.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/square/mongo-lock v0.0.0-20230808145049-cfcf499f6bf0 // indirect github.com/stretchr/testify v1.11.1 // indirect github.com/testcontainers/testcontainers-go v0.40.0 // indirect github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0 // indirect github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 // indirect - github.com/tidwall/gjson v1.18.0 // indirect - github.com/tidwall/match v1.2.0 // indirect - github.com/tidwall/pretty v1.2.1 // indirect - github.com/tidwall/sjson v1.2.5 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect @@ -171,19 +156,18 @@ require ( github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect go.opentelemetry.io/contrib/propagators/autoprop v0.64.0 // indirect go.opentelemetry.io/contrib/propagators/aws v1.39.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.39.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.39.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.39.0 // indirect - go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/sdk v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/sdk v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.47.0 // indirect diff --git a/go.sum b/go.sum index e4cb47e..bc7fba2 100644 --- a/go.sum +++ b/go.sum @@ -8,54 +8,28 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ONSdigital/dis-redirect-api v0.3.8 h1:2DusxFW4iNZLsOHuWehqPFtfkD/ZIMJFNozTd4W+Evw= github.com/ONSdigital/dis-redirect-api v0.3.8/go.mod h1:3WDOU92pqZP5ohbkP2shBNsS+/mbioyhcXqZq1pWMy4= -github.com/ONSdigital/dp-api-clients-go v1.28.0/go.mod h1:iyJy6uRL4B6OYOJA0XMr5UHt6+Q8XmN9uwmURO+9Oj4= -github.com/ONSdigital/dp-api-clients-go v1.34.3/go.mod h1:kX+YKuoLYLfkeLHMvQKRRydZVxO7ZEYyYiwG2xhV51E= -github.com/ONSdigital/dp-api-clients-go v1.41.1/go.mod h1:Ga1+ANjviu21NFJI9wp5NctJIdB4TJLDGbpQFl2V8Wc= -github.com/ONSdigital/dp-api-clients-go v1.43.0 h1:0982P/YxnYXvba1RhEcFmwF3xywC4eXokWQ8YH3Mm24= -github.com/ONSdigital/dp-api-clients-go v1.43.0/go.mod h1:V5MfINik+o3OAF985UXUoMjXIfrZe3JKYa5AhZn5jts= -github.com/ONSdigital/dp-api-clients-go/v2 v2.273.0 h1:iCn+HYkqYDTrzBuwDuvbUYCrIpOrRcVaRsMaATEpxJs= -github.com/ONSdigital/dp-api-clients-go/v2 v2.273.0/go.mod h1:bLseTP21r8LCStUEeOdVPyqtrTomOFP/azPjKWW4deA= -github.com/ONSdigital/dp-api-clients-go/v2 v2.273.1 h1:DRd3zE6PtaCFajnhu05hssnxC2zL0t54xUNZdgatXiM= -github.com/ONSdigital/dp-api-clients-go/v2 v2.273.1/go.mod h1:bLseTP21r8LCStUEeOdVPyqtrTomOFP/azPjKWW4deA= +github.com/ONSdigital/dp-api-clients-go/v2 v2.274.0 h1:UioN1/jWyZfyJZx3ExUjlXjMHVD54bPSq+E8Zvgk9OY= +github.com/ONSdigital/dp-api-clients-go/v2 v2.274.0/go.mod h1:bLseTP21r8LCStUEeOdVPyqtrTomOFP/azPjKWW4deA= github.com/ONSdigital/dp-authorisation/v2 v2.33.1 h1:v2KrjTB1EA4EC3lmfYw1yWpt4+y3dg7dtGkfBCR1KT4= github.com/ONSdigital/dp-authorisation/v2 v2.33.1/go.mod h1:3kHKB5LsTc4kqsdp4Ei8oEhBC5hOto2UJHRQTYRpgQM= github.com/ONSdigital/dp-cache v0.6.1 h1:wiYvxRIef9nu3Uk1//il5vNLTeKgajZ2kYacwkF4Xig= github.com/ONSdigital/dp-cache v0.6.1/go.mod h1:d95JP5gBSNE0L206aNvFDtI1m0oTEGWN9ZetmnN/7hk= github.com/ONSdigital/dp-component-test v1.2.4-alpha h1:DyFG6rMNIucFX3kjZ71PvPaMyE53qFQkiSyWNp5CH0M= github.com/ONSdigital/dp-component-test v1.2.4-alpha/go.mod h1:5IYvqiKqKTfkN5lTbIH8NzUxjclsztWgEroKxY/MVEE= -github.com/ONSdigital/dp-dataset-api v1.97.0 h1:4wJZ3qdypSLIGt2i07et1FD5h8sKyKQev3zq62N0ATw= -github.com/ONSdigital/dp-dataset-api v1.97.0/go.mod h1:+D4zJBRgyWbIgUmGf1QWfct9h/DWsF1osizcEuybbQ4= github.com/ONSdigital/dp-dataset-api v1.99.0 h1:dUlhHT5ut2oDXaqvIyNHcniOG0K6vDzaDxmHhzkia0I= github.com/ONSdigital/dp-dataset-api v1.99.0/go.mod h1:LsQGeVyPGPmeIJ83HfWghZdYQbpBGEju7G8wAd6Ep2M= github.com/ONSdigital/dp-files-api v1.16.0 h1:fpQlVJbPsXT5rSJCctVrG7A2wNXIUL/b1H8+vDY3RM4= github.com/ONSdigital/dp-files-api v1.16.0/go.mod h1:ywXbYQ06tvy8/UqD87sCXUj1FK70Npm6pZBSdMEu/0M= -github.com/ONSdigital/dp-graph/v2 v2.18.0 h1:sp5B78/ueRC7raYh1OuGr6XBvCJhXx7vtH68U84rcCk= -github.com/ONSdigital/dp-graph/v2 v2.18.0/go.mod h1:MfsJMrZIUDcIkPLvVDLzHOB+FxQpVbnvE34w79rghiM= -github.com/ONSdigital/dp-healthcheck v1.0.5/go.mod h1:2wbVAUHMl9+4tWhUlxYUuA1dnf2+NrwzC+So5f5BMLk= -github.com/ONSdigital/dp-healthcheck v1.1.0/go.mod h1:vZwyjMJiCHjp/sJ2R1ZEqzZT0rJ0+uHVGwxqdP4J5vg= github.com/ONSdigital/dp-healthcheck v1.6.4 h1:FhWOuVmob36dYq7AzCdbgyf0Vk58IFitSl8y8pWJ8ck= github.com/ONSdigital/dp-healthcheck v1.6.4/go.mod h1:j3UNbGT4ZJg1chrRkPLE6YUVYCg1su3AAQ8frcBrvgc= github.com/ONSdigital/dp-kafka/v3 v3.11.0 h1:A2rOt2c+7unDM7McFvZEl4EsS0FbT9GJhGHR40iOM1M= github.com/ONSdigital/dp-kafka/v3 v3.11.0/go.mod h1:46LmGU5t2MrVITblJHsQp3kj24FzJV6c2huzxRHMnI4= -github.com/ONSdigital/dp-kafka/v4 v4.2.0 h1:nkBJ1enfYAIOXfp59//vzwkZfnAcw77JPRAW75Fv2Zs= -github.com/ONSdigital/dp-kafka/v4 v4.2.0/go.mod h1:aLlTVci5Br9se+aLtIde+zq3rZbOUeQAZCTCm/TCmrI= -github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9/go.mod h1:BcIRgitUju//qgNePRBmNjATarTtynAgc0yV29VpLEk= github.com/ONSdigital/dp-mocking v0.11.0 h1:laln6e2JD4vtsYbg0cTw9ur1Xf390AUYdd85cG2UNQw= github.com/ONSdigital/dp-mocking v0.11.0/go.mod h1:oHkuukWnURnK7epY5TD5oYVkOwldR2La1D5LQBTxY0A= -github.com/ONSdigital/dp-mongodb-in-memory v1.8.1 h1:yCz6BfjA0bvesA0JjyBIA6nsOzNquBNS7FQP5pbnZKU= -github.com/ONSdigital/dp-mongodb-in-memory v1.8.1/go.mod h1:YyTE7QBdV+Fzz5vGnmcPI1nVGCkMcaqsO4TCUlRe6Pc= github.com/ONSdigital/dp-mongodb/v3 v3.11.0 h1:mwdc0MeFf0cQRW7Lwux2QWM4CA3NdvZK9koghjiGbZ8= github.com/ONSdigital/dp-mongodb/v3 v3.11.0/go.mod h1:0trEUsRcn7m46G8yUUjEf5V4xU9VArh3B27V+JNh2D4= -github.com/ONSdigital/dp-net v1.0.5-0.20200805082802-e518bc287596/go.mod h1:wDVhk2pYosQ1q6PXxuFIRYhYk2XX5+1CeRRnXpSczPY= -github.com/ONSdigital/dp-net v1.0.5-0.20200805145012-9227a11caddb/go.mod h1:MrSZwDUvp8u1VJEqa+36Gwq4E7/DdceW+BDCvGes6Cs= -github.com/ONSdigital/dp-net v1.0.5-0.20200805150805-cac050646ab5/go.mod h1:de3LB9tedE0tObBwa12dUOt5rvTW4qQkF5rXtt4b6CE= -github.com/ONSdigital/dp-net v1.0.7/go.mod h1:1QFzx32FwPKD2lgZI6MtcsUXritsBdJihlzIWDrQ/gc= -github.com/ONSdigital/dp-net v1.0.12/go.mod h1:2lvIKOlD4T3BjWQwjHhBUO2UNWDk82u/+mHRn0R3C9A= -github.com/ONSdigital/dp-net v1.2.0/go.mod h1:NinlaqcsPbIR+X7j5PXCl3UI5G2zCL041SDF6WIiiO4= github.com/ONSdigital/dp-net/v2 v2.22.0 h1:LY9C5x1+sfK9QyjNpB2G3TPvAtSuOFR9FRcXsR9twqs= github.com/ONSdigital/dp-net/v2 v2.22.0/go.mod h1:F6yL3jjuVwBLVMFIKgHF3zhMRbmZysAxBiu+aIAi3Z0= -github.com/ONSdigital/dp-net/v3 v3.9.0 h1:UJsNd36qXhvuGVJIfd6xdKhhUyZlTsxEuPP/2bk1rTs= -github.com/ONSdigital/dp-net/v3 v3.9.0/go.mod h1:ur4LLCvd2xW2jpa785pElE6HB2bPvszZxdAjqv0XFGg= github.com/ONSdigital/dp-net/v3 v3.10.0 h1:/IF6dKThaHNpy6e+bdpkrKJQGSIjyfKSwdRXoAhOAu4= github.com/ONSdigital/dp-net/v3 v3.10.0/go.mod h1:ur4LLCvd2xW2jpa785pElE6HB2bPvszZxdAjqv0XFGg= github.com/ONSdigital/dp-otel-go v0.0.8 h1:jSX32oDmOUKlHyH3FPCBkBpiBKYmWKELoNo6jontKRM= @@ -68,21 +42,6 @@ github.com/ONSdigital/dp-topic-api v1.3.1 h1:rnYeiZPRM7vrA1k75OCw4a7ZGAfsM9gbqew github.com/ONSdigital/dp-topic-api v1.3.1/go.mod h1:5mUuCGbkfpTFpTY2Nt1PeJ4CGvvPzS5A74/3kN34A1w= github.com/ONSdigital/dp-upload-service v1.13.0 h1:9fuLCRf+NbVTh5zSX4XS9VT3WbvESOtzzUhhMFD6LwQ= github.com/ONSdigital/dp-upload-service v1.13.0/go.mod h1:GV5oTh26j4M2SAssA0tBQ8bYujmXs9evDRelgOaeqAE= -github.com/ONSdigital/go-ns v0.0.0-20191104121206-f144c4ec2e58/go.mod h1:iWos35il+NjbvDEqwtB736pyHru0MPFE/LqcwkV1wDc= -github.com/ONSdigital/golang-neo4j-bolt-driver v0.0.0-20241121114036-9f4b82bb9d37 h1:AAp2YSXveTMb6iqi7qd88D0iCMbHbnWMVgMGWrDYyaU= -github.com/ONSdigital/golang-neo4j-bolt-driver v0.0.0-20241121114036-9f4b82bb9d37/go.mod h1:75Sxr59AMz2RiPskqSymLFxdeaIEhnkNaJE5lonMS3M= -github.com/ONSdigital/graphson v0.3.0 h1:s59Qx6RgC4IlqIvjXLD1NWpMGQ8tKIUaJGqnjuy8Zxk= -github.com/ONSdigital/graphson v0.3.0/go.mod h1:CimOliQd5p4nPFoncFl4uXguGczYzjHIfdf8YxFm1BQ= -github.com/ONSdigital/gremgo-neptune v1.1.0 h1:M//UNGPy+OBatFBGBVWFVzdJD/UkRmNbwR/cV4b9lb0= -github.com/ONSdigital/gremgo-neptune v1.1.0/go.mod h1:OC8qe/RRo+5Grdmrb3bcjEKntBEKH+SGmp1yG3twn5Y= -github.com/ONSdigital/log.go v1.0.0/go.mod h1:UnGu9Q14gNC+kz0DOkdnLYGoqugCvnokHBRBxFRpVoQ= -github.com/ONSdigital/log.go v1.0.1-0.20200805084515-ee61165ea36a/go.mod h1:dDnQATFXCBOknvj6ZQuKfmDhbOWf3e8mtV+dPEfWJqs= -github.com/ONSdigital/log.go v1.0.1-0.20200805145532-1f25087a0744/go.mod h1:y4E9MYC+cV9VfjRD0UBGj8PA7H3wABqQi87/ejrDhYc= -github.com/ONSdigital/log.go v1.0.1/go.mod h1:dIwSXuvFB5EsZG5x44JhsXZKMd80zlb0DZxmiAtpL4M= -github.com/ONSdigital/log.go v1.1.0/go.mod h1:0hOVuYR3bDUI30VRo48d5KHfJIoe+spuPXqgt6UF78o= -github.com/ONSdigital/log.go/v2 v2.0.0/go.mod h1:PR7vXrv9dZKUc7SI/0toxBbStk84snmybBnWpe+xY2o= -github.com/ONSdigital/log.go/v2 v2.0.5/go.mod h1:PR7vXrv9dZKUc7SI/0toxBbStk84snmybBnWpe+xY2o= -github.com/ONSdigital/log.go/v2 v2.0.9/go.mod h1:VyTDkL82FtiAkaNFaT+bURBhLbP7NsIx4rkVbdpiuEg= github.com/ONSdigital/log.go/v2 v2.5.1 h1:GCM270UHSP5+mv4OaQ2oHiWp0FiSgfnM6imW2zpAslw= github.com/ONSdigital/log.go/v2 v2.5.1/go.mod h1:KZNEweCUHD8dKwhlvoRvgd2Y2aUIuU3H9/MmbFyVzW8= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= @@ -145,8 +104,6 @@ github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZ github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= -github.com/cloudflare/cloudflare-go/v6 v6.4.0 h1:uigzhmfDfve+zFAYYWIBOAMEuDoPEJXdPS3NBrEm8/Q= -github.com/cloudflare/cloudflare-go/v6 v6.4.0/go.mod h1:Lj3MUqjvKctXRpdRhLQxZYRrNZHuRs0XYuH8JtQGyoI= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -189,10 +146,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A= github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= -github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -233,9 +186,6 @@ github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= -github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -243,16 +193,12 @@ github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E= @@ -282,8 +228,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= -github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -299,8 +243,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= -github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo= @@ -326,22 +268,14 @@ github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIi github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/maxcnunes/httpfake v1.2.4 h1:l7s/N7zuG6XpzG+5dUolg5SSoR3hANQxqzAkv+lREko= github.com/maxcnunes/httpfake v1.2.4/go.mod h1:rWVxb0bLKtOUM/5hN3UO1VEdEitz1hfcTXs7UyiK6r0= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= @@ -397,13 +331,8 @@ github.com/slack-go/slack v0.17.3 h1:zV5qO3Q+WJAQ/XwbGfNFrRMaJ5T/naqaonyPV/1TP4g github.com/slack-go/slack v0.17.3/go.mod h1:X+UqOufi3LYQHDnMG1vxf0J8asC6+WllXrVrhl8/Prk= github.com/smarty/assertions v1.16.0 h1:EvHNkdRA4QHMrn75NZSoUQ/mAUXAYWfatfB01yTCzfY= github.com/smarty/assertions v1.16.0/go.mod h1:duaaFdCS0K9dnoM50iyek/eYINOZ64gbh1Xlf6LG7AI= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= -github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= -github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -430,18 +359,7 @@ github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0 h1:z/1qHeliT github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0/go.mod h1:GaunAWwMXLtsMKG3xn2HYIBDbKddGArfcGsF2Aog81E= github.com/testcontainers/testcontainers-go/modules/redis v0.40.0 h1:OG4qwcxp2O0re7V7M9lY9w0v6wWgWf7j7rtkpAnGMd0= github.com/testcontainers/testcontainers-go/modules/redis v0.40.0/go.mod h1:Bc+EDhKMo5zI5V5zdBkHiMVzeAXbtI4n5isS/nzf6zw= -github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= -github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= -github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= -github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= @@ -467,8 +385,6 @@ go.mongodb.org/mongo-driver/v2 v2.3.0 h1:sh55yOXA2vUjW1QYw/2tRlHSQViwDyPnW61AwpZ go.mongodb.org/mongo-driver/v2 v2.3.0/go.mod h1:jHeEDJHJq7tm6ZF45Issun9dbogjfnPySb1vXA7EeAI= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0 h1:/RxdhdIi0HrKSzdWHLjureinjnGL5YQEYevaC/EAg1k= -go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0/go.mod h1:BKzh9a9EE+vHuq99EwD2cEa+T+Ts1fQ6W3ovO80mjkY= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.64.0 h1:vwZaYp+EEiPUQD1rYKPT0vLfGD7XMv2WypO/59ySpwM= go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.64.0/go.mod h1:D96L6/izMrfhIlFm1sFiyEC8zVyMcDzC8dwqUoTmGT8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y= @@ -483,8 +399,8 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.39.0 h1:Gz3yKzfMSEFzF0Vy5eIpu9 go.opentelemetry.io/contrib/propagators/jaeger v1.39.0/go.mod h1:2D/cxxCqTlrday0rZrPujjg5aoAdqk1NaNyoXn8FJn8= go.opentelemetry.io/contrib/propagators/ot v1.39.0 h1:vKTve1W/WKPVp1fzJamhCDDECt+5upJJ65bPyWoddGg= go.opentelemetry.io/contrib/propagators/ot v1.39.0/go.mod h1:FH5VB2N19duNzh1Q8ks6CsZFyu3LFhNLiA9lPxyEkvU= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 h1:in9O8ESIOlwJAEGTkkf34DesGRAc/Pn8qJ7k3r/42LM= @@ -493,14 +409,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMey go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0 h1:8UPA4IbVZxpsD76ihGOQiFml99GPAEZLohDXvqHdi6U= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0/go.mod h1:MZ1T/+51uIVKlRzGw1Fo46KEWThjlCBZKl2LzY5nv4g= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -508,9 +424,7 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= @@ -520,11 +434,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -537,21 +447,12 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210414055047-fe65e336abe0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -568,7 +469,6 @@ golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -577,14 +477,11 @@ golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3 h1:X9z6obt+cWRX8XjDVOn+SZWhWe5kZHm46TThU9j+jss= diff --git a/mapper/dataset.go b/mapper/dataset.go index 5a8220a..0ab64b2 100644 --- a/mapper/dataset.go +++ b/mapper/dataset.go @@ -3,6 +3,7 @@ package mapper import ( "context" "errors" + "fmt" "strings" "github.com/ONSdigital/dis-migration-service/cache" @@ -90,3 +91,11 @@ func getQMILink(methodologyLinks []zebedee.Link) *datasetModels.GeneralDetails { } return &datasetModels.GeneralDetails{} } + +// CreateDatasetLink creates a link to the dataset in the new location, +// which is added to the migration link field in the source dataset +// landing page. +func CreateDatasetLink(dataset *datasetModels.Dataset) string { + //TODO: add topics in here + return fmt.Sprintf("/datasets/%s", dataset.ID) +} diff --git a/mapper/dataset_test.go b/mapper/dataset_test.go index 6326788..d8c1436 100644 --- a/mapper/dataset_test.go +++ b/mapper/dataset_test.go @@ -6,6 +6,7 @@ import ( "github.com/ONSdigital/dis-migration-service/cache" "github.com/ONSdigital/dp-api-clients-go/v2/zebedee" + datasetModels "github.com/ONSdigital/dp-dataset-api/models" . "github.com/smartystreets/goconvey/convey" ) @@ -289,6 +290,22 @@ func TestMapDatasetLandingPageToDatasetAPI(t *testing.T) { }) } +func TestCreateDatasetLink(t *testing.T) { + Convey("Given a dataset with an ID", t, func() { + dataset := &datasetModels.Dataset{ + ID: "test-dataset-id", + } + + Convey("When CreateDatasetLink is called", func() { + link := CreateDatasetLink(dataset) + + Convey("Then the link is created in the expected format", func() { + So(link, ShouldEqual, "/datasets/test-dataset-id") + }) + }) + }) +} + func getTestDatasetLandingPage() zebedee.DatasetLandingPage { return zebedee.DatasetLandingPage{ Type: zebedee.PageTypeDatasetLandingPage, diff --git a/mapper/version.go b/mapper/version.go index 00a778d..de87767 100644 --- a/mapper/version.go +++ b/mapper/version.go @@ -2,6 +2,7 @@ package mapper import ( "errors" + "fmt" "github.com/ONSdigital/dis-migration-service/clients" "github.com/ONSdigital/dp-api-clients-go/v2/zebedee" @@ -95,3 +96,11 @@ func getCorrectionNotice(editionData zebedee.Dataset, version int) string { } return "" } + +// CreateDatasetVersionLink creates a link to the dataset version in +// the new location, which is added to the migration link field in the +// source dataset landing page. +func CreateDatasetVersionLink(version *datasetModels.Version) string { + //TODO: add topics in here + return fmt.Sprintf("/datasets/%s/editions/%s/versions/%d", version.DatasetID, version.Edition, version.Version) +} diff --git a/mapper/version_test.go b/mapper/version_test.go index 4bb4909..ee7f294 100644 --- a/mapper/version_test.go +++ b/mapper/version_test.go @@ -110,6 +110,24 @@ func TestMapDatasetVersionToDatasetAPI(t *testing.T) { }) } +func TestCreateDatasetVersionLink(t *testing.T) { + Convey("Given a dataset version with an edition and version number", t, func() { + version := &datasetModels.Version{ + DatasetID: "test-dataset-id", + Edition: "test-edition-id", + Version: 3, + } + + Convey("When a dataset version link is created", func() { + link := CreateDatasetVersionLink(version) + + Convey("Then the link is in the expected format", func() { + So(link, ShouldEqual, "/datasets/test-dataset-id/editions/test-edition-id/versions/3") + }) + }) + }) +} + func getTestDatasetVersionPage() zebedee.Dataset { return zebedee.Dataset{ Type: zebedee.PageTypeDataset, diff --git a/migrator/jobs.go b/migrator/jobs.go index 3e43157..e554d44 100644 --- a/migrator/jobs.go +++ b/migrator/jobs.go @@ -8,6 +8,7 @@ import ( "github.com/ONSdigital/dis-migration-service/application" "github.com/ONSdigital/dis-migration-service/clients" + "github.com/ONSdigital/dis-migration-service/config" "github.com/ONSdigital/dis-migration-service/domain" "github.com/ONSdigital/dis-migration-service/executor" "github.com/ONSdigital/dis-migration-service/slack" @@ -28,9 +29,9 @@ const ( EventUpdateTaskStateFailed = "Failed to update task state" ) -var getJobExecutors = func(jobService application.JobService, appClients *clients.ClientList) map[domain.JobType]executor.JobExecutor { +var getJobExecutors = func(jobService application.JobService, appClients *clients.ClientList, cfg *config.Config) map[domain.JobType]executor.JobExecutor { jobExecutors := make(map[domain.JobType]executor.JobExecutor) - jobExecutors[domain.JobTypeStaticDataset] = executor.NewStaticDatasetJobExecutor(jobService, appClients) + jobExecutors[domain.JobTypeStaticDataset] = executor.NewStaticDatasetJobExecutor(jobService, appClients, cfg.ServiceAuthToken) return jobExecutors } diff --git a/migrator/jobs_test.go b/migrator/jobs_test.go index 6973eb1..34e66c1 100644 --- a/migrator/jobs_test.go +++ b/migrator/jobs_test.go @@ -48,7 +48,7 @@ func TestMigratorExecuteJob(t *testing.T) { }, } - getJobExecutors = func(_ application.JobService, _ *clients.ClientList) map[domain.JobType]executor.JobExecutor { + getJobExecutors = func(_ application.JobService, _ *clients.ClientList, _ *config.Config) map[domain.JobType]executor.JobExecutor { return map[domain.JobType]executor.JobExecutor{ fakeJobType: mockJobExecutor, } @@ -111,7 +111,7 @@ func TestMigratorExecuteJob(t *testing.T) { }) Convey("Given a migrator with no executor for a job type", t, func() { - getJobExecutors = func(_ application.JobService, _ *clients.ClientList) map[domain.JobType]executor.JobExecutor { + getJobExecutors = func(_ application.JobService, _ *clients.ClientList, _ *config.Config) map[domain.JobType]executor.JobExecutor { return map[domain.JobType]executor.JobExecutor{} } @@ -162,7 +162,7 @@ func TestMigratorExecuteJob(t *testing.T) { }, } - getJobExecutors = func(_ application.JobService, _ *clients.ClientList) map[domain.JobType]executor.JobExecutor { + getJobExecutors = func(_ application.JobService, _ *clients.ClientList, _ *config.Config) map[domain.JobType]executor.JobExecutor { return map[domain.JobType]executor.JobExecutor{ fakeJobType: mockJobExecutor, } @@ -398,7 +398,7 @@ func TestGetJobExecutor(t *testing.T) { mockJobExecutor := &executorMocks.JobExecutorMock{} - getJobExecutors = func(_ application.JobService, _ *clients.ClientList) map[domain.JobType]executor.JobExecutor { + getJobExecutors = func(_ application.JobService, _ *clients.ClientList, _ *config.Config) map[domain.JobType]executor.JobExecutor { return map[domain.JobType]executor.JobExecutor{ fakeJobType: mockJobExecutor, } @@ -519,7 +519,7 @@ func TestMonitorJobs(t *testing.T) { }, } - getJobExecutors = func(_ application.JobService, _ *clients.ClientList) map[domain.JobType]executor.JobExecutor { + getJobExecutors = func(_ application.JobService, _ *clients.ClientList, _ *config.Config) map[domain.JobType]executor.JobExecutor { return map[domain.JobType]executor.JobExecutor{ fakeJobType: mockJobExecutor, } diff --git a/migrator/migrator.go b/migrator/migrator.go index 4b2b8d8..ac68430 100644 --- a/migrator/migrator.go +++ b/migrator/migrator.go @@ -37,7 +37,7 @@ func NewDefaultMigrator(cfg *config.Config, jobService application.JobService, a return nil, fmt.Errorf("topicCache is required but was nil - cannot initialize migrator without topic cache") } - jobExecutors := getJobExecutors(jobService, appClients) + jobExecutors := getJobExecutors(jobService, appClients, cfg) taskExecutors := getTaskExecutors(jobService, appClients, cfg, topicCache) mig := &migrator{ diff --git a/swagger.yaml b/swagger.yaml index 27e0b23..3484b12 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -476,6 +476,9 @@ definitions: MigrationJobConfig: type: object properties: + collection_id: + type: string + example: "migration-collection-20" source_id: type: string example: "/economy/inflationandpriceindices"