Skip to content

Commit 78883e5

Browse files
kushsharmakushsharma
committed
feature: adding optimus runtime protos
1 parent 32dc735 commit 78883e5

File tree

8 files changed

+1732
-0
lines changed

8 files changed

+1732
-0
lines changed

‎odpf/optimus/RuntimeService.proto‎

Lines changed: 383 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,383 @@
1+
syntax = "proto3";
2+
package odpf.optimus;
3+
4+
import "odpf/third_party/googleapis/google/api/annotations.proto";
5+
import "odpf/third_party/grpc-gateway/protoc-gen-openapiv2/options/annotations.proto";
6+
import "google/protobuf/timestamp.proto";
7+
import "google/protobuf/struct.proto";
8+
9+
option go_package = "github.com/odpf/proton/optimus";
10+
option java_multiple_files = true;
11+
option java_package = "io.odpf.optimus";
12+
option java_outer_classname = "RuntimeServiceManager";
13+
14+
// These annotations are used when generating the OpenAPI file.
15+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
16+
info: {
17+
version: "0.1";
18+
};
19+
external_docs: {
20+
description: "Optimus server";
21+
}
22+
schemes: HTTP;
23+
};
24+
25+
// WARNING: This is still in active development and can have breaking changes
26+
service RuntimeService {
27+
// server ping with version
28+
rpc Version(VersionRequest) returns (VersionResponse) {
29+
option (google.api.http) = {
30+
post: "/api/v1/version"
31+
body: "*"
32+
};
33+
}
34+
35+
// DeployJobSpecification schedules jobs for execution
36+
// returns a stream of messages which can be used to track the progress
37+
// of deployments. Message containing ack are status events other are progress
38+
// events
39+
rpc DeployJobSpecification(DeployJobSpecificationRequest) returns (stream DeployJobSpecificationResponse) {}
40+
41+
// ListJobSpecification returns list of jobs created in a project
42+
rpc ListJobSpecification(ListJobSpecificationRequest) returns (ListJobSpecificationResponse) {
43+
option (google.api.http) = {
44+
get: "/api/v1/project/{project_name}/job"
45+
};
46+
}
47+
48+
// DumpJobSpecification returns compiled representation of the job in a scheduler
49+
// consumable form
50+
rpc DumpJobSpecification(DumpJobSpecificationRequest) returns (DumpJobSpecificationResponse) {
51+
option (google.api.http) = {
52+
get: "/api/v1/project/{project_name}/job/{job_name}/dump"
53+
};
54+
}
55+
56+
// RegisterProject creates a new project
57+
rpc RegisterProject(RegisterProjectRequest) returns (RegisterProjectResponse) {
58+
option (google.api.http) = {
59+
post: "/api/v1/project"
60+
body: "*"
61+
};
62+
}
63+
// RegisterSecret creates a new secret of a project
64+
rpc RegisterSecret(RegisterSecretRequest) returns (RegisterSecretResponse) {
65+
option (google.api.http) = {
66+
post: "/api/v1/project/{project_name}/secret/{secret_name}"
67+
body: "*"
68+
};
69+
}
70+
// ListProjects returns list of registered projects and configurations
71+
rpc ListProjects(ListProjectsRequest) returns (ListProjectsResponse) {
72+
option (google.api.http) = {
73+
get: "/api/v1/project"
74+
};
75+
}
76+
77+
// RegisterInstance is an internal admin command used during task execution
78+
rpc RegisterInstance(RegisterInstanceRequest) returns (RegisterInstanceResponse) {
79+
option (google.api.http) = {
80+
post: "/api/v1/instance"
81+
body: "*"
82+
};
83+
}
84+
// JobStatus returns the current and past run status of jobs
85+
rpc JobStatus(JobStatusRequest) returns (JobStatusResponse) {
86+
option (google.api.http) = {
87+
get: "/api/v1/project/{project_name}/job/{job_name}/status"
88+
};
89+
}
90+
// GetWindow provides the start and end dates provided a scheduled date
91+
// of the execution window
92+
rpc GetWindow(GetWindowRequest) returns (GetWindowResponse) {
93+
option (google.api.http) = {
94+
get: "/api/v1/window"
95+
};
96+
}
97+
98+
// DeployResourceSpecification migrate all resource specifications of a datastore in project
99+
rpc DeployResourceSpecification(DeployResourceSpecificationRequest) returns (stream DeployResourceSpecificationResponse) {}
100+
// ListResourceSpecification lists all resource specifications of a datastore in project
101+
rpc ListResourceSpecification(ListResourceSpecificationRequest) returns (ListResourceSpecificationResponse) {
102+
option (google.api.http) = {
103+
get: "/api/v1/project/{project_name}/datastore/{datastore_name}/resource"
104+
};
105+
}
106+
107+
// Datastore CRUD
108+
rpc CreateResource(CreateResourceRequest) returns (CreateResourceResponse) {
109+
option (google.api.http) = {
110+
post: "/api/v1/project/{project_name}/datastore/{datastore_name}/resource"
111+
body: "*"
112+
};
113+
}
114+
rpc ReadResource(ReadResourceRequest) returns (ReadResourceResponse) {
115+
option (google.api.http) = {
116+
get: "/api/v1/project/{project_name}/datastore/{datastore_name}/resource/{resource_name}"
117+
};
118+
}
119+
rpc UpdateResource(UpdateResourceRequest) returns (UpdateResourceResponse) {
120+
option (google.api.http) = {
121+
put: "/api/v1/project/{project_name}/datastore/{datastore_name}/resource"
122+
body: "*"
123+
};
124+
}
125+
// TODO(kush.sharma): unsupported ATM
126+
//rpc DeleteResource(DeleteResourceRequest) returns (DeleteResourceResponse) {}
127+
128+
}
129+
130+
// models
131+
132+
message ProjectSpecification {
133+
string name = 1;
134+
map<string, string> config = 2;
135+
}
136+
137+
message JobSpecHook {
138+
string name = 1;
139+
repeated JobConfigItem config = 2;
140+
}
141+
142+
message JobSpecification {
143+
int32 version = 1;
144+
string name = 2;
145+
string owner = 3;
146+
147+
string start_date = 4;
148+
string end_date = 5; // optional
149+
string interval = 6;
150+
151+
bool depends_on_past = 7; // should only execute today if yesterday was completed with success?
152+
bool catch_up = 8; // should backfill till today?
153+
154+
string task_name = 9;
155+
repeated JobConfigItem config = 10;
156+
157+
string window_size = 11;
158+
string window_offset = 12;
159+
string window_truncate_to = 13;
160+
161+
repeated JobDependency dependencies = 14; // static dependencies
162+
map<string, string> assets = 15;
163+
164+
repeated JobSpecHook hooks = 16; // optional
165+
166+
string description = 17; // optional
167+
map<string, string> labels = 18;
168+
}
169+
170+
message JobConfigItem {
171+
string name = 1;
172+
string value = 2;
173+
}
174+
175+
message JobDependency {
176+
string name = 1;
177+
string type = 2;
178+
}
179+
180+
message ProjectSecret {
181+
string name = 1;
182+
string value = 2;
183+
}
184+
185+
message InstanceSpec {
186+
string state = 1;
187+
google.protobuf.Timestamp scheduled_at = 2;
188+
repeated InstanceSpecData data = 3;
189+
string job_name = 4;
190+
}
191+
192+
message InstanceSpecData {
193+
string name = 1;
194+
string value = 2;
195+
string task = 3;
196+
string type = 4;
197+
}
198+
199+
message JobStatus {
200+
string state = 1;
201+
google.protobuf.Timestamp scheduled_at = 2;
202+
}
203+
204+
// ResourceSpecification are datastore specification representation of a resource
205+
message ResourceSpecification {
206+
int32 version = 1;
207+
string name = 2;
208+
209+
string datastore = 3;
210+
string type = 4;
211+
212+
google.protobuf.Struct spec = 5;
213+
map<string, string> assets = 6;
214+
map<string, string> labels = 7;
215+
}
216+
217+
// rpc req/res
218+
219+
message VersionRequest {
220+
string client = 1;
221+
}
222+
223+
message VersionResponse {
224+
string server = 1;
225+
}
226+
227+
message DeployJobSpecificationRequest {
228+
string project_name = 1; // unique project identifier
229+
repeated JobSpecification jobs = 2;
230+
231+
// TODO: not implemented yet, default behaviour is to treat this true always till then
232+
// bool synchronize = 3; // deletes job that are not sent as part of this deployment
233+
}
234+
235+
message DeployJobSpecificationResponse {
236+
bool success = 1;
237+
238+
// non ack responses are more of a progress/info response
239+
// and not really success or failure statuses
240+
bool ack = 2;
241+
242+
string message = 3;
243+
string job_name = 4;
244+
}
245+
246+
message ListJobSpecificationRequest {
247+
string project_name = 1;
248+
}
249+
250+
message ListJobSpecificationResponse{
251+
repeated JobSpecification jobs = 1;
252+
}
253+
254+
message DumpJobSpecificationRequest {
255+
string project_name = 1;
256+
string job_name = 2;
257+
}
258+
259+
message DumpJobSpecificationResponse {
260+
bool success = 1;
261+
string content = 2;
262+
}
263+
264+
message RegisterProjectRequest {
265+
ProjectSpecification project = 1;
266+
}
267+
268+
message RegisterProjectResponse {
269+
bool success = 1;
270+
string message = 2;
271+
}
272+
273+
message RegisterSecretRequest {
274+
string project_name = 1;
275+
string secret_name = 2;
276+
string value = 3; // base64 encoded secret value
277+
}
278+
279+
message RegisterSecretResponse {
280+
bool success = 1;
281+
string message = 2;
282+
}
283+
284+
message ListProjectsRequest {}
285+
286+
message ListProjectsResponse {
287+
repeated ProjectSpecification projects = 1;
288+
}
289+
290+
message RegisterInstanceRequest {
291+
string project_name = 1;
292+
string job_name = 2;
293+
string type = 3;
294+
google.protobuf.Timestamp scheduled_at = 4;
295+
}
296+
297+
message RegisterInstanceResponse {
298+
ProjectSpecification project = 1;
299+
JobSpecification job = 2;
300+
InstanceSpec instance = 3;
301+
}
302+
303+
message JobStatusRequest {
304+
string project_name = 1;
305+
string job_name = 2;
306+
}
307+
308+
message JobStatusResponse {
309+
repeated JobStatus statuses = 1;
310+
}
311+
312+
message GetWindowRequest {
313+
google.protobuf.Timestamp scheduled_at = 1;
314+
string size = 2;
315+
string offset = 3;
316+
string truncate_to = 4;
317+
}
318+
319+
message GetWindowResponse {
320+
google.protobuf.Timestamp start = 1;
321+
google.protobuf.Timestamp end = 2;
322+
}
323+
324+
message DeployResourceSpecificationRequest {
325+
string project_name = 1;
326+
string datastore_name = 2;
327+
repeated ResourceSpecification resources = 3;
328+
}
329+
330+
message DeployResourceSpecificationResponse {
331+
bool success = 1;
332+
333+
// non ack responses are more of a progress/info response
334+
// and not success or failure statuses
335+
bool ack = 2;
336+
337+
string message = 3;
338+
string resource_name = 4;
339+
}
340+
341+
// ListResourceSpecificationRequest lists all resource specifications of a datastore in project
342+
message ListResourceSpecificationRequest {
343+
string project_name = 1;
344+
string datastore_name = 2;
345+
}
346+
347+
message ListResourceSpecificationResponse {
348+
repeated ResourceSpecification resources = 1;
349+
}
350+
351+
message CreateResourceRequest {
352+
string project_name = 1;
353+
string datastore_name = 2;
354+
ResourceSpecification resource = 3;
355+
}
356+
357+
message CreateResourceResponse {
358+
bool success = 1;
359+
string message = 2;
360+
}
361+
362+
message ReadResourceRequest {
363+
string project_name = 1;
364+
string datastore_name = 2;
365+
string resource_name = 3;
366+
}
367+
368+
message ReadResourceResponse {
369+
bool success = 1;
370+
string message = 2;
371+
ResourceSpecification resource = 3;
372+
}
373+
374+
message UpdateResourceRequest {
375+
string project_name = 1;
376+
string datastore_name = 2;
377+
ResourceSpecification resource = 3;
378+
}
379+
380+
message UpdateResourceResponse {
381+
bool success = 1;
382+
string message = 2;
383+
}

0 commit comments

Comments
 (0)