Skip to content

Commit d5dbb5c

Browse files
kiruxaspbkiruxaspb
authored
Merge pull request #541 from singnet/stub-generating-fix
Stub generating fix
2 parents f07d146 + 854787d commit d5dbb5c

File tree

7 files changed

+259
-92
lines changed

7 files changed

+259
-92
lines changed

‎MANIFEST.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
include snet/cli/resources/*
1+
recursive-include snet/cli/resources *
22

33
recursive-exclude * .git
44
recursive-exclude * node_modules

‎snet/cli/commands/mpe_channel.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717
from snet.cli.utils.token2cogs import cogs2strtoken
1818
from snet.cli.utils.ipfs_utils import get_from_ipfs_and_checkhash
1919
from snet.cli.utils.utils import abi_decode_struct_to_dict, abi_get_element_by_name, \
20-
compile_proto, type_converter, bytesuri_to_hash, get_file_from_filecoin, download_and_safe_extract_proto
20+
compile_proto, type_converter, bytesuri_to_hash, get_file_from_filecoin, download_and_safe_extract_proto, \
21+
check_training_in_proto
2122

2223

2324
# we inherit MPEServiceCommand because we need _get_service_metadata_from_registry
@@ -602,9 +603,10 @@ def _init_or_update_service_if_needed(self, metadata, service_registration):
602603
os.makedirs(spec_dir, mode=0o700)
603604
service_api_source = metadata.get("service_api_source") or metadata.get("model_ipfs_hash")
604605
download_and_safe_extract_proto(service_api_source, spec_dir, self._get_ipfs_client())
606+
training_added = check_training_in_proto(spec_dir)
605607

606608
# compile .proto files
607-
if not compile_proto(Path(spec_dir), service_dir):
609+
if not compile_proto(Path(spec_dir), service_dir, add_training = training_added):
608610
raise Exception("Fail to compile %s/*.proto" % spec_dir)
609611

610612
# save service_metadata.json in channel_dir

‎snet/cli/commands/sdk_command.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import os
22
from pathlib import Path, PurePath
33

4-
from snet.cli.utils.utils import compile_proto, download_and_safe_extract_proto
4+
from snet.cli.utils.utils import compile_proto, download_and_safe_extract_proto, check_training_in_proto
55
from snet.cli.commands.mpe_service import MPEServiceCommand
66

77

@@ -28,8 +28,10 @@ def generate_client_library(self):
2828
# Receive proto files
2929
download_and_safe_extract_proto(service_api_source, library_dir_path, self._get_ipfs_client())
3030

31+
training_added = check_training_in_proto(library_dir_path)
32+
3133
# Compile proto files
32-
compile_proto(Path(library_dir_path), library_dir_path)
34+
compile_proto(Path(library_dir_path), library_dir_path, add_training = training_added)
3335

3436
self._printout(
3537
'client libraries for service with id "{}" in org with id "{}" generated at {}'.format(library_service_id,

‎snet/cli/resources/proto/training.proto‎

Lines changed: 99 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,112 +1,128 @@
11
syntax = "proto3";
2-
import "google/protobuf/descriptor.proto";
2+
import "google/protobuf/descriptor.proto"; // Required for indicators to work
33
package training;
4-
option go_package = "../training";
5-
//Please note that the AI developers need to provide a server implementation of the gprc server of this proto.
6-
message ModelDetails {
7-
//This Id will be generated when you invoke the create_model method and hence doesnt need to be filled when you
8-
//invoke the create model
9-
string model_id = 1;
10-
//define the training method name
11-
string grpc_method_name = 2;
12-
//define the grpc service name , under which the method is defined
13-
string grpc_service_name = 3;
14-
string description = 4;
4+
option go_package = "github.com/singnet/snet-daemon/v5/training";
155

16-
string status = 6;
17-
string updated_date = 7;
18-
//List of all the addresses that will have access to this model
19-
repeated string address_list = 8;
20-
// this is optional
21-
string training_data_link = 9;
22-
string model_name = 10;
6+
// Methods that the service provider must implement
7+
service Model {
238

9+
// Free
10+
// Can pass the address of the model creator
11+
rpc create_model(NewModel) returns (ModelID) {}
2412

25-
string organization_id = 11;
26-
string service_id = 12 ;
27-
string group_id = 13;
13+
// Free
14+
rpc validate_model_price(ValidateRequest) returns (PriceInBaseUnit) {}
2815

29-
//set this to true if you want your model to be used by other AI consumers
30-
bool is_publicly_accessible = 14;
16+
// Paid
17+
rpc upload_and_validate(stream UploadInput) returns (StatusResponse) {}
3118

32-
}
19+
// Paid
20+
rpc validate_model(ValidateRequest) returns (StatusResponse) {}
3321

34-
message AuthorizationDetails {
35-
uint64 current_block = 1;
36-
//Signer can fill in any message here
37-
string message = 2;
38-
//signature of the following message:
39-
//("user specified message", user_address, current_block_number)
40-
bytes signature = 3;
41-
string signer_address = 4;
22+
// Free, one signature for both train_model_price & train_model methods
23+
rpc train_model_price(ModelID) returns (PriceInBaseUnit) {}
4224

43-
}
25+
// Paid
26+
rpc train_model(ModelID) returns (StatusResponse) {}
4427

45-
enum Status {
46-
CREATED = 0;
47-
IN_PROGRESS = 1;
48-
ERRORED = 2;
49-
COMPLETED = 3;
50-
DELETED = 4;
51-
}
28+
// Free
29+
rpc delete_model(ModelID) returns (StatusResponse) {
30+
// After model deletion, the status becomes DELETED in etcd
31+
}
5232

53-
message CreateModelRequest {
54-
AuthorizationDetails authorization = 1;
55-
ModelDetails model_details = 2;
33+
// Free
34+
rpc get_model_status(ModelID) returns (StatusResponse) {}
5635
}
5736

58-
//the signer address will get to know all the models associated with this address.
59-
message AccessibleModelsRequest {
60-
string grpc_method_name = 1;
61-
string grpc_service_name = 2;
62-
AuthorizationDetails authorization = 3;
63-
}
37+
message ModelResponse {
38+
string model_id = 1;
39+
Status status = 2;
40+
string created_date = 3;
41+
string updated_date = 4;
42+
string name = 5;
43+
string description = 6;
44+
string grpc_method_name = 7;
45+
string grpc_service_name = 8;
6446

65-
message AccessibleModelsResponse {
66-
repeated ModelDetails list_of_models = 1;
67-
}
47+
// List of all addresses that will have access to this model
48+
repeated string address_list = 9;
49+
50+
// Access to the model is granted only for use and viewing
51+
bool is_public = 10;
52+
53+
string training_data_link = 11;
6854

69-
message ModelDetailsRequest {
70-
ModelDetails model_details = 1 ;
71-
AuthorizationDetails authorization = 2;
55+
string created_by_address = 12;
56+
string updated_by_address = 13;
7257
}
7358

74-
//helps determine which service end point to call for model training
75-
//format is of type "packageName/serviceName/MethodName", Example :"/example_service.Calculator/estimate_add"
76-
//Daemon will invoke the model training end point , when the below method option is specified
77-
message TrainingMethodOption {
78-
string trainingMethodIndicator = 1;
59+
// Used as input for new_model requests
60+
// The service provider decides whether to use these fields; returning model_id is mandatory
61+
message NewModel {
62+
string name = 1;
63+
string description = 2;
64+
string grpc_method_name = 3;
65+
string grpc_service_name = 4;
66+
67+
// List of all addresses that will have access to this model
68+
repeated string address_list = 5;
69+
70+
// Set this to true if you want your model to be accessible by other AI consumers
71+
bool is_public = 6;
72+
73+
// These parameters will be passed by the daemon
74+
string organization_id = 7;
75+
string service_id = 8;
76+
string group_id = 9;
7977
}
8078

81-
extend google.protobuf.MethodOptions {
82-
TrainingMethodOption my_method_option = 9999197;
79+
// This structure must be used by the service provider
80+
message ModelID {
81+
string model_id = 1;
8382
}
8483

85-
message UpdateModelRequest {
86-
ModelDetails update_model_details = 1 ;
87-
AuthorizationDetails authorization = 2;
84+
// This structure must be used by the service provider
85+
// Used in the train_model_price method to get the training/validation price
86+
message PriceInBaseUnit {
87+
uint64 price = 1; // cogs, weis, afet, aasi, etc.
8888
}
8989

90+
enum Status {
91+
CREATED = 0;
92+
VALIDATING = 1;
93+
VALIDATED = 2;
94+
TRAINING = 3;
95+
READY_TO_USE = 4; // After training is completed
96+
ERRORED = 5;
97+
DELETED = 6;
98+
}
9099

91-
message ModelDetailsResponse {
100+
message StatusResponse {
92101
Status status = 1;
93-
ModelDetails model_details = 2;
94-
95102
}
96103

97-
service Model {
98-
99-
// The AI developer needs to Implement this service and Daemon will call these
100-
// There will be no cost borne by the consumer in calling these methods,
101-
// Pricing will apply when you actually call the training methods defined.
102-
// AI consumer will call all these methods
103-
rpc create_model(CreateModelRequest) returns (ModelDetailsResponse) {}
104-
rpc delete_model(UpdateModelRequest) returns (ModelDetailsResponse) {}
105-
rpc get_model_status(ModelDetailsRequest) returns (ModelDetailsResponse) {}
106-
107-
// Daemon will implement , however the AI developer should skip implementing these and just provide dummy code.
108-
rpc update_model_access(UpdateModelRequest) returns (ModelDetailsResponse) {}
109-
rpc get_all_models(AccessibleModelsRequest) returns (AccessibleModelsResponse) {}
104+
message UploadInput {
105+
string model_id = 1;
106+
bytes data = 2;
107+
string file_name = 3;
108+
uint64 file_size = 4; // in bytes
109+
uint64 batch_size = 5;
110+
uint64 batch_number = 6;
111+
uint64 batch_count = 7;
112+
}
110113

114+
message ValidateRequest {
115+
string model_id = 2;
116+
string training_data_link = 3;
117+
}
111118

112-
}
119+
extend google.protobuf.MethodOptions {
120+
string default_model_id = 50001;
121+
uint64 max_models_per_user = 50002; // max models per method & user
122+
uint64 dataset_max_size_mb = 50003; // max size of dataset
123+
uint64 dataset_max_count_files = 50004; // maximum number of files in the dataset
124+
uint64 dataset_max_size_single_file_mb = 50005; // maximum size of a single file in the dataset
125+
string dataset_files_type = 50006; // allowed files types in dataset. string with array or single value, example: jpg, png, mp3
126+
string dataset_type = 50007; // string with array or single value, example: zip, tar.gz, tar
127+
string dataset_description = 50008; // additional free-form requirements
128+
}

‎snet/cli/resources/proto/training/training.proto‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
syntax = "proto3";
2+
import "google/protobuf/descriptor.proto"; // Required for indicators to work
3+
package training;
4+
option go_package = "github.com/singnet/snet-daemon/v5/training";
5+
6+
// Methods that the service provider must implement
7+
service Model {
8+
9+
// Free
10+
// Can pass the address of the model creator
11+
rpc create_model(NewModel) returns (ModelID) {}
12+
13+
// Free
14+
rpc validate_model_price(ValidateRequest) returns (PriceInBaseUnit) {}
15+
16+
// Paid
17+
rpc upload_and_validate(stream UploadInput) returns (StatusResponse) {}
18+
19+
// Paid
20+
rpc validate_model(ValidateRequest) returns (StatusResponse) {}
21+
22+
// Free, one signature for both train_model_price & train_model methods
23+
rpc train_model_price(ModelID) returns (PriceInBaseUnit) {}
24+
25+
// Paid
26+
rpc train_model(ModelID) returns (StatusResponse) {}
27+
28+
// Free
29+
rpc delete_model(ModelID) returns (StatusResponse) {
30+
// After model deletion, the status becomes DELETED in etcd
31+
}
32+
33+
// Free
34+
rpc get_model_status(ModelID) returns (StatusResponse) {}
35+
}
36+
37+
message ModelResponse {
38+
string model_id = 1;
39+
Status status = 2;
40+
string created_date = 3;
41+
string updated_date = 4;
42+
string name = 5;
43+
string description = 6;
44+
string grpc_method_name = 7;
45+
string grpc_service_name = 8;
46+
47+
// List of all addresses that will have access to this model
48+
repeated string address_list = 9;
49+
50+
// Access to the model is granted only for use and viewing
51+
bool is_public = 10;
52+
53+
string training_data_link = 11;
54+
55+
string created_by_address = 12;
56+
string updated_by_address = 13;
57+
}
58+
59+
// Used as input for new_model requests
60+
// The service provider decides whether to use these fields; returning model_id is mandatory
61+
message NewModel {
62+
string name = 1;
63+
string description = 2;
64+
string grpc_method_name = 3;
65+
string grpc_service_name = 4;
66+
67+
// List of all addresses that will have access to this model
68+
repeated string address_list = 5;
69+
70+
// Set this to true if you want your model to be accessible by other AI consumers
71+
bool is_public = 6;
72+
73+
// These parameters will be passed by the daemon
74+
string organization_id = 7;
75+
string service_id = 8;
76+
string group_id = 9;
77+
}
78+
79+
// This structure must be used by the service provider
80+
message ModelID {
81+
string model_id = 1;
82+
}
83+
84+
// This structure must be used by the service provider
85+
// Used in the train_model_price method to get the training/validation price
86+
message PriceInBaseUnit {
87+
uint64 price = 1; // cogs, weis, afet, aasi, etc.
88+
}
89+
90+
enum Status {
91+
CREATED = 0;
92+
VALIDATING = 1;
93+
VALIDATED = 2;
94+
TRAINING = 3;
95+
READY_TO_USE = 4; // After training is completed
96+
ERRORED = 5;
97+
DELETED = 6;
98+
}
99+
100+
message StatusResponse {
101+
Status status = 1;
102+
}
103+
104+
message UploadInput {
105+
string model_id = 1;
106+
bytes data = 2;
107+
string file_name = 3;
108+
uint64 file_size = 4; // in bytes
109+
uint64 batch_size = 5;
110+
uint64 batch_number = 6;
111+
uint64 batch_count = 7;
112+
}
113+
114+
message ValidateRequest {
115+
string model_id = 2;
116+
string training_data_link = 3;
117+
}
118+
119+
extend google.protobuf.MethodOptions {
120+
string default_model_id = 50001;
121+
uint64 max_models_per_user = 50002; // max models per method & user
122+
uint64 dataset_max_size_mb = 50003; // max size of dataset
123+
uint64 dataset_max_count_files = 50004; // maximum number of files in the dataset
124+
uint64 dataset_max_size_single_file_mb = 50005; // maximum size of a single file in the dataset
125+
string dataset_files_type = 50006; // allowed files types in dataset. string with array or single value, example: jpg, png, mp3
126+
string dataset_type = 50007; // string with array or single value, example: zip, tar.gz, tar
127+
string dataset_description = 50008; // additional free-form requirements
128+
}

0 commit comments

Comments
 (0)