Skip to content

Commit 89d6dbb

Browse files
authored
Async predictions++ (#115)
* Add possibility to add output and error to patch transition execution call * Update changelog and versions * Add max-document limit to documents get, statistics, predictions get, and run-async support * Add changelog * update version and dependencies * fixes after review * fix changelog * remove input and output schema from transition
1 parent affc416 commit 89d6dbb

File tree

8 files changed

+32
-23
lines changed

8 files changed

+32
-23
lines changed

CHANGELOG.md

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
# Changelog
22

3-
## Version 13.2.3 - 2024-10-09
3+
## Version 13.3.0 - 2024-12-03
44

55
- Bugfix `transitions update-execution` now supports additional keyword arguments
6+
- Added optional parameter `--async` to `predictions create`
7+
- Added `predictions get`
8+
- Added optional parameter `--statistics-last-n-days` to `models get-training`
9+
- Added optional parameter `--max-elements` to `models get-training`
10+
- Remove optional parameter `--in-schema` and `--out-schema` from `transitions create` and `transitions update`
611

712
## Version 13.2.2 - 2024-06-13
813

lascli/__version__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@
77
__maintainer_email__ = '[email protected]'
88
__title__ = 'lucidtech-las-cli'
99
__url__ = 'https://github.com/LucidtechAI/las-cli'
10-
__version__ = '13.2.3'
10+
__version__ = '13.3.0'

lascli/parser/datasets.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ def create_documents(
321321
return dict(counter)
322322

323323

324-
def get_documents(las_client: Client, dataset_id, output_dir, num_threads, chunk_size):
324+
def get_documents(las_client: Client, dataset_id, output_dir, num_threads, chunk_size, max_results):
325325
already_downloaded = set()
326326
if output_dir.exists():
327327
for path in output_dir.iterdir():
@@ -337,6 +337,8 @@ def get_documents(las_client: Client, dataset_id, output_dir, num_threads, chunk
337337
already_downloaded_from_dataset.add(document['documentId'])
338338
else:
339339
documents.append(document)
340+
if max_results and max_results <= len(documents):
341+
break
340342
print(f'Found {len(already_downloaded_from_dataset)} documents already downloaded')
341343

342344
start_time = time()
@@ -468,6 +470,7 @@ def create_datasets_parser(subparsers):
468470
get_documents_parser.add_argument('output_dir', type=Path, help='Path to download directory')
469471
get_documents_parser.add_argument('--num-threads', default=32, type=int, help='Number of threads to use')
470472
get_documents_parser.add_argument('--chunk-size', default=100, type=int)
473+
get_documents_parser.add_argument('--max-results', default=0, type=int)
471474
get_documents_parser.set_defaults(cmd=get_documents)
472475

473476
create_transformation_parser = subparsers.add_parser('create-transformation')
@@ -480,7 +483,7 @@ def create_datasets_parser(subparsers):
480483
"options": {} (optional)
481484
},
482485
...
483-
]
486+
]
484487
Examples:
485488
[{"type": "remove-duplicates", "options": {}}]
486489
'''))

lascli/parser/models.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,12 @@ def create_training(las_client: Client, model_id, data_bundle_ids, data_scientis
7171
)
7272

7373

74-
def get_training(las_client: Client, model_id, training_id):
75-
return las_client.get_training(model_id=model_id, training_id=training_id)
74+
def get_training(las_client: Client, model_id, training_id, statistics_last_n_days):
75+
return las_client.get_training(
76+
model_id=model_id,
77+
training_id=training_id,
78+
statistics_last_n_days=statistics_last_n_days,
79+
)
7680

7781

7882
def list_trainings(las_client: Client, model_id, max_results, next_token):
@@ -254,6 +258,7 @@ def create_models_parser(subparsers):
254258
get_training_parser = subparsers.add_parser('get-training')
255259
get_training_parser.add_argument('model_id')
256260
get_training_parser.add_argument('training_id')
261+
get_training_parser.add_argument('--statistics-last-n-days', type=int_range(1, 30))
257262
get_training_parser.set_defaults(cmd=get_training)
258263

259264
list_trainings_parser = subparsers.add_parser('list-trainings')

lascli/parser/predictions.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,14 @@
88

99

1010
def create_prediction(las_client: Client, document_id, model_id, **optional_args):
11+
optional_args['run_async'] = optional_args.pop('async', None)
1112
return las_client.create_prediction(document_id, model_id, **optional_args)
1213

1314

15+
def get_prediction(las_client: Client, prediction_id, **optional_args):
16+
return las_client.get_prediction(prediction_id, **optional_args)
17+
18+
1419
def list_predictions(las_client: Client, **optional_args):
1520
return las_client.list_predictions(**optional_args)
1621

@@ -51,6 +56,7 @@ def create_predictions_parser(subparsers):
5156
{"strategy": "BEST_N_PAGES", "parameters": {"n": 3}}
5257
{"strategy": "BEST_N_PAGES", "parameters": {"n": 3, "collapse": true}}
5358
'''))
59+
create_predicton_parser.add_argument('--async', action='store_true', help='Create prediction async')
5460
create_predicton_parser.set_defaults(cmd=create_prediction)
5561

5662
list_predictions_parser = subparsers.add_parser('list')
@@ -61,4 +67,8 @@ def create_predictions_parser(subparsers):
6167
list_predictions_parser.add_argument('--model-id')
6268
list_predictions_parser.set_defaults(cmd=list_predictions)
6369

70+
get_prediction_parser = subparsers.add_parser('get')
71+
get_prediction_parser.add_argument('prediction_id')
72+
get_prediction_parser.set_defaults(cmd=get_prediction)
73+
6474
return parser

lascli/parser/transitions.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,6 @@ def create_transitions_parser(subparsers):
6767
create_parser = subparsers.add_parser('create')
6868
create_parser.add_argument('transition_type', choices=["docker", "manual"])
6969
create_parser.add_argument('--parameters', '-p', type=json_path, help='path to parameters to the docker image')
70-
create_parser.add_argument('--in-schema', type=json_path, help='path to input jsonschema')
71-
create_parser.add_argument('--out-schema', type=json_path, help='path to output jsonschema')
7270
create_parser.add_argument('--name')
7371
create_parser.add_argument('--description')
7472
create_parser.set_defaults(cmd=create_transition)
@@ -87,8 +85,6 @@ def create_transitions_parser(subparsers):
8785
update_parser.add_argument('transition_id')
8886
update_parser.add_argument('--name', type=nullable(str), default=NotProvided)
8987
update_parser.add_argument('--description', type=nullable(str), default=NotProvided)
90-
update_parser.add_argument('--in-schema', type=json_path, help='Path to input jsonschema')
91-
update_parser.add_argument('--out-schema', type=json_path, help='Path to output jsonschema')
9288
update_parser.add_argument(
9389
'--assets',
9490
type=json_path,

requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ PyYAML>=6.0.0
22
argcomplete>=2.0.0
33
dateparser>=1.1.1
44
filetype>=1.0.13
5-
lucidtech-las~=11.2
5+
lucidtech-las~=11.4

tests/test_transitions.py

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,36 +6,26 @@
66
('docker', ('--parameters', str(util.transition_parameters_path()))),
77
('manual', ()),
88
])
9-
@pytest.mark.parametrize('in_schema', [('--in-schema', str(util.schema_path())), ()])
10-
@pytest.mark.parametrize('out_schema', [('--out-schema', str(util.schema_path())), ()])
11-
def test_transitions_create(parser, client, transition_type, in_schema, out_schema, name_and_description, parameters):
9+
def test_transitions_create(parser, client, transition_type, name_and_description, parameters):
1210
args = [
1311
'transitions',
1412
'create',
1513
transition_type,
16-
*in_schema,
17-
*out_schema,
1814
*name_and_description,
1915
]
2016
util.main_parser(parser, client, args)
2117

2218

2319

24-
@pytest.mark.parametrize('in_schema', [('--in-schema', str(util.schema_path())), ()])
25-
@pytest.mark.parametrize('out_schema', [('--out-schema', str(util.schema_path())), ()])
2620
def test_transitions_update(
2721
parser,
2822
client,
29-
in_schema,
30-
out_schema,
3123
name_and_description,
3224
):
3325
args = [
3426
'transitions',
3527
'update',
3628
service.create_transition_id(),
37-
*in_schema,
38-
*out_schema,
3929
*name_and_description,
4030
]
4131

@@ -64,7 +54,7 @@ def test_transitions_update_manual(
6454
util.main_parser(parser, client, args)
6555
else:
6656
util.main_parser(parser, client, args)
67-
57+
6858

6959
@pytest.mark.parametrize('image_url', [('--image-url', 'image:url'), ()])
7060
@pytest.mark.parametrize('secret_id', [

0 commit comments

Comments
 (0)