Version 6.4.2 bump

Author: minknowbot

README.md

@@ -86,10 +86,15 @@ See [AUTH.md](AUTH.md) for information on authenticating with the MinKNOW API.
 The files in [`proto/minknow_api`](proto/minknow_api/) describe MinKNOW's APIs. Each file
 describes a single *service*, which covers a specific area of MinKNOW's functionality.
 
-There are two global services, the [manager service](proto/minknow_api/manager.proto) and
+**Prior to 6.3**, there are two global services, the [manager service](proto/minknow_api/manager.proto) and
 [basecaller service](proto/minknow_api/basecaller.proto). There is only one instance of each of
-these services: see below for how to connect to them. All other services are provided by each flow
-cell position independently. For example, if you are using a GridION X5, which has 5 flow cell
+these services: see below for how to connect to them.
+
+**For 6.3 and above** we have begun to construct a 'consolidated' API which is intended to simplify control and interrogation of protocol/acquisition runs.
+This new API is also present on the 'manager' executable, and proxies requests to individual `control_server` instances.
+As such, it only requires a single gRPC client for _any number_ of flow cell positions.
+
+The original per-flow-cell-position APIs are still maintained on each flow cell position independently. For example, if you are using a GridION X5, which has 5 flow cell
 positions, there will be 5 ports (or sets of ports - secure, gRPC Web, etc), each of which
 will provide *all* the other services.
 
@@ -127,6 +132,17 @@ information about protocol runs from before the last restart is not available vi
 See the [`start_protocol` example](python/minknow_api/examples/start_protocol.py) for an example of how to use
 this service to start a protocol.
 
+### v2/protocol.proto
+
+**6.3+ only!**
+
+[This is the first 'consolidated' API.](proto/minknow_api/v2/protocol.proto) It is (like the endpoints on `manager.proto`) always available on port `9501` and `9502`.
+
+It allows for protocols to be started on multiple positions simultaneously.
+Once started, multiple protocols may also be stopped using only their `protocol_run_ids`, without reference to the flow cell positions on which they are running.
+
+This pattern is intended to be extended to all methods of interacting with protocols - 'start' should be the only call that requires the user to provide information about a specific position.
+
 ### acquisition.proto
 
 The main work of a protocol is acquiring data, and this is managed in
@@ -209,8 +225,9 @@ If you are using the `minknow_api` Python package, this is all handled for you.
 gRPC client libraries directly (for example, if you are connecting from a language other than
 Python), you will need to tell the library about MinKNOW's TLS certificates.
 
-Within the MinKNOW installation, you can find the CA certificate at `conf/rpc-certs/ca.crt`. This
-can be passed to most gRPC client libraries as part of creating a secure/SSL channel.
+You can find the CA certificate by default at the location `<data_dir>/rpc-certs/minknow/ca.crt`,
+where `<data_dir>` maps to `C:\\data` on Windows and `/data` on Ubuntu and MacOS platforms.
+This can be passed to most gRPC client libraries as part of creating a secure/SSL channel.
 
 Note that this certificate is only valid for the "localhost" name - connecting to `127.0.0.1`
 directly will not work, nor will connecting across the network. You can work around this by

proto/minknow_api/acquisition.proto

@@ -3,8 +3,10 @@ syntax="proto3";
 package minknow_api.acquisition;
 
 option java_package = "com.nanoporetech.minknow_api";
+option go_package ="github.com/nanoporetech/minknow_api/go/gen/acquisition";
 option objc_class_prefix = "MKAPI";
 
+import "minknow_api/analysis_configuration.proto";
 import "minknow_api/run_until.proto";
 import "minknow_api/rpc_options.proto";
 import "google/protobuf/timestamp.proto";
@@ -674,8 +676,18 @@ message AcquisitionConfigSummary {
 
     // Was basecalling enabled for the run.
     bool basecalling_enabled = 1;
-    // Basecalling configuration filename (if basecalling enabled)
-    string basecalling_config_filename = 16;
+    oneof basecall_model {
+        // Basecalling configuration filename (if basecalling enabled)
+        string basecalling_config_filename = 16;
+
+        // Specify the models to run by name (see find_basecall_configurations in manager.proto)
+        //
+        // Model names should be taken from the `name` field of the above RPC directly.
+        //
+        // Since 6.3
+        analysis_configuration.BasecallerConfiguration.ModelNames basecalling_model_names = 25;
+    }
+
     // Basecalling model version (empty if basecalling not enabled)
     // Since 6.0
     string basecalling_model_version = 24;

proto/minknow_api/analysis_configuration.proto

@@ -2,12 +2,14 @@ syntax="proto3";
 
 package minknow_api.analysis_configuration;
 
+option go_package ="github.com/nanoporetech/minknow_api/go/gen/analysis_configuration";
 option java_package = "com.nanoporetech.minknow_api";
 option objc_class_prefix = "MKAPI";
 
 import "google/protobuf/duration.proto";
 import "google/protobuf/empty.proto";
 import "google/protobuf/wrappers.proto";
+import "minknow_api/read_end_reason.proto";
 import "minknow_api/rpc_options.proto";
 
 service AnalysisConfigurationService {
@@ -184,6 +186,14 @@ service AnalysisConfigurationService {
     // The new analysis parameters will be used after any data already received has been processe.
     rpc set_dynamic_analysis_configuration(DynamicAnalysisConfiguration) returns (SetDynamicAnalysisConfigurationResponse) { }
 
+    // Basecall configuration defaults
+    //
+    // These defaults will be based on the connected flow cell type and the selected base calling substrate.
+    //
+    // Since 6.3
+    rpc find_basecall_configuration_defaults(FindBasecallConfigurationDefaultsRequest) returns (FindBasecallConfigurationDefaultsResponse) {
+        option (experimental) = true;
+    }
 }
 
 message EventDetection {
@@ -238,6 +248,13 @@ message EventDetection {
     //
     // Note: no longer used.
     uint32 max_mux_change_back_shift = 18;
+
+    // Force minknow to detect events even when channels are disconnected
+    //
+    // If set to false (the default) events are not detected when channels are set outside well 1-4.
+    //
+    // If set to true events are always detected.
+    bool force_detect_events_when_disconnected = 20;
 }
 
 message ReadDetectionParams {
@@ -834,16 +851,44 @@ message BasecallerConfiguration
         google.protobuf.UInt64Value max_bases = 3;
     }
 
+    // A set of model names to run in the basecaller.
+    message ModelNames {
+        // Simplex model to use when basecalling.
+        //
+        // This field is not optional - empty values are invalid.
+        string simplex_model = 1;
+
+        // An optional list of modified bases to use, the caller is responsible
+        // for ensuring these are compatible modifications.
+        //
+        // An empty list will lead to no modified base calling.
+        repeated string modified_models = 2;
+
+        // An optional stereo model to use for calling.
+        //
+        // And empty string will perform no stereo step.
+        string stereo_model = 3;
+    }
+
     // Choose if the basecaller is enabled or disabled.
     //
     // If set to false then no basecalling will take place, and the rest of the config is ignored.
     bool enable = 2;
 
-    // The basecaller cfg file with all the settings.
-    //
-    // Filename can be absolute, or a basename (eg dna_r9.4_450bps.cfg)
-    // which the basecaller should locate (see basecaller application config entry: "data_path")
-    string config_filename = 1;
+    oneof basecall_model {
+        // The basecaller cfg file with all the settings.
+        //
+        // Filename can be absolute, or a basename (eg dna_r9.4_450bps.cfg)
+        // which the basecaller should locate (see basecaller application config entry: "data_path")
+        string config_filename = 1;
+
+        // Specify the models to run by name (see find_basecall_configurations in manager.proto)
+        //
+        // Model names should be taken from the `name` field of the above RPC directly.
+        //
+        // Since 6.3
+        ModelNames model_names = 12;
+    }
 
     // Enable or disable pass/fail filtering based on alignment.  When enabled, reads which
     // do not align to any references will be marked as "failed", and written to the folder
@@ -915,6 +960,11 @@ message BasecallerConfiguration
     //
     // Note: Since 5.9 this option has no effect, the basecaller is responsible for deciding read splitting score.
     google.protobuf.FloatValue min_score_read_splitting = 9;
+
+    // Enable poly a tail estimation.
+    //
+    // Since 6.3
+    bool estimate_poly_a = 11;
 }
 
 message SetBasecallerConfigurationRequest
@@ -1575,7 +1625,31 @@ message DynamicAnalysisConfiguration {
 
     // Parameters for read scale tracking:
     ReadScaleTracking read_scale_tracking = 1;
+
+    // Force MinKNOW to not write or basecall all data from the live
+    // sequencing stream.
+    //
+    // This may be useful if there is a section of data that is not
+    // wanted in the final output files and reports.
+    bool force_veto_all_reads = 2;
+
+    // Force MinKNOW to not write or basecall data with specific end reasons.
+    //
+    // This may be useful if there is a section of data that is not
+    // wanted in the final output files and reports.
+    repeated read_end_reason.ReadEndReason force_veto_end_reasons = 3;
 }
 
 message GetDynamicAnalysisConfigurationRequest {}
 message SetDynamicAnalysisConfigurationResponse {}
+
+message FindBasecallConfigurationDefaultsRequest {
+}
+
+message FindBasecallConfigurationDefaultsResponse {
+    // The default variant to be used.
+    //
+    // This can be used to search the results of manager.find_basecall_configurations
+    // for a simplex model with a matching variant.
+    string default_variant = 1;
+}

proto/minknow_api/analysis_workflows.proto

@@ -2,6 +2,7 @@ syntax="proto3";
 
 package minknow_api.analysis_workflows;
 
+option go_package ="github.com/nanoporetech/minknow_api/go/gen/analysis_workflow";
 option java_package = "com.nanoporetech.minknow_api";
 option objc_class_prefix = "MKAPI";
 
@@ -36,3 +37,11 @@ message ProxyResponse {
     uint32 status_code = 1;
     string response_body = 2;
 }
+
+message AnalysisWorkflowRequest {
+    // The id of the workflow to start
+    string workflow_id = 1;
+
+    // The parameters to send to the workflow, represented as a stringified JSON object
+    string parameters = 2;
+}

proto/minknow_api/basecaller.proto

@@ -2,6 +2,7 @@ syntax="proto3";
 
 package minknow_api.basecaller;
 
+option go_package ="github.com/nanoporetech/minknow_api/go/gen/basecaller";
 option java_package = "com.nanoporetech.minknow_api";
 option objc_class_prefix = "MKAPI";
 
@@ -171,8 +172,17 @@ message StartBasecallingRequest {
     // Reads will be sorted into subdirectories based on the sequencing run they came from.
     string output_reads_directory = 2;
 
-    // The name of the basecalling configuration to use.
-    string configuration = 3;
+    oneof basecall_model {
+        // The name of the basecalling configuration to use.
+        string configuration = 3;
+
+        // Specify the models to run by name (see find_basecall_configurations in manager.proto)
+        //
+        // Model names should be taken from the `name` field of the above RPC directly.
+        //
+        // Since 6.3
+        analysis_configuration.BasecallerConfiguration.ModelNames model_names = 16;
+    }
 
     // Enable output of .fast5 files containing original raw reads, event data/trace table from
     // basecall and basecall result sequence.
@@ -208,7 +218,7 @@ message StartBasecallingRequest {
     // Minimum Q-Score for a read to be passed by the basecaller.
     //
     // Since 6.1.
-    optional double min_qscore = 15;
+    double min_qscore = 15;
 
     // Enable read splitting in the basecaller
     //
@@ -220,6 +230,11 @@ message StartBasecallingRequest {
     //
     // Note: Since 5.9 this option has no effect, the basecaller is responsible for deciding when read splitting should be enabled.
     google.protobuf.FloatValue min_score_read_splitting = 14;
+
+    // Enable poly a tail estimation.
+    //
+    // Since 6.3
+    bool estimate_poly_a = 156;
 }
 
 message StartBasecallingResponse {
@@ -333,7 +348,12 @@ message StartRequest {
         basecaller.StartPostProcessingProtocolRequest start_post_processing_protocol_request = 5;
 
         // Since 5.8
+        //
+        // DEPRECATED: use 'analysis_workflow_request'
         analysis_workflows.ProxyRequest proxy_request = 6;
+
+        // Since 6.4
+        analysis_workflows.AnalysisWorkflowRequest analysis_workflow_request = 7;
     }
 }
 

proto/minknow_api/data.proto

@@ -3,11 +3,12 @@ syntax="proto3";
 package minknow_api.data;
 
 option java_package = "com.nanoporetech.minknow_api";
+option go_package ="github.com/nanoporetech/minknow_api/go/gen/data";
 option objc_class_prefix = "MKAPI";
 
 import "minknow_api/rpc_options.proto";
 import "minknow_api/device.proto";
-import "minknow_api/statistics.proto";
+import "minknow_api/read_end_reason.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/wrappers.proto";
 
@@ -206,8 +207,12 @@ service DataService {
 
     // Returns various points of yield information for the ongoing experiment, such as complete
     // read information and basecaller progress.
+    //
+    // Note: This API does not guarantee between versions and may change as internal MinKNOW changes
+    // require different data is monitored.
     rpc get_experiment_yield_info (GetExperimentYieldInfoRequest) returns (GetExperimentYieldInfoResponse) {
         option idempotency_level = NO_SIDE_EFFECTS;
+        option (experimental) = true;
     }
 }
 
@@ -662,7 +667,7 @@ message GetLiveReadsResponse {
         int32 previous_read_classification = 10;
 
         // The classification of the chunk prior to this read starting.
-        statistics.ReadEndReason previous_read_end_reason = 11;
+        read_end_reason.ReadEndReason previous_read_end_reason = 11;
     };
 
     message ActionResponse {
@@ -913,14 +918,16 @@ message GetExperimentYieldInfoResponse{
         uint64 pending_memory_reads = 2;
         uint64 pending_disk_reads = 15;
         uint64 pending_multi_disk_reads = 3;
-        uint64 pending_skipped_reads = 4;
+        uint64 pending_skipped_reads = 4; // No longer used
         uint64 pending_force_skipped_reads = 5;
+        uint64 pending_retry_reads = 18;
         uint64 processed_memory_reads = 6;
         uint64 processed_disk_reads = 7;
-        uint64 processed_skipped_reads = 8;
+        uint64 processed_skipped_reads = 8; // No longer used
         uint64 processed_force_skipped_reads = 9;
         uint64 stored_reads_bytes_memory = 10;
-        uint64 stored_read_supporting_bytes_memory = 16;
+        uint64 stored_read_supporting_bytes_memory = 16; // No longer used
+        uint64 stored_reads_bytes_memory_including_in_flight = 17;
         uint64 stored_reads_bytes_disk = 11;
         uint64 discarded_error_bytes = 12;
         uint64 channels_writing_to_disk = 13;
@@ -948,21 +955,23 @@ message GetExperimentYieldInfoResponse{
     }
 
     message HdfWriterInfo {
-        uint64 pending_compressions = 1;
-        uint64 pending_writes = 2;
+        uint64 pending_compressions = 1; // No longer used
+        uint64 pending_writes = 2; // No longer used
         uint64 pending_hdf_tasks = 3;
-        uint64 completed_writes = 4;
+        uint64 completed_writes = 4; // No longer used
         uint64 dataset_bytes_data_in_flight = 5;
     }
 
     message BasecallStatistics {
         uint64 reads_in_progress = 1;
+        uint64 samples_in_progress = 14;
         uint64 reads_processed = 2;
         uint64 reads_skipped = 3;
         uint64 reads_force_skipped = 4;
         uint64 reads_failed_calling_filtered = 5;
         uint64 reads_failed_calling = 6;
         uint64 reads_called = 7;
+        uint64 samples_in_client_buffer = 13;
         uint64 samples_called = 8;
         uint64 samples_skipped = 9;
         uint64 bases_passed_called = 10;