From 940742dbaf2e6b248e5fbf3174af38a34d2747a2 Mon Sep 17 00:00:00 2001 From: Chris Billington Date: Tue, 10 Dec 2019 12:04:54 -0500 Subject: [PATCH 1/7] compilation-queue: Working on a compilation queue, in order to facilitate just-in-time compilation of shots. --- runmanager/__init__.py | 44 ++- runmanager/__main__.py | 220 +++++++++-- runmanager/main.ui | 857 ++++++++++++++++++++++++++++++++--------- 3 files changed, 892 insertions(+), 229 deletions(-) diff --git a/runmanager/__init__.py b/runmanager/__init__.py index c2bb430..81be88b 100644 --- a/runmanager/__init__.py +++ b/runmanager/__init__.py @@ -737,17 +737,47 @@ def make_run_files( created at some point, simply convert the returned generator to a list. The filenames the run files are given is simply the sequence_id with increasing integers appended.""" - basename = os.path.join(output_folder, filename_prefix) - nruns = len(shots) - ndigits = int(np.ceil(np.log10(nruns))) if shuffle: random.shuffle(shots) - for i, shot_globals in enumerate(shots): - runfilename = ('%s_%0' + str(ndigits) + 'd.h5') % (basename, i) + run_filenames = make_run_filenames(output_folder, filename_prefix, len(shots)) + for run_no, (filename, shot_globals) in enumerate(zip(run_filenames, shots)): make_single_run_file( - runfilename, sequence_globals, shot_globals, sequence_attrs, i, nruns + filename, sequence_globals, shot_globals, sequence_attrs, run_no, len(shots) ) - yield runfilename + yield filename + + +def make_run_filenames(output_folder, filename_prefix, nruns): + """Like make_run_files(), but return the filenames instead of creating the the files + (and instead of creating a generator that creates them). These filenames can then be + passed to make_single_run_file(). So instead of: + + run_files = make_run_files( + output_folder, + sequence_globals, + shots, + sequence_attrs, + filename_prefix, + shuffle, + ) + + You may do: + + if shuffle: + random.shuffle(shots) + run_filenames = make_run_filenames(output_folder, filename_prefix, len(shots)) + for run_no, (filename, shot_globals) in enumerate(zip(run_filenames, shots)): + make_single_run_file( + filename, sequence_globals, shot_globals, sequence_attrs, run_no, len(shots) + ) + """ + basename = os.path.join(output_folder, filename_prefix) + ndigits = int(np.ceil(np.log10(nruns))) + filenames = [] + for i in range(nruns): + filename = ('%s_%0' + str(ndigits) + 'd.h5') % (basename, i) + filenames.append(filename) + return filenames def make_single_run_file(filename, sequenceglobals, runglobals, sequence_attrs, run_no, n_runs): diff --git a/runmanager/__main__.py b/runmanager/__main__.py index 0c38710..d46e3f4 100644 --- a/runmanager/__main__.py +++ b/runmanager/__main__.py @@ -37,6 +37,7 @@ import traceback import signal from pathlib import Path +import random splash.update_text('importing matplotlib') # Evaluation of globals happens in a thread with the pylab module imported. @@ -1352,6 +1353,10 @@ class RunManager(object): AXES_COL_SHUFFLE = 2 AXES_ROLE_NAME = QtCore.Qt.UserRole + 1 + # Constants for the model in the queue tab: + QUEUE_COL_NAME = 0 + QUEUE_ROLE_FULL_FILENAME = QtCore.Qt.UserRole + 1 + # Constants for the model in the groups tab: GROUPS_COL_NAME = 0 GROUPS_COL_ACTIVE = 1 @@ -1393,6 +1398,7 @@ def __init__(self): self.output_box_window.resize(800, 1000) self.setup_config() self.setup_axes_tab() + self.setup_queue_tab() self.setup_groups_tab() self.connect_signals() @@ -1436,8 +1442,13 @@ def __init__(self): # The prospective number of shots resulting from compilation self.n_shots = None - # Start the loop that allows compilations to be queued up: - self.compile_queue = queue.Queue() + # An event to tell the compilation queue to check if the next queued shot (if + # any) can be compiled: + self.compilation_potentially_required = threading.Event() + + # Create data structures for the compilation queue and start the compilation + # thread: + self.queued_shots = {} self.compile_queue_thread = threading.Thread(target=self.compile_loop) self.compile_queue_thread.daemon = True self.compile_queue_thread.start() @@ -1532,6 +1543,16 @@ def setup_axes_tab(self): # setup header widths self.ui.treeView_axes.header().setStretchLastSection(False) self.ui.treeView_axes.header().setSectionResizeMode(self.AXES_COL_NAME, QtWidgets.QHeaderView.Stretch) + + def setup_queue_tab(self): + self.queue_model = QtGui.QStandardItemModel() + + # Setup the model columns and link to the treeview + name_header_item = QtGui.QStandardItem('Sequence id/filename') + name_header_item.setToolTip('The sequence id and filenames of shot files queued for compilation') + self.queue_model.setHorizontalHeaderItem(self.QUEUE_COL_NAME, name_header_item) + self.ui.treeView_queue.setModel(self.queue_model) + self.ui.treeView_queue.header().setStretchLastSection(True) def setup_groups_tab(self): self.groups_model = QtGui.QStandardItemModel() @@ -1815,11 +1836,17 @@ def on_engage_clicked(self): sequenceglobals, shots, evaled_globals, global_hierarchy, expansions = self.parse_globals(active_groups, expansion_order=expansion_order) except Exception as e: raise Exception('Error parsing globals:\n%s\nCompilation aborted.' % str(e)) - logger.info('Making h5 files') - labscript_file, run_files = self.make_h5_files( - labscript_file, output_folder, sequenceglobals, shots, shuffle) - self.ui.pushButton_abort.setEnabled(True) - self.compile_queue.put([labscript_file, run_files, send_to_BLACS, BLACS_host, send_to_runviewer]) + logger.info('Queueing new sequence for compilation') + self.queue_new_sequence( + labscript_file, + output_folder, + sequenceglobals, + shots, + shuffle, + send_to_BLACS, + BLACS_host, + send_to_runviewer, + ) except Exception as e: self.output_box.output('%s\n\n' % str(e), red=True) logger.info('end engage') @@ -3211,37 +3238,98 @@ def warning(message): self.ui.actionSave_configuration_as.setEnabled(True) self.ui.actionRevert_configuration.setEnabled(True) + @inmain_decorator() + def next_queued_shot(self): + """Get the details of the next shot to be compiled, removing it from the + queue""" + if not self.queue_model.rowCount(): + return None, None + sequence_item = self.queue_model.item(0, self.QUEUE_COL_NAME) + shot_item = sequence_item.takeRow(0)[self.QUEUE_COL_NAME] + filename = shot_item.data(self.QUEUE_ROLE_FULL_FILENAME) + details = self.queued_shots.pop(filename) + if not sequence_item.rowCount(): + self.queue_model.takeRow(0) + return filename, details + def compile_loop(self): while True: try: - labscript_file, run_files, send_to_BLACS, BLACS_host, send_to_runviewer = self.compile_queue.get() - run_files = iter(run_files) # Should already be in iterator but just in case + self.compilation_potentially_required.wait() + self.compilation_potentially_required.clear() while True: if self.compilation_aborted.is_set(): self.output_box.output('Compilation aborted.\n\n', red=True) break + # If we're JIT and BLACS' queue isn't empty enough yet, also break. + # Print to the output box that that's what you're doing. Otherwise, + # go on: + run_file, shot_details = self.next_queued_shot() + if run_file is None: + self.output_box.output('Ready.\n\n') + break + ( + labscript_file, + sequence_globals, + shot_globals, + sequence_attrs, + run_no, + nshots, + send_to_BLACS, + BLACS_host, + send_to_runviewer, + ) = shot_details + try: - try: - # We do next() instead of looping over run_files - # so that if compilation is aborted we won't - # create an extra file unnecessarily. - run_file = next(run_files) - except StopIteration: - self.output_box.output('Ready.\n\n') - break - else: - self.to_child.put(['compile', [labscript_file, run_file]]) - signal, success = self.from_child.get() - assert signal == 'done' - if not success: - self.compilation_aborted.set() - continue - if send_to_BLACS: - self.send_to_BLACS(run_file, BLACS_host) - if send_to_runviewer: - self.send_to_runviewer(run_file) - except Exception as e: - self.output_box.output(str(e) + '\n', red=True) + # TODO: likely remove this if the UI for this changes. Do we + # want to store with the queued shot itself what the dynamic + # globals should be? Do we want to store a copy of the shot + # file...? Probably, yes. Complicated! + dynamic = inmain(self.ui.lineEdit_dynamic_globals.text) + dynamic = [s.strip() for s in dynamic.split(',') if s.strip()] + + if dynamic: + print(repr(dynamic)) + active_groups = inmain( + app.get_active_groups, interactive=False + ) + sequence_globals = runmanager.get_globals(active_groups) + all_globals = {} + evaled_globals, _, _ = runmanager.evaluate_globals( + sequence_globals, raise_exceptions=True + ) + for group_globals in evaled_globals.values(): + all_globals.update(group_globals) + + # Update the shot globals for this shot to the new values: + for name in dynamic: + shot_globals[name] = all_globals[name] + + runmanager.make_single_run_file( + run_file, + sequence_globals, + shot_globals, + sequence_attrs, + run_no, + nshots, + ) + self.to_child.put(['compile', [labscript_file, run_file]]) + signal, success = self.from_child.get() + assert signal == 'done' + if not success: + self.compilation_aborted.set() + # TODO: can probably think of something more sensible here, + # like prepending to the queue unless the user deletes the + # shot: + self.queue_model.clear() + self.queued_shots.clear() + continue + if send_to_BLACS: + self.send_to_BLACS(run_file, BLACS_host) + if send_to_runviewer: + self.send_to_runviewer(run_file) + except Exception: + self.output_box.output(traceback.format_exc() + '\n', red=True) self.compilation_aborted.set() inmain(self.ui.pushButton_abort.setEnabled, False) self.compilation_aborted.clear() @@ -3440,27 +3528,75 @@ def set_expansion_type_guess(expansion_types, expansions, global_name, expansion return expansion_types_changed - def make_h5_files(self, labscript_file, output_folder, sequence_globals, shots, shuffle): - sequence_attrs, default_output_dir, filename_prefix = runmanager.new_sequence_details( + def queue_new_sequence( + self, + labscript_file, + output_folder, + sequence_globals, + shots, + shuffle, + send_to_BLACS, + BLACS_host, + send_to_runviewer, + ): + details = runmanager.new_sequence_details( labscript_file, config=self.exp_config, increment_sequence_index=True ) + sequence_attrs, default_output_dir, filename_prefix = details if output_folder == self.previous_default_output_folder: - # The user is using dthe efault output folder. Just in case the sequence + # The user is using the default output folder. Just in case the sequence # index has been updated or the date has changed, use the default_output dir # obtained from new_sequence_details, as it is race-free, whereas the one # from the UI may be out of date since we only update it once a second. output_folder = default_output_dir self.check_output_folder_update() - run_files = runmanager.make_run_files( - output_folder, - sequence_globals, - shots, - sequence_attrs, - filename_prefix, - shuffle, + + if shuffle: + random.shuffle(shots) + + run_filenames = runmanager.make_run_filenames( + output_folder, filename_prefix, len(shots) ) - logger.debug(run_files) - return labscript_file, run_files + + sequence_item = QtGui.QStandardItem( + filename_prefix + + ' (%d shot%s)' % (len(shots), 's' if len(shots) > 1 else '') + ) + sequence_item.setToolTip(output_folder) + sequence_item.setEditable(False) + + self.queue_model.appendRow([sequence_item]) + + for run_no, (filename, shot_globals) in enumerate(zip(run_filenames, shots)): + + self.queued_shots[filename] = ( + labscript_file, + sequence_globals, + shot_globals, + sequence_attrs, + run_no, + len(shots), + send_to_BLACS, + BLACS_host, + send_to_runviewer, + ) + + item = QtGui.QStandardItem(os.path.basename(filename)) + item.setToolTip(filename) + item.setData(filename, self.QUEUE_ROLE_FULL_FILENAME) + item.setEditable(False) + + sequence_item.appendRow([item]) + + self.ui.treeView_queue.setExpanded(sequence_item.index(), True) + + logger.debug(run_filenames) + + # Tell the compilation queue to wake up: + self.compilation_potentially_required.set() + + self.ui.pushButton_abort.setEnabled(True) + def send_to_BLACS(self, run_file, BLACS_hostname): port = int(self.exp_config.get('ports', 'BLACS')) diff --git a/runmanager/main.ui b/runmanager/main.ui index 3f6f374..98d38e0 100644 --- a/runmanager/main.ui +++ b/runmanager/main.ui @@ -14,7 +14,7 @@ runmanager - the labscript suite - + :/qtutils/custom/runmanager.png:/qtutils/custom/runmanager.png @@ -94,7 +94,16 @@ QToolButton:hover:checked { 0 - + + 0 + + + 0 + + + 0 + + 0 @@ -120,6 +129,47 @@ QToolButton:hover:checked { 0 + + + + Qt::NoFocus + + + <html><head/><body><p>Send compiled shots to BLACS in a random order, to prevent scanned parameters correlating with temporal drifts.</p></body></html> + + + Shuffle + + + + :/qtutils/fugue/arrow-switch.png:/qtutils/fugue/arrow-switch.png + + + true + + + false + + + + + + + Qt::NoFocus + + + <html><head/><body><p>Forcefully restart the compilation subprocess, stopping any compilation in process.</p><p>This can be useful if the child process is misbehaving for any reason.</p></body></html> + + + Restart +subprocess + + + + :/qtutils/fugue/arrow-circle-135-left.png:/qtutils/fugue/arrow-circle-135-left.png + + + @@ -136,7 +186,7 @@ QToolButton:hover:checked { Abort - + :/qtutils/fugue/cross-octagon.png:/qtutils/fugue/cross-octagon.png @@ -157,8 +207,8 @@ QToolButton:hover:checked { QPushButton { - background-color: rgba(238,96,96,192); - border: 1px solid rgba(238,96,96,128); + background-color: rgb(238,96,96,192); + border: 1px solid rgb(238,96,96,128); border-radius: 3px; padding: 4px; } @@ -181,7 +231,7 @@ QPushButton:hover { Engage - + :/qtutils/fugue/control.png:/qtutils/fugue/control.png @@ -192,47 +242,6 @@ QPushButton:hover { - - - - Qt::NoFocus - - - <html><head/><body><p>Forcefully restart the compilation subprocess, stopping any compilation in process.</p><p>This can be useful if the child process is misbehaving for any reason.</p></body></html> - - - Restart -subprocess - - - - :/qtutils/fugue/arrow-circle-135-left.png:/qtutils/fugue/arrow-circle-135-left.png - - - - - - - Qt::NoFocus - - - <html><head/><body><p>Send compiled shots to BLACS in a random order, to prevent scanned parameters correlating with temporal drifts.</p></body></html> - - - Shuffle - - - - :/qtutils/fugue/arrow-switch.png:/qtutils/fugue/arrow-switch.png - - - true - - - false - - - @@ -307,100 +316,68 @@ subprocess 6 - - - - <html><head/><body><p>The host computer running BLACS.</p></body></html> - - - localhost - - - true - - - - - - - labscript file - - - - - + + false - <html><head/><body><p>Reset to default output folder.</p><p>If the folder does not exist it will be created at compile-time.</p></body></html> + <html><head/><body><p>Edit this labscript file in a text editor.</p></body></html> ... - - :/qtutils/fugue/arrow-turn-180-left.png:/qtutils/fugue/arrow-turn-180-left.png + + :/qtutils/fugue/document--pencil.png:/qtutils/fugue/document--pencil.png - - - - Shot output folder - - - - - - - BLACS hostname - - - - - - - false - + + - <html><head/><body><p>Edit this labscript file in a text editor.</p></body></html> + <html><head/><body><p>The host computer running BLACS.</p></body></html> - ... + localhost - - - :/qtutils/custom/python-document.png:/qtutils/custom/python-document.png + + true - + - - - 0 - 0 - + + false - - - 0 - 0 - + + QToolButton{ + background: rgb(224,224,224); + padding: 3px; +} QFrame::StyledPanel - + 0 - + + 0 + + + 0 + + + 0 + + 0 - + 0 @@ -408,7 +385,7 @@ subprocess - <html><head/><body><p>The folder in which newly compiled shot files will be placed.</p><p>If the folder does not exist, it will be created at compile time.</p></body></html> + The labscript file to compile. @@ -422,32 +399,16 @@ subprocess - - - <html><head/><body><p>Non-default output folder chosen: all shots will be produced in this folder.</p><p>New folders will not automatically be created for new sequences/dates/etc.</p><p>To reset to the default output folder, click the 'reset to default output folder' button.</p></body></html> - - - - - - :/qtutils/fugue/exclamation-white.png - - - - - + Qt::Vertical - - - false - + - Select folder ... + Select a file ... QToolButton{ @@ -470,37 +431,66 @@ QToolButton:hover { ... - - :/qtutils/fugue/folder-horizontal-open.png:/qtutils/fugue/folder-horizontal-open.png + + :/qtutils/custom/python-document.png:/qtutils/custom/python-document.png - - - + + + false - - QToolButton{ - background: rgb(224,224,224); - padding: 3px; -} + + <html><head/><body><p>Reset to default output folder.</p><p>If the folder does not exist it will be created at compile-time.</p></body></html> + + + ... + + + + :/qtutils/fugue/arrow-turn-180-left.png:/qtutils/fugue/arrow-turn-180-left.png + + + + + + + + 0 + 0 + + + + + 0 + 0 + QFrame::StyledPanel - + 0 - + + 0 + + + 0 + + + 0 + + 0 - + 0 @@ -508,7 +498,7 @@ QToolButton:hover { - The labscript file to compile. + <html><head/><body><p>The folder in which newly compiled shot files will be placed.</p><p>If the folder does not exist, it will be created at compile time.</p></body></html> @@ -522,16 +512,32 @@ QToolButton:hover { - + + + <html><head/><body><p>Non-default output folder chosen: all shots will be produced in this folder.</p><p>New folders will not automatically be created for new sequences/dates/etc.</p><p>To reset to the default output folder, click the 'reset to default output folder' button.</p></body></html> + + + + + + :/qtutils/fugue/exclamation-white.png + + + + + Qt::Vertical - + + + false + - Select a file ... + Select folder ... QToolButton{ @@ -554,14 +560,35 @@ QToolButton:hover { ... - - :/qtutils/fugue/folder-open-document-text.png:/qtutils/fugue/folder-open-document-text.png + + :/qtutils/fugue/folder-horizontal-open.png:/qtutils/fugue/folder-horizontal-open.png + + + + BLACS hostname + + + + + + + Shot output folder + + + + + + + labscript file + + + @@ -588,7 +615,7 @@ QToolButton:hover { QTabWidget::Rounded - 0 + 1 Qt::ElideRight @@ -607,7 +634,7 @@ QToolButton:hover { - + :/qtutils/fugue/terminal.png:/qtutils/fugue/terminal.png @@ -620,7 +647,16 @@ QToolButton:hover { 0 - + + 0 + + + 0 + + + 0 + + 0 @@ -628,9 +664,425 @@ QToolButton:hover { + + + + :/qtutils/fugue/edit-list-order.png:/qtutils/fugue/edit-list-order.png + + + Queue + + + + + + 6 + + + 6 + + + 6 + + + 6 + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + + + Ahead-of-time compilation + + + true + + + + + + + Just-in-time compilation + + + + + + + + + Dynamic globals: (comma-separated list) + + + + + + + + 300 + 0 + + + + + + + + + + + + Compile next when BLACS has + + + + + + + + + + shot(s) left + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + QFrame::StyledPanel + + + + 6 + + + 6 + + + 6 + + + 6 + + + + + 6 + + + 0 + + + 0 + + + 0 + + + + + Qt::NoFocus + + + Pause the queue + + + Pause + + + + :/qtutils/fugue/control-pause.png:/qtutils/fugue/control-pause.png + + + true + + + false + + + + + + + 0 + + + 0 + + + + + Qt::NoFocus + + + Re-add shots to the end of the queue after completion + + + Repeat + + + + :/qtutils/fugue/arrow-repeat.png:/qtutils/fugue/arrow-repeat.png + + + true + + + + + + + Select repeat mode + + + + + + + :/qtutils/fugue/control-270.png:/qtutils/fugue/control-270.png + + + + 8 + 19 + + + + QToolButton::InstantPopup + + + Qt::NoArrow + + + + + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + 0 + + + + + 6 + + + 3 + + + 3 + + + + + Remove selected sequences/shots + + + ... + + + + :/qtutils/fugue/minus.png:/qtutils/fugue/minus.png + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Move selected to top + + + ... + + + + :/qtutils/fugue/arrow-stop-090.png:/qtutils/fugue/arrow-stop-090.png + + + + + + + Move selected up + + + ... + + + + :/qtutils/fugue/arrow-090.png:/qtutils/fugue/arrow-090.png + + + + + + + Move selected down + + + ... + + + + :/qtutils/fugue/arrow-270.png:/qtutils/fugue/arrow-270.png + + + + + + + Move selected to bottom + + + ... + + + + :/qtutils/fugue/arrow-stop-270.png:/qtutils/fugue/arrow-stop-270.png + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + true + + + QAbstractItemView::ExtendedSelection + + + Qt::ElideLeft + + + + + + + + + - + :/qtutils/custom/outer.png:/qtutils/custom/outer.png @@ -643,7 +1095,16 @@ QToolButton:hover { 0 - + + 0 + + + 0 + + + 0 + + 0 @@ -679,7 +1140,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-stop-090.png:/qtutils/fugue/arrow-stop-090.png @@ -693,7 +1154,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-090.png:/qtutils/fugue/arrow-090.png @@ -707,7 +1168,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-270.png:/qtutils/fugue/arrow-270.png @@ -721,7 +1182,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-stop-270.png:/qtutils/fugue/arrow-stop-270.png @@ -754,15 +1215,24 @@ QToolButton:hover { 0 0 - 926 - 486 + 941 + 521 0 - + + 0 + + + 0 + + + 0 + + 0 @@ -770,15 +1240,15 @@ QToolButton:hover { QAbstractItemView::NoEditTriggers + + true + QAbstractItemView::ExtendedSelection QAbstractItemView::SelectRows - - true - @@ -789,7 +1259,7 @@ QToolButton:hover { - + :/qtutils/fugue/tables-stacks.png:/qtutils/fugue/tables-stacks.png @@ -802,7 +1272,16 @@ QToolButton:hover { 0 - + + 0 + + + 0 + + + 0 + + 0 @@ -810,7 +1289,16 @@ QToolButton:hover { 9 - + + 3 + + + 3 + + + 3 + + 3 @@ -825,7 +1313,7 @@ QToolButton:hover { Open globals file - + :/qtutils/fugue/folder-open-table.png:/qtutils/fugue/folder-open-table.png @@ -842,7 +1330,7 @@ QToolButton:hover { New globals file - + :/qtutils/fugue/table--plus.png:/qtutils/fugue/table--plus.png @@ -859,7 +1347,7 @@ QToolButton:hover { Diff globals file - + :/qtutils/fugue/tables.png:/qtutils/fugue/tables.png @@ -892,15 +1380,24 @@ QToolButton:hover { 0 0 - 963 - 448 + 970 + 490 0 - + + 0 + + + 0 + + + 0 + + 0 @@ -926,7 +1423,7 @@ QToolButton:hover { 0 0 1000 - 31 + 22 @@ -943,7 +1440,7 @@ QToolButton:hover { - + :/qtutils/fugue/folder-open.png:/qtutils/fugue/folder-open.png @@ -958,7 +1455,7 @@ QToolButton:hover { false - + :/qtutils/fugue/disk--plus.png:/qtutils/fugue/disk--plus.png @@ -970,7 +1467,7 @@ QToolButton:hover { - + :/qtutils/fugue/cross-button.png:/qtutils/fugue/cross-button.png @@ -982,7 +1479,7 @@ QToolButton:hover { - + :/qtutils/fugue/disk.png:/qtutils/fugue/disk.png @@ -997,7 +1494,7 @@ QToolButton:hover { false - + :/qtutils/fugue/arrow-curve-180-left.png:/qtutils/fugue/arrow-curve-180-left.png @@ -1044,7 +1541,7 @@ QToolButton:hover { scrollArea_2 - + From 9b1d35e16fd3160399e2d331e6711fc00b49f13f Mon Sep 17 00:00:00 2001 From: Chris Billington Date: Tue, 10 Dec 2019 21:19:15 -0500 Subject: [PATCH 2/7] compilation-queue: Repeat and pause working. Abort still does not have sensible behaviour. --- runmanager/__init__.py | 31 +++++- runmanager/__main__.py | 221 +++++++++++++++++++++++++++++------------ 2 files changed, 189 insertions(+), 63 deletions(-) diff --git a/runmanager/__init__.py b/runmanager/__init__.py index 81be88b..357ec12 100644 --- a/runmanager/__init__.py +++ b/runmanager/__init__.py @@ -780,7 +780,9 @@ def make_run_filenames(output_folder, filename_prefix, nruns): return filenames -def make_single_run_file(filename, sequenceglobals, runglobals, sequence_attrs, run_no, n_runs): +def make_single_run_file( + filename, sequenceglobals, runglobals, sequence_attrs, run_no, n_runs, rep_no=0 +): """Does what it says. runglobals is a dict of this run's globals, the format being the same as that of one element of the list returned by expand_globals. sequence_globals is a nested dictionary of the type returned by get_globals. @@ -793,6 +795,7 @@ def make_single_run_file(filename, sequenceglobals, runglobals, sequence_attrs, f.attrs.update(sequence_attrs) f.attrs['run number'] = run_no f.attrs['n_runs'] = n_runs + f.attrs['run repeat'] = rep_no f.create_group('globals') if sequenceglobals is not None: for groupname, groupvars in sequenceglobals.items(): @@ -1126,3 +1129,29 @@ def globals_diff_shots(file1, file2, max_cols=100): print('Globals diff between:\n%s\n%s\n\n' % (file1, file2)) return globals_diff_groups(active_groups, other_groups, max_cols=max_cols, return_string=False) + + +def new_rep_name(h5_filepath): + """Extract the rep number, if any, from the filepath of the given shot file, and + return a filepath for a repetition of that shot, using the lowest rep number greater + than it, not already corresponding to a file on the filesystem. Create an empty file + with that filepath such that it then exists in the filesystem, such that making + multiple calls to this funciton by different applications is race-free. Return the + filepath of the new file, and the rep number as an integer. The file should be + overwritten by opening a h5 file with that path in 'w' mode, which will truncate the + original file. Otherwise it should be deleted, as it is not a valid HDF5 file.""" + path, ext = os.path.splitext(h5_filepath) + if '_rep' in path and ext == '.h5': + repno = path.split('_rep')[-1] + try: + repno = int(repno) + except ValueError: + # not a rep + repno = 0 + else: + repno = 0 + while True: + new_path = path.rsplit('_rep', 1)[0] + '_rep%05d.h5' % (repno + 1) + if not os.path.exists(new_path): + return new_path, repno + repno += 1 diff --git a/runmanager/__main__.py b/runmanager/__main__.py index d46e3f4..ac13065 100644 --- a/runmanager/__main__.py +++ b/runmanager/__main__.py @@ -1356,6 +1356,8 @@ class RunManager(object): # Constants for the model in the queue tab: QUEUE_COL_NAME = 0 QUEUE_ROLE_FULL_FILENAME = QtCore.Qt.UserRole + 1 + QUEUE_ROLE_FILENAME_PREFIX = QtCore.Qt.UserRole + 2 + QUEUE_ROLE_OUTPUT_FOLDER = QtCore.Qt.UserRole + 3 # Constants for the model in the groups tab: GROUPS_COL_NAME = 0 @@ -1368,6 +1370,11 @@ class RunManager(object): GROUPS_ROLE_GROUP_IS_OPEN = QtCore.Qt.UserRole + 4 GROUPS_DUMMY_ROW_TEXT = '' + REPEAT_ALL = 0 + REPEAT_LAST = 1 + ICON_REPEAT = ':qtutils/fugue/arrow-repeat' + ICON_REPEAT_LAST = ':qtutils/fugue/arrow-repeat-once' + def __init__(self): splash.update_text('loading graphical interface') loader = UiLoader() @@ -1449,6 +1456,7 @@ def __init__(self): # Create data structures for the compilation queue and start the compilation # thread: self.queued_shots = {} + self.queue_repeat_mode = self.REPEAT_ALL self.compile_queue_thread = threading.Thread(target=self.compile_loop) self.compile_queue_thread.daemon = True self.compile_queue_thread.start() @@ -1553,6 +1561,22 @@ def setup_queue_tab(self): self.queue_model.setHorizontalHeaderItem(self.QUEUE_COL_NAME, name_header_item) self.ui.treeView_queue.setModel(self.queue_model) self.ui.treeView_queue.header().setStretchLastSection(True) + + # Set up repeat mode button menu: + self.repeat_mode_menu = QtWidgets.QMenu(self.ui) + self.action_repeat_all = QtWidgets.QAction( + QtGui.QIcon(self.ICON_REPEAT), 'Repeat all', self.ui + ) + self.action_repeat_last = QtWidgets.QAction( + QtGui.QIcon(self.ICON_REPEAT_LAST), 'Repeat last', self.ui + ) + + self.repeat_mode_menu.addAction(self.action_repeat_all) + self.repeat_mode_menu.addAction(self.action_repeat_last) + self.ui.repeat_mode_select_button.setMenu(self.repeat_mode_menu) + + # The button already has an arrow indicating a menu, don't draw another one: + self.ui.repeat_mode_select_button.setStyleSheet("QToolButton::menu-indicator{width: 0;}") def setup_groups_tab(self): self.groups_model = QtGui.QStandardItemModel() @@ -1634,6 +1658,15 @@ def connect_signals(self): # Tab closebutton clicked: self.ui.tabWidget.tabCloseRequested.connect(self.on_tabCloseRequested) + # Queue tab: + self.ui.queue_pause_button.toggled.connect(self.on_queue_paused_toggled) + self.action_repeat_all.triggered.connect( + lambda: self.set_queue_repeat_mode(self.REPEAT_ALL) + ) + self.action_repeat_last.triggered.connect( + lambda: self.set_queue_repeat_mode(self.REPEAT_LAST) + ) + # Axes tab; right click menu, menu actions, reordering # self.ui.treeView_axes.customContextMenuRequested.connect(self.on_treeView_axes_context_menu_requested) self.action_axes_check_selected.triggered.connect(self.on_axes_check_selected_triggered) @@ -3242,25 +3275,61 @@ def warning(message): def next_queued_shot(self): """Get the details of the next shot to be compiled, removing it from the queue""" - if not self.queue_model.rowCount(): + if not self.queue_model.rowCount() or self.ui.queue_pause_button.isChecked(): return None, None + elif self.compilation_aborted.is_set(): + self.output_box.output('Compilation aborted.\n\n', red=True) + return None, None + self.ui.pushButton_abort.setEnabled(True) sequence_item = self.queue_model.item(0, self.QUEUE_COL_NAME) shot_item = sequence_item.takeRow(0)[self.QUEUE_COL_NAME] filename = shot_item.data(self.QUEUE_ROLE_FULL_FILENAME) details = self.queued_shots.pop(filename) if not sequence_item.rowCount(): self.queue_model.takeRow(0) + else: + self.update_sequence_item_text(sequence_item) + self.update_queue_tab_label() return filename, details + @inmain_decorator() + def check_repeat(self, run_file, shot_details): + """If we are in repeat-all mode, or repeat-last mode and the queue is empty, + create details of a new shot that is a rep of the given shot, and add it to the + queue. The shot file must already exist, as the lowest rep number not already + existing in the filesystem will be used for the rep shot's name""" + if self.ui.queue_repeat_button.isChecked() and ( + self.queue_repeat_mode == self.REPEAT_ALL or not self.queue_model.rowCount() + ): + new_run_file, rep_no = runmanager.new_rep_name(run_file) + shot_details['rep_no'] = rep_no + self.queued_shots[new_run_file] = shot_details + filename_prefix = shot_details['filename_prefix'] + output_folder = shot_details['output_folder'] + # Use the existing sequence item in the model if it matches our filename + # prefix and output, otherwise make a new one: + n_seq = self.queue_model.rowCount() + sequence_item = None + if n_seq: + print(f"{n_seq=}") + item = self.queue_model.item(n_seq - 1) + prefix = item.data(self.QUEUE_ROLE_FILENAME_PREFIX) + folder = item.data(self.QUEUE_ROLE_OUTPUT_FOLDER) + if prefix == filename_prefix and folder == output_folder: + sequence_item = item + if sequence_item is None: + sequence_item = self.append_sequence_item_to_queue( + filename_prefix, output_folder + ) + self.append_shot_item_to_queue(sequence_item, new_run_file) + self.update_queue_tab_label() + def compile_loop(self): while True: try: self.compilation_potentially_required.wait() self.compilation_potentially_required.clear() while True: - if self.compilation_aborted.is_set(): - self.output_box.output('Compilation aborted.\n\n', red=True) - break # If we're JIT and BLACS' queue isn't empty enough yet, also break. # Print to the output box that that's what you're doing. Otherwise, # go on: @@ -3268,35 +3337,23 @@ def compile_loop(self): if run_file is None: self.output_box.output('Ready.\n\n') break - ( - labscript_file, - sequence_globals, - shot_globals, - sequence_attrs, - run_no, - nshots, - send_to_BLACS, - BLACS_host, - send_to_runviewer, - ) = shot_details - + shot_globals = shot_details['shot_globals'] + labscript_file = shot_details['labscript_file'] + # TODO: likely remove this if the UI for this changes. Do we + # want to store with the queued shot itself what the dynamic + # globals should be? Do we want to store a copy of the shot + # file...? Probably, yes. Complicated! + dynamic = inmain(self.ui.lineEdit_dynamic_globals.text) + dynamic = [s.strip() for s in dynamic.split(',') if s.strip()] try: - # TODO: likely remove this if the UI for this changes. Do we - # want to store with the queued shot itself what the dynamic - # globals should be? Do we want to store a copy of the shot - # file...? Probably, yes. Complicated! - dynamic = inmain(self.ui.lineEdit_dynamic_globals.text) - dynamic = [s.strip() for s in dynamic.split(',') if s.strip()] - if dynamic: - print(repr(dynamic)) active_groups = inmain( app.get_active_groups, interactive=False ) - sequence_globals = runmanager.get_globals(active_groups) all_globals = {} evaled_globals, _, _ = runmanager.evaluate_globals( - sequence_globals, raise_exceptions=True + runmanager.get_globals(active_groups), + raise_exceptions=True, ) for group_globals in evaled_globals.values(): all_globals.update(group_globals) @@ -3307,11 +3364,12 @@ def compile_loop(self): runmanager.make_single_run_file( run_file, - sequence_globals, + shot_details['sequence_globals'], shot_globals, - sequence_attrs, - run_no, - nshots, + shot_details['sequence_attrs'], + shot_details['run_no'], + shot_details['n_runs'], + rep_no=shot_details['rep_no'], ) self.to_child.put(['compile', [labscript_file, run_file]]) signal, success = self.from_child.get() @@ -3324,9 +3382,10 @@ def compile_loop(self): self.queue_model.clear() self.queued_shots.clear() continue - if send_to_BLACS: - self.send_to_BLACS(run_file, BLACS_host) - if send_to_runviewer: + self.check_repeat(run_file, shot_details) + if shot_details['send_to_BLACS']: + self.send_to_BLACS(run_file, shot_details['BLACS_host']) + if shot_details['send_to_runviewer']: self.send_to_runviewer(run_file) except Exception: self.output_box.output(traceback.format_exc() + '\n', red=True) @@ -3528,6 +3587,54 @@ def set_expansion_type_guess(expansion_types, expansions, global_name, expansion return expansion_types_changed + def on_queue_paused_toggled(self, paused): + self.update_queue_tab_label() + if not paused: + self.compilation_potentially_required.set() + + def set_queue_repeat_mode(self, mode): + if mode == self.REPEAT_ALL: + self.ui.queue_repeat_button.setIcon(QtGui.QIcon(self.ICON_REPEAT)) + elif mode == self.REPEAT_LAST: + self.ui.queue_repeat_button.setIcon(QtGui.QIcon(self.ICON_REPEAT_LAST)) + else: + raise ValueError(mode) + self.queue_repeat_mode = mode + + @inmain_decorator() + def update_queue_tab_label(self): + nsequences = self.queue_model.rowCount() + nshots = sum(self.queue_model.item(i).rowCount() for i in range(nsequences)) + paused = ', paused' if self.ui.queue_pause_button.isChecked() else '' + label = 'Queue (%d%s)' % (nshots, paused) + index = self.ui.tabWidget.indexOf(self.ui.tab_queue) + self.ui.tabWidget.setTabText(index, label) + + def append_sequence_item_to_queue(self, filename_prefix, output_folder): + sequence_item = QtGui.QStandardItem(filename_prefix) + sequence_item.setToolTip(output_folder) + sequence_item.setData(filename_prefix, self.QUEUE_ROLE_FILENAME_PREFIX) + sequence_item.setData(output_folder, self.QUEUE_ROLE_OUTPUT_FOLDER) + sequence_item.setEditable(False) + self.queue_model.appendRow([sequence_item]) + self.ui.treeView_queue.setExpanded(sequence_item.index(), True) + return sequence_item + + def append_shot_item_to_queue(self, sequence_item, filename): + item = QtGui.QStandardItem(os.path.basename(filename)) + item.setToolTip(filename) + item.setData(filename, self.QUEUE_ROLE_FULL_FILENAME) + item.setEditable(False) + sequence_item.appendRow([item]) + self.update_sequence_item_text(sequence_item) + + def update_sequence_item_text(self, sequence_item): + nshots = sequence_item.rowCount() + filename_prefix = sequence_item.data(self.QUEUE_ROLE_FILENAME_PREFIX) + sequence_item.setText( + filename_prefix + ' (%d shot%s)' % (nshots, 's' if nshots > 1 else '') + ) + def queue_new_sequence( self, labscript_file, @@ -3558,44 +3665,34 @@ def queue_new_sequence( output_folder, filename_prefix, len(shots) ) - sequence_item = QtGui.QStandardItem( - filename_prefix - + ' (%d shot%s)' % (len(shots), 's' if len(shots) > 1 else '') + sequence_item = self.append_sequence_item_to_queue( + filename_prefix, output_folder ) - sequence_item.setToolTip(output_folder) - sequence_item.setEditable(False) - - self.queue_model.appendRow([sequence_item]) for run_no, (filename, shot_globals) in enumerate(zip(run_filenames, shots)): - self.queued_shots[filename] = ( - labscript_file, - sequence_globals, - shot_globals, - sequence_attrs, - run_no, - len(shots), - send_to_BLACS, - BLACS_host, - send_to_runviewer, - ) - - item = QtGui.QStandardItem(os.path.basename(filename)) - item.setToolTip(filename) - item.setData(filename, self.QUEUE_ROLE_FULL_FILENAME) - item.setEditable(False) - - sequence_item.appendRow([item]) - - self.ui.treeView_queue.setExpanded(sequence_item.index(), True) + self.queued_shots[filename] = { + 'labscript_file': labscript_file, + 'sequence_globals': sequence_globals, + 'shot_globals': shot_globals, + 'sequence_attrs': sequence_attrs, + 'run_no': run_no, + 'n_runs': len(shots), + 'rep_no': 0, + 'send_to_BLACS': send_to_BLACS, + 'BLACS_host': BLACS_host, + 'send_to_runviewer': send_to_runviewer, + 'filename_prefix': filename_prefix, + 'output_folder': output_folder + } + + self.append_shot_item_to_queue(sequence_item, filename) logger.debug(run_filenames) # Tell the compilation queue to wake up: self.compilation_potentially_required.set() - - self.ui.pushButton_abort.setEnabled(True) + self.update_queue_tab_label() def send_to_BLACS(self, run_file, BLACS_hostname): From d14f4695503bada586d72380f286c22655ed3608 Mon Sep 17 00:00:00 2001 From: Chris Billington Date: Wed, 11 Dec 2019 16:25:26 -0500 Subject: [PATCH 3/7] compilation-queue: Most functionality imlplemented. Niceties yet to come, e.g. queue is simply cleared if there is a compilation error, that could be improved. Still need deletion of shots, reordering of shots, status labels. --- runmanager/__main__.py | 48 +++++++++--- runmanager/main.ui | 171 +++++++++++++++++++++++------------------ runmanager/remote.py | 7 ++ 3 files changed, 142 insertions(+), 84 deletions(-) diff --git a/runmanager/__main__.py b/runmanager/__main__.py index ac13065..b7411e6 100644 --- a/runmanager/__main__.py +++ b/runmanager/__main__.py @@ -1386,6 +1386,10 @@ def __init__(self): self.output_box = OutputBox(self.ui.verticalLayout_output_tab) + # An event to tell the compilation queue to check if the next queued shot (if + # any) can be compiled: + self.compilation_potentially_required = threading.Event() + # Add a 'pop-out' button to the output tab: output_tab_index = self.ui.tabWidget.indexOf(self.ui.tab_output) self.output_popout_button = TabToolButton(self.ui.tabWidget.parent()) @@ -1449,13 +1453,10 @@ def __init__(self): # The prospective number of shots resulting from compilation self.n_shots = None - # An event to tell the compilation queue to check if the next queued shot (if - # any) can be compiled: - self.compilation_potentially_required = threading.Event() - # Create data structures for the compilation queue and start the compilation # thread: self.queued_shots = {} + self.BLACS_shots_remaining_events = queue.Queue() self.queue_repeat_mode = self.REPEAT_ALL self.compile_queue_thread = threading.Thread(target=self.compile_loop) self.compile_queue_thread.daemon = True @@ -1660,6 +1661,8 @@ def connect_signals(self): # Queue tab: self.ui.queue_pause_button.toggled.connect(self.on_queue_paused_toggled) + self.ui.button_delay_compilation.toggled.connect(self.on_delay_compilation_toggled) + self.ui.spinBox_delayed_num_shots.valueChanged.connect(self.compilation_potentially_required.set) self.action_repeat_all.triggered.connect( lambda: self.set_queue_repeat_mode(self.REPEAT_ALL) ) @@ -3275,9 +3278,29 @@ def warning(message): def next_queued_shot(self): """Get the details of the next shot to be compiled, removing it from the queue""" - if not self.queue_model.rowCount() or self.ui.queue_pause_button.isChecked(): + BLACS_shots_remaining = None + while True: + # If multiple events, get only the most recent: + try: + BLACS_shots_remaining = self.BLACS_shots_remaining_events.get_nowait() + except queue.Empty: + break + if self.ui.button_delay_compilation.isChecked(): + if ( + BLACS_shots_remaining is None + or BLACS_shots_remaining > self.ui.spinBox_delayed_num_shots.value() + ): + # TODO: set a status saying "waiting for BLACS queue to reach target" or + # similar + return None, None + if not self.queue_model.rowCount(): + # TODO: Set a status saying "idle" + return None, None + if self.ui.queue_pause_button.isChecked(): + # TODO:Set a status label to "paused" return None, None elif self.compilation_aborted.is_set(): + # TODO: Pause the queue self.output_box.output('Compilation aborted.\n\n', red=True) return None, None self.ui.pushButton_abort.setEnabled(True) @@ -3335,7 +3358,6 @@ def compile_loop(self): # go on: run_file, shot_details = self.next_queued_shot() if run_file is None: - self.output_box.output('Ready.\n\n') break shot_globals = shot_details['shot_globals'] labscript_file = shot_details['labscript_file'] @@ -3377,9 +3399,9 @@ def compile_loop(self): if not success: self.compilation_aborted.set() # TODO: can probably think of something more sensible here, - # like prepending to the queue unless the user deletes the - # shot: - self.queue_model.clear() + # like prepending to the queue and pausing it, unless the + # user deletes the shot: + inmain(self.queue_model.clear) self.queued_shots.clear() continue self.check_repeat(run_file, shot_details) @@ -3592,6 +3614,10 @@ def on_queue_paused_toggled(self, paused): if not paused: self.compilation_potentially_required.set() + def on_delay_compilation_toggled(self, delay): + self.ui.delay_compilation_options.setVisible(delay) + self.compilation_potentially_required.set() + def set_queue_repeat_mode(self, mode): if mode == self.REPEAT_ALL: self.ui.queue_repeat_button.setIcon(QtGui.QIcon(self.ICON_REPEAT)) @@ -3914,6 +3940,10 @@ def handle_is_output_folder_default(self): def handle_reset_shot_output_folder(self): app.on_reset_shot_output_folder_clicked(None) + def advise_BLACS_shots_remaining(self, value): + app.BLACS_shots_remaining_events.put(value) + app.compilation_potentially_required.set() + def handler(self, request_data): cmd, args, kwargs = request_data if cmd == 'hello': diff --git a/runmanager/main.ui b/runmanager/main.ui index 98d38e0..eebdcf7 100644 --- a/runmanager/main.ui +++ b/runmanager/main.ui @@ -689,21 +689,11 @@ QToolButton:hover { - - - - Qt::Horizontal - - - - 40 - 20 - - - - + + <html><head/><body><p>Delay shot compilation until BLACS has the given number of shots remaining. In this case, the values of the globals listed as &quot;dynamic globals&quot; will be determined from their values in runmanager at the time the shot is compiled (instead of being locked in when 'engage' was clicked). This allows feedback from analysis routines or FunctionRunner devices (or any other code) that may call the runmanager.remote.set_globals() function to influence the values of globals in shots yet to be compiled. This requires the BLACS delay_compilation plugin to be enabled, to send events to runmanager advising it of how many shots BLACS has remaining.</p></body></html> + QFrame::StyledPanel @@ -712,64 +702,26 @@ QToolButton:hover { - - - Ahead-of-time compilation - - - true - - - - - - - Just-in-time compilation + + + 0 - - - - - + - Dynamic globals: (comma-separated list) + delay compilation - - - - - - - 300 - 0 - - - - - - - - - - - - Compile next when BLACS has + + + :/qtutils/fugue/alarm-clock--arrow.png:/qtutils/fugue/alarm-clock--arrow.png - - - - - - - - - shot(s) left + + true - + Qt::Horizontal @@ -783,22 +735,91 @@ QToolButton:hover { + + + + true + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + 0 + + + + + until BLACS has + + + + + + + + + + shot(s) left + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + true + + + Dynamic globals: (comma-separated list) + + + + + + + + 300 + 0 + + + + + + + + + - - - - Qt::Horizontal - - - - 40 - 20 - - - - diff --git a/runmanager/remote.py b/runmanager/remote.py index 42018f4..8811277 100644 --- a/runmanager/remote.py +++ b/runmanager/remote.py @@ -113,6 +113,13 @@ def reset_shot_output_folder(self): """Reset the shot output folder to the default path""" return self.request('reset_shot_output_folder') + def advise_BLACS_shots_remaining(self, value): + """Tell runmanager how many shots BLACS has left before its queue is empty - + that is, the number of shots in its queue, plus one if a shot has been removed + from the queue but has not finished running. This is so that if runmanager's + compilation queue is configured for just-in-time compilation, it can compile and + submit the next shot when BLACS has a certain number of shots left.""" + return self.request('advise_BLACS_queue_size', value) _default_client = Client() From 35f874bd292d4bc5847f6f5dcd90ede43432778d Mon Sep 17 00:00:00 2001 From: Chris Billington Date: Fri, 13 Dec 2019 16:06:48 -0500 Subject: [PATCH 4/7] compilation-queue: Make runmanager remote client take a default_timeout instantiation argument --- runmanager/__main__.py | 14 ++++++++++++-- runmanager/remote.py | 6 ++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/runmanager/__main__.py b/runmanager/__main__.py index b7411e6..fd871dc 100644 --- a/runmanager/__main__.py +++ b/runmanager/__main__.py @@ -1571,7 +1571,11 @@ def setup_queue_tab(self): self.action_repeat_last = QtWidgets.QAction( QtGui.QIcon(self.ICON_REPEAT_LAST), 'Repeat last', self.ui ) - + + # Hide this initially, TODO remove this line once we are restoring from settings + # whether delayed compilation is enabled or not. + self.ui.delay_compilation_options.setVisible(False) + self.repeat_mode_menu.addAction(self.action_repeat_all) self.repeat_mode_menu.addAction(self.action_repeat_last) self.ui.repeat_mode_select_button.setMenu(self.repeat_mode_menu) @@ -3412,6 +3416,11 @@ def compile_loop(self): except Exception: self.output_box.output(traceback.format_exc() + '\n', red=True) self.compilation_aborted.set() + # TODO: can probably think of something more sensible here, + # like prepending to the queue and pausing it, unless the + # user deletes the shot: + inmain(self.queue_model.clear) + self.queued_shots.clear() inmain(self.ui.pushButton_abort.setEnabled, False) self.compilation_aborted.clear() except Exception: @@ -3940,7 +3949,8 @@ def handle_is_output_folder_default(self): def handle_reset_shot_output_folder(self): app.on_reset_shot_output_folder_clicked(None) - def advise_BLACS_shots_remaining(self, value): + def handle_advise_BLACS_shots_remaining(self, value): + print("BLACS said how many shots it has:", value) app.BLACS_shots_remaining_events.put(value) app.compilation_potentially_required.set() diff --git a/runmanager/remote.py b/runmanager/remote.py index 8811277..8775e71 100644 --- a/runmanager/remote.py +++ b/runmanager/remote.py @@ -22,8 +22,9 @@ def __init__(self, host=None, port=None, timeout=None): self.timeout = timeout def request(self, command, *args, **kwargs): + timeout = kwargs.pop('timeout', self.timeout) return self.get( - self.port, self.host, data=[command, args, kwargs], timeout=self.timeout + self.port, self.host, data=[command, args, kwargs], timeout=timeout ) def say_hello(self): @@ -119,7 +120,7 @@ def advise_BLACS_shots_remaining(self, value): from the queue but has not finished running. This is so that if runmanager's compilation queue is configured for just-in-time compilation, it can compile and submit the next shot when BLACS has a certain number of shots left.""" - return self.request('advise_BLACS_queue_size', value) + return self.request('advise_BLACS_shots_remaining', value) _default_client = Client() @@ -145,6 +146,7 @@ def advise_BLACS_shots_remaining(self, value): error_in_globals = _default_client.error_in_globals is_output_folder_default = _default_client.is_output_folder_default reset_shot_output_folder = _default_client.reset_shot_output_folder +advise_BLACS_shots_remaining = _default_client.advise_BLACS_shots_remaining if __name__ == '__main__': # Test From b6f328565badf4c33bfad96a1366d5c09e1ce35b Mon Sep 17 00:00:00 2001 From: chrisjbillington Date: Fri, 7 Jan 2022 16:27:31 +1100 Subject: [PATCH 5/7] Update path to qtutils in .ui file Is now appropriate for if runmanager and qtutils are cloned into the same directory. Is a relative path, so finally this might be sensible for all developers! --- runmanager/main.ui | 80 +++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/runmanager/main.ui b/runmanager/main.ui index eebdcf7..4ccdaf1 100644 --- a/runmanager/main.ui +++ b/runmanager/main.ui @@ -14,7 +14,7 @@ runmanager - the labscript suite - + :/qtutils/custom/runmanager.png:/qtutils/custom/runmanager.png @@ -141,7 +141,7 @@ QToolButton:hover:checked { Shuffle - + :/qtutils/fugue/arrow-switch.png:/qtutils/fugue/arrow-switch.png @@ -165,7 +165,7 @@ QToolButton:hover:checked { subprocess - + :/qtutils/fugue/arrow-circle-135-left.png:/qtutils/fugue/arrow-circle-135-left.png @@ -186,7 +186,7 @@ subprocess Abort - + :/qtutils/fugue/cross-octagon.png:/qtutils/fugue/cross-octagon.png @@ -231,7 +231,7 @@ QPushButton:hover { Engage - + :/qtutils/fugue/control.png:/qtutils/fugue/control.png @@ -328,7 +328,7 @@ QPushButton:hover { ... - + :/qtutils/fugue/document--pencil.png:/qtutils/fugue/document--pencil.png @@ -431,7 +431,7 @@ QToolButton:hover { ... - + :/qtutils/custom/python-document.png:/qtutils/custom/python-document.png @@ -451,7 +451,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-turn-180-left.png:/qtutils/fugue/arrow-turn-180-left.png @@ -520,7 +520,7 @@ QToolButton:hover { - :/qtutils/fugue/exclamation-white.png + :/qtutils/fugue/exclamation-white.png @@ -560,7 +560,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/folder-horizontal-open.png:/qtutils/fugue/folder-horizontal-open.png @@ -634,7 +634,7 @@ QToolButton:hover { - + :/qtutils/fugue/terminal.png:/qtutils/fugue/terminal.png @@ -666,7 +666,7 @@ QToolButton:hover { - + :/qtutils/fugue/edit-list-order.png:/qtutils/fugue/edit-list-order.png @@ -712,7 +712,7 @@ QToolButton:hover { delay compilation - + :/qtutils/fugue/alarm-clock--arrow.png:/qtutils/fugue/alarm-clock--arrow.png @@ -881,7 +881,7 @@ QToolButton:hover { Pause - + :/qtutils/fugue/control-pause.png:/qtutils/fugue/control-pause.png @@ -912,7 +912,7 @@ QToolButton:hover { Repeat - + :/qtutils/fugue/arrow-repeat.png:/qtutils/fugue/arrow-repeat.png @@ -929,7 +929,7 @@ QToolButton:hover { - + :/qtutils/fugue/control-270.png:/qtutils/fugue/control-270.png @@ -993,7 +993,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/minus.png:/qtutils/fugue/minus.png @@ -1020,7 +1020,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-stop-090.png:/qtutils/fugue/arrow-stop-090.png @@ -1034,7 +1034,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-090.png:/qtutils/fugue/arrow-090.png @@ -1048,7 +1048,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-270.png:/qtutils/fugue/arrow-270.png @@ -1062,7 +1062,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-stop-270.png:/qtutils/fugue/arrow-stop-270.png @@ -1103,7 +1103,7 @@ QToolButton:hover { - + :/qtutils/custom/outer.png:/qtutils/custom/outer.png @@ -1161,7 +1161,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-stop-090.png:/qtutils/fugue/arrow-stop-090.png @@ -1175,7 +1175,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-090.png:/qtutils/fugue/arrow-090.png @@ -1189,7 +1189,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-270.png:/qtutils/fugue/arrow-270.png @@ -1203,7 +1203,7 @@ QToolButton:hover { ... - + :/qtutils/fugue/arrow-stop-270.png:/qtutils/fugue/arrow-stop-270.png @@ -1236,8 +1236,8 @@ QToolButton:hover { 0 0 - 941 - 521 + 70 + 70 @@ -1280,7 +1280,7 @@ QToolButton:hover { - + :/qtutils/fugue/tables-stacks.png:/qtutils/fugue/tables-stacks.png @@ -1334,7 +1334,7 @@ QToolButton:hover { Open globals file - + :/qtutils/fugue/folder-open-table.png:/qtutils/fugue/folder-open-table.png @@ -1351,7 +1351,7 @@ QToolButton:hover { New globals file - + :/qtutils/fugue/table--plus.png:/qtutils/fugue/table--plus.png @@ -1368,7 +1368,7 @@ QToolButton:hover { Diff globals file - + :/qtutils/fugue/tables.png:/qtutils/fugue/tables.png @@ -1401,8 +1401,8 @@ QToolButton:hover { 0 0 - 970 - 490 + 86 + 70 @@ -1461,7 +1461,7 @@ QToolButton:hover { - + :/qtutils/fugue/folder-open.png:/qtutils/fugue/folder-open.png @@ -1476,7 +1476,7 @@ QToolButton:hover { false - + :/qtutils/fugue/disk--plus.png:/qtutils/fugue/disk--plus.png @@ -1488,7 +1488,7 @@ QToolButton:hover { - + :/qtutils/fugue/cross-button.png:/qtutils/fugue/cross-button.png @@ -1500,7 +1500,7 @@ QToolButton:hover { - + :/qtutils/fugue/disk.png:/qtutils/fugue/disk.png @@ -1515,7 +1515,7 @@ QToolButton:hover { false - + :/qtutils/fugue/arrow-curve-180-left.png:/qtutils/fugue/arrow-curve-180-left.png @@ -1562,7 +1562,7 @@ QToolButton:hover { scrollArea_2 - + From 5899d81fc1b17a8036d596bb14e0c7c4f7d2b2c6 Mon Sep 17 00:00:00 2001 From: chrisjbillington Date: Fri, 7 Jan 2022 18:20:58 +1100 Subject: [PATCH 6/7] Restore changes to main.ui from master branch This compilation-queue branch was rebased onto master, but merging the .ui file was nontrivial, so we reimplement the changes in the master branch here to keep it consistent. --- runmanager/main.ui | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runmanager/main.ui b/runmanager/main.ui index 4ccdaf1..01dcabb 100644 --- a/runmanager/main.ui +++ b/runmanager/main.ui @@ -207,8 +207,8 @@ subprocess QPushButton { - background-color: rgb(238,96,96,192); - border: 1px solid rgb(238,96,96,128); + background-color: rgba(238,96,96,192); + border: 1px solid rgba(238,96,96,128); border-radius: 3px; padding: 4px; } @@ -329,7 +329,7 @@ QPushButton:hover { - :/qtutils/fugue/document--pencil.png:/qtutils/fugue/document--pencil.png + :/qtutils/custom/python-document.png:/qtutils/custom/python-document.png @@ -432,7 +432,7 @@ QToolButton:hover { - :/qtutils/custom/python-document.png:/qtutils/custom/python-document.png + :/qtutils/fugue/folder-open-document-text.png:/qtutils/fugue/folder-open-document-text.png From cc5019c41d84b7ac7e3b8eac2f739df95a70e14c Mon Sep 17 00:00:00 2001 From: chrisjbillington Date: Fri, 7 Jan 2022 18:49:08 +1100 Subject: [PATCH 7/7] Edit labels and tooltips Yet to do is add a tooltip for how to make BLACS tell runmanager how many shots it has remaining. --- runmanager/main.ui | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runmanager/main.ui b/runmanager/main.ui index 01dcabb..93e67f2 100644 --- a/runmanager/main.ui +++ b/runmanager/main.ui @@ -692,7 +692,7 @@ QToolButton:hover { - <html><head/><body><p>Delay shot compilation until BLACS has the given number of shots remaining. In this case, the values of the globals listed as &quot;dynamic globals&quot; will be determined from their values in runmanager at the time the shot is compiled (instead of being locked in when 'engage' was clicked). This allows feedback from analysis routines or FunctionRunner devices (or any other code) that may call the runmanager.remote.set_globals() function to influence the values of globals in shots yet to be compiled. This requires the BLACS delay_compilation plugin to be enabled, to send events to runmanager advising it of how many shots BLACS has remaining.</p></body></html> + <html><head/><body><p>Delay shot compilation until BLACS has the given number of shots remaining. In this case, the values of the globals listed as &quot;dynamic globals&quot; will be determined from their values in runmanager at the time the shot is compiled (instead of being locked in when 'engage' was clicked). This allows feedback from analysis routines or FunctionRunner devices (or any other code) that may call the runmanager.remote.set_globals() function to influence the values of globals in shots yet to be compiled.</p></body></html> QFrame::StyledPanel @@ -709,7 +709,7 @@ QToolButton:hover { - delay compilation + Delay compilation @@ -761,7 +761,7 @@ QToolButton:hover { - until BLACS has + Until BLACS has