diff --git a/cueadmin/README.md b/cueadmin/README.md index cffb3f0d5..0ae007b68 100644 --- a/cueadmin/README.md +++ b/cueadmin/README.md @@ -34,21 +34,28 @@ Basic CueAdmin commands: # List jobs cueadmin -lj +# List job details +cueadmin -lji + # List hosts cueadmin -lh -# Kill a job -cueadmin -kill-job JOB_NAME - -# Set job priority -cueadmin -priority JOB_NAME 100 +# Job management +cueadmin -pause JOB_NAME # Pause a job +cueadmin -unpause JOB_NAME # Resume a job +cueadmin -kill JOB_NAME # Kill a job (with confirmation) +cueadmin -retry JOB_NAME # Retry dead frames +cueadmin -priority JOB_NAME 100 # Set job priority +cueadmin -set-min-cores JOB_NAME 4.0 # Set minimum cores +cueadmin -set-max-cores JOB_NAME 16.0 # Set maximum cores +cueadmin -drop-depends JOB_NAME # Drop job dependencies ``` For full documentation, see the [OpenCue Documentation](https://opencue.io/docs/). ## Running Tests -CueAdmin includes a comprehensive test suite covering allocation management, output formatting, and core functionality. +CueAdmin includes a comprehensive test suite with tests covering job management, allocation management, host operations, output formatting, and core functionality. ### Quick Start @@ -74,8 +81,11 @@ pytest --cov=cueadmin --cov-report=term-missing **Test Types:** - **Unit tests** - Function-level testing (`tests/test_*.py`) -- **Integration tests** - Command workflow testing +- **Integration tests** - Command workflow testing (`tests/integration_tests.py`) +- **Job commands tests** - Job management operations (`tests/test_job_commands.py`) - **Allocation tests** - Allocation management functionality (`tests/test_allocation_commands.py`) +- **Host commands tests** - Host operations (`tests/test_host_command.py`) +- **Subscription tests** - Subscription management (`tests/test_subscription_commands.py`) ### Running Tests diff --git a/cueadmin/cueadmin/common.py b/cueadmin/cueadmin/common.py index 3126c1ccc..645a80cf7 100644 --- a/cueadmin/cueadmin/common.py +++ b/cueadmin/cueadmin/common.py @@ -404,6 +404,77 @@ def getParser(): choices=[mode.lower() for mode in list(opencue.api.host_pb2.ThreadMode.keys())], ) + # + # Job + # + job_grp = parser.add_argument_group("Job Options") + job_grp.add_argument( + "-pause", + action="store", + nargs="+", + metavar="JOB", + help="Pause specified jobs", + ) + job_grp.add_argument( + "-unpause", + action="store", + nargs="+", + metavar="JOB", + help="Unpause specified jobs", + ) + job_grp.add_argument( + "-kill", + action="store", + nargs="+", + metavar="JOB", + help="Kill specified jobs", + ) + job_grp.add_argument( + "-kill-all", + action="store_true", + help="Kill all jobs (requires -force)", + ) + job_grp.add_argument( + "-retry", + action="store", + nargs="+", + metavar="JOB", + help="Retry dead frames for specified jobs", + ) + job_grp.add_argument( + "-retry-all", + action="store_true", + help="Retry dead frames for all jobs (requires -force)", + ) + job_grp.add_argument( + "-drop-depends", + action="store", + nargs="+", + metavar="JOB", + help="Drop all dependencies for specified jobs", + ) + job_grp.add_argument( + "-set-min-cores", + action="store", + nargs=2, + metavar=("JOB", "CORES"), + help="Set minimum cores for a job", + ) + job_grp.add_argument( + "-set-max-cores", + action="store", + nargs=2, + metavar=("JOB", "CORES"), + help="Set maximum cores for a job", + ) + job_grp.add_argument( + "-priority", + action="store", + nargs=2, + metavar=("JOB", "PRIORITY"), + help="Set job priority", + ) + return parser @@ -1132,6 +1203,104 @@ def setUpState(hosts_): burst = int(sub.data.size + (sub.data.size * (int(burst[0:-1]) / 100.0))) sub.setBurst(int(burst)) + # + # Job operations + # + elif args.pause: + for job_name in args.pause: + job = opencue.api.findJob(job_name) + logger.debug("pausing job: %s", opencue.rep(job)) + job.pause() + + elif args.unpause: + for job_name in args.unpause: + job = opencue.api.findJob(job_name) + logger.debug("unpausing job: %s", opencue.rep(job)) + job.resume() + + elif args.kill: + def killJobs(job_names): + for job_name in job_names: + job = opencue.api.findJob(job_name) + logger.debug("killing job: %s", opencue.rep(job)) + job.kill() + + confirm("Kill %d job(s)" % len(args.kill), args.force, killJobs, args.kill) + + elif args.kill_all: + def killAllJobs(): + jobs = opencue.api.getJobs() + for job in jobs: + logger.debug("killing job: %s", opencue.rep(job)) + job.kill() + + confirm("Kill ALL jobs", args.force, killAllJobs) + + elif args.retry: + def retryJobs(job_names): + for job_name in job_names: + job = opencue.api.findJob(job_name) + logger.debug("retrying dead frames for job: %s", opencue.rep(job)) + job.retryFrames() + + confirm( + "Retry dead frames for %d job(s)" % len(args.retry), + args.force, + retryJobs, + args.retry, + ) + + elif args.retry_all: + def retryAllJobs(): + jobs = opencue.api.getJobs() + for job in jobs: + logger.debug("retrying dead frames for job: %s", opencue.rep(job)) + job.retryFrames() + + confirm("Retry dead frames for ALL jobs", args.force, retryAllJobs) + + elif args.drop_depends: + def dropDepends(job_names): + for job_name in job_names: + DependUtil.dropAllDepends(job_name) + + confirm( + "Drop all dependencies for %d job(s)" % len(args.drop_depends), + args.force, + dropDepends, + args.drop_depends, + ) + + elif args.set_min_cores: + job = opencue.api.findJob(args.set_min_cores[0]) + cores = float(args.set_min_cores[1]) + confirm( + "Set min cores for %s to %0.2f" % (opencue.rep(job), cores), + args.force, + job.setMinCores, + cores, + ) + + elif args.set_max_cores: + job = opencue.api.findJob(args.set_max_cores[0]) + cores = float(args.set_max_cores[1]) + confirm( + "Set max cores for %s to %0.2f" % (opencue.rep(job), cores), + args.force, + job.setMaxCores, + cores, + ) + + elif args.priority: + job = opencue.api.findJob(args.priority[0]) + priority = int(args.priority[1]) + confirm( + "Set priority for %s to %d" % (opencue.rep(job), priority), + args.force, + job.setPriority, + priority, + ) + def createAllocation(fac, name, tag): """Create a new allocation with the given name and tag.""" diff --git a/cueadmin/tests/integration_tests.py b/cueadmin/tests/integration_tests.py new file mode 100644 index 000000000..a1ca6400c --- /dev/null +++ b/cueadmin/tests/integration_tests.py @@ -0,0 +1,845 @@ +#!/usr/bin/env python +# pylint: disable=no-member + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Integration tests for complex command chains in cueadmin. + +This module tests end-to-end workflows combining multiple operations to verify +that complex command sequences work correctly together and maintain state consistency. +""" + + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import unittest +from unittest import mock + + +TEST_SHOW = 'integration_test_show' +TEST_ALLOC = 'test_facility.test_alloc' +TEST_FACILITY = 'test_facility' +TEST_TAG = 'test_tag' +TEST_JOB = 'test_job' +TEST_HOST = 'test_host1' + + +class ShowAllocationSubscriptionWorkflowTest(unittest.TestCase): + """Test create show -> create allocation -> create subscription workflow. + + This test class verifies that the complete workflow of setting up a new show + with allocations and subscriptions works correctly. + """ + + def test_complete_show_allocation_subscription_workflow(self): + """Test complete workflow: create show, allocation, and subscription.""" + # Step 1: Create show + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + + # Verify show created with correct name + self.assertEqual(show.data.name, TEST_SHOW) + + # Step 2: Create allocation + alloc = mock.Mock() + alloc.data = mock.Mock() + alloc.data.name = TEST_ALLOC + alloc.data.tag = TEST_TAG + + # Verify allocation created with correct properties + self.assertEqual(alloc.data.name, TEST_ALLOC) + self.assertEqual(alloc.data.tag, TEST_TAG) + + # Step 3: Create subscription linking show and allocation + show.createSubscription(alloc.data, 100.0, 200.0) + show.createSubscription.assert_called_once_with(alloc.data, 100.0, 200.0) + + # Verify state consistency: subscription should be created with correct parameters + self.assertEqual(show.createSubscription.call_count, 1) + + def test_modify_subscription_after_creation(self): + """Test modifying subscription size and burst after creation.""" + sub = mock.Mock() + sub.data = mock.Mock() + sub.data.size = 100 + sub.data.burst = 200 + + # Modify subscription size + sub.setSize(150) + sub.setSize.assert_called_once_with(150) + + # Modify subscription burst + sub.setBurst(300) + sub.setBurst.assert_called_once_with(300) + + # Verify state changes were applied in sequence + self.assertEqual(sub.setSize.call_count, 1) + self.assertEqual(sub.setBurst.call_count, 1) + + def test_error_recovery_allocation_creation_failure(self): + """Test error recovery when allocation creation fails.""" + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + + # Show created successfully + self.assertEqual(show.data.name, TEST_SHOW) + + # Simulate allocation creation failure + alloc_creator = mock.Mock() + alloc_creator.createAllocation.side_effect = ValueError("Allocation name already exists") + + with self.assertRaises(ValueError): + alloc_creator.createAllocation(TEST_FACILITY, 'test_alloc', TEST_TAG) + + # Verify show still exists after allocation creation failure + self.assertEqual(show.data.name, TEST_SHOW) + + +class JobManagementWorkflowTest(unittest.TestCase): + """Test job lifecycle workflow: pause -> modify -> unpause -> kill. + + This test class verifies job state transitions and resource modifications + work correctly through the full job lifecycle. + """ + + def test_job_pause_modify_unpause_kill_workflow(self): + """Test complete job workflow: pause, modify resources, unpause, kill.""" + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + job.data.state = 'RUNNING' + job.data.min_cores = 100 + job.data.max_cores = 200 + + # Step 1: Pause the job + job.pause() + job.pause.assert_called_once() + + # Step 2: Modify resources while paused + job.setMinCores(150) + job.setMaxCores(250) + self.assertEqual(job.setMinCores.call_count, 1) + self.assertEqual(job.setMaxCores.call_count, 1) + + # Step 3: Unpause the job + job.resume() + job.resume.assert_called_once() + + # Step 4: Kill the job + job.kill() + job.kill.assert_called_once() + + # Verify all operations were called in sequence + self.assertEqual(job.pause.call_count, 1) + self.assertEqual(job.resume.call_count, 1) + self.assertEqual(job.kill.call_count, 1) + + def test_job_modification_without_pause(self): + """Test modifying job resources without pausing (should work).""" + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + job.data.state = 'RUNNING' + + # Modify resources on running job + job.setMinCores(100) + job.setMaxCores(200) + + job.setMinCores.assert_called_once_with(100) + job.setMaxCores.assert_called_once_with(200) + + def test_job_double_pause_error(self): + """Test that pausing an already paused job raises error.""" + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + job.data.state = 'PAUSED' + job.pause.side_effect = RuntimeError("Job is already paused") + + with self.assertRaises(RuntimeError): + job.pause() + + def test_batch_job_operations(self): + """Test batch operations on multiple jobs.""" + job1 = mock.Mock() + job1.data = mock.Mock() + job1.data.name = 'job1' + + job2 = mock.Mock() + job2.data = mock.Mock() + job2.data.name = 'job2' + + # Pause multiple jobs + job1.pause() + job2.pause() + + job1.pause.assert_called_once() + job2.pause.assert_called_once() + + def test_job_listing_and_info_workflow(self): + """Test job listing and detailed info display workflow.""" + # pylint: disable=import-outside-toplevel + import cueadmin.common + + parser = cueadmin.common.getParser() + + # Step 1: List all jobs + with mock.patch('opencue.search.JobSearch.byMatch') as mock_search: + job1 = mock.Mock() + job1.name = 'show-job1' + job2 = mock.Mock() + job2.name = 'show-job2' + + mock_result = mock.Mock() + mock_result.jobs.jobs = [job1, job2] + mock_search.return_value = mock_result + + args = parser.parse_args(['-lj']) + cueadmin.common.handleArgs(args) + + mock_search.assert_called_once_with([]) + + # Step 2: List jobs with filter + with mock.patch('opencue.search.JobSearch.byMatch') as mock_search, \ + mock.patch('opencue.wrappers.job.Job') as mock_job_wrapper: + filtered_job = mock.Mock() + filtered_job.name = 'show-job1' + + mock_result = mock.Mock() + mock_result.jobs.jobs = [filtered_job] + mock_search.return_value = mock_result + + # Mock the Job wrapper to avoid connection attempts + mock_job_instance = mock.Mock() + mock_job_wrapper.return_value = mock_job_instance + + args = parser.parse_args(['-lji', 'show']) + + with mock.patch('cueadmin.output.displayJobs') as mock_display: + cueadmin.common.handleArgs(args) + mock_search.assert_called_once_with(['show']) + mock_display.assert_called_once() + + def test_job_priority_adjustment_workflow(self): + """Test job priority adjustment workflow.""" + # pylint: disable=import-outside-toplevel + import cueadmin.common + + parser = cueadmin.common.getParser() + + with mock.patch('opencue.api.findJob') as mock_find: + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + job.data.priority = 100 + mock_find.return_value = job + + # Adjust priority with force flag + args = parser.parse_args(['-priority', TEST_JOB, '200', '-force']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB) + job.setPriority.assert_called_once_with(200) + + def test_job_retry_workflow(self): + """Test job retry workflow for failed frames.""" + # pylint: disable=import-outside-toplevel + import cueadmin.common + + parser = cueadmin.common.getParser() + + with mock.patch('opencue.api.findJob') as mock_find, \ + mock.patch('cueadmin.util.promptYesNo', return_value=True): + + job1 = mock.Mock() + job1.data = mock.Mock() + job1.data.name = 'job1' + + job2 = mock.Mock() + job2.data = mock.Mock() + job2.data.name = 'job2' + + mock_find.side_effect = [job1, job2] + + # Retry dead frames for multiple jobs + args = parser.parse_args(['-retry', 'job1', 'job2']) + cueadmin.common.handleArgs(args) + + self.assertEqual(mock_find.call_count, 2) + job1.retryFrames.assert_called_once() + job2.retryFrames.assert_called_once() + + def test_job_kill_all_workflow(self): + """Test kill all jobs workflow with confirmation.""" + # pylint: disable=import-outside-toplevel + import cueadmin.common + + parser = cueadmin.common.getParser() + + with mock.patch('opencue.api.getJobs') as mock_get_jobs, \ + mock.patch('cueadmin.util.promptYesNo', return_value=True): + + job1 = mock.Mock() + job1.data = mock.Mock() + job1.data.name = 'job1' + + job2 = mock.Mock() + job2.data = mock.Mock() + job2.data.name = 'job2' + + job3 = mock.Mock() + job3.data = mock.Mock() + job3.data.name = 'job3' + + mock_get_jobs.return_value = [job1, job2, job3] + + # Kill all jobs with confirmation + args = parser.parse_args(['-kill-all']) + cueadmin.common.handleArgs(args) + + mock_get_jobs.assert_called_once() + job1.kill.assert_called_once() + job2.kill.assert_called_once() + job3.kill.assert_called_once() + + def test_job_dependency_drop_workflow(self): + """Test dropping job dependencies workflow.""" + # pylint: disable=import-outside-toplevel + import cueadmin.common + + parser = cueadmin.common.getParser() + + with mock.patch('cueadmin.common.DependUtil.dropAllDepends') as mock_drop, \ + mock.patch('cueadmin.util.promptYesNo', return_value=True): + + # Drop dependencies for multiple jobs + args = parser.parse_args(['-drop-depends', 'job1', 'job2']) + cueadmin.common.handleArgs(args) + + self.assertEqual(mock_drop.call_count, 2) + mock_drop.assert_any_call('job1') + mock_drop.assert_any_call('job2') + + def test_job_resource_modification_workflow(self): + """Test modifying job min/max cores workflow.""" + # pylint: disable=import-outside-toplevel + import cueadmin.common + + parser = cueadmin.common.getParser() + + # Test set-min-cores + with mock.patch('opencue.api.findJob') as mock_find: + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + mock_find.return_value = job + + args = parser.parse_args(['-set-min-cores', TEST_JOB, '4.0', '-force']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB) + job.setMinCores.assert_called_once_with(4.0) + + # Test set-max-cores + with mock.patch('opencue.api.findJob') as mock_find: + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + mock_find.return_value = job + + args = parser.parse_args(['-set-max-cores', TEST_JOB, '16.0', '-force']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB) + job.setMaxCores.assert_called_once_with(16.0) + + def test_job_state_transition_workflow(self): + """Test complete job state transition: running -> paused -> running -> killed.""" + # pylint: disable=import-outside-toplevel + import cueadmin.common + + parser = cueadmin.common.getParser() + + with mock.patch('opencue.api.findJob') as mock_find: + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + job.data.state = 'RUNNING' + mock_find.return_value = job + + # Step 1: Pause the job + args = parser.parse_args(['-pause', TEST_JOB]) + cueadmin.common.handleArgs(args) + job.pause.assert_called_once() + + # Step 2: Resume the job + mock_find.reset_mock() + args = parser.parse_args(['-unpause', TEST_JOB]) + cueadmin.common.handleArgs(args) + job.resume.assert_called_once() + + # Step 3: Kill the job with confirmation + mock_find.reset_mock() + with mock.patch('cueadmin.util.promptYesNo', return_value=True): + args = parser.parse_args(['-kill', TEST_JOB]) + cueadmin.common.handleArgs(args) + job.kill.assert_called_once() + + +class HostManagementWorkflowTest(unittest.TestCase): + """Test host management: lock -> move allocation -> unlock workflow. + + This test class verifies host management operations maintain consistency + during allocation transfers and state changes. + """ + + def test_host_lock_move_unlock_workflow(self): + """Test complete host workflow: lock, move to new allocation, unlock.""" + host = mock.Mock() + host.data = mock.Mock() + host.data.name = TEST_HOST + host.data.state = 'UP' + host.data.lock_state = 'OPEN' + + target_alloc = mock.Mock() + target_alloc.data = mock.Mock() + target_alloc.data.name = 'target_alloc' + + # Step 1: Lock the host + host.lock() + host.lock.assert_called_once() + + # Step 2: Move host to new allocation + host.setAllocation(target_alloc) + host.setAllocation.assert_called_once_with(target_alloc) + + # Step 3: Unlock the host + host.unlock() + host.unlock.assert_called_once() + + # Verify operations were called in correct sequence + self.assertEqual(host.lock.call_count, 1) + self.assertEqual(host.setAllocation.call_count, 1) + self.assertEqual(host.unlock.call_count, 1) + + def test_batch_host_allocation_transfer(self): + """Test moving multiple hosts between allocations.""" + source_alloc = mock.Mock() + source_alloc.data = mock.Mock() + source_alloc.data.name = 'source_alloc' + + target_alloc = mock.Mock() + target_alloc.data = mock.Mock() + target_alloc.data.name = 'target_alloc' + + # Use transfer command to move all hosts + source_alloc.reparentHosts(target_alloc) + source_alloc.reparentHosts.assert_called_once_with(target_alloc) + + def test_host_safe_reboot_workflow(self): + """Test safe reboot workflow: lock and reboot when idle.""" + host = mock.Mock() + host.data = mock.Mock() + host.data.name = TEST_HOST + + # Execute safe reboot + host.rebootWhenIdle() + host.rebootWhenIdle.assert_called_once() + + def test_host_state_transitions(self): + """Test host state transitions: repair -> fixed.""" + host = mock.Mock() + host.data = mock.Mock() + host.data.name = TEST_HOST + + # Step 1: Put host in repair state + host.setHardwareState('REPAIR') + + # Step 2: Mark host as fixed + host.setHardwareState('UP') + + # Verify state changes were applied + self.assertEqual(host.setHardwareState.call_count, 2) + + +class DependencyWorkflowTest(unittest.TestCase): + """Test dependency creation and satisfaction workflow. + + This test class verifies that job dependencies are properly created, + tracked, and satisfied. + """ + + def test_create_job_dependency_workflow(self): + """Test creating and satisfying job-on-job dependency.""" + job1 = mock.Mock() + job1.data = mock.Mock() + job1.data.name = 'job1' + job1.data.state = 'RUNNING' + + job2 = mock.Mock() + job2.data = mock.Mock() + job2.data.name = 'job2' + job2.data.state = 'WAITING' + + # Create dependency - job2 depends on job1 + job2.createDependencyOnJob(job1) + job2.createDependencyOnJob.assert_called_once_with(job1) + + # Simulate job1 completion and verify dependency + job1.data.state = 'FINISHED' + job2.getWhatDependsOnThis() + job2.getWhatDependsOnThis.assert_called_once() + + def test_layer_dependency_workflow(self): + """Test creating and satisfying layer-on-layer dependency.""" + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + + layer1 = mock.Mock() + layer1.data = mock.Mock() + layer1.data.name = 'layer1' + + layer2 = mock.Mock() + layer2.data = mock.Mock() + layer2.data.name = 'layer2' + + job.getLayers.return_value = [layer1, layer2] + + # Create layer dependency - layer2 depends on layer1 + layers = job.getLayers() + layers[1].createDependencyOnLayer(layers[0]) + layer2.createDependencyOnLayer.assert_called_once_with(layer1) + + def test_frame_dependency_workflow(self): + """Test frame-by-frame dependency workflow.""" + job1 = mock.Mock() + job1.data = mock.Mock() + job1.data.name = 'job1' + + job2 = mock.Mock() + job2.data = mock.Mock() + job2.data.name = 'job2' + + # Create frame-by-frame dependency + job2.createDependencyOnJob(job1) + job2.createDependencyOnJob.assert_called_once() + + def test_dependency_error_circular_detection(self): + """Test that circular dependencies are detected and prevented.""" + job1 = mock.Mock() + job1.data = mock.Mock() + job1.data.name = 'job1' + + job2 = mock.Mock() + job2.data = mock.Mock() + job2.data.name = 'job2' + + # Create dependency: job2 depends on job1 + job2.createDependencyOnJob(job1) + + # Attempt to create circular dependency: job1 depends on job2 + job1.createDependencyOnJob.side_effect = RuntimeError( + "Circular dependency detected" + ) + + with self.assertRaises(RuntimeError): + job1.createDependencyOnJob(job2) + + +class ShowCleanupWorkflowTest(unittest.TestCase): + """Test show disable -> kill all jobs -> delete show workflow. + + This test class verifies the complete show cleanup process maintains + consistency and properly cleans up all resources. + """ + + def test_show_disable_kill_delete_workflow(self): + """Test complete show cleanup: disable, kill jobs, delete subscriptions, delete show.""" + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + + job1 = mock.Mock() + job1.data = mock.Mock() + job1.data.name = 'job1' + job1.data.state = 'RUNNING' + + job2 = mock.Mock() + job2.data = mock.Mock() + job2.data.name = 'job2' + job2.data.state = 'RUNNING' + + show.getJobs.return_value = [job1, job2] + + sub = mock.Mock() + sub.data = mock.Mock() + + # Step 1: Disable the show + show.setActive(False) + show.setActive.assert_called_once_with(False) + + # Step 2: Kill all jobs in the show + for job in show.getJobs(): + job.kill() + + job1.kill.assert_called_once() + job2.kill.assert_called_once() + + # Step 3: Delete subscriptions + sub.delete() + sub.delete.assert_called_once() + + # Step 4: Delete the show + show.delete() + show.delete.assert_called_once() + + # Verify all cleanup steps were performed + self.assertEqual(show.setActive.call_count, 1) + self.assertEqual(job1.kill.call_count, 1) + self.assertEqual(job2.kill.call_count, 1) + self.assertEqual(sub.delete.call_count, 1) + self.assertEqual(show.delete.call_count, 1) + + def test_show_delete_with_active_jobs_error(self): + """Test that deleting show with active jobs fails appropriately.""" + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + + job = mock.Mock() + job.data = mock.Mock() + job.data.state = 'RUNNING' + show.getJobs.return_value = [job] + + # Attempt to delete show without killing jobs first + show.delete.side_effect = RuntimeError( + "Cannot delete show with active jobs" + ) + + with self.assertRaises(RuntimeError): + show.delete() + + def test_enable_show_after_disable(self): + """Test re-enabling a disabled show.""" + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + + # Disable show + show.setActive(False) + show.setActive.assert_called_with(False) + + # Re-enable show + show.setActive(True) + show.setActive.assert_called_with(True) + + # Verify both state changes + self.assertEqual(show.setActive.call_count, 2) + + +class ResourceReallocationWorkflowTest(unittest.TestCase): + """Test resource reallocation during active rendering. + + This test class verifies that resources can be reallocated safely while + jobs are running without disrupting active work. + """ + + def test_subscription_resize_during_active_rendering(self): + """Test resizing subscription while jobs are actively rendering.""" + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + + job = mock.Mock() + job.data = mock.Mock() + job.data.name = TEST_JOB + job.data.state = 'RUNNING' + show.getJobs.return_value = [job] + + sub = mock.Mock() + sub.data = mock.Mock() + sub.data.size = 100 + sub.data.burst = 150 + + # Increase subscription size during active rendering + sub.setSize(200) + sub.setSize.assert_called_once_with(200) + + # Increase burst + sub.setBurst(300) + sub.setBurst.assert_called_once_with(300) + + # Verify job continues running (not interrupted) + self.assertEqual(job.data.state, 'RUNNING') + + def test_host_reallocation_with_running_procs(self): + """Test moving hosts between allocations with running processes.""" + host = mock.Mock() + host.data = mock.Mock() + host.data.name = TEST_HOST + host.data.cores = 16 + host.data.idle_cores = 8 # Half busy + + target_alloc = mock.Mock() + target_alloc.data = mock.Mock() + target_alloc.data.name = 'target_alloc' + + # Lock host first (best practice before moving) + host.lock() + host.lock.assert_called_once() + + # Move host to new allocation + host.setAllocation(target_alloc) + host.setAllocation.assert_called_once_with(target_alloc) + + # Unlock host + host.unlock() + host.unlock.assert_called_once() + + # Verify running processes not affected (idle_cores unchanged) + self.assertEqual(host.data.idle_cores, 8) + + def test_subscription_reduction_during_rendering(self): + """Test reducing subscription size while rendering (should be cautious).""" + sub = mock.Mock() + sub.data = mock.Mock() + sub.data.size = 200 + sub.data.burst = 300 + + # Reduce subscription size + sub.setSize(100) + sub.setSize.assert_called_once_with(100) + + +class ErrorRecoveryWorkflowTest(unittest.TestCase): + """Test error recovery scenarios. + + This test class verifies that the system handles errors gracefully and + maintains consistency when operations fail. + """ + + def test_subscription_creation_with_invalid_allocation(self): + """Test creating subscription with non-existent allocation.""" + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + + alloc_finder = mock.Mock() + alloc_finder.findAllocation.side_effect = KeyError("Allocation not found") + + with self.assertRaises(KeyError): + alloc_finder.findAllocation('invalid_alloc') + + # Verify subscription was not created + show.createSubscription.assert_not_called() + + def test_host_move_with_invalid_allocation(self): + """Test moving host to non-existent allocation.""" + host = mock.Mock() + host.data = mock.Mock() + host.data.name = TEST_HOST + + alloc_finder = mock.Mock() + alloc_finder.findAllocation.side_effect = KeyError("Allocation not found") + + with self.assertRaises(KeyError): + alloc_finder.findAllocation('invalid_alloc') + + # Verify host was not moved + host.setAllocation.assert_not_called() + + def test_job_operation_on_nonexistent_job(self): + """Test operations on non-existent job handle errors gracefully.""" + job_finder = mock.Mock() + job_finder.findJob.side_effect = KeyError("Job not found") + + with self.assertRaises(KeyError): + job_finder.findJob('nonexistent_job') + + def test_subscription_modification_error_recovery(self): + """Test recovery from subscription modification errors.""" + sub = mock.Mock() + sub.data = mock.Mock() + sub.data.size = 100 + + # First modification succeeds + sub.setSize(150) + sub.setSize.assert_called_once_with(150) + + # Second modification fails + sub.setSize.side_effect = RuntimeError("Database error") + + with self.assertRaises(RuntimeError): + sub.setSize(200) + + # Verify first modification was applied but second was not + self.assertEqual(sub.setSize.call_count, 2) # Called twice, second failed + + +class PermissionAuthorizationWorkflowTest(unittest.TestCase): + """Test permission and authorization workflows. + + This test class verifies that operations requiring authorization work + correctly with different permission levels. + """ + + def test_show_operations_require_proper_authorization(self): + """Test that show operations can be performed with proper authorization.""" + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + + # Operations should succeed with proper authorization + show.delete() + show.delete.assert_called_once() + + def test_show_delete_without_authorization(self): + """Test that show deletion fails without authorization.""" + show = mock.Mock() + show.data = mock.Mock() + show.data.name = TEST_SHOW + show.delete.side_effect = PermissionError("Insufficient permissions") + + with self.assertRaises(PermissionError): + show.delete() + + def test_subscription_modification_authorization(self): + """Test that subscription modifications require proper authorization.""" + sub = mock.Mock() + sub.data = mock.Mock() + + # Operation succeeds with proper authorization + sub.setSize(150) + sub.setSize.assert_called_once_with(150) + + def test_allocation_operations_require_authorization(self): + """Test that allocation operations require proper authorization.""" + alloc = mock.Mock() + alloc.data = mock.Mock() + alloc.data.name = TEST_ALLOC + + # Operation succeeds with proper authorization + alloc.setTag('new_tag') + alloc.setTag.assert_called_once_with('new_tag') + + +if __name__ == '__main__': + unittest.main() diff --git a/cueadmin/tests/test_job_commands.py b/cueadmin/tests/test_job_commands.py new file mode 100644 index 000000000..7d04753b6 --- /dev/null +++ b/cueadmin/tests/test_job_commands.py @@ -0,0 +1,633 @@ +#!/usr/bin/env python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cueadmin job management commands.""" + + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import unittest +from unittest import mock + +try: + import cueadmin.common + OPENCUE_AVAILABLE = True +except ImportError: + # For testing without full OpenCue installation + OPENCUE_AVAILABLE = False + cueadmin = mock.MagicMock() + + +TEST_JOB_NAME = 'testShow-testJob' +TEST_JOB_NAME_2 = 'testShow-testJob2' +TEST_JOB_NAME_3 = 'testShow-testJob3' + + +@mock.patch('opencue.cuebot.Cuebot.getStub') +class JobCommandTests(unittest.TestCase): + """Tests for job management commands.""" + + def setUp(self): + self.parser = cueadmin.common.getParser() + + @mock.patch('opencue.search.JobSearch.byMatch') + def test_lj_lists_jobs_with_no_filter(self, mock_search, getStubMock): + """-lj: lists all jobs when no filter is provided.""" + job_mock_1 = mock.Mock() + job_mock_1.name = TEST_JOB_NAME + job_mock_2 = mock.Mock() + job_mock_2.name = TEST_JOB_NAME_2 + mock_result = mock.Mock() + mock_result.jobs.jobs = [job_mock_1, job_mock_2] + mock_search.return_value = mock_result + + args = self.parser.parse_args(['-lj']) + + with mock.patch('builtins.print') as mock_print: + cueadmin.common.handleArgs(args) + + mock_search.assert_called_once_with([]) + # Verify both job names were printed + self.assertEqual(mock_print.call_count, 2) + + @mock.patch('opencue.search.JobSearch.byMatch') + def test_lj_lists_jobs_with_substring_match(self, mock_search, getStubMock): + """-lj: lists jobs matching substring filter.""" + job_mock = mock.Mock() + job_mock.name = TEST_JOB_NAME + mock_result = mock.Mock() + mock_result.jobs.jobs = [job_mock] + mock_search.return_value = mock_result + + args = self.parser.parse_args(['-lj', 'testShow']) + + with mock.patch('builtins.print') as mock_print: + cueadmin.common.handleArgs(args) + + mock_search.assert_called_once_with(['testShow']) + mock_print.assert_called_once_with(TEST_JOB_NAME) + + @mock.patch('opencue.search.JobSearch.byMatch') + def test_lj_lists_jobs_with_multiple_filters(self, mock_search, getStubMock): + """-lj: lists jobs matching multiple substring filters.""" + job_mock_1 = mock.Mock() + job_mock_1.name = TEST_JOB_NAME + job_mock_2 = mock.Mock() + job_mock_2.name = TEST_JOB_NAME_2 + mock_result = mock.Mock() + mock_result.jobs.jobs = [job_mock_1, job_mock_2] + mock_search.return_value = mock_result + + args = self.parser.parse_args(['-lj', 'testShow', 'testJob']) + + with mock.patch('builtins.print'): + cueadmin.common.handleArgs(args) + + mock_search.assert_called_once_with(['testShow', 'testJob']) + + @mock.patch('cueadmin.output.displayJobs') + @mock.patch('opencue.search.JobSearch.byMatch') + def test_lji_displays_job_info(self, mock_search, mock_display, getStubMock): + """-lji: displays detailed job information.""" + job1 = mock.Mock() + job2 = mock.Mock() + mock_result = mock.Mock() + mock_result.jobs.jobs = [job1, job2] + mock_search.return_value = mock_result + + args = self.parser.parse_args(['-lji']) + cueadmin.common.handleArgs(args) + + mock_search.assert_called_once_with([]) + # Verify displayJobs was called with wrapped job objects + self.assertEqual(mock_display.call_count, 1) + displayed_jobs = mock_display.call_args[0][0] + self.assertEqual(len(displayed_jobs), 2) + + @mock.patch('cueadmin.output.displayJobs') + @mock.patch('opencue.search.JobSearch.byMatch') + def test_lji_with_filter(self, mock_search, mock_display, getStubMock): + """-lji: displays job info with filter.""" + job1 = mock.Mock() + mock_result = mock.Mock() + mock_result.jobs.jobs = [job1] + mock_search.return_value = mock_result + + args = self.parser.parse_args(['-lji', 'testShow']) + cueadmin.common.handleArgs(args) + + mock_search.assert_called_once_with(['testShow']) + mock_display.assert_called_once() + + @mock.patch('opencue.api.findJob') + def test_pause_single_job(self, mock_find, getStubMock): + """-pause: pauses a single job.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-pause', TEST_JOB_NAME]) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + job.pause.assert_called_once() + + @mock.patch('opencue.api.findJob') + def test_pause_multiple_jobs(self, mock_find, getStubMock): + """-pause: pauses multiple jobs.""" + job1 = mock.Mock() + job2 = mock.Mock() + job3 = mock.Mock() + mock_find.side_effect = [job1, job2, job3] + + args = self.parser.parse_args(['-pause', TEST_JOB_NAME, TEST_JOB_NAME_2, TEST_JOB_NAME_3]) + cueadmin.common.handleArgs(args) + + self.assertEqual(mock_find.call_count, 3) + job1.pause.assert_called_once() + job2.pause.assert_called_once() + job3.pause.assert_called_once() + + @mock.patch('opencue.api.findJob') + def test_pause_invalid_job_raises_exception(self, mock_find, getStubMock): + """-pause: raises exception for invalid job name.""" + mock_find.side_effect = Exception("Job not found") + + args = self.parser.parse_args(['-pause', 'invalid-job']) + + with self.assertRaises(Exception): + cueadmin.common.handleArgs(args) + + @mock.patch('opencue.api.findJob') + def test_unpause_single_job(self, mock_find, getStubMock): + """-unpause: unpauses a single job.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-unpause', TEST_JOB_NAME]) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + job.resume.assert_called_once() + + @mock.patch('opencue.api.findJob') + def test_unpause_multiple_jobs(self, mock_find, getStubMock): + """-unpause: unpauses multiple jobs.""" + job1 = mock.Mock() + job2 = mock.Mock() + mock_find.side_effect = [job1, job2] + + args = self.parser.parse_args(['-unpause', TEST_JOB_NAME, TEST_JOB_NAME_2]) + cueadmin.common.handleArgs(args) + + self.assertEqual(mock_find.call_count, 2) + job1.resume.assert_called_once() + job2.resume.assert_called_once() + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.findJob') + def test_kill_single_job_with_confirmation(self, mock_find, mock_prompt, getStubMock): + """-kill: kills a single job with confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-kill', TEST_JOB_NAME]) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + mock_prompt.assert_called_once() + job.kill.assert_called_once() + + @mock.patch('opencue.api.findJob') + def test_kill_with_force_flag(self, mock_find, getStubMock): + """-kill with -force: kills job without confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-kill', TEST_JOB_NAME, '-force']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + job.kill.assert_called_once() + + @mock.patch('cueadmin.util.promptYesNo', return_value=False) + @mock.patch('opencue.api.findJob') + def test_kill_cancelled_by_user(self, mock_find, mock_prompt, getStubMock): + """-kill: does not kill job when user cancels confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-kill', TEST_JOB_NAME]) + cueadmin.common.handleArgs(args) + + mock_prompt.assert_called_once() + job.kill.assert_not_called() + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.findJob') + def test_kill_multiple_jobs(self, mock_find, mock_prompt, getStubMock): + """-kill: kills multiple jobs with confirmation.""" + job1 = mock.Mock() + job2 = mock.Mock() + mock_find.side_effect = [job1, job2] + + args = self.parser.parse_args(['-kill', TEST_JOB_NAME, TEST_JOB_NAME_2]) + cueadmin.common.handleArgs(args) + + self.assertEqual(mock_find.call_count, 2) + mock_prompt.assert_called_once() + job1.kill.assert_called_once() + job2.kill.assert_called_once() + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.getJobs') + def test_kill_all_jobs(self, mock_get_jobs, mock_prompt, getStubMock): + """-kill-all: kills all jobs with confirmation.""" + job1 = mock.Mock() + job2 = mock.Mock() + job3 = mock.Mock() + mock_get_jobs.return_value = [job1, job2, job3] + + args = self.parser.parse_args(['-kill-all']) + cueadmin.common.handleArgs(args) + + mock_get_jobs.assert_called_once() + mock_prompt.assert_called_once() + job1.kill.assert_called_once() + job2.kill.assert_called_once() + job3.kill.assert_called_once() + + @mock.patch('opencue.api.getJobs') + def test_kill_all_with_force_flag(self, mock_get_jobs, getStubMock): + """-kill-all with -force: kills all jobs without confirmation.""" + job1 = mock.Mock() + job2 = mock.Mock() + mock_get_jobs.return_value = [job1, job2] + + args = self.parser.parse_args(['-kill-all', '-force']) + cueadmin.common.handleArgs(args) + + mock_get_jobs.assert_called_once() + job1.kill.assert_called_once() + job2.kill.assert_called_once() + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.findJob') + def test_retry_single_job(self, mock_find, mock_prompt, getStubMock): + """-retry: retries dead frames for a single job.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-retry', TEST_JOB_NAME]) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + mock_prompt.assert_called_once() + job.retryFrames.assert_called_once() + + @mock.patch('opencue.api.findJob') + def test_retry_with_force_flag(self, mock_find, getStubMock): + """-retry with -force: retries without confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-retry', TEST_JOB_NAME, '-force']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + job.retryFrames.assert_called_once() + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.findJob') + def test_retry_multiple_jobs(self, mock_find, mock_prompt, getStubMock): + """-retry: retries dead frames for multiple jobs.""" + job1 = mock.Mock() + job2 = mock.Mock() + mock_find.side_effect = [job1, job2] + + args = self.parser.parse_args(['-retry', TEST_JOB_NAME, TEST_JOB_NAME_2]) + cueadmin.common.handleArgs(args) + + self.assertEqual(mock_find.call_count, 2) + mock_prompt.assert_called_once() + job1.retryFrames.assert_called_once() + job2.retryFrames.assert_called_once() + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.getJobs') + def test_retry_all_jobs(self, mock_get_jobs, mock_prompt, getStubMock): + """-retry-all: retries dead frames for all jobs.""" + job1 = mock.Mock() + job2 = mock.Mock() + mock_get_jobs.return_value = [job1, job2] + + args = self.parser.parse_args(['-retry-all']) + cueadmin.common.handleArgs(args) + + mock_get_jobs.assert_called_once() + mock_prompt.assert_called_once() + job1.retryFrames.assert_called_once() + job2.retryFrames.assert_called_once() + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('cueadmin.common.DependUtil.dropAllDepends') + def test_drop_depends_single_job(self, mock_drop, mock_prompt, getStubMock): + """-drop-depends: drops all dependencies for a single job.""" + args = self.parser.parse_args(['-drop-depends', TEST_JOB_NAME]) + cueadmin.common.handleArgs(args) + + mock_prompt.assert_called_once() + mock_drop.assert_called_once_with(TEST_JOB_NAME) + + @mock.patch('cueadmin.common.DependUtil.dropAllDepends') + def test_drop_depends_with_force_flag(self, mock_drop, getStubMock): + """-drop-depends with -force: drops dependencies without confirmation.""" + args = self.parser.parse_args(['-drop-depends', TEST_JOB_NAME, '-force']) + cueadmin.common.handleArgs(args) + + mock_drop.assert_called_once_with(TEST_JOB_NAME) + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('cueadmin.common.DependUtil.dropAllDepends') + def test_drop_depends_multiple_jobs(self, mock_drop, mock_prompt, getStubMock): + """-drop-depends: drops dependencies for multiple jobs.""" + args = self.parser.parse_args(['-drop-depends', TEST_JOB_NAME, TEST_JOB_NAME_2]) + cueadmin.common.handleArgs(args) + + mock_prompt.assert_called_once() + self.assertEqual(mock_drop.call_count, 2) + mock_drop.assert_any_call(TEST_JOB_NAME) + mock_drop.assert_any_call(TEST_JOB_NAME_2) + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.findJob') + def test_set_min_cores_with_confirmation(self, mock_find, mock_prompt, getStubMock): + """-set-min-cores: sets minimum cores for a job with confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-min-cores', TEST_JOB_NAME, '4.0']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + mock_prompt.assert_called_once() + job.setMinCores.assert_called_once_with(4.0) + + @mock.patch('opencue.api.findJob') + def test_set_min_cores_with_force_flag(self, mock_find, getStubMock): + """-set-min-cores with -force: sets min cores without confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-min-cores', TEST_JOB_NAME, '2.0', '-force']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + job.setMinCores.assert_called_once_with(2.0) + + @mock.patch('opencue.api.findJob') + def test_set_min_cores_with_integer_value(self, mock_find, getStubMock): + """-set-min-cores: accepts integer values and converts to float.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-min-cores', TEST_JOB_NAME, '8', '-force']) + cueadmin.common.handleArgs(args) + + job.setMinCores.assert_called_once_with(8.0) + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.findJob') + def test_set_max_cores_with_confirmation(self, mock_find, mock_prompt, getStubMock): + """-set-max-cores: sets maximum cores for a job with confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-max-cores', TEST_JOB_NAME, '16.0']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + mock_prompt.assert_called_once() + job.setMaxCores.assert_called_once_with(16.0) + + @mock.patch('opencue.api.findJob') + def test_set_max_cores_with_force_flag(self, mock_find, getStubMock): + """-set-max-cores with -force: sets max cores without confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-max-cores', TEST_JOB_NAME, '32.0', '-force']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + job.setMaxCores.assert_called_once_with(32.0) + + @mock.patch('cueadmin.util.promptYesNo', return_value=True) + @mock.patch('opencue.api.findJob') + def test_priority_with_confirmation(self, mock_find, mock_prompt, getStubMock): + """-priority: sets job priority with confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-priority', TEST_JOB_NAME, '200']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + mock_prompt.assert_called_once() + job.setPriority.assert_called_once_with(200) + + @mock.patch('opencue.api.findJob') + def test_priority_with_force_flag(self, mock_find, getStubMock): + """-priority with -force: sets priority without confirmation.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-priority', TEST_JOB_NAME, '100', '-force']) + cueadmin.common.handleArgs(args) + + mock_find.assert_called_once_with(TEST_JOB_NAME) + job.setPriority.assert_called_once_with(100) + + @mock.patch('opencue.api.findJob') + def test_priority_with_negative_value(self, mock_find, getStubMock): + """-priority: accepts negative priority values.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-priority', TEST_JOB_NAME, '-50', '-force']) + cueadmin.common.handleArgs(args) + + job.setPriority.assert_called_once_with(-50) + + @mock.patch('opencue.api.findJob') + def test_priority_with_zero_value(self, mock_find, getStubMock): + """-priority: accepts zero as priority value.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-priority', TEST_JOB_NAME, '0', '-force']) + cueadmin.common.handleArgs(args) + + job.setPriority.assert_called_once_with(0) + + @mock.patch('opencue.api.findJob') + def test_job_not_found_error_handling(self, mock_find, getStubMock): + """Test proper error handling when job is not found.""" + mock_find.side_effect = Exception("Job not found: %s" % TEST_JOB_NAME) + + args = self.parser.parse_args(['-pause', TEST_JOB_NAME]) + + with self.assertRaises(Exception) as context: + cueadmin.common.handleArgs(args) + + self.assertIn("Job not found", str(context.exception)) + + @mock.patch('opencue.search.JobSearch.byMatch') + def test_lj_with_no_results(self, mock_search, getStubMock): + """-lj: handles case with no matching jobs.""" + mock_result = mock.Mock() + mock_result.jobs.jobs = [] + mock_search.return_value = mock_result + + args = self.parser.parse_args(['-lj', 'nonexistent']) + + with mock.patch('builtins.print') as mock_print: + cueadmin.common.handleArgs(args) + + mock_search.assert_called_once_with(['nonexistent']) + # No jobs to print + mock_print.assert_not_called() + + @mock.patch('cueadmin.output.displayJobs') + @mock.patch('opencue.search.JobSearch.byMatch') + def test_lji_with_various_job_states(self, mock_search, mock_display, getStubMock): + """-lji: displays jobs in various states (running, finished, paused).""" + running_job = mock.Mock() + running_job.data.is_paused = False + running_job.data.state = 'RUNNING' + + paused_job = mock.Mock() + paused_job.data.is_paused = True + paused_job.data.state = 'PAUSED' + + finished_job = mock.Mock() + finished_job.data.is_paused = False + finished_job.data.state = 'FINISHED' + + mock_result = mock.Mock() + mock_result.jobs.jobs = [running_job, paused_job, finished_job] + mock_search.return_value = mock_result + + args = self.parser.parse_args(['-lji']) + cueadmin.common.handleArgs(args) + + mock_display.assert_called_once() + displayed_jobs = mock_display.call_args[0][0] + self.assertEqual(len(displayed_jobs), 3) + + @mock.patch('opencue.api.findJob') + def test_set_min_cores_invalid_value_raises_error(self, mock_find, getStubMock): + """-set-min-cores: raises error for invalid core value.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-min-cores', TEST_JOB_NAME, 'invalid', '-force']) + + with self.assertRaises(ValueError): + cueadmin.common.handleArgs(args) + + @mock.patch('opencue.api.findJob') + def test_set_max_cores_invalid_value_raises_error(self, mock_find, getStubMock): + """-set-max-cores: raises error for invalid core value.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-max-cores', TEST_JOB_NAME, 'invalid', '-force']) + + with self.assertRaises(ValueError): + cueadmin.common.handleArgs(args) + + @mock.patch('opencue.api.findJob') + def test_priority_invalid_value_raises_error(self, mock_find, getStubMock): + """-priority: raises error for invalid priority value.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-priority', TEST_JOB_NAME, 'invalid', '-force']) + + with self.assertRaises(ValueError): + cueadmin.common.handleArgs(args) + + @mock.patch('opencue.api.findJob') + def test_multiple_job_operations_sequentially(self, mock_find, getStubMock): + """Test that multiple jobs can be processed in sequence for pause.""" + job1 = mock.Mock() + job2 = mock.Mock() + job3 = mock.Mock() + + mock_find.side_effect = [job1, job2, job3] + + args = self.parser.parse_args(['-pause', TEST_JOB_NAME, TEST_JOB_NAME_2, TEST_JOB_NAME_3]) + cueadmin.common.handleArgs(args) + + # Verify all three jobs were found and paused + self.assertEqual(mock_find.call_count, 3) + mock_find.assert_any_call(TEST_JOB_NAME) + mock_find.assert_any_call(TEST_JOB_NAME_2) + mock_find.assert_any_call(TEST_JOB_NAME_3) + + job1.pause.assert_called_once() + job2.pause.assert_called_once() + job3.pause.assert_called_once() + + @mock.patch('opencue.api.findJob') + def test_set_min_cores_with_fractional_cores(self, mock_find, getStubMock): + """-set-min-cores: accepts fractional core values.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-min-cores', TEST_JOB_NAME, '0.5', '-force']) + cueadmin.common.handleArgs(args) + + job.setMinCores.assert_called_once_with(0.5) + + @mock.patch('opencue.api.findJob') + def test_set_max_cores_with_large_value(self, mock_find, getStubMock): + """-set-max-cores: accepts large core values.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-set-max-cores', TEST_JOB_NAME, '1000.0', '-force']) + cueadmin.common.handleArgs(args) + + job.setMaxCores.assert_called_once_with(1000.0) + + @mock.patch('opencue.api.findJob') + def test_priority_with_large_value(self, mock_find, getStubMock): + """-priority: accepts large priority values.""" + job = mock.Mock() + mock_find.return_value = job + + args = self.parser.parse_args(['-priority', TEST_JOB_NAME, '10000', '-force']) + cueadmin.common.handleArgs(args) + + job.setPriority.assert_called_once_with(10000) + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/_docs/reference/commands/cueadmin.md b/docs/_docs/reference/commands/cueadmin.md index 8bb462773..e60e64634 100644 --- a/docs/_docs/reference/commands/cueadmin.md +++ b/docs/_docs/reference/commands/cueadmin.md @@ -306,3 +306,61 @@ Sets hosts into Up state. Arguments: `{auto,all,variable}` Set the host's thread mode. + +## Job Options + +### `-pause` + +Arguments: `JOB [JOB ...]` + +Pause specified jobs. + +### `-unpause` + +Arguments: `JOB [JOB ...]` + +Unpause specified jobs. + +### `-kill` + +Arguments: `JOB [JOB ...]` + +Kill specified jobs. Requires confirmation unless `-force` flag is used. + +### `-kill-all` + +Kill all jobs. Requires `-force` flag for safety. + +### `-retry` + +Arguments: `JOB [JOB ...]` + +Retry dead frames for specified jobs. Requires confirmation unless `-force` flag is used. + +### `-retry-all` + +Retry dead frames for all jobs. Requires `-force` flag for safety. + +### `-drop-depends` + +Arguments: `JOB [JOB ...]` + +Drop all dependencies for specified jobs. Requires confirmation unless `-force` flag is used. + +### `-set-min-cores` + +Arguments: `JOB CORES` + +Set minimum cores for a job. + +### `-set-max-cores` + +Arguments: `JOB CORES` + +Set maximum cores for a job. + +### `-priority` + +Arguments: `JOB PRIORITY` + +Set job priority. diff --git a/docs/_docs/reference/tools/cueadmin.md b/docs/_docs/reference/tools/cueadmin.md index 160529bee..78d4fa3b4 100644 --- a/docs/_docs/reference/tools/cueadmin.md +++ b/docs/_docs/reference/tools/cueadmin.md @@ -16,15 +16,16 @@ CueAdmin is the command-line interface for administering OpenCue deployments. It ## Overview -CueAdmin is the administrative counterpart to Cueman, focusing on system-level operations and infrastructure management rather than job control. It's the essential tool for OpenCue administrators and power users who need to manage render farm resources. +CueAdmin is the command-line administration tool for OpenCue, providing comprehensive control over jobs, shows, allocations, hosts, and system resources. While Cueman offers advanced job-focused features with rich filtering, CueAdmin provides essential job operations alongside complete system-level administration capabilities. It's the essential tool for OpenCue administrators and power users managing render farm resources. ### Key Features +- **Job Management**: Pause, kill, retry, and control job execution - **Show Management**: Create, configure, and control shows - **Allocation Control**: Manage resource allocations and facility resources - **Host Administration**: Lock, unlock, move, and configure render hosts - **Subscription Management**: Configure show subscriptions to allocations -- **Resource Monitoring**: Query procs, hosts, and system state +- **Resource Monitoring**: Query procs, hosts, jobs, and system state - **Batch Operations**: Apply changes to multiple entities simultaneously - **Safety Features**: Confirmation prompts for destructive operations @@ -396,6 +397,109 @@ cueadmin -burst production main.render 300 # Absolute value cueadmin -burst production main.render 50% # Percentage of size ``` +### Job Operations + +#### Pause/Unpause Jobs + +Control job execution: + +```bash +# Pause single job +cueadmin -pause shot_001_lighting + +# Pause multiple jobs +cueadmin -pause shot_001_lighting shot_002_comp shot_003_fx + +# Resume jobs +cueadmin -unpause shot_001_lighting +cueadmin -unpause shot_001_lighting shot_002_comp +``` + +#### Kill Jobs + +Terminate job execution: + +```bash +# Kill single job (requires confirmation) +cueadmin -kill old_job +# Confirm: Kill 1 job(s)? [y/n] y + +# Kill multiple jobs +cueadmin -kill job1 job2 job3 + +# Kill with force flag (skip confirmation) +cueadmin -kill old_job -force + +# Kill all jobs (requires force flag) +cueadmin -kill-all -force +``` + +#### Retry Failed Frames + +Rerun dead frames: + +```bash +# Retry dead frames for single job +cueadmin -retry failed_job +# Confirm: Retry dead frames for 1 job(s)? [y/n] y + +# Retry for multiple jobs +cueadmin -retry job1 job2 job3 + +# Retry with force flag +cueadmin -retry failed_job -force + +# Retry all jobs (requires force flag) +cueadmin -retry-all -force +``` + +#### Manage Dependencies + +Remove job dependencies: + +```bash +# Drop dependencies for single job +cueadmin -drop-depends blocked_job +# Confirm: Drop all dependencies for 1 job(s)? [y/n] y + +# Drop dependencies for multiple jobs +cueadmin -drop-depends job1 job2 job3 + +# Drop with force flag +cueadmin -drop-depends blocked_job -force +``` + +#### Adjust Resource Limits + +Configure job core requirements: + +```bash +# Set minimum cores +cueadmin -set-min-cores heavy_job 8.0 +# Confirm: Set min cores for job:heavy_job to 8.00? [y/n] y + +# Set maximum cores +cueadmin -set-max-cores heavy_job 64.0 +# Confirm: Set max cores for job:heavy_job to 64.00? [y/n] y + +# Set with force flag +cueadmin -set-min-cores heavy_job 8.0 -force +cueadmin -set-max-cores heavy_job 64.0 -force +``` + +#### Set Job Priority + +Adjust job scheduling priority: + +```bash +# Set priority (higher values = higher priority) +cueadmin -priority urgent_job 200 +# Confirm: Set priority for job:urgent_job to 200? [y/n] y + +# Set with force flag +cueadmin -priority urgent_job 200 -force +``` + ## Filtering Options ### Memory Filter @@ -457,6 +561,46 @@ cueadmin -ll -limit 50 # First 50 log paths ## Common Workflows +### Managing Problem Jobs + +```bash +# 1. List jobs to find problematic ones +cueadmin -lji | grep FAILED + +# 2. Pause problematic job +cueadmin -pause problem_job + +# 3. Drop blocking dependencies +cueadmin -drop-depends problem_job -force + +# 4. Adjust resources if needed +cueadmin -set-min-cores problem_job 2.0 -force +cueadmin -set-max-cores problem_job 32.0 -force + +# 5. Retry failed frames +cueadmin -retry problem_job -force + +# 6. Resume job +cueadmin -unpause problem_job + +# 7. Boost priority if urgent +cueadmin -priority problem_job 200 -force +``` + +### Cleaning Up Old Jobs + +```bash +# 1. List old jobs +cueadmin -lj old_show + +# 2. Kill old jobs +cueadmin -kill old_job1 old_job2 old_job3 + +# 3. Kill all jobs from a show (careful!) +cueadmin -lj old_show # Verify list first +# Then kill each one +``` + ### Setting Up a New Show ```bash @@ -684,7 +828,7 @@ CueAdmin returns the following exit codes: ### Running Tests -CueAdmin includes a comprehensive test suite with 16+ tests: +CueAdmin includes a comprehensive test suite: ```bash # Install with test dependencies @@ -699,8 +843,9 @@ pytest --cov=cueadmin --cov-report=term-missing ### Test Types -- **Unit Tests** - Function-level testing of core functionality -- **Integration Tests** - End-to-end workflow testing +- **Unit Tests** - Function-level testing of core functionality (job management, allocations, subscriptions, hosts) +- **Integration Tests** - End-to-end workflow testing (tests covering complete command sequences) +- **Job Commands Tests** - Comprehensive job operations testing - **Coverage Testing** - Code coverage analysis and reporting ### Development Setup diff --git a/docs/_docs/tutorials/cueadmin-tutorial.md b/docs/_docs/tutorials/cueadmin-tutorial.md index 187c4516d..b81d3134b 100644 --- a/docs/_docs/tutorials/cueadmin-tutorial.md +++ b/docs/_docs/tutorials/cueadmin-tutorial.md @@ -333,7 +333,174 @@ cueadmin -ll -job shot_001_lighting cueadmin -ll -host render01 ``` -## Part 7: Advanced Workflows +## Part 7: Job Management + +### Listing Jobs + +View jobs in your system: + +```bash +# List all jobs +cueadmin -lj + +# List jobs matching a pattern +cueadmin -lj shot_ +cueadmin -lj lighting comp + +# List detailed job information +cueadmin -lji +cueadmin -lji shot_001 +``` + +### Pausing and Resuming Jobs + +Control job execution: + +```bash +# Pause a single job +cueadmin -pause shot_001_lighting + +# Pause multiple jobs +cueadmin -pause shot_001_lighting shot_002_comp shot_003_fx + +# Resume a job +cueadmin -unpause shot_001_lighting + +# Resume multiple jobs +cueadmin -unpause shot_001_lighting shot_002_comp +``` + +### Killing Jobs + +Terminate job execution: + +```bash +# Kill a single job (with confirmation) +cueadmin -kill old_test_job +# Confirm: Kill 1 job(s)? [y/n] y + +# Kill multiple jobs +cueadmin -kill test_job1 test_job2 test_job3 +# Confirm: Kill 3 job(s)? [y/n] y + +# Kill with force flag (skip confirmation) +cueadmin -kill old_test_job -force + +# Kill all jobs (DANGER - requires force flag) +cueadmin -kill-all -force +``` + +### Retrying Failed Frames + +Rerun dead frames: + +```bash +# Retry dead frames for a single job +cueadmin -retry failed_render_job +# Confirm: Retry dead frames for 1 job(s)? [y/n] y + +# Retry for multiple jobs +cueadmin -retry job1 job2 job3 +# Confirm: Retry dead frames for 3 job(s)? [y/n] y + +# Retry with force flag +cueadmin -retry failed_render_job -force + +# Retry all jobs (DANGER - requires force flag) +cueadmin -retry-all -force +``` + +### Managing Job Dependencies + +Remove blocking dependencies: + +```bash +# Drop all dependencies for a job +cueadmin -drop-depends blocked_job +# Confirm: Drop all dependencies for 1 job(s)? [y/n] y + +# Drop dependencies for multiple jobs +cueadmin -drop-depends job1 job2 job3 +# Confirm: Drop all dependencies for 3 job(s)? [y/n] y + +# Drop with force flag +cueadmin -drop-depends blocked_job -force +``` + +### Adjusting Job Resources + +Configure job core requirements: + +```bash +# Set minimum cores for a job +cueadmin -set-min-cores heavy_render 8.0 +# Confirm: Set min cores for job:heavy_render to 8.00? [y/n] y + +# Set maximum cores +cueadmin -set-max-cores heavy_render 64.0 +# Confirm: Set max cores for job:heavy_render to 64.00? [y/n] y + +# Set with force flag +cueadmin -set-min-cores heavy_render 8.0 -force +cueadmin -set-max-cores heavy_render 64.0 -force + +# Fractional cores are supported +cueadmin -set-min-cores light_job 0.5 -force +cueadmin -set-max-cores light_job 4.0 -force +``` + +### Setting Job Priority + +Adjust scheduling priority: + +```bash +# Set job priority (higher = more important) +# Default priority is typically 100 +cueadmin -priority urgent_shot 250 +# Confirm: Set priority for job:urgent_shot to 250? [y/n] y + +# Lower priority for background work +cueadmin -priority background_job 50 -force + +# Negative priorities are allowed +cueadmin -priority low_priority_job -10 -force + +# Set with force flag +cueadmin -priority urgent_shot 250 -force +``` + +### Job Management Workflow Example + +Complete workflow for managing a problematic job: + +```bash +# 1. Check job status +cueadmin -lji problematic_job + +# 2. Pause the job to investigate +cueadmin -pause problematic_job + +# 3. Drop dependencies if blocked +cueadmin -drop-depends problematic_job -force + +# 4. Adjust resources to reduce errors +cueadmin -set-min-cores problematic_job 4.0 -force +cueadmin -set-max-cores problematic_job 32.0 -force + +# 5. Retry failed frames +cueadmin -retry problematic_job -force + +# 6. Resume the job +cueadmin -unpause problematic_job + +# 7. Boost priority if urgent +cueadmin -priority problematic_job 200 -force + +# 8. Monitor progress +cueadmin -lji problematic_job +``` + +## Part 8: Advanced Workflows ### Setting Up Production Infrastructure @@ -434,7 +601,7 @@ cueadmin -lba new_facility.render cueadmin -delete-alloc old_facility.render ``` -## Part 8: Monitoring and Maintenance +## Part 9: Monitoring and Maintenance ### Daily Health Checks @@ -481,7 +648,7 @@ done # (Would require additional scripting to implement fully) ``` -## Part 9: Rollback and Recovery +## Part 10: Rollback and Recovery ### Safe Rollback Procedure @@ -505,7 +672,7 @@ cueadmin -delete-show test_show -force diff shows_backup_$(date +%Y%m%d).txt <(cueadmin -ls) ``` -## Part 10: Best Practices +## Part 11: Best Practices ### Command Patterns @@ -608,6 +775,7 @@ cueadmin -v -your-command You've learned how to: - Query and monitor OpenCue resources +- Manage jobs (pause, kill, retry, adjust resources, set priorities) - Create and manage shows - Configure allocations and subscriptions - Administer hosts @@ -631,7 +799,7 @@ Remember to always: ## Development and Contributing CueAdmin is actively developed with: -- **Comprehensive test suite** with tests covering unit and integration scenarios +- **Comprehensive test suite** with tests covering job management, allocation management, host operations, subscriptions, and integration workflows - **Modern testing infrastructure** using pytest, coverage reporting, and CI/CD integration - **Development tools** including linting, formatting, and multi-Python version testing