Implement slapcomp workflow

colorbleed/fusion/lib.py

@@ -1,10 +1,15 @@
-import sys
+import os
+import re
+import logging
+import traceback
 
 import avalon.fusion
+from avalon import api, io, pipeline
 
+import colorbleed.lib as colorbleed
 
-self = sys.modules[__name__]
-self._project = None
+
+log = logging.getLogger(__name__)
 
 
 def update_frame_range(start, end, comp=None, set_render_range=True):
@@ -38,3 +43,119 @@ def update_frame_range(start, end, comp=None, set_render_range=True):
 
     with avalon.fusion.comp_lock_and_undo_chunk(comp):
         comp.SetAttrs(attrs)
+
+
+def get_next_version_folder(folder):
+    """Format a version folder based on the filepath
+
+    Assumption here is made that, if the path does not exists the folder
+    will be "v001"
+
+    Args:
+        folder: file path to a folder
+
+    Returns:
+        str: new version folder name
+    """
+
+    version_int = 1
+    if os.path.isdir(folder):
+        re_version = re.compile("v\d+")
+        versions = [i for i in os.listdir(folder) if re_version.match(i)
+                    and os.path.isdir(os.path.join(folder, i))]
+        if versions:
+            # ensure the "v" is not included and convert to ints
+            int_versions = [int(v[1:]) for v in versions]
+            version_int += max(int_versions)
+
+    return "v{:03d}".format(version_int)
+
+
+def update_savers(comp, session, project):
+    """Update all savers of the current comp to ensure the output is correct
+
+    Args:
+        comp (object): current comp instance
+        session (dict): the current Avalon session
+        project (dict): the project document from the database
+
+    Returns:
+         None
+    """
+
+    template = project["config"]["template"]["work"]
+    template_work = pipeline._format_work_template(template, session)
+
+    render_dir = os.path.join(os.path.normpath(template_work), "renders")
+    version_folder = get_next_version_folder(render_dir)
+    renders_version = os.path.join(render_dir, version_folder)
+
+    comp.Print("New renders to: %s\n" % render_dir)
+
+    with avalon.fusion.comp_lock_and_undo_chunk(comp):
+        savers = comp.GetToolList(False, "Saver").values()
+        for saver in savers:
+            filepath = saver.GetAttrs("TOOLST_Clip_Name")[1.0]
+            filename = os.path.basename(filepath)
+            new_path = os.path.join(renders_version, filename)
+            saver["Clip"] = new_path
+
+
+def switch(asset_name):
+    """Switch the current containers of the comp to the other asset (shot)
+
+    Args:
+        asset_name (str): name of the asset (shot)
+
+    Returns:
+        comp (PyObject): the comp instance
+
+    """
+
+    assert asset_name, "Function requires asset name"
+
+    host = api.registered_host()
+    assert host, "Host must be installed"
+
+    # Get current project
+    project = io.find_one({"type": "project",
+                           "name": api.Session["AVALON_PROJECT"]})
+
+    # Assert asset name exists
+    # It is better to do this here then to wait till switch_shot does it
+    asset = io.find_one({"type": "asset", "name": asset_name})
+    assert asset, "Could not find '%s' in the database" % asset_name
+
+    # Use the current open comp
+    current_comp = avalon.fusion.get_current_comp()
+    assert current_comp is not None, "Could not find current comp"
+
+    containers = list(host.ls())
+    assert containers, "Nothing to update"
+
+    representations = []
+    for container in containers:
+        try:
+            representation = colorbleed.switch_item(container,
+                                                    asset_name=asset_name)
+            representations.append(representation)
+            log.debug(str(representation["_id"]) + "\n")
+        except Exception as e:
+            msg = traceback.format_stack(e)
+            log.debug("Error in switching! %s\n" % msg)
+
+    log.info("Switched %i Loaders of the %i\n" % (len(representations),
+                                                  len(containers)))
+    # Updating frame range
+    log.debug("Updating frame range ..\n")
+    version_ids = [r["parent"] for r in representations]
+    versions = io.find({"type": "version", "_id": {"$in": version_ids}})
+    versions = list(versions)
+
+    start = min(v["data"]["startFrame"] for v in versions)
+    end = max(v["data"]["endFrame"] for v in versions)
+
+    update_frame_range(start, end, comp=current_comp)
+    update_savers(current_comp, api.Session, project)
+
+    return current_comp

colorbleed/launcher_actions.py

@@ -1,10 +1,13 @@
 import os
-from avalon import api, lib, pipeline
+
+from avalon import api, pipeline, lib
+
+import acre
 
 
 class FusionRenderNode(api.Action):
 
-    name = "fusionrendernode9"
+    name = "fusionrendernode"
     label = "F9 Render Node"
     icon = "object-group"
     order = 997
@@ -26,16 +29,17 @@ def process(self, session, **kwargs):
 
         """
 
-        # Update environment with session
-        env = os.environ.copy()
-        env.update(session)
+        # Update environment with session and the current environment
+        tools_env = acre.get_tools(["global", "fusionnode9"])
+        env = acre.compute(tools_env)
+        env = acre.merge(env, current_env=dict(os.environ))
 
         # Get executable by name
-        app = lib.get_application(self.name)
-        env.update(app["environment"])
-        executable = lib.which(app["executable"])
+        executable = acre.which(self.name, env)
+        if not executable:
+            raise ValueError("Unable to find executable %s" % self.name)
 
-        return lib.launch(executable=executable, args=[], environment=env)
+        return acre.launch(executable=executable, args=[], environment=env)
 
 
 class VrayRenderSlave(api.Action):
@@ -63,20 +67,16 @@ def process(self, session, **kwargs):
         """
 
         # Update environment with session
-        env = os.environ.copy()
-        env.update(session)
+        tools_env = acre.get_tools(["global", "maya2018", "vrayrenderslave"])
+        env = acre.compute(tools_env)
+        env = acre.merge(env, current_env=dict(os.environ))
 
         # Get executable by name
-        app = lib.get_application(self.name)
-        env.update(app["environment"])
-        executable = lib.which(app["executable"])
+        executable = acre.which("vray", env)
 
         # Run as server
-        arguments = ["-server", "-portNumber=20207"]
-
-        return lib.launch(executable=executable,
-                          args=arguments,
-                          environment=env)
+        args = ["-server", "-portNumber=20207"]
+        return acre.launch(executable=executable, args=args, environment=env)
 
 
 def register_launcher_actions():

colorbleed/plugins/fusion/publish/collect_comp.py

@@ -1,5 +1,3 @@
-import os
-
 import pyblish.api
 
 from avalon import fusion

colorbleed/plugins/fusion/publish/publish_image_sequences.py

@@ -8,22 +8,19 @@
 from colorbleed.action import get_errored_plugins_from_data
 
 
-def _get_script():
+def _get_script_dir():
     """Get path to the image sequence script"""
-
-    # todo: use a more elegant way to get the python script
-
     try:
-        from colorbleed.scripts import publish_filesequence
-    except Exception:
-        raise RuntimeError("Expected module 'publish_imagesequence'"
-                           "to be available")
+        import colorbleed
+        config_dir = os.path.dirname(colorbleed.__file__)
+        script_dir = os.path.join(config_dir, "scripts")
+    except ImportError:
+        raise RuntimeError("This is a bug")
 
-    module_path = publish_filesequence.__file__
-    if module_path.endswith(".pyc"):
-        module_path = module_path[:-len(".pyc")] + ".py"
+    assert os.path.isdir(script_dir), "Config is incomplete"
+    script_dir = script_dir.replace(os.sep, "/")
 
-    return module_path
+    return script_dir
 
 
 class PublishImageSequence(pyblish.api.InstancePlugin):
@@ -73,7 +70,13 @@ def process(self, instance):
         startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
         startupinfo.wShowWindow = subprocess.SW_HIDE
 
-        process = subprocess.Popen(["python", _get_script(),
+        # Get script
+        script_dir = _get_script_dir()
+        script = os.path.join(script_dir, "publish_imagesequence.py")
+        assert os.path.isfile(script), ("Config incomplete, missing "
+                                        "`script/publish_imagesequence.py`")
+
+        process = subprocess.Popen(["python", script,
                                     "--paths", path],
                                    bufsize=1,
                                    stdout=subprocess.PIPE,

colorbleed/plugins/fusion/publish/submit_deadline.py

@@ -39,7 +39,7 @@ def process(self, instance):
 
         # Collect all saver instances in context that are to be rendered
         saver_instances = []
-        for instance in context[:]:
+        for instance in list(context):
             if not self.families[0] in instance.data.get("families"):
                 # Allow only saver family instances
                 continue
@@ -117,22 +117,18 @@ def process(self, instance):
             payload["JobInfo"]["OutputFilename%d" % index] = filename
 
         # Include critical variables with submission
-        keys = [
-            # TODO: This won't work if the slaves don't have accesss to
-            # these paths, such as if slaves are running Linux and the
-            # submitter is on Windows.
-            "PYTHONPATH",
-            "OFX_PLUGIN_PATH",
-            "FUSION9_MasterPrefs"
-        ]
-        environment = dict({key: os.environ[key] for key in keys
-                            if key in os.environ}, **api.Session)
+        TOOL_ENV = os.getenv("TOOL_ENV")
+        assert TOOL_ENV, "No environment directory found"
+
+        env = api.Session.copy()
+        env["AVALON_TOOLS"] = "global;fusion9"
+        env["TOOL_ENV"] = TOOL_ENV
 
         payload["JobInfo"].update({
             "EnvironmentKeyValue%d" % index: "{key}={value}".format(
                 key=key,
-                value=environment[key]
-            ) for index, key in enumerate(environment)
+                value=env[key]
+            ) for index, key in enumerate(env)
         })
 
         self.log.info("Submitting..")

colorbleed/plugins/fusion/publish/validate_unique_subsets.py

@@ -11,10 +11,9 @@ class ValidateUniqueSubsets(pyblish.api.InstancePlugin):
 
     @classmethod
     def get_invalid(cls, instance):
-
         context = instance.context
         subset = instance.data["subset"]
-        for other_instance in context[:]:
+        for other_instance in list(context):
             if other_instance == instance:
                 continue
 
@@ -26,4 +25,4 @@ def get_invalid(cls, instance):
     def process(self, instance):
         invalid = self.get_invalid(instance)
         if invalid:
-            raise RuntimeError("Animation content is invalid. See log.")
+            raise RuntimeError("Content is invalid. See log.")

colorbleed/plugins/global/publish/submit_publish_job.py

@@ -10,17 +10,15 @@
 
 def _get_script():
     """Get path to the image sequence script"""
-    try:
-        from colorbleed.scripts import publish_filesequence
-    except Exception as e:
-        raise RuntimeError("Expected module 'publish_imagesequence'"
-                           "to be available")
 
-    module_path = publish_filesequence.__file__
-    if module_path.endswith(".pyc"):
-        module_path = module_path[:-len(".pyc")] + ".py"
+    import colorbleed
+    configpath = os.path.dirname(colorbleed.__file__)
+    script = os.path.join(configpath, "scripts", "publish_filesequence.py")
+    # verify script exists
+    assert os.path.isfile(script), ("Script `publish_filesequence.py` does not"
+                                    "exists")
 
-    return module_path
+    return script
 
 
 class SubmitDependentImageSequenceJobDeadline(pyblish.api.InstancePlugin):
@@ -70,7 +68,7 @@ def process(self, instance):
                                "submission prior to this plug-in.")
 
         subset = instance.data["subset"]
-        state = instance.data.get("publishJobState", "Suspended")
+        state = instance.data.get("suspendPublishJob", "Active")
         job_name = "{batch} - {subset} [publish image sequence]".format(
             batch=job["Props"]["Name"],
             subset=subset
@@ -144,7 +142,13 @@ def process(self, instance):
 
         # Transfer the environment from the original job to this dependent
         # job so they use the same environment
+        TOOL_ENV = os.getenv("TOOL_ENV")
+        assert TOOL_ENV, "No environment directory found"
+
         environment = job["Props"].get("Env", {})
+        environment["AVALON_TOOLS"] = "global;python36"
+        environment["TOOL_ENV"] = TOOL_ENV
+
         payload["JobInfo"].update({
             "EnvironmentKeyValue%d" % index: "{key}={value}".format(
                 key=key,
@@ -159,3 +163,7 @@ def process(self, instance):
         response = requests.post(url, json=payload)
         if not response.ok:
             raise Exception(response.text)
+
+        # Temporary key name, deadlineSubmissionJob was already taken
+        if instance.data.get("runSlapComp", False):
+            instance.data["deadlineDependJob"] = response.json()

colorbleed/plugins/maya/create/colorbleed_renderglobals.py

@@ -30,6 +30,8 @@ def __init__(self, *args, **kwargs):
         data["priority"] = 50
         data["whitelist"] = False
         data["machineList"] = ""
+        data["runSlapComp"] = False
+        data["flowFile"] = ""
 
         self.data = data
         self.options = {"useSelection": False}  # Force no content