diff --git a/CHANGES b/CHANGES index 557faff0..ec4181f0 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,15 @@ pyblish Changelog This contains all major version changes between pyblish releases. +Version 1.0.13 +-------------- + +- Added pyblish.api.sort_plugins +- Added ordered output to pyblish.api.discover +- pyblish.api.plugins_by_family now yields correct results + for plug-ins with wildcard families. +- Refactored main.py into util.py + Version 1.0.12 -------------- diff --git a/pyblish/api.py b/pyblish/api.py index 41fb7360..c640a78a 100644 --- a/pyblish/api.py +++ b/pyblish/api.py @@ -28,6 +28,8 @@ plugins_by_family, plugins_by_host, instances_by_plugin) from .plugin import Config as __Config +from .plugin import sort as sort_plugins + from .lib import log, format_filename from .error import ( PyblishError, SelectionError, ValidationError, @@ -60,40 +62,41 @@ __all__ = [ # Base objects - 'Context', - 'Instance', + "Context", + "Instance", # Plug-ins - 'Selector', - 'Validator', - 'Extractor', - 'Conformer', + "Selector", + "Validator", + "Extractor", + "Conformer", # Plug-in utilities - 'discover', - 'plugin_paths', - 'registered_paths', - 'configured_paths', - 'environment_paths', - 'register_plugin_path', - 'deregister_plugin_path', - 'deregister_all', - 'plugins_by_family', - 'plugins_by_host', - 'instances_by_plugin', + "discover", + "plugin_paths", + "registered_paths", + "configured_paths", + "environment_paths", + "register_plugin_path", + "deregister_plugin_path", + "deregister_all", + "plugins_by_family", + "plugins_by_host", + "instances_by_plugin", + "sort_plugins", # Configuration - 'config', + "config", # Decorators - 'log', - 'format_filename', + "log", + "format_filename", # Exceptions - 'PyblishError', - 'SelectionError', - 'ValidationError', - 'ExtractionError', - 'ConformError', - 'NoInstancesError' + "PyblishError", + "SelectionError", + "ValidationError", + "ExtractionError", + "ConformError", + "NoInstancesError" ] diff --git a/pyblish/main.py b/pyblish/main.py index 99262758..0fdb1087 100644 --- a/pyblish/main.py +++ b/pyblish/main.py @@ -1,521 +1,4 @@ -"""Entry-point of Pyblish +import warnings +from util import * -Attributes: - TAB: Number of spaces for a tab - LOG_TEMPATE: Template used for logging coming from - plug-ins - SCREEN_WIDTH: Default width at which logging and printing - will (attempt to) restrain to. - logging_handlers: Record of handlers at the start of - importing this module. This module will modify the - currently handlers and restore then once finished. - log: Current logger - intro_message: Message printed upon initiating a publish. -""" - -from __future__ import absolute_import - -# Standard library -import time -import logging - -# Local library -import pyblish.api - -TAB = " " -LOG_TEMPATE = " %(levelname)-8s %(message)s" -SCREEN_WIDTH = 80 - -# Messages -NO_INSTANCES_ERROR = "Cancelled due to not finding any instances" -SELECTION_ERROR = "Selection failed" -VALIDATION_ERROR = "Validation failed" - -# Templates -AUTO_REPAIR = "There were errors, attempting to auto-repair.." -COMMITTED_TEMPLATE = "{tab}Committed to: {dir}" -CONFORMED_TEMPLATE = "{tab}Conformed to: {dir}" -FAILED_VALIDATION_TEMPLATE = "{tab}- \"{instance}\": {exception} ({plugin})" -SUCCESS_TEMPLATE = "Processed {num} instance{s} {status} in {seconds}s" -TRACEBACK_TEMPLATE = "(Line {line} in \"{file}\" @ \"{func}\")" -ERROR_TEMPLATE = "{tab}{instance}: {error} {traceback}" -VALIDATIONS_FAILED_TEMPLATE = """ -These validations failed: -{failures}""" - -__all__ = ['select', - 'validate', - 'extract', - 'conform', - 'publish', - 'publish_all'] - - -def publish(context=None, - auto_repair=False, - include_optional=True, - logging_level=logging.INFO, - **kwargs): - """Publish everything - - This function will process all available plugins of the - currently running host, publishing anything picked up - during selection. - - Arguments: - context (pyblish.api.Context): Optional Context. - Defaults to creating a new context each time. - auto_repair (bool): Whether or not to attempt to automatically - repair instances which fail validation. - include_optional (bool): Should validation include plug-ins - which has been defined as optional? - logging_level (logging level): Optional level with which - to log messages. Default is logging.INFO. - - Usage: - >> publish() - >> publish(context=Context()) - - """ - - # Hidden argument - _orders = kwargs.pop('orders', None) or (0, 1, 2, 3) - assert not kwargs # There are no more arguments - - obj = Publish(context, - auto_repair=auto_repair, - include_optional=include_optional, - logging_level=logging_level) - - obj.orders = _orders - obj.process() - - return obj.context - - -def validate_all(*args, **kwargs): - if not 'orders' in kwargs: - kwargs['orders'] = (0, 1) - return publish(*args, **kwargs) - - -def select(*args, **kwargs): - """Convenience function for selection""" - if not 'orders' in kwargs: - kwargs['orders'] = (0,) - return publish(*args, **kwargs) - - -def validate(*args, **kwargs): - """Convenience function for validation""" - if not 'orders' in kwargs: - kwargs['orders'] = (1,) - return publish(*args, **kwargs) - - -def extract(*args, **kwargs): - """Convenience function for extraction""" - if not 'orders' in kwargs: - kwargs['orders'] = (2,) - return publish(*args, **kwargs) - - -def conform(*args, **kwargs): - """Convenience function for conform""" - if not 'orders' in kwargs: - kwargs['orders'] = (3,) - return publish(*args, **kwargs) - - -class Publish(object): - """Publishing operator - - Arguments: - context (pyblish.api.Context): Optional Context. - Defaults to creating a new context each time. - auto_repair (bool): Whether or not to attempt to automatically - repair instances which fail validation. - include_optional (bool): Should validation include plug-ins - which has been defined as optional? - logging_level (logging level): Optional level with which - to log messages. Default is logging.INFO. - - """ - - log = logging.getLogger() - - @property - def duration(self): - return "%.2f" % (self._time['end'] - self._time['start']) - - def __init__(self, - context=None, - auto_repair=False, - include_optional=True, - logging_level=logging.INFO): - - if context is None: - context = pyblish.api.Context() - - self.context = context - self.orders = (0, 1, 2, 3) - self.repair = auto_repair - self.optional = include_optional - self.logging_level = logging_level - - self._plugins = pyblish.plugin.Manager() - self._conf = pyblish.api.config - self._time = {'start': None, 'end': None} - self._errors = list() - - def process(self): - """Process all instances within the given context""" - self._time['start'] = time.time() - self._log_intro() - - log_summary = False - - # Initialise pretty-printing for plug-ins - self._init_log() - - try: - for order in self.orders: - self.process_order(order) - - except pyblish.api.NoInstancesError as exc: - self.log.warning(NO_INSTANCES_ERROR) - - except pyblish.api.SelectionError: - self.log.error(SELECTION_ERROR) - - except pyblish.api.ValidationError as exc: - self.log.error(VALIDATION_ERROR) - - failures = list() - for error in exc.errors: - failures.append(FAILED_VALIDATION_TEMPLATE.format( - instance=error.instance, - tab=TAB, - exception=error, - plugin=error.plugin.__name__)) - print VALIDATIONS_FAILED_TEMPLATE.format( - failures="\n".join(failures)) - - except Exception as exc: - self.log.error("Unhandled exception: %s" % exc) - - else: - log_summary = True - - # Clear context - self._time['end'] = time.time() - - print # newline - print "-" * 80 - - self._log_time() - - if log_summary: - self._log_summary() - - self._reset_log() - self._log_success() - - def process_order(self, order): - """Process context using plug-ins with the specified `order` - - Arguments: - order (int): Order of plug-ins with which to process context. - - Raises: - pyblish.api.SelectionError: When selection fails - pyblish.api.ValidationError: When validation fails - - """ - - if order != 0 and not self.context: - # If there aren't any instances after selection, - # there is no point in going on. - raise pyblish.api.NoInstancesError - - order_errors = list() - for plugin in self._plugins: - if plugin.order != order: - continue - - plugin_errors = self.process_plugin(plugin) - order_errors.extend(plugin_errors) - - if not plugin_errors: - continue - - # Before proceeding with extraction, ensure - # that there are no failed validators. - self.log.warning("There were errors:") - for error in plugin_errors: - self._log_error(error.instance, error) - - if not order_errors: - return - - # If the error occurred during selection or validation, - # we don't want to continue. - if order == 0: - raise pyblish.api.SelectionError - - if order == 1: - exception = pyblish.api.ValidationError - exception.errors = order_errors - raise exception - - def process_plugin(self, plugin): - """Process context using a single plugin - - Arguments: - plugin (Plugin): Plug-in used to process context - - Returns: - List of errors occurred for `plugin` - - """ - - errors = list() - - # Do not include optional plug-ins - if plugin.optional and self.optional is False: - self._log_plugin(plugin, suffix="(optional and skipped)") - return errors - - self._log_plugin(plugin) - - for instance, error in plugin().process(self.context): - if instance is None and error is None: - self.log.debug("Skipped, no compatible instances.") - continue - - if error is None: - continue - - repaired = False - if plugin.order == 1 and self.repair: - repaired = self._repair(plugin, instance) - - if not repaired: - errors.append(error) - - # Inject data for logging - error.instance = instance - error.plugin = plugin - - # Store global reference for self._report() - self._errors.append(error) - - return errors - - def _repair(self, plugin, instance): - if hasattr(plugin, 'repair_instance'): - self.log.warning(AUTO_REPAIR) - try: - plugin().repair_instance(instance) - - except Exception as err: - self.log.warning("Could not auto-repair..") - self._log_error(instance, err) - - else: - self.log.info("Auto-repair successful") - return True - - return False - - def _init_log(self): - self.log._handlers = list(self.log.handlers) - self.log.handlers[:] = [] - - formatter = logging.Formatter(LOG_TEMPATE) - - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - self.log.addHandler(stream_handler) - - self.log.setLevel(self.logging_level) - - def _reset_log(self): - self.log.handlers[:] = self.log._handlers - self.log.setLevel(logging.INFO) - - def _log_plugin(self, plugin, suffix=''): - if hasattr(plugin, 'name'): - name = "%s (%s) %s" % (plugin.__name__, plugin.name, suffix) - else: - name = plugin.__name__ - - print "{plugin}...".format( - tab=TAB, - plugin=name) - - def _log_intro(self): - """Provide a preface to what is about to happen - - Including: - - Pyblish version - - User configuration - - Available paths - - Available plug-ins - - """ - - intro = """ -{line} -pyblish version {version} -{line} - -Available plugin paths: -{paths} - -Available plugins: -{plugins} -""" - - message = intro.format( - line="-" * SCREEN_WIDTH, - version=pyblish.__version__, - paths=_format_paths(self._plugins.paths), - plugins=_format_plugins(self._plugins)) - - message += "\n{line}\nProcessing\n".format(line="-" * 80) - print message - - def _log_error(self, instance, error): - """Format outputted error message - - Including: - - Instance involved in error - - File name in which the error occurred - - Function/method of error - - Line number of error - - Arguments: - instance (pyblish.api.Instance): Instance involved in error - error (Exception): Error to format - - Returns: - Error as pretty-formatted string - - """ - - traceback = getattr(error, 'traceback', None) - - if traceback: - fname, line_number, func, exc = traceback - traceback = (TRACEBACK_TEMPLATE.format(line=line_number, - file=fname, - func=func)) - - self.log.error(ERROR_TEMPLATE.format( - tab=TAB, - instance=instance, - error=error, - traceback=traceback if traceback else '')) - - def _log_time(self): - """Return time-taken message""" - message = 'Time taken: %s' % self.duration - print message.rjust(SCREEN_WIDTH) - - def _log_success(self): - """Log a success message""" - processed_instances = list() - - for instance in self.context: - if not instance.data('__is_processed__'): - continue - processed_instances.append(instance) - - if self.context and not processed_instances: - self.log.warning("Instances were found, but none were processed") - return - - if not self.context: - self.log.warning("No instances were found") - return - - status = "successfully without errors" - if self._errors: - status = "with errors" - - num_processed_instances = len(processed_instances) - (self.log.warning if self._errors else self.log.info)( - SUCCESS_TEMPLATE.format( - num=num_processed_instances, - s="s" if num_processed_instances > 1 else "", - status=status, - seconds=self.duration)) - - def _log_summary(self): - """Layout summary for `context`""" - message = "Summary:\n" - - for instance in self.context: - is_processed = instance.data('__is_processed__') - processed_by = instance.data('__processed_by__') - commit_dir = instance.data('commit_dir') - conform_dir = instance.data('conform_dir') - - _message = "{tab}- \"{inst}\" ".format( - tab=TAB, - inst=instance) - - _message += "processed by:" - - if is_processed: - for _plugin in processed_by or list(): - _message += " \"%s\"," % _plugin.__name__ - _message = _message[:-1] - - else: - _message += " None" - - message += _message + "\n" - - if commit_dir: - message += COMMITTED_TEMPLATE.format( - tab=TAB*2, dir=commit_dir) + "\n" - - if conform_dir: - message += CONFORMED_TEMPLATE.format( - tab=TAB*2, dir=conform_dir) + "\n" - - print # newline - print message - - -# For backwards compatibility -publish_all = publish - - -def _format_paths(paths): - """Return paths at one new each""" - message = '' - for path in paths: - message += "{0}- {1}\n".format(TAB, path) - return message[:-1] # Discard last newline - - -def _format_plugins(plugins): - message = '' - for plugin in sorted(plugins, key=lambda p: p.__name__): - line = "{tab}- {plug}".format( - tab=TAB, plug=plugin.__name__) - - if hasattr(plugin, 'families'): - line = line.ljust(50) + " " - for family in plugin.families: - line += "%s, " % family - line = line[:-2] - - line += "\n" - - message += line - - return message[:-1] +warnings.warn("main.py deprecated; use util.py") diff --git a/pyblish/plugin.py b/pyblish/plugin.py index 63272b3e..2dedb5c0 100644 --- a/pyblish/plugin.py +++ b/pyblish/plugin.py @@ -31,62 +31,24 @@ from .vendor import iscompatible -__all__ = ['Plugin', - 'Selector', - 'Validator', - 'Extractor', - 'Conformer', - 'Context', - 'Instance', - 'discover', - 'plugin_paths', - 'registered_paths', - 'environment_paths', - 'configured_paths', - 'register_plugin_path', - 'deregister_plugin_path', - 'deregister_all'] - - -log = logging.getLogger('pyblish.plugin') - - -class Manager(list): - """Plug-in manager""" - def __init__(self, paths=None): - self.paths = paths or plugin_paths() - self.discover() - - def discover(self, paths=None): - self[:] = discover(paths=paths) - - def lookup_paths(self): - self.paths = plugin_paths() - - def by_order(self, order): - plugins = list() - for plugin in self: - if plugins.order == order: - plugins.append() - return plugins - - def by_type(self, type): - types = {'selector': 0, - 'validator': 1, - 'extractor': 2, - 'conformer': 3} - plugins = list() - for plugin in self: - if plugins.order == types.get(type): - plugins.append() - return plugins - - def by_host(self, host): - plugins = list() - for plugin in self: - if "*" in plugin.hosts or current_host() in plugins.hosts: - plugins.append(plugin) - return plugins +__all__ = ["Plugin", + "Selector", + "Validator", + "Extractor", + "Conformer", + "Context", + "Instance", + "discover", + "plugin_paths", + "registered_paths", + "environment_paths", + "configured_paths", + "register_plugin_path", + "deregister_plugin_path", + "deregister_all"] + + +log = logging.getLogger("pyblish.plugin") class Config(dict): @@ -94,27 +56,12 @@ class Config(dict): .. note:: Config is a singleton. - Configuration is cascading in the following order; - - .. code-block:: bash - - _________ ________ ______ - | | | | | | - | Default | + | Custom | + | User | - |_________| |________| |______| - - In which `User` is being added last and thus overwrites any - previous configuration. - Attributes: DEFAULTCONFIG: Name of default configuration file - HOMEDIR: Absolute path to user's home directory PACKAGEDIR: Absolute path to parent package of Config DEFAULTCONFIGPATH: Absolute path to default configuration file default: Access to default configuration - custom: Access to custom configuration - user: Access to user configuration Usage: >>> config = Config() @@ -191,7 +138,6 @@ class Plugin(object): Plug-ins requiring a version newer than the current version will not be loaded. 1.0.8 was when :attr:`Plugin.requires` was first introduced. - """ hosts = list() # Hosts compatible with plugin @@ -211,7 +157,8 @@ def process(self, context, instances=None): Arguments: context (Context): Context to process - instances (list): Limit which instances to process + instances (list, optional): Names of instances to process, + names not in list will not be processed. .. note:: If an instance contains the data "publish" and that data is `False` the instance will not be processed. @@ -234,10 +181,11 @@ def process(self, context, instances=None): self.process_context(context) except Exception as err: - self.log.error("Could not process context: {0}".format(context)) + self.log.error("Could not process context: " + "%s: %s" % (context, err)) yield None, err - else: + finally: compatible_instances = instances_by_plugin( instances=context, plugin=self) @@ -246,35 +194,35 @@ def process(self, context, instances=None): else: for instance in compatible_instances: - if instance.has_data('publish'): - if instance.data('publish', default=True) is False: - self.log.info("Skipping %s, " - "publish-flag was false" % instance) - continue - - elif not pyblish.config['publish_by_default']: - self.log.info("Skipping %s, " - "no publish-flag was " - "set, and publishing " - "by default is False" % instance) - continue - # Limit instances to those specified in `instances` if instances is not None and \ instance.name not in instances: - self.log.info("Skipping %s, " - "not included in " - "exclusive list (%s)" % (instance, - instances)) + self.log.debug("Skipping %s, " + "not included in " + "exclusive list (%s)" % (instance, + instances)) + continue + + if instance.has_data("publish"): + if instance.data("publish", default=True) is False: + self.log.debug("Skipping %s, " + "publish-flag was false" % instance) + continue + + elif not pyblish.config["publish_by_default"]: + self.log.debug("Skipping %s, " + "no publish-flag was " + "set, and publishing " + "by default is False" % instance) continue self.log.info("Processing instance: \"%s\"" % instance) # Inject data - processed_by = instance.data('__processed_by__') or list() + processed_by = instance.data("__processed_by__") or list() processed_by.append(type(self)) - instance.set_data('__processed_by__', processed_by) - instance.set_data('__is_processed__', True) + instance.set_data("__processed_by__", processed_by) + instance.set_data("__is_processed__", True) try: self.process_instance(instance) @@ -288,8 +236,6 @@ def process(self, context, instances=None): except: pass - # err.traceback = traceback.format_exc() - finally: yield instance, err @@ -314,7 +260,7 @@ def process_instance(self, instance): Implement this method in your subclasses to handle processing of compatible instances. It is run once per instance and - distinguishes between instances compatible with the plugin's + distinguishes between instances compatible with the plugin"s family and host automatically. Returns: @@ -381,7 +327,7 @@ def compute_commit_directory(self, instance): run-time: - pyblish: With absolute path to pyblish package directory - - prefix: With Config['prefix'] + - prefix: With Config["prefix"] - date: With date embedded into `instance` - family: With instance embedded into `instance` - instance: Name of `instance` @@ -398,14 +344,14 @@ def compute_commit_directory(self, instance): """ - workspace_dir = instance.context.data('workspace_dir') + workspace_dir = instance.context.data("workspace_dir") if not workspace_dir: # Project has not been set. Files will # instead end up next to the working file. - current_file = instance.context.data('current_file') + current_file = instance.context.data("current_file") workspace_dir = os.path.dirname(current_file) - date = instance.context.data('date') + date = instance.context.data("date") # This is assumed from default plugins assert date @@ -413,11 +359,11 @@ def compute_commit_directory(self, instance): if not workspace_dir: raise pyblish.error.ExtractorError( "Could not determine commit directory. " - "Instance MUST supply either 'current_file' or " - "'workspace_dir' as data prior to commit") + "Instance MUST supply either \"current_file\" or " + "\"workspace_dir\" as data prior to commit") # Remove invalid characters from output name - name = instance.data('name') + name = instance.data("name") valid_name = pyblish.lib.format_filename(name) if name != valid_name: self.log.info("Formatting instance name: " @@ -425,16 +371,16 @@ def compute_commit_directory(self, instance): % (name, valid_name)) name = valid_name - variables = {'pyblish': pyblish.lib.main_package_path(), - 'prefix': pyblish.config['prefix'], - 'date': date, - 'family': instance.data('family'), - 'instance': name, - 'user': instance.data('user')} + variables = {"pyblish": pyblish.lib.main_package_path(), + "prefix": pyblish.config["prefix"], + "date": date, + "family": instance.data("family"), + "instance": name, + "user": instance.data("user")} # Restore separators to those native to the current OS - commit_template = pyblish.config['commit_template'] - commit_template = commit_template.replace('/', os.sep) + commit_template = pyblish.config["commit_template"] + commit_template = commit_template.replace("/", os.sep) commit_dir = commit_template.format(**variables) commit_dir = os.path.join(workspace_dir, commit_dir) @@ -466,7 +412,7 @@ def commit(self, path, instance): shutil.copytree(path, commit_dir) # Persist path of commit within instance - instance.set_data('commit_dir', value=commit_dir) + instance.set_data("commit_dir", value=commit_dir) return commit_dir @@ -575,12 +521,12 @@ def create_instance(self, name): """Convenience method of the following. >>> ctx = Context() - >>> inst = Instance('name', parent=ctx) + >>> inst = Instance("name", parent=ctx) >>> ctx.add(inst) Example: >>> ctx = Context() - >>> inst = ctx.create_instance(name='Name') + >>> inst = ctx.create_instance(name="Name") """ @@ -608,13 +554,13 @@ class Instance(AbstractEntity): """ def __eq__(self, other): - return self.name == getattr(other, 'name', None) + return self.name == getattr(other, "name", None) def __ne__(self, other): - return self.name != getattr(other, 'name', None) + return self.name != getattr(other, "name", None) def __repr__(self): - return u"%s.%s('%s')" % (__name__, type(self).__name__, self) + return u"%s.%s(\"%s\")" % (__name__, type(self).__name__, self) def __str__(self): return self.name @@ -648,16 +594,16 @@ def data(self, key=None, default=None): That way, names may be overridden via data. Example: - >>> inst = Instance(name='test') - >>> assert inst.data('name') == 'test' - >>> inst.set_data('name', 'newname') - >>> assert inst.data('name') == 'newname' + >>> inst = Instance(name="test") + >>> assert inst.data("name") == "test" + >>> inst.set_data("name", "newname") + >>> assert inst.data("name") == "newname" """ value = super(Instance, self).data(key, default) - if key == 'name' and value is None: + if key == "name" and value is None: return self.name return value @@ -672,38 +618,38 @@ def current_host(): Example: >> # Running within Autodesk Maya >> current_host() - 'maya' + "maya" >> # Running within Sidefx Houdini >> current_host() - 'houdini' + "houdini" """ executable = os.path.basename(sys.executable).lower() - if 'python' in executable: + if "python" in executable: # Running from standalone Python - return 'python' + return "python" - if 'maya' in executable: + if "maya" in executable: # Maya is distinguished by looking at the currently running # executable of the Python interpreter. It will be something # like: "maya.exe" or "mayapy.exe"; without suffix for # posix platforms. - return 'maya' + return "maya" - if 'nuke' in executable: + if "nuke" in executable: # Nuke typically includes a version number, e.g. Nuke8.0.exe # and mixed-case letters. - return 'nuke' + return "nuke" # ..note:: The following are guesses, feel free to correct - if 'modo' in executable: - return 'modo' + if "modo" in executable: + return "modo" - if 'houdini' in executable: - return 'houdini' + if "houdini" in executable: + return "houdini" raise ValueError("Could not determine host") @@ -712,11 +658,11 @@ def register_plugin_path(path): """Plug-ins are looked up at run-time from directories registered here To register a new directory, run this command along with the absolute - path to where you're plug-ins are located. + path to where you"re plug-ins are located. Example: >>> import os - >>> my_plugins = os.path.expanduser('~') + >>> my_plugins = os.path.expanduser("~") >>> register_plugin_path(my_plugins) >>> deregister_plugin_path(my_plugins) @@ -734,7 +680,7 @@ def deregister_plugin_path(path): """Remove a pyblish._registered_paths path Raises: - KeyError if `path` isn't registered + KeyError if `path` isn"t registered """ @@ -760,8 +706,8 @@ def configured_paths(): """Return paths added via configuration""" paths = list() - for path_template in pyblish.config['paths']: - variables = {'pyblish': pyblish.lib.main_package_path()} + for path_template in pyblish.config["paths"]: + variables = {"pyblish": pyblish.lib.main_package_path()} plugin_path = path_template.format(**variables) @@ -777,7 +723,7 @@ def environment_paths(): paths = list() - env_var = pyblish.config['paths_environment_variable'] + env_var = pyblish.config["paths_environment_variable"] env_val = os.environ.get(env_var) if env_val: env_paths = env_val.split(os.pathsep) @@ -844,10 +790,10 @@ def discover(type=None, regex=None, paths=None): """ - patterns = {'validators': pyblish.config['validators_regex'], - 'extractors': pyblish.config['extractors_regex'], - 'selectors': pyblish.config['selectors_regex'], - 'conformers': pyblish.config['conformers_regex']} + patterns = {"validators": pyblish.config["validators_regex"], + "extractors": pyblish.config["extractors_regex"], + "selectors": pyblish.config["selectors_regex"], + "conformers": pyblish.config["conformers_regex"]} if type is not None and type not in patterns: raise ValueError("Type not recognised: %s" % type) @@ -886,15 +832,15 @@ def plugin_is_valid(plugin): try: if (issubclass(plugin, Selector) - and not getattr(plugin, 'hosts')): + and not getattr(plugin, "hosts")): raise Exception(0) if (issubclass(plugin, (Validator, Extractor)) - and not getattr(plugin, 'families') - and not getattr(plugin, 'hosts')): + and not getattr(plugin, "families") + and not getattr(plugin, "hosts")): raise Exception(1) if (issubclass(plugin, Conformer) - and not getattr(plugin, 'families')): + and not getattr(plugin, "families")): raise Exception(2) except Exception as e: @@ -939,7 +885,7 @@ def host_is_compatible(plugin): reload(module) except Exception as err: - log.warning('Skipped: "%s" (%s)', mod_name, err) + log.warning("Skipped: \"%s\" (%s)", mod_name, err) continue finally: @@ -969,7 +915,31 @@ def host_is_compatible(plugin): if regex is None or re.match(regex, obj.__name__): discovered_plugins[obj.__name__] = obj - return discovered_plugins.values() + plugins = discovered_plugins.values() + sort(plugins) # In-place + return plugins + + +def sort(plugins): + """Sort `plugins` in-place + + Their order is determined by their `order` attribute, + which defaults to their standard execution order: + + 1. Selection + 2. Validation + 3. Extraction + 4. Conform + + *But may be overridden. + + Arguments: + plugins (list): Plug-ins to sort + + """ + + plugins.sort(key=lambda p: p.order) + return plugins def plugins_by_family(plugins, family): @@ -1014,8 +984,8 @@ def plugins_by_host(plugins, host): if not hasattr(plugin, "hosts"): continue - # TODO(marcus): Expand to take partial wildcards e.g. '*Mesh' - if any(x in plugin.hosts for x in (host, '*')): + # TODO(marcus): Expand to take partial wildcards e.g. "*Mesh" + if any(x in plugin.hosts for x in (host, "*")): compatible.append(plugin) return compatible @@ -1039,7 +1009,7 @@ def instances_by_plugin(instances, plugin): compatible = list() for instance in instances: - if not hasattr(plugin, 'families'): + if not hasattr(plugin, "families"): continue family = instance.data("family") diff --git a/pyblish/tests/test_main.py b/pyblish/tests/test_main.py index f11ea944..a517aef7 100644 --- a/pyblish/tests/test_main.py +++ b/pyblish/tests/test_main.py @@ -1,5 +1,5 @@ -import pyblish.main +import pyblish.util import pyblish.plugin from pyblish.vendor import mock @@ -8,12 +8,12 @@ teardown, FAMILY, HOST, setup_failing, setup_full) -@mock.patch('pyblish.main.Publish.log') +@mock.patch('pyblish.util.Publish.log') @with_setup(setup_full, teardown) def test_publish_all(_): """publish() calls upon each convenience function""" ctx = pyblish.plugin.Context() - pyblish.main.publish(context=ctx) + pyblish.util.publish(context=ctx) for inst in ctx: assert inst.data('selected') is True @@ -22,18 +22,18 @@ def test_publish_all(_): assert inst.data('conformed') is True -@mock.patch('pyblish.main.Publish.log') +@mock.patch('pyblish.util.Publish.log') def test_publish_all_no_instances(mock_log): """Having no instances is fine, a warning is logged""" ctx = pyblish.plugin.Context() - pyblish.main.publish(ctx) + pyblish.util.publish(ctx) assert mock_log.warning.called @with_setup(setup_full, teardown) def test_publish_all_no_context(): """Not passing a context is fine""" - ctx = pyblish.main.publish() + ctx = pyblish.util.publish() for inst in ctx: assert inst.data('selected') is True @@ -42,12 +42,12 @@ def test_publish_all_no_context(): assert inst.data('conformed') is True -@mock.patch('pyblish.main.Publish.log') +@mock.patch('pyblish.util.Publish.log') @with_setup(setup_full, teardown) def test_validate_all(_): """validate_all() calls upon two of the convenience functions""" ctx = pyblish.plugin.Context() - pyblish.main.validate_all(context=ctx) + pyblish.util.validate_all(context=ctx) for inst in ctx: assert inst.data('selected') is True @@ -56,13 +56,13 @@ def test_validate_all(_): assert inst.data('conformed') is False -@mock.patch('pyblish.main.Publish.log') +@mock.patch('pyblish.util.Publish.log') @with_setup(setup_full, teardown) def test_convenience(_): """Convenience function work""" ctx = pyblish.plugin.Context() - pyblish.main.select(context=ctx) + pyblish.util.select(context=ctx) for inst in ctx: assert inst.data('selected') is True @@ -70,7 +70,7 @@ def test_convenience(_): assert inst.data('extracted') is False assert inst.data('conformed') is False - pyblish.main.validate(context=ctx) + pyblish.util.validate(context=ctx) for inst in ctx: assert inst.data('selected') is True @@ -78,7 +78,7 @@ def test_convenience(_): assert inst.data('extracted') is False assert inst.data('conformed') is False - pyblish.main.extract(context=ctx) + pyblish.util.extract(context=ctx) for inst in ctx: assert inst.data('selected') is True @@ -86,7 +86,7 @@ def test_convenience(_): assert inst.data('extracted') is True assert inst.data('conformed') is False - pyblish.main.conform(context=ctx) + pyblish.util.conform(context=ctx) for inst in ctx: assert inst.data('selected') is True @@ -95,17 +95,17 @@ def test_convenience(_): assert inst.data('conformed') is True -@mock.patch('pyblish.main.Publish.log') +@mock.patch('pyblish.util.Publish.log') @with_setup(setup_failing, teardown) def test_main_safe_processes_fail(_): """Failing selection, extraction or conform merely logs a message""" ctx = pyblish.plugin.Context() - pyblish.main.select(ctx) + pyblish.util.select(ctx) # Give plugins something to process inst = ctx.create_instance(name='TestInstance') inst.set_data('family', value=FAMILY) inst.set_data('host', value=HOST) - pyblish.main.extract(ctx) - pyblish.main.conform(ctx) + pyblish.util.extract(ctx) + pyblish.util.conform(ctx) diff --git a/pyblish/tests/test_plugins.py b/pyblish/tests/test_plugins.py index 67f04f29..36b54a1c 100644 --- a/pyblish/tests/test_plugins.py +++ b/pyblish/tests/test_plugins.py @@ -3,6 +3,7 @@ import os import time import shutil +import random import tempfile # Local library @@ -13,8 +14,7 @@ from pyblish.tests.lib import ( setup, teardown, setup_failing, HOST, FAMILY, setup_duplicate, setup_invalid, setup_wildcard) -from pyblish.vendor.nose.tools import ( - raises, with_setup, assert_raises) +from pyblish.vendor.nose.tools import * config = pyblish.plugin.Config() @@ -454,3 +454,43 @@ def test_plugins_by_family_wildcard(): assert Plugin2 in pyblish.api.plugins_by_family( [Plugin1, Plugin2], "myFamily") + + +def test_failing_context_processing(): + """Plug-in should not skip processing of Instance if Context fails""" + + value = {"a": False} + + class MyPlugin(pyblish.api.Validator): + families = ["myFamily"] + hosts = ["python"] + + def process_context(self, context): + raise Exception("Failed") + + def process_instance(self, instance): + value["a"] = True + + ctx = pyblish.api.Context() + inst = ctx.create_instance(name="MyInstance") + inst.set_data("family", "myFamily") + + for instance, error in MyPlugin().process(ctx): + pass + + assert_true(value["a"]) + + +@with_setup(setup, teardown) +def test_plugins_sorted(): + """Plug-ins are returned sorted by their `order` attribute""" + plugins = pyblish.api.discover() + random.shuffle(plugins) # Randomise their order + pyblish.api.sort_plugins(plugins) + + order = 0 + for plugin in plugins: + assert_true(plugin.order >= order) + order = plugin.order + + assert order > 0, plugins diff --git a/pyblish/util.py b/pyblish/util.py new file mode 100644 index 00000000..d224430a --- /dev/null +++ b/pyblish/util.py @@ -0,0 +1,527 @@ +"""Conveinence function for Pyblish + +Attributes: + TAB: Number of spaces for a tab + LOG_TEMPATE: Template used for logging coming from + plug-ins + SCREEN_WIDTH: Default width at which logging and printing + will (attempt to) restrain to. + logging_handlers: Record of handlers at the start of + importing this module. This module will modify the + currently handlers and restore then once finished. + log: Current logger + intro_message: Message printed upon initiating a publish. + +""" + +from __future__ import absolute_import + +# Standard library +import time +import logging + +# Local library +import pyblish.api + +TAB = " " +LOG_TEMPATE = " %(levelname)-8s %(message)s" +SCREEN_WIDTH = 80 + +# Messages +NO_INSTANCES_ERROR = "Cancelled due to not finding any instances" +SELECTION_ERROR = "Selection failed" +VALIDATION_ERROR = "Validation failed" + +# Templates +AUTO_REPAIR = "There were errors, attempting to auto-repair.." +COMMITTED_TEMPLATE = "{tab}Committed to: {dir}" +CONFORMED_TEMPLATE = "{tab}Conformed to: {dir}" +FAILED_VALIDATION_TEMPLATE = "{tab}- \"{instance}\": {exception} ({plugin})" +SUCCESS_TEMPLATE = "Processed {num} instance{s} {status} in {seconds}s" +TRACEBACK_TEMPLATE = "(Line {line} in \"{file}\" @ \"{func}\")" +ERROR_TEMPLATE = "{tab}{instance}: {error} {traceback}" +VALIDATIONS_FAILED_TEMPLATE = """ +These validations failed: +{failures}""" + +__all__ = ['select', + 'validate', + 'extract', + 'conform', + 'publish', + 'publish_all'] + + +def publish(context=None, + auto_repair=False, + include_optional=True, + instances=None, + logging_level=logging.INFO, + **kwargs): + """Publish everything + + This function will process all available plugins of the + currently running host, publishing anything picked up + during selection. + + Arguments: + context (pyblish.api.Context): Optional Context. + Defaults to creating a new context each time. + auto_repair (bool): Whether or not to attempt to automatically + repair instances which fail validation. + include_optional (bool): Should validation include plug-ins + which has been defined as optional? + logging_level (logging level): Optional level with which + to log messages. Default is logging.INFO. + + Usage: + >> publish() + >> publish(context=Context()) + + """ + + # Hidden argument + _orders = kwargs.pop('orders', None) or (0, 1, 2, 3) + assert not kwargs # There are no more arguments + + obj = Publish(context, + auto_repair=auto_repair, + include_optional=include_optional, + instances=instances, + logging_level=logging_level) + + obj.orders = _orders + obj.process() + + return obj.context + + +def validate_all(*args, **kwargs): + if 'orders' not in kwargs: + kwargs['orders'] = (0, 1) + return publish(*args, **kwargs) + + +def select(*args, **kwargs): + """Convenience function for selection""" + if 'orders' not in kwargs: + kwargs['orders'] = (0,) + return publish(*args, **kwargs) + + +def validate(*args, **kwargs): + """Convenience function for validation""" + if 'orders' not in kwargs: + kwargs['orders'] = (1,) + return publish(*args, **kwargs) + + +def extract(*args, **kwargs): + """Convenience function for extraction""" + if 'orders' not in kwargs: + kwargs['orders'] = (2,) + return publish(*args, **kwargs) + + +def conform(*args, **kwargs): + """Convenience function for conform""" + if 'orders' not in kwargs: + kwargs['orders'] = (3,) + return publish(*args, **kwargs) + + +class Publish(object): + """Publishing operator + + Arguments: + context (pyblish.api.Context): Optional Context. + Defaults to creating a new context each time. + auto_repair (bool): Whether or not to attempt to automatically + repair instances which fail validation. + include_optional (bool): Should validation include plug-ins + which has been defined as optional? + logging_level (logging level): Optional level with which + to log messages. Default is logging.INFO. + + """ + + log = logging.getLogger() + + @property + def duration(self): + return "%.2f" % (self._time['end'] - self._time['start']) + + def __init__(self, + context=None, + auto_repair=False, + include_optional=True, + instances=None, + logging_level=logging.INFO): + + if context is None: + context = pyblish.api.Context() + + self.context = context + self.orders = (0, 1, 2, 3) + self.repair = auto_repair + self.optional = include_optional + self.logging_level = logging_level + self.instances = None + + self._plugins = pyblish.plugin.discover() + self._conf = pyblish.api.config + self._time = {'start': None, 'end': None} + self._errors = list() + + def process(self): + """Process all instances within the given context""" + self._time['start'] = time.time() + self._log_intro() + + log_summary = False + + # Initialise pretty-printing for plug-ins + self._init_log() + + try: + for order in self.orders: + self.process_order(order) + + except pyblish.api.NoInstancesError as exc: + self.log.warning(NO_INSTANCES_ERROR) + + except pyblish.api.SelectionError: + self.log.error(SELECTION_ERROR) + + except pyblish.api.ValidationError as exc: + self.log.error(VALIDATION_ERROR) + + failures = list() + for error in exc.errors: + failures.append(FAILED_VALIDATION_TEMPLATE.format( + instance=error.instance, + tab=TAB, + exception=error, + plugin=error.plugin.__name__)) + print VALIDATIONS_FAILED_TEMPLATE.format( + failures="\n".join(failures)) + + except Exception as exc: + self.log.error("Unhandled exception: %s" % exc) + + else: + log_summary = True + + # Clear context + self._time['end'] = time.time() + + print # newline + print "-" * 80 + + self._log_time() + + if log_summary: + self._log_summary() + + self._reset_log() + self._log_success() + + def process_order(self, order): + """Process context using plug-ins with the specified `order` + + Arguments: + order (int): Order of plug-ins with which to process context. + + Raises: + pyblish.api.SelectionError: When selection fails + pyblish.api.ValidationError: When validation fails + + """ + + if order != 0 and not self.context: + # If there aren't any instances after selection, + # there is no point in going on. + raise pyblish.api.NoInstancesError + + order_errors = list() + for plugin in self._plugins: + + if plugin.order != order: + continue + + plugin_errors = self.process_plugin(plugin) + order_errors.extend(plugin_errors) + + if not plugin_errors: + continue + + # Before proceeding with extraction, ensure + # that there are no failed validators. + self.log.warning("There were errors:") + for error in plugin_errors: + self._log_error(error.instance, error) + + if not order_errors: + return + + # If the error occurred during selection or validation, + # we don't want to continue. + if order == 0: + raise pyblish.api.SelectionError + + if order == 1: + exception = pyblish.api.ValidationError + exception.errors = order_errors + raise exception + + def process_plugin(self, plugin): + """Process context using a single plugin + + Arguments: + plugin (Plugin): Plug-in used to process context + + Returns: + List of errors occurred for `plugin` + + """ + + errors = list() + + # Do not include optional plug-ins + if plugin.optional and self.optional is False: + self._log_plugin(plugin, suffix="(optional and skipped)") + return errors + + self._log_plugin(plugin) + + for instance, error in plugin().process(self.context): + if instance is None and error is None: + self.log.debug("Skipped, no compatible instances.") + continue + + if error is None: + continue + + repaired = False + if plugin.order == 1 and self.repair: + repaired = self._repair(plugin, instance) + + if not repaired: + errors.append(error) + + # Inject data for logging + error.instance = instance + error.plugin = plugin + + # Store global reference for self._report() + self._errors.append(error) + + return errors + + def _repair(self, plugin, instance): + if hasattr(plugin, 'repair_instance'): + self.log.warning(AUTO_REPAIR) + try: + plugin().repair_instance(instance) + + except Exception as err: + self.log.warning("Could not auto-repair..") + self._log_error(instance, err) + + else: + self.log.info("Auto-repair successful") + return True + + return False + + def _init_log(self): + self.log._handlers = list(self.log.handlers) + self.log.handlers[:] = [] + + formatter = logging.Formatter(LOG_TEMPATE) + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + self.log.addHandler(stream_handler) + + self.log.setLevel(self.logging_level) + + def _reset_log(self): + self.log.handlers[:] = self.log._handlers + self.log.setLevel(logging.INFO) + + def _log_plugin(self, plugin, suffix=''): + if hasattr(plugin, 'name'): + name = "%s (%s) %s" % (plugin.__name__, plugin.name, suffix) + else: + name = plugin.__name__ + + print "{plugin}...".format( + tab=TAB, + plugin=name) + + def _log_intro(self): + """Provide a preface to what is about to happen + + Including: + - Pyblish version + - User configuration + - Available paths + - Available plug-ins + + """ + + intro = """ +{line} +pyblish version {version} +{line} + +Available plugin paths: +{paths} + +Available plugins: +{plugins} +""" + + message = intro.format( + line="-" * SCREEN_WIDTH, + version=pyblish.__version__, + paths=_format_paths(pyblish.api.plugin_paths()), + plugins=_format_plugins(self._plugins)) + + message += "\n{line}\nProcessing\n".format(line="-" * 80) + print message + + def _log_error(self, instance, error): + """Format outputted error message + + Including: + - Instance involved in error + - File name in which the error occurred + - Function/method of error + - Line number of error + + Arguments: + instance (pyblish.api.Instance): Instance involved in error + error (Exception): Error to format + + Returns: + Error as pretty-formatted string + + """ + + traceback = getattr(error, 'traceback', None) + + if traceback: + fname, line_number, func, exc = traceback + traceback = (TRACEBACK_TEMPLATE.format(line=line_number, + file=fname, + func=func)) + + self.log.error(ERROR_TEMPLATE.format( + tab=TAB, + instance=instance, + error=error, + traceback=traceback if traceback else '')) + + def _log_time(self): + """Return time-taken message""" + message = 'Time taken: %s' % self.duration + print message.rjust(SCREEN_WIDTH) + + def _log_success(self): + """Log a success message""" + processed_instances = list() + + for instance in self.context: + if not instance.data('__is_processed__'): + continue + processed_instances.append(instance) + + if self.context and not processed_instances: + self.log.warning("Instances were found, but none were processed") + return + + if not self.context: + self.log.warning("No instances were found") + return + + status = "successfully without errors" + if self._errors: + status = "with errors" + + num_processed_instances = len(processed_instances) + (self.log.warning if self._errors else self.log.info)( + SUCCESS_TEMPLATE.format( + num=num_processed_instances, + s="s" if num_processed_instances > 1 else "", + status=status, + seconds=self.duration)) + + def _log_summary(self): + """Layout summary for `context`""" + message = "Summary:\n" + + for instance in self.context: + is_processed = instance.data('__is_processed__') + processed_by = instance.data('__processed_by__') + commit_dir = instance.data('commit_dir') + conform_dir = instance.data('conform_dir') + + _message = "{tab}- \"{inst}\" ".format( + tab=TAB, + inst=instance) + + _message += "processed by:" + + if is_processed: + for _plugin in processed_by or list(): + _message += " \"%s\"," % _plugin.__name__ + _message = _message[:-1] + + else: + _message += " None" + + message += _message + "\n" + + if commit_dir: + message += COMMITTED_TEMPLATE.format( + tab=TAB*2, dir=commit_dir) + "\n" + + if conform_dir: + message += CONFORMED_TEMPLATE.format( + tab=TAB*2, dir=conform_dir) + "\n" + + print # newline + print message + + +# For backwards compatibility +publish_all = publish + + +def _format_paths(paths): + """Return paths at one new each""" + message = '' + for path in paths: + message += "{0}- {1}\n".format(TAB, path) + return message[:-1] # Discard last newline + + +def _format_plugins(plugins): + message = '' + for plugin in sorted(plugins, key=lambda p: p.__name__): + line = "{tab}- {plug}".format( + tab=TAB, plug=plugin.__name__) + + if hasattr(plugin, 'families'): + line = line.ljust(50) + " " + for family in plugin.families: + line += "%s, " % family + line = line[:-2] + + line += "\n" + + message += line + + return message[:-1] diff --git a/pyblish/version.py b/pyblish/version.py index c1d53442..2aabcba2 100644 --- a/pyblish/version.py +++ b/pyblish/version.py @@ -1,7 +1,7 @@ VERSION_MAJOR = 1 VERSION_MINOR = 0 -VERSION_PATCH = 12 +VERSION_PATCH = 13 version_info = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) version = '%i.%i.%i' % version_info