From 62d7dd4b0f9ddec415e8ab7c9356a7ab47a543bd Mon Sep 17 00:00:00 2001 From: Joseph Zhong Date: Mon, 6 May 2019 14:12:49 -0400 Subject: [PATCH] initial commit --- cmd_line.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++ utility.py | 95 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 cmd_line.py create mode 100644 utility.py diff --git a/cmd_line.py b/cmd_line.py new file mode 100644 index 0000000..fa0c429 --- /dev/null +++ b/cmd_line.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +""" +cmd_line.py +--- + +CMD Line parsing utilities. + +""" +import argparse +import inspect +import logging +import subprocess +from types import GeneratorType + +import src.utils.utility as _util + +_logger = _util.getLogger("CMD Line") + +def runCmd(cmd, logger=None, stopOnFail=True): + if logger is None: + logger = _logger + else: + assert isinstance(logger, logging.Logger) + + logger.info("Running '%s'", cmd) + # output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, shell=True) + # print output + # ret = subprocess.call(cmd.split(), shell=True) + # if ret != 0: + # logger.error("'%s' returned with error code: '%s'", cmd, ret) + # logger.debug("Traceback: '%s'", traceback.format_exc()) + # if stopOnFail: + # sys.exit(ret) + # else: + # logger.info("'{}' Success!".format(cmd)) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=False) + proc_stdout = process.communicate()[0].strip() + print(proc_stdout) + logger.info("Completed running '%s", cmd) + +def _str_to_bool(s): + """Convert string to bool (in argparse context).""" + if s.lower() not in ['true', 'false']: + raise ValueError('Need bool; got %r' % s) + return {'true': True, 'false': False}[s.lower()] + +def add_boolean_argument(parser, name, default=False): + group = parser.add_mutually_exclusive_group() + group.add_argument( + '--' + name, + nargs='?', + default=default, + const=True, + type=_str_to_bool) + group.add_argument('--no' + name, + dest=name, + action='store_false') + +def parseArgsForClassOrScript(fn): + assert inspect.isfunction(fn) or inspect.ismethod(fn) + + spec = inspect.getargspec(fn) + + parser = argparse.ArgumentParser() + for i, arg in enumerate(spec.args): + if arg == 'self' or arg == 'logger': + continue + + # If index is greater than the last var with a default, it's required. + numReq = len(spec.args) - len(spec.defaults) + required = i < numReq + default = spec.defaults[i - numReq] if not required else None + # By default, args are parsed as strings if not otherwise specified. + if isinstance(default, bool): + parser.add_argument("--" + arg, default=default, action='store_true') + elif isinstance(default, (tuple, list, GeneratorType)): + parser.add_argument("--" + arg, default=default, nargs="+", help="Tuple of " + arg, required=False) + else: + parser.add_argument("--" + arg, default=default, type=type(default) if default is not None else str) + + parser.add_argument("-v", "--verbosity", + default=_util.DEFAULT_VERBOSITY, + type=int, + help="Verbosity mode. Default is 4. " + "Set as " + "0 for CRITICAL level logs only. " + "1 for ERROR and above level logs " + "2 for WARNING and above level logs " + "3 for INFO and above level logs " + "4 for DEBUG and above level logs") + argv = parser.parse_args() + argsToVals = vars(argv) + + # Print args for any verbosity greater than CRITICAL. + if argv.verbosity > 0 or argv.help: + docstr = inspect.getdoc(fn) + if docstr is None: + print( +"""WARNING: + Please write documentation :) +""") + print() + print(docstr.strip()) + print() + print("Arguments and corresponding default or set values") + for arg in spec.args: + if arg == 'self' or arg == 'logger' or arg not in argsToVals: + continue + print("\t{}={}".format(arg, argsToVals[arg] if argsToVals[arg] is not None else "")) + print() + + # parser.print_help() + + return argv diff --git a/utility.py b/utility.py new file mode 100644 index 0000000..f23a6c3 --- /dev/null +++ b/utility.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +""" +Uility functions in logging and IO. +""" + +import os +import shutil +import logging + +DEFAULT_VERBOSITY = 4 + +_ws_dir = None + +_logger = None +_LOGGING_FORMAT = "[%(asctime)s %(levelname)5s %(filename)s %(funcName)s:%(lineno)s] %(message)s" +# REVIEW josephz: How do I enable file-logging as well? +logging.basicConfig(format=_LOGGING_FORMAT, datefmt="%Y-%m-%d %H:%M:%S") + +# REVIEW josephz: The logger is pretty broken right now -- in particular, the verbosity doesn't transfer recursively. +def getLogger(name, level=logging.DEBUG, verbosity=DEFAULT_VERBOSITY): + level = max(level, logging.CRITICAL - 10 * verbosity) + + logger = logging.getLogger(name) + logger.setLevel(level) + return logger + +def _getUtilityLogger(): + global _logger + if _logger is None: + _logger = getLogger("Utility") + return _logger + +def _getPathFromEnv(envVar): + path = os.getenv(envVar, None) + assert path is not None, \ + "Environment variable '{}' not found: " \ + "please check project installation and ~/.bashrc".format(envVar) + return path + +# REVIEW josephz: This should be configurable without having to touch code. +def getWsDir(envName="WS_PATH"): + global _ws_dir + if _ws_dir is None: + _ws_dir = _getPathFromEnv(envName) + return _ws_dir + +def getRelDataPath(*relPath): + return os.path.join(getWsDir(), "data", *relPath) + +def getRelRawPath(*relPath): + return getRelDataPath("raw", *relPath) + +def getRelWeightsPath(*relPath): + return getRelDataPath("weights", *relPath) + +def getRelDatasetsPath(*relPath): + return getRelDataPath("datasets", *relPath) + +def getRelPicklesPath(*relPath): + return getRelDataPath("pickles", *relPath) + +def mkdirP(path): + if not os.path.exists(path): + os.makedirs(path) + +def touch(path): + with open(path, 'a'): + os.utime(path, None) + +def mv(src, dst, mkdirMode=True, force=False): + """ Moves src to dst as if `mv` was used. Both src and dst are relative to root, + which is set to the 'data path' by default. With the mkdir option, we enforce + the dst to be a path and we support "move to dir" behavior. Otherwise we support + "move to dir and rename file" behavior. + """ + assert os.path.exists(src), "'{}' not found".format(src) + + # In mkdir mode, we enforce the dst to be a path to allow "move to dir" behavior. + # Otherwise we are supporting "move to dir and rename" behavior. + if not dst.endswith('/') and mkdirMode: + dst += '/' + dstHeadPath, _ = os.path.split(dst) + mkdirP(dstHeadPath) + + if os.path.isdir(dst): + _getUtilityLogger().info("Moving '{}' into directory '{}'".format(src, dst)) + else: + _getUtilityLogger().info("Renaming '{}' to '{}'".format(src, dst)) + + if force: + shutil.copy(src, dst) + else: + shutil.move(src, dst) +