initial commit

cmd_line.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""
+cmd_line.py
+---
+
+CMD Line parsing utilities.
+
+"""
+import argparse
+import inspect
+import logging
+import subprocess
+from types import GeneratorType
+
+import src.utils.utility as _util
+
+_logger = _util.getLogger("CMD Line")
+
+def runCmd(cmd, logger=None, stopOnFail=True):
+    if logger is None:
+        logger = _logger
+    else:
+        assert isinstance(logger, logging.Logger)
+
+    logger.info("Running '%s'", cmd)
+    # output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, shell=True)
+    # print output
+    # ret = subprocess.call(cmd.split(), shell=True)
+    # if ret != 0:
+    #     logger.error("'%s' returned with error code: '%s'", cmd, ret)
+    #     logger.debug("Traceback: '%s'", traceback.format_exc())
+    #     if stopOnFail:
+    #         sys.exit(ret)
+    # else:
+    #     logger.info("'{}' Success!".format(cmd))
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=False)
+    proc_stdout = process.communicate()[0].strip()
+    print(proc_stdout)
+    logger.info("Completed running '%s", cmd)
+
+def _str_to_bool(s):
+    """Convert string to bool (in argparse context)."""
+    if s.lower() not in ['true', 'false']:
+        raise ValueError('Need bool; got %r' % s)
+    return {'true': True, 'false': False}[s.lower()]
+
+def add_boolean_argument(parser, name, default=False):
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--' + name,
+        nargs='?',
+        default=default,
+        const=True,
+        type=_str_to_bool)
+    group.add_argument('--no' + name,
+        dest=name,
+        action='store_false')
+
+def parseArgsForClassOrScript(fn):
+    assert inspect.isfunction(fn) or inspect.ismethod(fn)
+
+    spec = inspect.getargspec(fn)
+
+    parser = argparse.ArgumentParser()
+    for i, arg in enumerate(spec.args):
+        if arg == 'self' or arg == 'logger':
+            continue
+
+        # If index is greater than the last var with a default, it's required.
+        numReq = len(spec.args) - len(spec.defaults)
+        required = i < numReq
+        default = spec.defaults[i - numReq] if not required else None
+        # By default, args are parsed as strings if not otherwise specified.
+        if isinstance(default, bool):
+            parser.add_argument("--" + arg, default=default, action='store_true')
+        elif isinstance(default, (tuple, list, GeneratorType)):
+            parser.add_argument("--" + arg, default=default, nargs="+", help="Tuple of " + arg, required=False)
+        else:
+            parser.add_argument("--" + arg, default=default, type=type(default) if default is not None else str)
+
+    parser.add_argument("-v", "--verbosity",
+        default=_util.DEFAULT_VERBOSITY,
+        type=int,
+        help="Verbosity mode. Default is 4. "
+                 "Set as "
+                 "0 for CRITICAL level logs only. "
+                 "1 for ERROR and above level logs "
+                 "2 for WARNING and above level logs "
+                 "3 for INFO and above level logs "
+                 "4 for DEBUG and above level logs")
+    argv = parser.parse_args()
+    argsToVals = vars(argv)
+
+    # Print args for any verbosity greater than CRITICAL.
+    if argv.verbosity > 0 or argv.help:
+        docstr = inspect.getdoc(fn)
+        if docstr is None:
+                print(
+"""WARNING:
+        Please write documentation :)
+""")
+        print()
+        print(docstr.strip())
+        print()
+        print("Arguments and corresponding default or set values")
+        for arg in spec.args:
+            if arg == 'self' or arg == 'logger' or arg not in argsToVals:
+                continue
+            print("\t{}={}".format(arg, argsToVals[arg] if argsToVals[arg] is not None else ""))
+        print()
+
+            # parser.print_help()
+
+    return argv

utility.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+"""
+Uility functions in logging and IO.
+"""
+
+import os
+import shutil
+import logging
+
+DEFAULT_VERBOSITY = 4
+
+_ws_dir = None
+
+_logger = None
+_LOGGING_FORMAT = "[%(asctime)s %(levelname)5s %(filename)s %(funcName)s:%(lineno)s] %(message)s"
+# REVIEW josephz: How do I enable file-logging as well?
+logging.basicConfig(format=_LOGGING_FORMAT, datefmt="%Y-%m-%d %H:%M:%S")
+
+# REVIEW josephz: The logger is pretty broken right now -- in particular, the verbosity doesn't transfer recursively.
+def getLogger(name, level=logging.DEBUG, verbosity=DEFAULT_VERBOSITY):
+    level = max(level, logging.CRITICAL - 10 * verbosity)
+
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    return logger
+
+def _getUtilityLogger():
+    global _logger
+    if _logger is None:
+        _logger = getLogger("Utility")
+    return _logger
+
+def _getPathFromEnv(envVar):
+    path = os.getenv(envVar, None)
+    assert path is not None, \
+        "Environment variable '{}' not found: " \
+        "please check project installation and ~/.bashrc".format(envVar)
+    return path
+
+# REVIEW josephz: This should be configurable without having to touch code.
+def getWsDir(envName="WS_PATH"):
+    global _ws_dir
+    if _ws_dir is None:
+        _ws_dir = _getPathFromEnv(envName)
+    return _ws_dir
+
+def getRelDataPath(*relPath):
+    return os.path.join(getWsDir(), "data", *relPath)
+
+def getRelRawPath(*relPath):
+    return getRelDataPath("raw", *relPath)
+
+def getRelWeightsPath(*relPath):
+    return getRelDataPath("weights", *relPath)
+
+def getRelDatasetsPath(*relPath):
+    return getRelDataPath("datasets", *relPath)
+
+def getRelPicklesPath(*relPath):
+    return getRelDataPath("pickles", *relPath)
+
+def mkdirP(path):
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+def touch(path):
+    with open(path, 'a'):
+        os.utime(path, None)
+
+def mv(src, dst, mkdirMode=True, force=False):
+    """ Moves src to dst as if `mv` was used. Both src and dst are relative to root,
+    which is set to the 'data path' by default. With the mkdir option, we enforce
+    the dst to be a path and we support "move to dir" behavior. Otherwise we support
+    "move to dir and rename file" behavior.
+    """
+    assert os.path.exists(src), "'{}' not found".format(src)
+
+    # In mkdir mode, we enforce the dst to be a path to allow "move to dir" behavior.
+    # Otherwise we are supporting "move to dir and rename" behavior.
+    if not dst.endswith('/') and mkdirMode:
+        dst += '/'
+    dstHeadPath, _ = os.path.split(dst)
+    mkdirP(dstHeadPath)
+
+    if os.path.isdir(dst):
+        _getUtilityLogger().info("Moving '{}' into directory '{}'".format(src, dst))
+    else:
+        _getUtilityLogger().info("Renaming '{}' to '{}'".format(src, dst))
+
+    if force:
+        shutil.copy(src, dst)
+    else:
+        shutil.move(src, dst)
+