diff --git a/conftest.py b/conftest.py index 2702d33e5..9a5037255 100644 --- a/conftest.py +++ b/conftest.py @@ -50,19 +50,25 @@ def pytest_addoption(parser): - """Add a command line option to pytest that enables a mode that invokes - `kdiff3` to display diffs and, after user confirmation, can automatically - update or write new test sample documents on mismatches. - - The diff should be studied carefully before updating the sample since there is - a risk of introducing errors. - - The implementation is in D1TestCase.assert_equals_sample() + """Add command line switches for pytest comstomization. See README.md for + info. """ parser.addoption( - '--update-samples', action='store_true', default=False, + '--sample-ask', action='store_true', default=False, help='Prompt to update or write new test sample files on failures' ) + parser.addoption( + '--sample-write', action='store_true', default=False, + help='Automatically update or write sample files on failures' + ) + parser.addoption( + '--sample-review', action='store_true', default=False, + help='Review samples (use after --sample-write)' + ) + parser.addoption( + '--sample-tidy', action='store_true', default=False, + help='Move unused sample files to test_docs_tidy' + ) parser.addoption( '--pycharm', action='store_true', default=False, help='Attempt to move the cursor in PyCharm to location of most recent test ' @@ -74,17 +80,14 @@ def pytest_addoption(parser): ) -def pytest_generate_tests(metafunc): - """Parameterize test functions via parameterize_dict class member""" - try: - func_arg_list = metafunc.cls.parameterize_dict[metafunc.function.__name__] - except (AttributeError, KeyError): - return - arg_names = sorted(func_arg_list[0]) - metafunc.parametrize( - arg_names, - [[func_args[name] for name in arg_names] for func_args in func_arg_list] - ) +# Hooks + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(items): + """Hook that runs as early as possible""" + if pytest.config.getoption('--sample-tidy'): + d1_test.sample.init_tidy() @pytest.hookimpl(tryfirst=True, hookwrapper=True) @@ -119,6 +122,19 @@ def pytest_runtest_makereport(item, call): ) +def pytest_generate_tests(metafunc): + """Parameterize test functions via parameterize_dict class member""" + try: + func_arg_list = metafunc.cls.parameterize_dict[metafunc.function.__name__] + except (AttributeError, KeyError): + return + arg_names = sorted(func_arg_list[0]) + metafunc.parametrize( + arg_names, + [[func_args[name] for name in arg_names] for func_args in func_arg_list] + ) + + # Fixtures for parameterizing tests over CN/MN and v1/v2 clients. MOCK_BASE_URL = 'http://mock/node' @@ -196,11 +212,11 @@ def django_db_setup(django_db_blocker): except psycopg2.DatabaseError as e: logging.debug(str(e)) logging.debug('Dropping test DB') - run_sql('postgres', "drop database if exists {};".format(test_db_name)) + run_sql('postgres', 'drop database if exists {};'.format(test_db_name)) logging.debug('Creating test DB from template') run_sql( 'postgres', - "create database {} template {};".format(test_db_name, template_db_name) + 'create database {} template {};'.format(test_db_name, template_db_name) ) # Haven't found out how to prevent transactions from being started, so # closing the implicit transaction here so that template fixture remains diff --git a/gmn/src/d1_gmn/tests/gmn_test_case.py b/gmn/src/d1_gmn/tests/gmn_test_case.py index 5657d1749..7bb43e4fc 100644 --- a/gmn/src/d1_gmn/tests/gmn_test_case.py +++ b/gmn/src/d1_gmn/tests/gmn_test_case.py @@ -21,7 +21,6 @@ from __future__ import absolute_import import datetime -import hashlib import json import logging import os @@ -49,6 +48,10 @@ import d1_common.xml import d1_test.d1_test_case +import d1_test.instance_generator.access_policy +import d1_test.instance_generator.identifier +import d1_test.instance_generator.random_data +import d1_test.instance_generator.sciobj import d1_test.mock_api.create import d1_test.mock_api.django_client import d1_test.mock_api.get @@ -60,15 +63,7 @@ from django.db import connection -DEFAULT_PERMISSION_LIST = [ - (['subj1'], ['read']), - (['subj2', 'subj3', 'subj4'], ['read', 'write']), - (['subj5', 'subj6', 'subj7', 'subj8'], ['read', 'changePermission']), - (['subj9', 'subj10', 'subj11', 'subj12'], ['changePermission']), -] - HTTPBIN_SERVER_STR = 'http://httpbin.org' -GMN_TEST_SUBJECT_PUBLIC = 'public' ENABLE_SQL_PROFILING = False # d1_common.util.log_setup(True) @@ -107,6 +102,10 @@ def setup_method(self, method): # explode... self.maxDiff = None + @property + def mock(self): + return d1_gmn.tests.gmn_mock + @classmethod def capture_exception(cls): """If GMN responds with something that cannot be parsed by d1_client as @@ -252,7 +251,9 @@ def create_revision_chain(self, client, chain_len, sid=None, *args, **kwargs): def did(idx, is_sid=False): return '#{:03d}_{}'.format( - idx, self.random_sid() if is_sid else self.random_pid() + idx, + d1_test.instance_generator.identifier.generate_sid() + if is_sid else d1_test.instance_generator.identifier.generate_pid() ) base_pid, sid, sciobj_str, sysmeta_pyxb = ( @@ -336,12 +337,13 @@ def create_obj( pid, sid, sciobj_str, sysmeta_pyxb = self.generate_sciobj_with_defaults( client, pid, sid, submitter, rights_holder, permission_list, now_dt ) - self.call_d1_client( - client.create, pid, - StringIO.StringIO(sciobj_str), sysmeta_pyxb, vendor_dict, - active_subj_list=active_subj_list, trusted_subj_list=trusted_subj_list, - disable_auth=disable_auth - ) + with d1_gmn.tests.gmn_mock.disable_sysmeta_sanity_checks(): + self.call_d1_client( + client.create, pid, + StringIO.StringIO(sciobj_str), sysmeta_pyxb, vendor_dict, + active_subj_list=active_subj_list, trusted_subj_list=trusted_subj_list, + disable_auth=disable_auth + ) assert self.get_pyxb_value(sysmeta_pyxb, 'identifier') == pid return pid, sid, sciobj_str, sysmeta_pyxb @@ -358,12 +360,13 @@ def update_obj( pid, sid, sciobj_str, sysmeta_pyxb = self.generate_sciobj_with_defaults( client, new_pid, sid, submitter, rights_holder, permission_list, now_dt ) - self.call_d1_client( - client.update, old_pid, - StringIO.StringIO(sciobj_str), pid, sysmeta_pyxb, vendor_dict, - active_subj_list=active_subj_list, trusted_subj_list=trusted_subj_list, - disable_auth=disable_auth - ) + with d1_gmn.tests.gmn_mock.disable_sysmeta_sanity_checks(): + self.call_d1_client( + client.update, old_pid, + StringIO.StringIO(sciobj_str), pid, sysmeta_pyxb, vendor_dict, + active_subj_list=active_subj_list, trusted_subj_list=trusted_subj_list, + disable_auth=disable_auth + ) assert self.get_pyxb_value(sysmeta_pyxb, 'identifier') == pid return pid, sid, sciobj_str, sysmeta_pyxb @@ -384,108 +387,31 @@ def get_obj( self.assert_sci_obj_checksum_matches_sysmeta(sciobj_str, sysmeta_pyxb) return sciobj_str, sysmeta_pyxb - # - # SysMeta - # - - def generate_sysmeta( - self, client, pid, sid=None, sciobj_str=None, submitter=None, - rights_holder=None, obsoletes=None, obsoleted_by=None, - permission_list=None, now_dt=None - ): - sysmeta_pyxb = client.bindings.systemMetadata() - sysmeta_pyxb.serialVersion = 1 - sysmeta_pyxb.identifier = pid - sysmeta_pyxb.seriesId = sid - sysmeta_pyxb.formatId = 'application/octet-stream' - sysmeta_pyxb.size = len(sciobj_str) - sysmeta_pyxb.submitter = submitter - sysmeta_pyxb.rightsHolder = rights_holder - sysmeta_pyxb.checksum = d1_common.types.dataoneTypes.checksum( - hashlib.md5(sciobj_str).hexdigest() - ) - sysmeta_pyxb.checksum.algorithm = 'MD5' - sysmeta_pyxb.dateUploaded = now_dt - sysmeta_pyxb.dateSysMetadataModified = now_dt - sysmeta_pyxb.originMemberNode = 'urn:node:GMNUnitTestOrigin' - sysmeta_pyxb.authoritativeMemberNode = 'urn:node:GMNUnitTestAuthoritative' - sysmeta_pyxb.obsoletes = obsoletes - sysmeta_pyxb.obsoletedBy = obsoleted_by - sysmeta_pyxb.accessPolicy = self.generate_access_policy( - client, permission_list - ) - sysmeta_pyxb.replicationPolicy = self.create_replication_policy_pyxb(client) - return sysmeta_pyxb - - def generate_access_policy(self, client, permission_list=None): - if permission_list is None: - return None - elif permission_list == 'default': - permission_list = DEFAULT_PERMISSION_LIST - access_policy_pyxb = client.bindings.accessPolicy() - for subject_list, action_list in permission_list: - subject_list = d1_gmn.tests.gmn_mock.expand_subjects(subject_list) - action_list = list(action_list) - access_rule_pyxb = client.bindings.AccessRule() - for subject_str in subject_list: - access_rule_pyxb.subject.append(subject_str) - for action_str in action_list: - permission_pyxb = client.bindings.Permission(action_str) - access_rule_pyxb.permission.append(permission_pyxb) - access_policy_pyxb.append(access_rule_pyxb) - return access_policy_pyxb - - def create_replication_policy_pyxb( - self, client, preferred_node_list=None, blocked_node_list=None, - is_replication_allowed=True, num_replicas=None - ): - """{preferred_node_list} and {preferred_node_list}: - None: No node list is generated - A list of strings: A node list is generated using the strings as node URNs - 'random': A short node list is generated from random strings - """ - preferred_node_list = self.prep_node_list(preferred_node_list, 'preferred') - blocked_node_list = self.prep_node_list(blocked_node_list, 'blocked') - rep_pyxb = client.bindings.ReplicationPolicy() - rep_pyxb.preferredMemberNode = preferred_node_list - rep_pyxb.blockedMemberNode = blocked_node_list - rep_pyxb.replicationAllowed = is_replication_allowed - rep_pyxb.numberReplicas = num_replicas or random.randint(10, 100) - return rep_pyxb - - def generate_sciobj( - self, client, pid, sid=None, submitter=None, rights_holder=None, - obsoletes=None, obsoleted_by=None, permission_list=None, now_dt=None - ): - """Generate the object bytes and system metadata for a test object - """ - sciobj_str = d1_test.d1_test_case.generate_reproducible_sciobj_str(pid) - sysmeta_pyxb = self.generate_sysmeta( - client, pid, sid, sciobj_str, submitter or GMN_TEST_SUBJECT_PUBLIC, - rights_holder or GMN_TEST_SUBJECT_PUBLIC, obsoletes, obsoleted_by, - permission_list, now_dt - ) - return sciobj_str, sysmeta_pyxb - def generate_sciobj_with_defaults( self, client, pid=True, sid=None, submitter=True, rights_holder=True, permission_list=True, now_dt=True ): - """Generate the object bytes and system metadata for a test object - Parameters: - True: Use a default or generate a value - Other: Use the supplied value - """ - pid = self.random_pid() if pid is True else pid - sid = self.random_sid() if sid is True else sid - sciobj_str, sysmeta_pyxb = self.generate_sciobj( - client, pid, sid, - 'submitter_subj' if submitter is True else submitter, - 'rights_holder_subj' if rights_holder is True else rights_holder, - None, None, - DEFAULT_PERMISSION_LIST if permission_list is True else permission_list, - datetime.datetime.now() if now_dt is True else now_dt, - ) # yapf: disable + permission_list = ( + d1_test.d1_test_case.DEFAULT_PERMISSION_LIST + if permission_list is True else permission_list + ) + sid = d1_test.instance_generator.identifier.generate_sid() if sid is True else sid + option_dict = { + k: v + for (k, v) in (('identifier', pid), + ('seriesId', sid), + ('submitter', submitter), + ('rightsHolder', rights_holder), ( + 'accessPolicy', d1_test.instance_generator.access_policy. + generate_from_permission_list(client, permission_list) + ), + ('dateUploaded', now_dt), + ('dateSysMetadataModified', now_dt),) if v is not True + } + pid, sid, sciobj_str, sysmeta_pyxb = \ + d1_test.instance_generator.sciobj.generate_reproducible( + client, None if pid is True else pid, option_dict + ) return pid, sid, sciobj_str, sysmeta_pyxb # @@ -501,17 +427,6 @@ def log_to_pid_list(self, log_record_list_pyxb): def vendor_proxy_mode(self, object_stream_url): return {'VENDOR-GMN-REMOTE-URL': object_stream_url} - def prep_node_list(self, node_list, tag_str, num_nodes=5): - if node_list is None: - return None - elif isinstance(node_list, list): - return node_list - elif node_list == 'random': - return [ - 'urn:node:{}'.format(self.random_tag(tag_str)) - for _ in range(num_nodes) - ] - def dump_permissions(self): logging.debug('Permissions:') for s in d1_gmn.app.models.Permission.objects.all(): @@ -525,20 +440,6 @@ def dump_subjects(self): for s in d1_gmn.app.models.Subject.objects.all(): logging.debug(' {}'.format(s.subject)) - def dump_pyxb(self, type_pyxb): - map(logging.debug, self.format_pyxb(type_pyxb).splitlines()) - - def format_pyxb(self, type_pyxb): - ss = StringIO.StringIO() - ss.write('PyXB object:\n') - ss.write( - '\n'.join([ - u' {}'.format(s) - for s in d1_common.xml.pretty_pyxb(type_pyxb).splitlines() - ]) - ) - return ss.getvalue() - def get_pid_list(self): """Get list of all PIDs in the DB fixture""" return json.loads(self.sample.load('db_fixture_pid.json', 'rb')) @@ -552,3 +453,9 @@ def get_total_log_records(self, client, **filters): def get_total_objects(self, client, **filters): return client.listObjects(start=0, count=0, **filters).total + + def get_random_pid_sample(self, n_pids): + return random.sample( + [v.pid.did for v in d1_gmn.app.models.ScienceObject.objects.all()], + n_pids, + ) diff --git a/test_utilities/src/d1_test/d1_test_case.py b/test_utilities/src/d1_test/d1_test_case.py index a03d5ac69..43d22b5b9 100644 --- a/test_utilities/src/d1_test/d1_test_case.py +++ b/test_utilities/src/d1_test/d1_test_case.py @@ -23,13 +23,10 @@ import contextlib import datetime -import hashlib import inspect import logging import os import random -import re -import string import StringIO import sys import xml @@ -47,6 +44,7 @@ import d1_common.util import d1_common.xml +import d1_test.instance_generator.date_time import d1_test.instance_generator.system_metadata import d1_test.sample @@ -58,6 +56,18 @@ SOLR_QUERY_ENDPOINT = '/cn/v1/query/solr/' +DEFAULT_PERMISSION_LIST = [ + (['subj1'], ['read']), + (['subj2', 'subj3', 'subj4'], ['read', 'write']), + (['subj5', 'subj6', 'subj7', 'subj8'], ['read', 'changePermission']), + (['subj9', 'subj10', 'subj11', 'subj12'], ['changePermission']), +] + +SUBJ_DICT = { + 'trusted': 'gmn_test_subject_trusted', + 'submitter': 'gmn_test_subject_submitter', +} + @contextlib.contextmanager def capture_std(): @@ -72,6 +82,10 @@ def capture_std(): @contextlib.contextmanager def capture_log(): + """Capture anything output by the logging module. Uses a handler that does not + not include any context, such as date-time or originating module to help keep + the result stable over time. + """ stream = StringIO.StringIO() logger = None stream_handler = None @@ -99,6 +113,15 @@ def _log(prompt_str): yield +@contextlib.contextmanager +def disable_debug_level_logging(): + try: + logging.disable(logging.DEBUG) + yield + finally: + logging.disable(logging.NOTSET) + + # reproducible_random # TODO: When we move to Py3, move this over to the simple wrapper supported @@ -123,10 +146,10 @@ def reproducible_random_decorator_real(cls_or_func): def _reproducible_random_class_decorator(cls, seed): for test_name, test_func in cls.__dict__.items(): if test_name.startswith('test_'): - logging.debug( - 'Decorating: {}.{}: reproducible_random()'. - format(cls.__name__, test_name) - ) + # logging.debug( + # 'Decorating: {}.{}: reproducible_random()'. + # format(cls.__name__, test_name) + # ) setattr( cls, test_name, _reproducible_random_func_decorator(test_func, seed) ) @@ -135,9 +158,9 @@ def _reproducible_random_class_decorator(cls, seed): def _reproducible_random_func_decorator(func, seed): def wrapper(func2, *args, **kwargs): - logging.debug( - 'Decorating: {}: reproducible_random()'.format(func2.__name__) - ) + # logging.debug( + # 'Decorating: {}: reproducible_random()'.format(func2.__name__) + # ) with reproducible_random_context(seed): return func2(*args, **kwargs) @@ -153,24 +176,6 @@ def reproducible_random_context(seed): random.setstate(state) -def generate_reproducible_sciobj_str(pid): - """Return a science object byte string that is always the same for a given PID - """ - # Ignore any decoration. - pid = re.sub(r'^<.*?>', '', pid) - pid_hash_int = int(hashlib.md5(pid.encode('utf-8')).hexdigest(), 16) - with reproducible_random_context(pid_hash_int): - return ( - 'These are the reproducible Science Object bytes for pid="{}". ' - 'What follows is 100 to 200 random bytes: '.format(pid.encode('utf-8')) + - str( - bytearray( - random.getrandbits(8) for _ in range(random.randint(100, 200)) - ) - ) - ) - - #=============================================================================== @@ -245,22 +250,6 @@ def mock_ssl_download(cert_obj): mock_getpeercert.return_value = cert_der yield mock_connect, mock_getpeercert - def create_random_sciobj(self, client, pid=True, sid=True): - pid = self.random_pid() if pid is True else pid - sid = self.random_sid() if sid is True else sid - options = { - # 'rightsHolder': 'fixture_rights_holder_subj', - 'identifier': client.bindings.Identifier(pid) if pid else None, - 'seriesId': client.bindings.Identifier(sid) if sid else None, - } - sciobj_str = generate_reproducible_sciobj_str(pid) - sysmeta_pyxb = ( - d1_test.instance_generator.system_metadata.generate_from_file( - client, StringIO.StringIO(sciobj_str), options - ) - ) - return pid, sid, sciobj_str, sysmeta_pyxb - def get_pyxb_value(self, inst_pyxb, inst_attr): try: return unicode(getattr(inst_pyxb, inst_attr).value()) @@ -270,19 +259,118 @@ def get_pyxb_value(self, inst_pyxb, inst_attr): def now_str(self): return datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") - def random_str(self, num_chars=10): - return ''.join([ - random.choice(string.ascii_lowercase) for _ in range(num_chars) - ]) + # def random_str(self, num_chars=10): + # return ''.join([ + # random.choice(string.ascii_lowercase) for _ in range(num_chars) + # ]) + + # def random_id(self): + # return '{}_{}'.format(self.random_str(), self.now_str()) + # + # def random_pid(self): + # return 'PID_{}'.format(self.random_id()) + # + # def random_sid(self): + # return 'SID_{}'.format(self.random_id()) + # + # def random_tag(self, tag_str): + # return '{}_{}'.format(tag_str, self.random_str()) + + def dump_pyxb(self, type_pyxb): + map(logging.debug, self.format_pyxb(type_pyxb).splitlines()) + + def format_pyxb(self, type_pyxb): + ss = StringIO.StringIO() + ss.write('PyXB object:\n') + ss.write( + '\n'.join([ + u' {}'.format(s) + for s in d1_common.xml.pretty_pyxb(type_pyxb).splitlines() + ]) + ) + return ss.getvalue() + + # + # SysMeta + # + + @staticmethod + def expand_subjects(subj): + if isinstance(subj, basestring): + subj = [subj] + return {SUBJ_DICT[v] if v in SUBJ_DICT else v for v in subj or []} + + def prep_node_list(self, node_list, tag_str, num_nodes=5): + if node_list is None: + return None + elif isinstance(node_list, list): + return node_list + elif node_list == 'random': + return [ + 'urn:node:{}'.format(self.random_tag(tag_str)) + for _ in range(num_nodes) + ] + - def random_id(self): - return '{}_{}'.format(self.random_str(), self.now_str()) +#=============================================================================== - def random_pid(self): - return 'PID_{}'.format(self.random_id()) +# import logging, logging.config, colorstreamhandler +# +# _LOGCONFIG = { +# "version": 1, +# "disable_existing_loggers": False, +# +# "handlers": { +# "console": { +# "class": "colorstreamhandler.ColorStreamHandler", +# "stream": "ext://sys.stderr", +# "level": "INFO" +# } +# }, +# +# "root": { +# "level": "INFO", +# "handlers": ["console"] +# } +# } +# +# logging.config.dictConfig(_LOGCONFIG) +# mylogger = logging.getLogger("mylogger") +# mylogger.warning("foobar") + + +class ColorStreamHandler(logging.StreamHandler): + DEFAULT = '\x1b[0m' + RED = '\x1b[31m' + GREEN = '\x1b[32m' + YELLOW = '\x1b[33m' + CYAN = '\x1b[36m' + + CRITICAL = RED + ERROR = RED + WARNING = YELLOW + INFO = GREEN + DEBUG = CYAN + + @classmethod + def _get_color(cls, level): + if level >= logging.CRITICAL: + return cls.CRITICAL + elif level >= logging.ERROR: + return cls.ERROR + elif level >= logging.WARNING: + return cls.WARNING + elif level >= logging.INFO: + return cls.INFO + elif level >= logging.DEBUG: + return cls.DEBUG + else: + return cls.DEFAULT - def random_sid(self): - return 'SID_{}'.format(self.random_id()) + def __init__(self, stream=None): + logging.StreamHandler.__init__(self, stream) - def random_tag(self, tag_str): - return '{}_{}'.format(tag_str, self.random_str()) + def format(self, record): + text = logging.StreamHandler.format(self, record) + color = self._get_color(record.levelno) + return color + text + self.DEFAULT diff --git a/test_utilities/src/d1_test/sample.py b/test_utilities/src/d1_test/sample.py index d78229973..3fbe0b0c7 100644 --- a/test_utilities/src/d1_test/sample.py +++ b/test_utilities/src/d1_test/sample.py @@ -28,6 +28,7 @@ import traceback import pytest +import requests.structures import d1_common import d1_common.types @@ -38,24 +39,64 @@ import d1_client.util +def init_tidy(): + """Call at start of test run to tidy the samples directory. + """ + sample_dir_path = os.path.join(d1_common.util.abs_path('test_docs')) + tidy_dir_path = os.path.join(d1_common.util.abs_path('test_docs_tidy')) + d1_common.util.ensure_dir_exists(sample_dir_path) + d1_common.util.ensure_dir_exists(tidy_dir_path) + for item_name in os.listdir(sample_dir_path): + sample_path = os.path.join(sample_dir_path, item_name) + tidy_path = os.path.join(tidy_dir_path, item_name) + if os.path.exists(tidy_path): + os.unlink(tidy_path) + os.rename(sample_path, tidy_path) + + def assert_equals( got_obj, name_postfix_str, client=None, extension_str='sample' ): filename = _format_file_name(client, name_postfix_str, extension_str) logging.info('Using sample file. filename="{}"'.format(filename)) - got_str = _obj_to_pretty_str(got_obj) exp_path = _get_or_create_path(filename) - diff_str = _get_sxs_diff(got_str, exp_path) + got_str = obj_to_pretty_str(got_obj) + + if pytest.config.getoption('--sample-review'): + _review_interactive(got_str, exp_path) + return + + diff_str = _get_sxs_diff_file(got_str, exp_path) + if diff_str is None: return - logging.error( + + if pytest.config.getoption('--sample-write'): + save(got_str, filename) + return + + if pytest.config.getoption('--sample-ask'): + _save_interactive(got_str, exp_path) + return + + raise AssertionError( '\nSample file: {0}\n{1} Sample mismatch. GOT <-> EXPECTED {1}\n{2}'. format(filename, '-' * 10, diff_str) ) - if pytest.config.getoption('--update-samples'): - _save_interactive(got_str, exp_path) + + +def assert_no_diff(a_obj, b_obj): + a_str = obj_to_pretty_str(a_obj) + b_str = obj_to_pretty_str(b_obj) + diff_str = _get_sxs_diff_str(a_str, b_str) + if diff_str is None: + return + err_msg = '\n{0} Diff mismatch. A <-> B {0}\n{1}'.format('-' * 10, diff_str) + if pytest.config.getoption('--sample-ask'): + logging.error(err_msg) + _diff_interactive(a_str, b_str) else: - raise AssertionError('Sample mismatch. filename="{}"'.format(filename)) + raise AssertionError(err_msg) def get_path(filename): @@ -88,12 +129,17 @@ def load_xml_to_pyxb(filename, mode_str='r'): return d1_common.types.dataoneTypes.CreateFromDocument(xml_str) -def save(filename, got_str, mode_str='wb'): +def save(got_str, filename, mode_str='wb'): logging.info('Writing sample file. filename="{}"'.format(filename)) with open(_get_or_create_path(filename), mode_str) as f: return f.write(got_str) +def save_obj(got_obj, filename, mode_str='wb'): + got_str = obj_to_pretty_str(got_obj) + save(got_str, filename, mode_str) + + def save_path(got_str, exp_path, mode_str='wb'): logging.info( 'Writing sample file. filename="{}"'.format(os.path.split(exp_path)[1]) @@ -102,6 +148,56 @@ def save_path(got_str, exp_path, mode_str='wb'): return f.write(got_str) +def get_sxs_diff(a_obj, b_obj): + return _get_sxs_diff_str( + obj_to_pretty_str(a_obj), + obj_to_pretty_str(b_obj), + ) + + +def obj_to_pretty_str(o): + # noinspection PyUnreachableCode + def serialize(o): + logging.debug('Serializing object. type="{}"'.format(type(o))) + if isinstance(o, unicode): + o = o.encode('utf-8') + if isinstance(o, requests.structures.CaseInsensitiveDict): + with ignore_exceptions(): + o = dict(o) + with ignore_exceptions(): + return d1_common.xml.pretty_xml(o) + with ignore_exceptions(): + return d1_common.xml.pretty_pyxb(o) + with ignore_exceptions(): + return '\n'.join(sorted(o.serialize(doc_format='nt').splitlines())) + with ignore_exceptions(): + if 'digraph' in o: + return '\n'.join( + sorted(str(re.sub(r'node\d+', 'nodeX', o)).splitlines()) + ) + with ignore_exceptions(): + if '\n' in str(o): + return str(o) + with ignore_exceptions(): + return json.dumps(o, sort_keys=True, indent=2, cls=SetToList) + with ignore_exceptions(): + return str(o) + return repr(o) + + return clobber_uncontrolled_volatiles(serialize(o)).rstrip() + '\n' + + +def clobber_uncontrolled_volatiles(o_str): + """Some volatile values in results are not controlled by freezing the time + and PRNG seed. We replace those with a fixed string here. + """ + # requests-toolbelt is using another prng for mmp docs + o_str = re.sub(r'(?<=boundary=)[0-9a-fA-F]+', '[volatile]', o_str) + # entryId is based on a db sequence type + o_str = re.sub(r'(?<=)\d+', '[volatile]', o_str) + return o_str + + def _get_or_create_path(filename): """Get the path to a sample file and enable cleaning out unused sample files. See the test docs for usage. @@ -137,10 +233,10 @@ def _get_sxs_diff_str(got_str, exp_str): with tempfile.NamedTemporaryFile(suffix='__EXPECTED') as exp_f: exp_f.write(exp_str) exp_f.seek(0) - return _get_sxs_diff(got_str, exp_f.name) + return _get_sxs_diff_file(got_str, exp_f.name) -def _get_sxs_diff(got_str, exp_path): +def _get_sxs_diff_file(got_str, exp_path): """Return a minimal formatted side by side diff if there are any none-whitespace changes, else None. """ @@ -148,8 +244,8 @@ def _get_sxs_diff(got_str, exp_path): sdiff_proc = subprocess.Popen( [ 'sdiff', '--ignore-blank-lines', '--ignore-all-space', '--minimal', - '--width=120', '--tabsize=2', '--strip-trailing-cr', '--expand-tabs', - exp_path, '-' + '--width=130', '--tabsize=2', '--strip-trailing-cr', '--expand-tabs', + '--text', '-', exp_path # '--suppress-common-lines' ], bufsize=-1, @@ -174,60 +270,54 @@ def _get_sxs_diff(got_str, exp_path): ) -def _display_diff_pyxb(got_pyxb, exp_pyxb): - return _display_diff_str( - d1_common.xml.pretty_pyxb(got_pyxb), - d1_common.xml.pretty_pyxb(exp_pyxb), - ) - - -def _display_diff_xml(got_xml, exp_xml): - return _display_diff_str( - d1_common.xml.pretty_xml(got_xml), - d1_common.xml.pretty_xml(exp_xml), - ) - - -def _display_diff_str(got_str, exp_path): +def _gui_diff_str_path(got_str, exp_path): with open(exp_path, 'rb') as exp_f: exp_str = exp_f.read() with _tmp_file_pair(got_str, exp_str) as (got_f, exp_f): subprocess.call(['kdiff3', got_f.name, exp_f.name]) +def _gui_diff_str_str(a_str, b_str, a_name='received', b_name='expected'): + with _tmp_file_pair(a_str, b_str, a_name, b_name) as (a_f, b_f): + subprocess.call(['kdiff3', a_f.name, b_f.name]) + + def _save_interactive(got_str, exp_path): - _display_diff_str(got_str, exp_path) + _gui_diff_str_path(got_str, exp_path) answer_str = None - while answer_str not in ('y', 'n', ''): + while answer_str not in ('', 'n', 'f'): answer_str = raw_input( - 'Update sample file "{}"? [Y/n] '.format(os.path.split(exp_path)[1]) + 'Update sample file "{}"? Yes/No/Fail [Enter/n/f] '. + format(os.path.split(exp_path)[1]) ).lower() - if answer_str in ('y', ''): + if answer_str == '': save_path(got_str, exp_path) + elif answer_str == 'f': + raise AssertionError('Failure triggered interactively') -# noinspection PyUnreachableCode -def _obj_to_pretty_str(o): - logging.debug('Serializing object. type="{}"'.format(type(o))) - if isinstance(o, unicode): - o = o.encode('utf-8') - with ignore_exceptions(): - return d1_common.xml.pretty_xml(o) - with ignore_exceptions(): - return d1_common.xml.pretty_pyxb(o) - with ignore_exceptions(): - return '\n'.join(sorted(o.serialize(doc_format='nt').splitlines())) - with ignore_exceptions(): - if 'digraph' in o: - return '\n'.join(sorted(str(re.sub(r'node\d+', 'nodeX', o)).splitlines())) - with ignore_exceptions(): - if '\n' in str(o): - return str(o) - with ignore_exceptions(): - return json.dumps(o, sort_keys=True, indent=2) - with ignore_exceptions(): - return str(o) - return repr(o) +def _diff_interactive(a_str, b_str): + _gui_diff_str_str(a_str, b_str, 'a' * 10, 'b' * 10) + answer_str = None + while answer_str not in ('', 'n'): + answer_str = raw_input('Fail? Yes/No [Enter/n] ').lower() + if answer_str == '': + raise AssertionError('Failure triggered interactively') + + +def _review_interactive(got_str, exp_path): + + _gui_diff_str_path(got_str, exp_path) + answer_str = None + while answer_str not in ('', 'n', 'f'): + answer_str = raw_input( + 'Update sample file "{}"? Yes/No/Fail [Enter/n/f] '. + format(os.path.split(exp_path)[1]) + ).lower() + if answer_str == '': + save_path(got_str, exp_path) + elif answer_str == 'f': + raise AssertionError('Failure triggered interactively') @contextlib.contextmanager @@ -240,9 +330,11 @@ def ignore_exceptions(*exception_list): @contextlib.contextmanager -def _tmp_file_pair(got_str, exp_str): - with tempfile.NamedTemporaryFile(suffix='__RECEIVED') as got_f: - with tempfile.NamedTemporaryFile(suffix='__EXPECTED') as exp_f: +def _tmp_file_pair(got_str, exp_str, a_name='a', b_name='b'): + with tempfile.NamedTemporaryFile(suffix='__{}'.format(a_name.upper()) + ) as got_f: + with tempfile.NamedTemporaryFile(suffix='__{}'.format(b_name.upper()) + ) as exp_f: got_f.write(got_str) exp_f.write(exp_str) got_f.seek(0) @@ -250,5 +342,18 @@ def _tmp_file_pair(got_str, exp_str): yield got_f, exp_f +# ============================================================================== + + class SampleException(Exception): pass + + +# ============================================================================== + + +class SetToList(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return sorted(list(obj)) + return json.JSONEncoder.default(self, obj) diff --git a/test_utilities/src/d1_test/utilities/generate_test_subject_certs.py b/test_utilities/src/d1_test/utilities/generate_test_subject_certs.py index a1b6cb008..459d79b4b 100644 --- a/test_utilities/src/d1_test/utilities/generate_test_subject_certs.py +++ b/test_utilities/src/d1_test/utilities/generate_test_subject_certs.py @@ -36,9 +36,9 @@ # Config. ca_cert_key_path = './ca_intermediate.key' -ca_cert_pem_path = './ca_intermediate.crt' +ca_cert_pub_path = './ca_intermediate.crt' #ca_cert_key_path = './ca.key' -#ca_cert_pem_path = './ca.crt' +#ca_cert_pub_path = './ca.crt' cert_dir = './test_subject_certs/' @@ -315,7 +315,7 @@ def main(): # Load the DataONE Test CA cert. try: - ca_cert_file = open(ca_cert_pem_path, 'r').read() + ca_cert_file = open(ca_cert_pub_path, 'r').read() except IOError: logger.error('Must set path to CA key in config section') raise @@ -353,10 +353,10 @@ def main(): ) # Write the cert to disk. - out_cert_pem_path = os.path.join( + out_cert_pub_path = os.path.join( cert_dir, '{}.crt'.format(urllib.quote(subject, '')) ) - out_cert_file = open(out_cert_pem_path, 'w') + out_cert_file = open(out_cert_pub_path, 'w') out_cert_file.write( OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) ) diff --git a/test_utilities/src/d1_test/utilities/list_effective_subjects.py b/test_utilities/src/d1_test/utilities/list_effective_subjects.py index f260e97e7..c85b024ab 100644 --- a/test_utilities/src/d1_test/utilities/list_effective_subjects.py +++ b/test_utilities/src/d1_test/utilities/list_effective_subjects.py @@ -128,15 +128,15 @@ def _deserialize_subject_info(self, subject_info_xml): # ============================================================================== -def read_cert_from_file_and_print_subjects(options, cert_pem_path): - cert = read_certificate_from_file(cert_pem_path) +def read_cert_from_file_and_print_subjects(options, cert_pub_path): + cert = read_certificate_from_file(cert_pub_path) primary_subject, subjects = get_subjects_from_certificate().get(cert) print_effective_subjects(primary_subject, subjects) -def read_certificate_from_file(cert_pem_path): +def read_certificate_from_file(cert_pub_path): try: - with open(cert_pem_path) as f: + with open(cert_pub_path) as f: return f.read() except EnvironmentError as e: print('Error reading certificate file: {}'.format(str(e))) @@ -170,8 +170,8 @@ def main(): else: logging.getLogger('').setLevel(logging.ERROR) - cert_pem_path = args[0] - read_cert_from_file_and_print_subjects(options, cert_pem_path) + cert_pub_path = args[0] + read_cert_from_file_and_print_subjects(options, cert_pub_path) if __name__ == '__main__': diff --git a/test_utilities/src/d1_test/utilities/populate_mn.py b/test_utilities/src/d1_test/utilities/populate_mn.py index 2cc9bcf5f..93535604d 100644 --- a/test_utilities/src/d1_test/utilities/populate_mn.py +++ b/test_utilities/src/d1_test/utilities/populate_mn.py @@ -138,13 +138,13 @@ def main(): if options.use_v1: mn_client = d1_client.mnclient.MemberNodeClient( options.mn_base_url, - cert_pem_path=options.cert_pub_path, + cert_pub_path=options.cert_pub_path, cert_key_path=options.cert_key_path, ) else: mn_client = d1_client.mnclient_2_0.MemberNodeClient_2_0( options.mn_base_url, - cert_pem_path=options.cert_pub_path, + cert_pub_path=options.cert_pub_path, cert_key_path=options.cert_key_path, ) diff --git a/tox.ini b/tox.ini index 83b212fb4..984274395 100644 --- a/tox.ini +++ b/tox.ini @@ -21,4 +21,5 @@ addopts = --reuse-db #--ds=d1_gmn.settings_test testpaths = ./lib_common ./lib_client ./test_utilities ./gmn ./client_cli ./client_onedrive python_files = test_*.py +norecursedirs = test_docs test_docs_tidy DJANGO_SETTINGS_MODULE = d1_gmn.settings_test