From bd707012bc4dc11751744cf5718653b3fc423639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20S=C3=B6derbergh?= Date: Mon, 3 Mar 2025 15:10:51 +0100 Subject: [PATCH 1/2] Remove deprecated functionality Co-authored-by: Zazmuz --- problemtools/verifyproblem.py | 302 +--------------------------------- 1 file changed, 2 insertions(+), 300 deletions(-) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 51d7a6c..9e696ed 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -99,7 +99,6 @@ def __init__(self, args: argparse.Namespace, executor: ThreadPoolExecutor|None) self.data_filter: Pattern[str] = args.data_filter self.submission_filter: Pattern[str] = args.submission_filter self.fixed_timelim: int|None = args.fixed_timelim - self.compile_generators: bool = ('compile_generators' not in args or args.compile_generators) self.executor = executor self._background_work: list[concurrent.futures.Future[object]] = [] @@ -729,7 +728,6 @@ def __init__(self, problem: Problem): elif param == 'interactive': pass - self._data['languages'] = self._data['languages'].split() def __str__(self) -> str: return 'problem configuration' @@ -819,299 +817,12 @@ def check(self, context: Context) -> bool: self.error('Limits key in problem.yaml must specify a dict') self._data['limits'] = ProblemConfig._OPTIONAL_CONFIG['limits'] - if self._data['languages'] != '': - for lang_id in self._data['languages']: - if lang_id != 'all' and self._problem.language_config.get(lang_id) is None: - self.error("Unrecognized language id '%s'" % lang_id) - # Some things not yet implemented if self._data['libraries'] != '': self.error("Libraries not yet supported") return self._check_res - -class Generators(ProblemAspect): - _TESTCASE_OPTIONS = ['input', 'solution', 'visualizer', 'random_salt'] - _NULLABLE_OPTIONS = ['input', 'solution', 'visualizer'] - _DATA_DIRECTORIES = {'sample', 'secret'} - _VISUALIZER_EXTENSIONS = ['png', 'jpg', 'jpeg', 'svg', 'interaction', 'desc', 'hint'] - - def __init__(self, problem: Problem): - super().__init__(f"{problem.shortname}.generators") - self.debug(' Loading generators') - self._problem = problem - self.configfile = os.path.join(problem.probdir, 'generators', 'generators.yaml') - self._data = None - self._generators: dict[str, str|list[str]|run.Program] = {} - - if os.path.isfile(self.configfile): - try: - with open(self.configfile) as f: - self._data = yaml.safe_load(f) - # Loading empty yaml yields None, for no apparent reason... - if self._data is None: - self._data = {} - except Exception as e: - self.error(str(e)) - - if isinstance(self._data, dict): - # The top-level dict always represents a directory, even if there - # is no type key - self._data['type'] = 'directory' - - def __str__(self) -> str: - return 'generators' - - def _parse_command(self, key: str, state: dict) -> tuple[str, list[str]]|None: - command = state[key] - name = os.path.basename(state['path']) - random_salt = str(state['random_salt']) - - def err() -> None: - self.error('Invalid %s key for path %s in generators.yaml' % (key, state['path'])) - - if not isinstance(command, str): - err() - return None - - seed = str(int(hashlib.sha512((random_salt + command).encode('utf-8')).hexdigest(), 16) % (2**31)) - - parts = shlex.split(command) - if not parts: - err() - return None - - for i, part in enumerate(parts): - new = '' - for j, group in enumerate(part.split('{')): - if group.count('}') != (0 if j == 0 else 1): - err() - return None - if j == 0: - new += group - else: - group, rest = group.split('}') - if group.startswith('seed'): - new += seed - elif group == 'name': - new += name - else: - err() - return None - new += rest - parts[i] = new - - program, arguments = parts[0], parts[1:] - if program not in self._generators: - self._generators[program] = program - - return (program, arguments) - - def _parse_testcase(self, data: dict, state: dict) -> None: - if state['input'] is None: - self.error('Path %s in generators.yaml must contain an input key' % state['path']) - for key in ['input', 'solution', 'visualizer']: - if state[key] is not None: - state[key] = self._parse_command(key, state) - - def _parse_directory(self, data: dict, state: dict) -> None: - # TODO: Process includes - - if 'testdata.yaml' in data: - content = data['testdata.yaml'] - if content is None: - content = {} - - cases = data.get('data', {}) - ordered = True - if not isinstance(cases, list): - ordered = False - cases = [cases] - - case_counter = 0 - case_format = '%%0%dd' % len(str(len(cases))) - for case in cases: - if not isinstance(case, dict): - self.error('Path %s/data in generators.yaml must contain a dict or a list of dicts' % state['path']) - continue - - if ordered: - case_counter += 1 - - for name, value in sorted(case.items(), key=lambda kv: str(kv[0])): - if ordered: - num = case_format % case_counter - name = num + ('' if name is None else '-' + str(name)) - else: - name = str(name) - - next_state = copy.deepcopy(state) - next_state['path'] = '%s/%s' % (state['path'], name) - self._parse_element(value, next_state) - - def _parse_element(self, data: dict, state: dict) -> None: - if data is None: - data = '/%s.in' % state['path'] - state['manual'] = True - if isinstance(data, str): - data = { 'input': data } - if not isinstance(data, dict): - self.error("Path %s in generators.yaml must specify a dict" % state['path']) - return - - state.update({ - key: data[key] - for key in Generators._TESTCASE_OPTIONS - if key in data - }) - - if data.get('type', 'testcase') == 'testcase': - self._parse_testcase(data, state) - else: - if data['type'] != 'directory': - self.error("Type of %s in generators.yaml must be 'directory'" % state['path']) - self._parse_directory(data, state) - - def _resolve_path(self, path: str) -> str: - base_path = self._problem.probdir - if path.startswith('/'): - path = path[1:] - else: - base_path = os.path.join(base_path, 'generators') - return os.path.join(*([base_path] + path.split('/'))) - - def _compile_generators(self) -> None: - for gen, files in list(self._generators.items()): - implicit = True - manual = False - if isinstance(files, str): - path = files - files = [] - implicit = False - if path.endswith('.in'): - manual = True - for ext in ['ans'] + Generators._VISUALIZER_EXTENSIONS: - other_path = path[:-2] + ext - if os.path.isfile(self._resolve_path(other_path)): - files.append(other_path) - # Always add original file last, to ensure it is chosen as - # the representative file - files.append(path) - if not isinstance(files, list) or not files: - self.error('Invalid generator %s in generators.yaml' % gen) - continue - tmpdir = tempfile.mkdtemp(prefix='generator', dir=self._problem.tmpdir) - ok = True - for opath in files: - if not isinstance(opath, str) or not opath: - self.error('Invalid generator %s in generators.yaml' % gen) - ok = False - break - - name = os.path.basename(opath) - if implicit and opath == files[0]: - # In implicit generators, the first listed file should - # be the entry point. problemtools usually picks the - # lexicographically smallest filename as the entry - # point, unless there exists a file that starts with - # "main.". Thus the following renames the file that - # should be the entry point to "main.old.extension". - # TODO: Make problemtools support passing a different - # entry point than "main.", and remove this hack. - name = 'main' + os.path.splitext(name)[1] - - fpath = self._resolve_path(opath) - dest = os.path.join(tmpdir, name) - if os.path.exists(dest): - self.error('Duplicate entry for filename %s in generator %s' % (name, gen)) - ok = False - elif not os.path.exists(fpath): - self.error('Generator %s does not exist' % opath) - ok = False - else: - try: - if os.path.isdir(fpath): - shutil.copytree(fpath, dest) - else: - shutil.copy2(fpath, dest) - except Exception as e: - self.error(str(e)) - ok = False - if ok: - if manual: - self._generators[gen] = dest - else: - prog = run.get_program(tmpdir if implicit else dest, - language_config=self._problem.language_config, - work_dir=self._problem.tmpdir) - if prog is None: - self.error('Could not load generator %s' % gen) - ok = False - else: - self._generators[gen] = prog - success, msg = prog.compile() - if not success: - self.error('Compile error for generator %s' % gen, msg) - ok = False - if not ok and gen in self._generators: - del self._generators[gen] - - def check(self, context: Context) -> bool: - if self._check_res is not None: - return self._check_res - self._check_res = True - - if self._data is None: - return self._check_res - if not isinstance(self._data, dict): - self.error('generators.yaml must specify a dict') - return self._check_res - - self._generators = self._data.get('generators') or {} - if not isinstance(self._generators, dict): - self.error('Generators key in generators.yaml must specify a dict') - self._generators = {} - - # Check the shape of the top-level data dict - if isinstance(self._data.get('data'), list): - self.error('Top-level data key in generators.yaml must specify a dict') - self._data['data'] = {} - - if isinstance(self._data.get('data'), dict): - invalid = [] - for key, value in self._data['data'].items(): - valid = False - if key not in Generators._DATA_DIRECTORIES: - self.warning("Invalid key '%s' in generators.yaml, expected one of %s" % (key, Generators._DATA_DIRECTORIES)) - elif not isinstance(value, dict): - self.warning("Key '%s' in generators.yaml must specify a dict" % key) - elif value.get('type') != 'directory': - self.warning("Type of %s in generators.yaml must be 'directory'" % key) - else: - valid = True - if not valid: - invalid.append(key) - for key in invalid: - del self._data['data'][key] - - # Run a depth-first search through generators.yaml and generate a - # flattened list of testcases - default_state: dict[str, str|bool|None] = { key: None for key in Generators._TESTCASE_OPTIONS } - default_state.update({ - 'path': 'data', - 'manual': False, - 'random_salt': '', - }) - - self._parse_element(self._data, default_state) - - if context.compile_generators: - self._compile_generators() - - return self._check_res - - class ProblemStatement(ProblemAspect): def __init__(self, problem: Problem): super().__init__(f"{problem.shortname}.statement") @@ -1949,7 +1660,7 @@ def check(self, context: Context) -> bool: return self._check_res -PROBLEM_PARTS = ['config', 'statement', 'validators', 'graders', 'generators', 'data', 'submissions'] +PROBLEM_PARTS = ['config', 'statement', 'validators', 'graders', 'data', 'submissions'] class Problem(ProblemAspect): def __init__(self, probdir: str): @@ -1968,14 +1679,7 @@ def __enter__(self) -> Problem: self.statement = ProblemStatement(self) self.attachments = Attachments(self) self.config = ProblemConfig(self) - available_languages = self.config.get('languages') - if 'all' not in available_languages: - language_config = languages.Languages() - for lang_id in available_languages: - lang_spec = self.language_config.get(lang_id) - if lang_spec is not None: - language_config.update({lang_id: self.language_config.get(lang_id)}) - self.language_config = language_config + self.available_languages = languages.load_language_config() self.is_interactive = 'interactive' in self.config.get('validation-params') self.is_scoring = (self.config.get('type') == 'scoring') @@ -1985,7 +1689,6 @@ def __enter__(self) -> Problem: self.testcase_by_infile: dict[str, TestCase] = {} self.testdata = TestCaseGroup(self, os.path.join(self.probdir, 'data')) self.submissions = Submissions(self) - self.generators = Generators(self) return self def __exit__(self, exc_type, exc_value, exc_traceback) -> None: @@ -2012,7 +1715,6 @@ def check(self, args: argparse.Namespace) -> tuple[int, int]: 'statement': [self.statement, self.attachments], 'validators': [self.input_validators, self.output_validators], 'graders': [self.graders], - 'generators': [self.generators], 'data': [self.testdata], 'submissions': [self.submissions], } From c7b03650a0a77c4421be8fadb292055414fa2b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20S=C3=B6derbergh?= Date: Mon, 3 Mar 2025 15:12:14 +0100 Subject: [PATCH 2/2] add build to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ddbe037..2f37220 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /problemtools.egg-info/ /support/default_validator/default_validator /support/interactive/interactive +build/