diff --git a/bin/build_clang.sh b/bin/build_clang.sh index bd5c5a7..f7665e1 100755 --- a/bin/build_clang.sh +++ b/bin/build_clang.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -. "${BASH_SOURCE%/*}/common.sh" +# shellcheck source=bin/common.sh +. "${BASH_SOURCE[0]%/*}/common.sh" yb_activate_virtualenv "$build_clang_project_root" set_pythonpath -python3 "$build_clang_project_root/src/build_clang/build_clang_main.py" "$@" \ No newline at end of file +python3 "$build_clang_project_root/src/build_clang/build_clang_main.py" "$@" diff --git a/bin/check.sh b/bin/check.sh index 502a7a6..4dc457d 100755 --- a/bin/check.sh +++ b/bin/check.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -euo pipefail -. "${BASH_SOURCE%/*}/common.sh" -yb_activate_virtualenv "$build_clang_project_root" +# shellcheck source=bin/common.sh +. "${BASH_SOURCE[0]%/*}/common.sh" -set_pythonpath +yb_activate_virtualenv "$build_clang_project_root" -"$build_clang_project_root/src/build_clang/check_python_code.py" \ No newline at end of file +codecheck diff --git a/bin/common.sh b/bin/common.sh index 54f33df..f65f4e1 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -1,18 +1,21 @@ +#!/usr/bin/env bash + set -euo pipefail set_pythonpath() { export PYTHONPATH=$build_clang_project_root/src } -if [[ $BASH_SOURCE == $0 ]]; then - echo "$BASH_SOURCE must be sourced, not executed" >&2 +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + echo "${BASH_SOURCE[0]} must be sourced, not executed" >&2 exit 1 fi -build_clang_project_root=$( cd "${BASH_SOURCE%/*}" && cd .. && pwd ) +build_clang_project_root=$( cd "${BASH_SOURCE[0]%/*}" && cd .. && pwd ) if [[ ! -d $build_clang_project_root/yugabyte-bash-common || -z "$( ls -A "$build_clang_project_root/yugabyte-bash-common" )" ]]; then ( cd "$build_clang_project_root"; git submodule update --init --recursive ) fi +# shellcheck source=yugabyte-bash-common/src/yugabyte-bash-common.sh . "$build_clang_project_root/yugabyte-bash-common/src/yugabyte-bash-common.sh" diff --git a/requirements.txt b/requirements.txt index 10efb23..eaf7413 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ mypy pycodestyle requests PyGithub +codecheck diff --git a/src/build_clang/build_clang_main.py b/src/build_clang/build_clang_main.py index 25ed08c..d3f8c99 100644 --- a/src/build_clang/build_clang_main.py +++ b/src/build_clang/build_clang_main.py @@ -203,7 +203,6 @@ def get_llvm_cmake_variables(self) -> Dict[str, str]: LLVM_TARGETS_TO_BUILD='X86', CLANG_DEFAULT_CXX_STDLIB='libc++', - #CLANG_DEFAULT_LINKER='lld', BUILD_SHARED_LIBS=ON, diff --git a/src/build_clang/check_python_code.py b/src/build_clang/check_python_code.py deleted file mode 100755 index 5759ba1..0000000 --- a/src/build_clang/check_python_code.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 - -""" -Checks Python code in this repository using various methods (MyPy, importing modules, syntax -checks, pycodestyle). Runs multiple checks in parallel. -""" - -import concurrent.futures -import urllib.request -import glob -import os -import sys -import subprocess -import time - -from typing import List, Union, Dict -from build_clang.helpers import BUILD_CLANG_SCRIPTS_ROOT_PATH - - -CHECK_TYPES = [ - 'mypy', - 'compile', - 'import', - 'pycodestyle', -] - - -def ensure_decoded(s: Union[str, bytes]) -> str: - if isinstance(s, bytes): - return s.decode('utf-8') - return s - - -def increment_counter(d: Dict[str, int], key: str) -> None: - if key in d: - d[key] += 1 - else: - d[key] = 1 - - -def print_stats(description: str, d: Dict[str, int]) -> None: - print("%s:\n %s" % ( - description, - '\n '.join('%s: %s' % (k, v) for k, v in sorted(d.items())) - )) - - -def rel_to_repo_root(file_path: str) -> str: - return os.path.relpath( - os.path.realpath(file_path), - os.path.realpath(BUILD_CLANG_SCRIPTS_ROOT_PATH)) - - -class CheckResult: - def __init__( - self, - check_type: str, - file_path: str, - cmd_args: List[str] = [], - stdout: str = '', - stderr: str = '', - returncode: int = 0): - self.check_type = check_type - self.cmd_args = cmd_args - self.file_path = file_path - self.stdout = stdout - self.stderr = stderr - self.returncode = returncode - - def get_description(self) -> str: - return "Check '%s' for %s" % (self.check_type, rel_to_repo_root(self.file_path)) - - -class Reporter: - def __init__(self, line_width: int): - self.line_width = line_width - - def write(self, line: str) -> None: - sys.stdout.write(line) - - def print(self, line: str) -> None: - self.write(line + '\n') - - def get_horizontal_line(self) -> str: - return '-' * self.line_width + '\n' - - def print_check_result(self, check_result: CheckResult) -> None: - if check_result.returncode == 0: - return - - s = '' - s += self.get_horizontal_line() - s += check_result.get_description() + '\n' - s += self.get_horizontal_line() - s += 'Exit code: %d\n' % check_result.returncode - - if check_result.stdout.strip(): - s += '\n' - s += 'Standard output:\n' - s += check_result.stdout - - if check_result.stderr.strip(): - s += '\n' - s += 'Standard error:\n' - s += '\n' - s += check_result.stderr - - s += '\n' - self.write(s) - - -def check_file(file_path: str, check_type: str) -> CheckResult: - assert check_type in CHECK_TYPES - - if check_type == 'mypy': - args = ['mypy', '--config-file', 'mypy.ini'] - elif check_type == 'compile': - args = ['python3', '-m', 'py_compile'] - elif check_type == 'import': - rel_path = rel_to_repo_root(file_path) - where_to_import_from = None - what_to_import = None - if rel_path.startswith('src/build_clang/'): - where_to_import_from = 'build_clang' - # Assuming a one-level hierarchy, i.e. that the build_clang directory does not contain - # any module subdirectories with Python files. - else: - return CheckResult(check_type=check_type, file_path=file_path) - - args = [ - 'python3', '-c', 'from %s import %s' % ( - where_to_import_from, - os.path.splitext(os.path.basename(file_path))[0] - ) - ] - elif check_type == 'pycodestyle': - args = ['pycodestyle', - '--config=%s' % os.path.join(BUILD_CLANG_SCRIPTS_ROOT_PATH, 'pycodestyle.cfg')] - else: - raise ValueError(f"Unknown check type: {check_type}") - args.append(file_path) - - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - return CheckResult( - check_type=check_type, - cmd_args=args, - file_path=file_path, - stdout=ensure_decoded(stdout), - stderr=ensure_decoded(stderr), - returncode=process.returncode) - - -def check_python_code() -> bool: - start_time = time.time() - input_file_paths = ( - glob.glob(os.path.join(BUILD_CLANG_SCRIPTS_ROOT_PATH, 'src', 'build_clang', '*.py')) + - glob.glob(os.path.join(BUILD_CLANG_SCRIPTS_ROOT_PATH, '*.py')) - ) - - for dirpath, dirnames, filenames in os.walk( - os.path.join(BUILD_CLANG_SCRIPTS_ROOT_PATH, 'python')): - for file_name in filenames: - if file_name.endswith('.py'): - input_file_paths.append(os.path.join(dirpath, file_name)) - - os.environ['MYPYPATH'] = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - reporter = Reporter(line_width=80) - checks_by_dir: Dict[str, int] = {} - checks_by_type: Dict[str, int] = {} - checks_by_result: Dict[str, int] = {} - - success = True - - with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: - future_to_check_input = { - executor.submit(check_file, file_path, check_type): (file_path, check_type) - for file_path in input_file_paths - for check_type in CHECK_TYPES if check_type != 'import' - } - for future in concurrent.futures.as_completed(future_to_check_input): - file_path, check_type = future_to_check_input[future] - try: - check_result = future.result() - except Exception as exc: - print("Check '%s' for %s generated an exception: %s" % (check_type, file_path, exc)) - success = False - else: - reporter.print_check_result(check_result) - if check_result.cmd_args: - rel_dir = os.path.dirname(rel_to_repo_root(file_path)) or 'root' - increment_counter(checks_by_dir, rel_dir) - increment_counter(checks_by_type, check_result.check_type) - if check_result.returncode == 0: - increment_counter(checks_by_result, 'success') - else: - increment_counter(checks_by_result, 'failure') - success = False - - print_stats("Checks by directory (relative to repo root)", checks_by_dir) - print_stats("Checks by type", checks_by_type) - print_stats("Checks by result", checks_by_result) - print("Elapsed time: %.1f seconds" % (time.time() - start_time)) - print() - if success: - print("All checks are successful") - else: - print("Some checks failed") - print() - return success - - -if __name__ == '__main__': - if check_python_code(): - sys.exit(0) - sys.exit(1)