diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 37af5529..406434df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,11 +7,11 @@ Build and test: services: - docker:dind before_script: - - apk add black=20.8_beta1-r0 docker-compose make + - apk add black=20.8_beta1-r0 py3-flake8 docker-compose make script: - cd contrib/docker - make build - make update-db - make test-all - cd ../.. - - black --check lib/debsources + - make check diff --git a/HACKING.md b/HACKING.md index 18f7f3df..bb0e9c93 100644 --- a/HACKING.md +++ b/HACKING.md @@ -103,7 +103,7 @@ go now? Running tests ------------- -See `doc/testing.txt`. +See [testing.md](doc/testing.md]. Coding conventions @@ -113,8 +113,11 @@ All new Debsources code should be [PEP8][1] compliant and pass [pyflakes][2] validation. Before submitting patches, please make sure that the lines of code they touch conform to such requirements. +Additionally, `black` [3] is used to format Python source files. + [1]: https://www.python.org/dev/peps/pep-0008/ [2]: https://pypi.python.org/pypi/pyflakes +[3]: https://black.readthedocs.io/en/stable/ If you develop on Debian(-based distros), a good way to check that this is the case is: @@ -122,11 +125,13 @@ case is: # apt-get install python-flake8 $ flake8 file1.py file2.py ... -You can check all Debsources Python source files flake8 compliance with: +You can check all Debsources Python source files black and flake8 compliance +with: $ make check You can add a pre-commit hook to automatically test PEP8 compliance: + $ ln -s ../../contrib/git-pre-commit .git/hooks/pre-commit If you stumble upon Debsources source files that are not flake8-compliant, diff --git a/Makefile b/Makefile index 29f22373..ec149b56 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ NOSE = nosetests3 -FLAKE = flake8 +FLAKE = flake8 --max-line-length 88 --ignore=E203,W503 +# E203 (whitespace before ':') conflicts with black formatting +# W503 (line break before binary operator), ditto +BLACK = black --check SRCDIR = lib/debsources BINDIR = bin @@ -29,6 +32,7 @@ test-coverage: check: $(FLAKE) $(SRCDIR)/ $(shell grep -H 'env python' $(BINDIR)/debsources-* | cut -f 1 -d :) + $(BLACK) $(SRCDIR) test-online-app: contrib/test-online-app diff --git a/bin/debsources-backup-db b/bin/debsources-backup-db index 4cb9deeb..2ad43f31 100755 --- a/bin/debsources-backup-db +++ b/bin/debsources-backup-db @@ -1,6 +1,7 @@ #!/bin/bash -# Copyright (C) 2014-2015 The Debsources developers . +# Copyright (C) 2014-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # diff --git a/bin/debsources-bulk-insert-checksums b/bin/debsources-bulk-insert-checksums index 24aca41b..05ec01d1 100755 --- a/bin/debsources-bulk-insert-checksums +++ b/bin/debsources-bulk-insert-checksums @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright (C) 2014-2015 The Debsources developers . +# Copyright (C) 2014-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # @@ -23,42 +24,48 @@ from debsources import mainlib from debsources import fs_storage from debsources.plugins.hook_checksums import parse_checksums -METADATA_SUFFIX = '.checksums' +METADATA_SUFFIX = ".checksums" METADATA_SUFFIX_LEN = len(METADATA_SUFFIX) BINDPARAMS = [ - bindparam('path', type_=LargeBinary), + bindparam("path", type_=LargeBinary), ] -PACKAGE_ID_Q = sql_query(""" +PACKAGE_ID_Q = sql_query( + """ SELECT versions.id FROM versions, packages WHERE versions.package_id = packages.id AND versions.version = :v AND packages.name = :p - """) + """ +) -CHECKSUMS_EXIST_Q = sql_query(""" +CHECKSUMS_EXIST_Q = sql_query( + """ SELECT 1 FROM checksums WHERE package_id = :v LIMIT 1 - """) + """ +) -INSERT_SHA_Q = sql_query(""" +INSERT_SHA_Q = sql_query( + """ INSERT INTO checksums (package_id, path, sha256) VALUES (:package_id, :path, :sha256) """, - bindparams=BINDPARAMS) + bindparams=BINDPARAMS, +) def insert_checksums(conn, sumsfile): - steps = sumsfile.split('/') + steps = sumsfile.split("/") package = steps[-2] version = steps[-1][:-METADATA_SUFFIX_LEN] - logging.debug('package %s/%s' % (package, version)) - logging.debug('INSERT INTO checksums %s/%s...' % (package, version)) + logging.debug("package %s/%s" % (package, version)) + logging.debug("INSERT INTO checksums %s/%s..." % (package, version)) package_id = conn.execute(PACKAGE_ID_Q, v=version, p=package).first()[0] assert package_id is not None @@ -68,9 +75,9 @@ def insert_checksums(conn, sumsfile): checksums = [] for (sha256, path) in parse_checksums(sumsfile): - checksums.append({'package_id': package_id, - 'path': str(path), - 'sha256': sha256}) + checksums.append( + {"package_id": package_id, "path": str(path), "sha256": sha256} + ) if checksums: conn.execute(INSERT_SHA_Q, checksums) @@ -80,27 +87,29 @@ def is_checksum(p): def main(conf, sumsfile=None): - conn = create_engine(conf['db_uri']) + conn = create_engine(conf["db_uri"]) if sumsfile: insert_checksums(conn, sumsfile) else: - for f in fs_storage.walk(conf['sources_dir'], test=is_checksum): + for f in fs_storage.walk(conf["sources_dir"], test=is_checksum): with conn.begin() as trans: try: insert_checksums(conn, f) - except: - logging.exception('cannot inject %s' % f) + except Exception: + logging.exception("cannot inject %s" % f) trans.rollback() -if __name__ == '__main__': - cmdline = argparse.ArgumentParser( - description='Debsources: bulk checksum insertion') - cmdline.add_argument('--config', '-c', dest='conffile', - default=mainlib.guess_conffile(), - help='alternate configuration file') - cmdline.add_argument('file', nargs='?', - help='.checksum metadata file') +if __name__ == "__main__": + cmdline = argparse.ArgumentParser(description="Debsources: bulk checksum insertion") + cmdline.add_argument( + "--config", + "-c", + dest="conffile", + default=mainlib.guess_conffile(), + help="alternate configuration file", + ) + cmdline.add_argument("file", nargs="?", help=".checksum metadata file") args = cmdline.parse_args() conf = mainlib.load_configuration(args.conffile) diff --git a/bin/debsources-bulk-insert-ctags b/bin/debsources-bulk-insert-ctags index 41f054ca..752f1d83 100755 --- a/bin/debsources-bulk-insert-ctags +++ b/bin/debsources-bulk-insert-ctags @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright (C) 2014-2015 The Debsources developers . +# Copyright (C) 2014-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # @@ -25,56 +26,61 @@ from sqlalchemy.sql import text as sql_query from debsources import fs_storage from debsources.plugins.hook_ctags import parse_ctags -METADATA_SUFFIX = '.ctags' +METADATA_SUFFIX = ".ctags" METADATA_SUFFIX_LEN = len(METADATA_SUFFIX) BINDPARAMS = [ - bindparam('path', type_=LargeBinary), + bindparam("path", type_=LargeBinary), ] -PACKAGE_ID_Q = sql_query(""" +PACKAGE_ID_Q = sql_query( + """ SELECT versions.id FROM versions, packages WHERE versions.package_id = packages.id AND versions.version = :v AND packages.name = :p - """) + """ +) -CTAGS_EXIST_Q = sql_query(""" +CTAGS_EXIST_Q = sql_query( + """ SELECT 1 FROM ctags WHERE package_id = :v LIMIT 1 - """) + """ +) -INSERT_CTAGS_Q = sql_query(""" +INSERT_CTAGS_Q = sql_query( + """ INSERT INTO ctags (package_id, tag, path, line, kind, language) VALUES (:package_id, :tag, :path, :line, :kind, :language) """, - bindparams=BINDPARAMS) + bindparams=BINDPARAMS, +) def insert_ctags(conn, ctagsfile): - steps = ctagsfile.split('/') + steps = ctagsfile.split("/") package = steps[-2] version = steps[-1][:-METADATA_SUFFIX_LEN] - logging.debug('package %s/%s' % (package, version)) + logging.debug("package %s/%s" % (package, version)) package_id = conn.execute(PACKAGE_ID_Q, v=version, p=package).first() if not package_id: - logging.warn('skipping %s/%s (does not exist in DB)' % - (package, version)) + logging.warn("skipping %s/%s (does not exist in DB)" % (package, version)) return has_ctags = conn.execute(CTAGS_EXIST_Q, v=package_id[0]).first() if has_ctags: - logging.warn('skipping %s/%s (already inserted)' % (package, version)) + logging.warn("skipping %s/%s (already inserted)" % (package, version)) return - logging.debug('INSERT INTO ctags %s/%s...' % (package, version)) + logging.debug("INSERT INTO ctags %s/%s..." % (package, version)) ctags = [] for tag in parse_ctags(ctagsfile): - tag['package_id'] = package_id + tag["package_id"] = package_id ctags.append(tag) if ctags: conn.execute(INSERT_CTAGS_Q, ctags) @@ -85,27 +91,29 @@ def is_ctags(p): def main(conf, ctagsfile=None): - conn = create_engine(conf['db_uri']).connect() + conn = create_engine(conf["db_uri"]).connect() if ctagsfile: insert_ctags(conn, ctagsfile) else: - for f in fs_storage.walk(conf['sources_dir'], test=is_ctags): + for f in fs_storage.walk(conf["sources_dir"], test=is_ctags): with conn.begin() as trans: try: insert_ctags(conn, f) - except: - logging.exception('cannot inject %s' % f) + except Exception: + logging.exception("cannot inject %s" % f) trans.rollback() -if __name__ == '__main__': - cmdline = argparse.ArgumentParser( - description='Debsources: bulk ctags insertion') - cmdline.add_argument('--config', '-c', dest='conffile', - default=mainlib.guess_conffile(), - help='alternate configuration file') - cmdline.add_argument('file', nargs='?', - help='.ctags metadata file') +if __name__ == "__main__": + cmdline = argparse.ArgumentParser(description="Debsources: bulk ctags insertion") + cmdline.add_argument( + "--config", + "-c", + dest="conffile", + default=mainlib.guess_conffile(), + help="alternate configuration file", + ) + cmdline.add_argument("file", nargs="?", help=".ctags metadata file") args = cmdline.parse_args() conf = mainlib.load_configuration(args.conffile) diff --git a/bin/debsources-dbadmin b/bin/debsources-dbadmin index b7884f12..0d665831 100755 --- a/bin/debsources-dbadmin +++ b/bin/debsources-dbadmin @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright (C) 2013-2014 The Debsources developers . +# Copyright (C) 2013-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # @@ -26,19 +27,20 @@ if __name__ == "__main__": start_time = time.time() parser = argparse.ArgumentParser(description="Manage Debsources DB") - parser.add_argument("dburi", - help="SQLAlchemy URI to the DB, e.g. postgres://" - "username:password@localhost:5432/debsources") - parser.add_argument("--createdb", - help="create DB schema", - action="store_true") - parser.add_argument("--dropdb", - help="destroy existing DB schema " - "(WARNING: you will lose all data)", - action="store_true") - parser.add_argument("--verbose", - help="verbose logging (default: be quiet)", - action="store_true") + parser.add_argument( + "dburi", + help="SQLAlchemy URI to the DB, e.g. postgres://" + "username:password@localhost:5432/debsources", + ) + parser.add_argument("--createdb", help="create DB schema", action="store_true") + parser.add_argument( + "--dropdb", + help="destroy existing DB schema " "(WARNING: you will lose all data)", + action="store_true", + ) + parser.add_argument( + "--verbose", help="verbose logging (default: be quiet)", action="store_true" + ) args = parser.parse_args() db = create_engine(args.dburi, echo=args.verbose) diff --git a/bin/debsources-debian-source-archive b/bin/debsources-debian-source-archive index 5ac5b557..98b000c1 100755 --- a/bin/debsources-debian-source-archive +++ b/bin/debsources-debian-source-archive @@ -1,6 +1,7 @@ #!/bin/bash -# Copyright (C) 2011-2014 The Debsources developers . +# Copyright (C) 2011-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # diff --git a/bin/debsources-debian-source-mirror b/bin/debsources-debian-source-mirror index 707de766..3c81b1e3 100755 --- a/bin/debsources-debian-source-mirror +++ b/bin/debsources-debian-source-mirror @@ -1,6 +1,7 @@ #!/bin/bash -# Copyright (C) 2011-2013 The Debsources developers . +# Copyright (C) 2011-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # diff --git a/bin/debsources-dev-fetch-test-data b/bin/debsources-dev-fetch-test-data index 47aed8f8..059ad72a 100755 --- a/bin/debsources-dev-fetch-test-data +++ b/bin/debsources-dev-fetch-test-data @@ -1,6 +1,7 @@ #!/bin/bash -# Copyright (C) 2013-2014 The Debsources developers . +# Copyright (C) 2013-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # diff --git a/bin/debsources-foreach b/bin/debsources-foreach index 69183c1d..f4c4193b 100755 --- a/bin/debsources-foreach +++ b/bin/debsources-foreach @@ -1,6 +1,7 @@ #!/bin/bash -# Copyright (C) 2011-2014 The Debsources developers . +# Copyright (C) 2011-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # diff --git a/bin/debsources-fsck b/bin/debsources-fsck index cd2631e6..afd5fbca 100755 --- a/bin/debsources-fsck +++ b/bin/debsources-fsck @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright (C) 2013-2014 The Debsources developers . +# Copyright (C) 2013-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # @@ -35,56 +36,54 @@ Session = sessionmaker() # Global. List of allowed extensions in FS storage (will be extended querying # plugin information) -file_extensions = ['.done', '.log'] +file_extensions = [".done", ".log"] def fs_check_missing(conf, session, fix=False): - logging.info('fs storage: check for missing data...') + logging.info("fs storage: check for missing data...") for version in session.query(Package).all(): pkg = SourcePackage.from_db_model(version) - pkgdir = pkg.extraction_dir(conf['sources_dir']) + pkgdir = pkg.extraction_dir(conf["sources_dir"]) if not pkgdir.is_dir(): - logging.warn('missing package directory: %s' % pkgdir) + logging.warn("missing package directory: %s" % pkgdir) for ext in file_extensions: metafile = Path(str(pkgdir) + ext) if not metafile.is_file(): - logging.warn('missing metadata file: %s' % metafile) + logging.warn("missing metadata file: %s" % metafile) def fs_check_stale(conf, session, fix=False): checked_versions = {} def have_version(package, version): - """check if a given package/version exists in the DB, with memoization - """ + """check if a given package/version exists in the DB, with memoization""" pkg_id = (package, version) if pkg_id not in checked_versions.has_key: - checked_versions[pkg_id] = \ - bool(db_storage.lookup_package(session, package, version)) + checked_versions[pkg_id] = bool( + db_storage.lookup_package(session, package, version) + ) return checked_versions[pkg_id] - logging.info('fs storage: check for stale data...') - for entry in fs_storage.walk(conf['sources_dir']): + logging.info("fs storage: check for stale data...") + for entry in fs_storage.walk(conf["sources_dir"]): path = fs_storage.parse_path(entry) if entry.is_dir(): - if not have_version(path['package'], path['version']): - logging.warn('orphan package directory: %s' % entry) + if not have_version(path["package"], path["version"]): + logging.warn("orphan package directory: %s" % entry) if fix: - logging.info('removing orphan package directory %s' % - entry) + logging.info("removing orphan package directory %s" % entry) shutil.rmtree(entry) elif entry.is_file(): - if path['ext'] in file_extensions: - if not have_version(path['package'], path['version']): - logging.warn('orphan metadata file: %s' % entry) + if path["ext"] in file_extensions: + if not have_version(path["package"], path["version"]): + logging.warn("orphan metadata file: %s" % entry) if fix: - logging.info('removing orphan metadata file %s' % - entry) + logging.info("removing orphan metadata file %s" % entry) entry.unlink() else: - logging.warn('unknown metadata file: %s' % entry) + logging.warn("unknown metadata file: %s" % entry) if fix: - logging.info('removing unknown metadata file %s' % entry) + logging.info("removing unknown metadata file %s" % entry) entry.unlink() @@ -93,21 +92,23 @@ def main(conf, session, fix): fs_check_stale(conf, session, fix) -if __name__ == '__main__': - cmdline = argparse.ArgumentParser(description='Debsources fsck') - cmdline.add_argument('--config', '-c', dest='conffile', - default=mainlib.guess_conffile(), - help='alternate configuration file') - cmdline.add_argument('--fix', '-f', - action='store_true', - help='fix errors') +if __name__ == "__main__": + cmdline = argparse.ArgumentParser(description="Debsources fsck") + cmdline.add_argument( + "--config", + "-c", + dest="conffile", + default=mainlib.guess_conffile(), + help="alternate configuration file", + ) + cmdline.add_argument("--fix", "-f", action="store_true", help="fix errors") args = cmdline.parse_args() conf = mainlib.load_configuration(args.conffile) mainlib.init_logging(conf, logging.INFO) (_observers, exts) = mainlib.load_hooks(conf) file_extensions.extend(exts.keys()) - db = create_engine(conf['db_uri']) + db = create_engine(conf["db_uri"]) session = Session(bind=db) main(conf, session, args.fix) diff --git a/bin/debsources-main b/bin/debsources-main index 16891fe6..e9b65422 100755 --- a/bin/debsources-main +++ b/bin/debsources-main @@ -1,6 +1,7 @@ #!/bin/bash -# Copyright (C) 2013-2015 The Debsources developers . +# Copyright (C) 2013-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # diff --git a/bin/debsources-reportbug b/bin/debsources-reportbug index 3d861aa7..0a19e41e 100755 --- a/bin/debsources-reportbug +++ b/bin/debsources-reportbug @@ -1,6 +1,7 @@ #!/bin/bash -# Copyright (C) 2014 The Debsources developers . +# Copyright (C) 2014-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # diff --git a/bin/debsources-run-app b/bin/debsources-run-app index 13820f07..9396658d 100755 --- a/bin/debsources-run-app +++ b/bin/debsources-run-app @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright (C) 2013-2015 The Debsources developers . +# Copyright (C) 2013-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # @@ -17,16 +18,21 @@ from __future__ import print_function import argparse if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Run Flask app, for dev/debug.') - parser.add_argument('--host', type=str, default='127.0.0.1', - required=False, - help='Host, use 0.0.0.0 to listen on all IPs.') - parser.add_argument('--port', type=int, default=5000, required=False, - help='Port in use') + parser = argparse.ArgumentParser(description="Run Flask app, for dev/debug.") + parser.add_argument( + "--host", + type=str, + default="127.0.0.1", + required=False, + help="Host, use 0.0.0.0 to listen on all IPs.", + ) + parser.add_argument( + "--port", type=int, default=5000, required=False, help="Port in use" + ) args = parser.parse_args() from debsources.app import app_wrapper + app_wrapper.go() - print('Running on %s:%d' % (args.host, args.port)) + print("Running on %s:%d" % (args.host, args.port)) app_wrapper.app.run(debug=True, host=args.host, port=args.port) diff --git a/bin/debsources-shell b/bin/debsources-shell index 890d0387..32a1a5df 100755 --- a/bin/debsources-shell +++ b/bin/debsources-shell @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright (C) 2013-2015 The Debsources developers . +# Copyright (C) 2013-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # @@ -26,19 +27,15 @@ from debsources.models import * # NOQA def main(): conf = mainlib.load_conf(mainlib.guess_conffile()) - engine, session = sqla_session._get_engine_session(conf['db_uri']) + engine, session = sqla_session._get_engine_session(conf["db_uri"]) readline.parse_and_bind("tab: complete") - os.environ['PYTHONINSPECT'] = 'True' - banner = ( - "Debsources interactive console\n" - "`engine` and `session` are available!" - ) + os.environ["PYTHONINSPECT"] = "True" + banner = "Debsources interactive console\n" "`engine` and `session` are available!" code.interact(banner=banner, local={**globals(), **locals()}) if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Run Debsources shell, for debugging.') + parser = argparse.ArgumentParser(description="Run Debsources shell, for debugging.") args = parser.parse_args() main() diff --git a/bin/debsources-sloccount b/bin/debsources-sloccount index 8d4a3e1c..41305030 100755 --- a/bin/debsources-sloccount +++ b/bin/debsources-sloccount @@ -1,6 +1,7 @@ #!/usr/bin/perl -# Copyright (C) 2013-2015 The Debsources developers . +# Copyright (C) 2013-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # diff --git a/bin/debsources-suite-archive b/bin/debsources-suite-archive index 8347f6ca..d4bc970c 100755 --- a/bin/debsources-suite-archive +++ b/bin/debsources-suite-archive @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright (C) 2014 The Debsources developers . +# Copyright (C) 2014-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # @@ -26,53 +27,58 @@ from debsources import mainlib def main(): - cmdline = argparse.ArgumentParser(description='Debsources suite ' - 'archive manager') - cmdline.add_argument('action', metavar='ACTION', - choices=['add', 'list', 'remove'], - help='action to perform on the archive of ' - 'sticky suites') - cmdline.add_argument('suite', metavar='SUITE', nargs='?', default=None, - help='name of the suite to act upon ' - '(for "add" and "remove")') + cmdline = argparse.ArgumentParser(description="Debsources suite " "archive manager") + cmdline.add_argument( + "action", + metavar="ACTION", + choices=["add", "list", "remove"], + help="action to perform on the archive of " "sticky suites", + ) + cmdline.add_argument( + "suite", + metavar="SUITE", + nargs="?", + default=None, + help="name of the suite to act upon " '(for "add" and "remove")', + ) mainlib.add_arguments(cmdline) args = cmdline.parse_args() - if args.action in ['add', 'remove'] and args.suite is None: - cmdline.error('%s requires a suite name' % args.action) + if args.action in ["add", "remove"] and args.suite is None: + cmdline.error("%s requires a suite name" % args.action) conf = mainlib.load_conf(args.conffile or mainlib.guess_conffile()) mainlib.override_conf(conf, args) mainlib.init_logging(conf, mainlib.log_level_of_verbosity(args.verbose)) - logging.debug('loaded configuration from %s' % conf['conffile']) - conf['observers'], conf['file_exts'] = mainlib.load_hooks(conf) + logging.debug("loaded configuration from %s" % conf["conffile"]) + conf["observers"], conf["file_exts"] = mainlib.load_hooks(conf) mainlib.conf_warnings(conf) - db = sqlalchemy.create_engine(conf['db_uri'], echo=args.verbose >= 4) + db = sqlalchemy.create_engine(conf["db_uri"], echo=args.verbose >= 4) Session = sqlalchemy.orm.sessionmaker() - session = Session(bind=db, autocommit=not conf['single_transaction']) + session = Session(bind=db, autocommit=not conf["single_transaction"]) try: - archive = debmirror.SourceMirrorArchive(conf['mirror_archive_dir']) - if args.action == 'add': + archive = debmirror.SourceMirrorArchive(conf["mirror_archive_dir"]) + if args.action == "add": archiver.add_suite(conf, session, args.suite, archive) - elif args.action == 'list': + elif args.action == "list": print("# name\tindexed\tavailable") - for (suite, present) in \ - archiver.list_suites(conf, session, archive).items(): - print('%s\t%s\t%s' % - (suite, present['db'], present['archive'])) - elif args.action == 'remove': + for (suite, present) in archiver.list_suites( + conf, session, archive + ).items(): + print("%s\t%s\t%s" % (suite, present["db"], present["archive"])) + elif args.action == "remove": archiver.remove_suite(conf, session, args.suite) - if conf['single_transaction']: + if conf["single_transaction"]: session.commit() except SystemExit: # exit as requested raise - except: # store trace in log, then exit - logging.exception('unhandled exception. Abort') + except Exception: # store trace in log, then exit + logging.exception("unhandled exception. Abort") sys.exit(2) finally: session.close() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/bin/debsources-update b/bin/debsources-update index e0734c7d..88db813c 100755 --- a/bin/debsources-update +++ b/bin/debsources-update @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# Copyright (C) 2013-2014 The Debsources developers . +# Copyright (C) 2013-2021 The Debsources developers +# . # See the AUTHORS file at the top-level directory of this distribution and at # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS # @@ -23,35 +24,35 @@ from debsources import updater def main(): - cmdline = argparse.ArgumentParser(description='Debsources updater') + cmdline = argparse.ArgumentParser(description="Debsources updater") mainlib.add_arguments(cmdline) args = cmdline.parse_args() conf = mainlib.load_conf(args.conffile or mainlib.guess_conffile()) mainlib.override_conf(conf, args) mainlib.init_logging(conf, mainlib.log_level_of_verbosity(args.verbose)) - logging.debug('loaded configuration from %s' % conf['conffile']) - conf['observers'], conf['file_exts'] = mainlib.load_hooks(conf) + logging.debug("loaded configuration from %s" % conf["conffile"]) + conf["observers"], conf["file_exts"] = mainlib.load_hooks(conf) mainlib.conf_warnings(conf) try: - db = sqlalchemy.create_engine(conf['db_uri'], echo=args.verbose >= 4) + db = sqlalchemy.create_engine(conf["db_uri"], echo=args.verbose >= 4) Session = sqlalchemy.orm.sessionmaker() - if conf['single_transaction']: + if conf["single_transaction"]: session = Session(bind=db, autocommit=False) - updater.update(conf, session, stages=conf['stages']) + updater.update(conf, session, stages=conf["stages"]) session.commit() else: session = Session(bind=db, autocommit=True) - updater.update(conf, session, stages=conf['stages']) + updater.update(conf, session, stages=conf["stages"]) except SystemExit: # exit as requested raise - except: # store trace in log, then exit - logging.exception('unhandled exception. Abort') + except Exception: # store trace in log, then exit + logging.exception("unhandled exception. Abort") sys.exit(2) finally: session.close() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/lib/debsources/app/copyright/views.py b/lib/debsources/app/copyright/views.py index 0bdeb580..fe44b689 100644 --- a/lib/debsources/app/copyright/views.py +++ b/lib/debsources/app/copyright/views.py @@ -106,16 +106,18 @@ def _get_license_dict(self, files): # parse file try: c = helper.parse_license(license_path) - l = helper.get_license(f["package"], f["version"], f["path"], c) + license = helper.get_license( + f["package"], f["version"], f["path"], c + ) except copyright.NotMachineReadableError: - l = None + license = None result.append( dict( oracle="debian", path=f["path"], package=f["package"], version=f["version"], - license=l, + license=license, origin=helper.license_url(f["package"], f["version"]), ) ) @@ -233,15 +235,15 @@ def _license_of_files(self, f): try: c = helper.parse_license(license_path) - l = helper.get_license(f.package, f.version, f.path, c) + license = helper.get_license(f.package, f.version, f.path, c) except copyright.NotMachineReadableError: - l = None + license = None return dict( oracle="debian", path=f.path, package=f.package, version=f.version, - license=l, + license=license, origin=helper.license_url(f.package, f.version), ) diff --git a/lib/debsources/app/helper.py b/lib/debsources/app/helper.py index 92420026..da4c81c8 100644 --- a/lib/debsources/app/helper.py +++ b/lib/debsources/app/helper.py @@ -17,7 +17,6 @@ from flask import request, url_for, render_template, redirect import debsources.query as qry -from debsources import consts from debsources.models import SuiteAlias from debsources.excepts import InvalidPackageOrVersionError, Http404Error from . import app_wrapper @@ -54,7 +53,7 @@ def format_big_num(num): """ try: res = "{:,}".format(num) - except: + except Exception: res = num return res diff --git a/lib/debsources/app/sourcecode.py b/lib/debsources/app/sourcecode.py index 2a382b87..52cce442 100644 --- a/lib/debsources/app/sourcecode.py +++ b/lib/debsources/app/sourcecode.py @@ -36,7 +36,7 @@ def __init__(self, filepath, hl=None, msg=None, encoding="utf8", lang=None): # we store the firstline (used to determine file language) try: self.firstline = next(self.file) - except: # empty file + except Exception: # empty file self.firstline = "" self.file.seek(0) @@ -61,7 +61,7 @@ def __init__(self, filepath, hl=None, msg=None, encoding="utf8", lang=None): else: # it's a single line try: self.hls.add(int(r)) - except: + except Exception: pass def __iter__(self): diff --git a/lib/debsources/app/sources/routes.py b/lib/debsources/app/sources/routes.py index 9a7bcb13..e0c3f278 100644 --- a/lib/debsources/app/sources/routes.py +++ b/lib/debsources/app/sources/routes.py @@ -15,9 +15,18 @@ from ..helper import bind_render, generic_before_request from ..views import ( - IndexView, SearchView, CtagView, ChecksumView, PrefixView, - ListPackagesView, InfoPackageView, Ping, ErrorHandler, - PackageVersionsView, NewsArchiveView) + IndexView, + SearchView, + CtagView, + ChecksumView, + PrefixView, + ListPackagesView, + InfoPackageView, + Ping, + ErrorHandler, + PackageVersionsView, + NewsArchiveView, +) from .views import StatsView, SourceView from . import bp_sources @@ -28,293 +37,332 @@ @bp_sources.context_processor def skeleton_variables(): site_name = bp_sources.name - return dict(site_name=site_name,) + return dict( + site_name=site_name, + ) # site errors # XXX 500 handler cannot be registered on a blueprint # TODO see debsources.app.view#errorhandler section -bp_sources.errorhandler(403)( - lambda e: (ErrorHandler()(e, http=403), 403)) -bp_sources.errorhandler(404)( - lambda e: (ErrorHandler()(e, http=404), 404)) +bp_sources.errorhandler(403)(lambda e: (ErrorHandler()(e, http=403), 403)) +bp_sources.errorhandler(404)(lambda e: (ErrorHandler()(e, http=404), 404)) # Before request @bp_sources.before_request def before_request(): try: - if 'embedded' in request.endpoint: + if "embedded" in request.endpoint: return generic_before_request(request, 3) - elif 'source' in request.endpoint.split('.')[1]: + elif "source" in request.endpoint.split(".")[1]: return generic_before_request(request, 2) except Http404Error: - return render_template('404.html'), 404 + return render_template("404.html"), 404 + # ping service bp_sources.add_url_rule( - '/api/ping/', + "/api/ping/", view_func=Ping.as_view( - 'ping',)) + "ping", + ), +) # INDEXVIEW bp_sources.add_url_rule( - '/', + "/", view_func=IndexView.as_view( - 'index', - render_func=bind_render('sources/index.html'), - err_func=ErrorHandler('sources'), - news_html='sources_news.html', - news_archive_html='sources_news_archive.html')) + "index", + render_func=bind_render("sources/index.html"), + err_func=ErrorHandler("sources"), + news_html="sources_news.html", + news_archive_html="sources_news_archive.html", + ), +) # NEWSARCHIVEVIEW bp_sources.add_url_rule( - '/news_archive', + "/news_archive", view_func=NewsArchiveView.as_view( - 'news_archive', - render_func=bind_render('news_archive.html'), - err_func=ErrorHandler('sources'), - news_archive_html='sources_news_archive.html')) + "news_archive", + render_func=bind_render("news_archive.html"), + err_func=ErrorHandler("sources"), + news_archive_html="sources_news_archive.html", + ), +) # STATSVIEW bp_sources.add_url_rule( - '/stats/', + "/stats/", view_func=StatsView.as_view( - 'stats', - render_func=bind_render('sources/stats.html'), - err_func=ErrorHandler('sources'), - get_objects='stats',)) + "stats", + render_func=bind_render("sources/stats.html"), + err_func=ErrorHandler("sources"), + get_objects="stats", + ), +) # api bp_sources.add_url_rule( - '/api/stats/', + "/api/stats/", view_func=StatsView.as_view( - 'api_stats', - err_func=ErrorHandler(mode='json'), - get_objects='stats',)) + "api_stats", + err_func=ErrorHandler(mode="json"), + get_objects="stats", + ), +) bp_sources.add_url_rule( - '/stats//', + "/stats//", view_func=StatsView.as_view( - 'stats_suite', - render_func=bind_render('sources/stats_suite.html'), - err_func=ErrorHandler('sources'), - get_objects='stats_suite',)) + "stats_suite", + render_func=bind_render("sources/stats_suite.html"), + err_func=ErrorHandler("sources"), + get_objects="stats_suite", + ), +) # api bp_sources.add_url_rule( - '/api/stats//', + "/api/stats//", view_func=StatsView.as_view( - 'api_stats_suite', + "api_stats_suite", render_func=jsonify, - err_func=ErrorHandler(mode='json'), - get_objects='stats_suite')) + err_func=ErrorHandler(mode="json"), + get_objects="stats_suite", + ), +) # SEARCHVIEW bp_sources.add_url_rule( - '/search/', + "/search/", view_func=SearchView.as_view( - 'recv_search', - render_func=bind_render('sources/index.html'), - err_func=ErrorHandler('sources'), - recv_search=True), - methods=['GET', 'POST']) + "recv_search", + render_func=bind_render("sources/index.html"), + err_func=ErrorHandler("sources"), + recv_search=True, + ), + methods=["GET", "POST"], +) bp_sources.add_url_rule( - '/advancedsearch/', + "/advancedsearch/", view_func=SearchView.as_view( - 'advanced_search', - render_func=bind_render('sources/search_advanced.html'), - err_func=ErrorHandler('sources'), - get_objects='advanced',)) + "advanced_search", + render_func=bind_render("sources/search_advanced.html"), + err_func=ErrorHandler("sources"), + get_objects="advanced", + ), +) # api bp_sources.add_url_rule( - '/api/advancedsearch/', + "/api/advancedsearch/", view_func=SearchView.as_view( - 'api_advanced_search', - render_func=jsonify, - err_func=ErrorHandler(mode='json'))) + "api_advanced_search", render_func=jsonify, err_func=ErrorHandler(mode="json") + ), +) bp_sources.add_url_rule( - '/search//', + "/search//", view_func=SearchView.as_view( - 'search', - render_func=bind_render('search.html'), - err_func=ErrorHandler('sources'), - get_objects='query',)) + "search", + render_func=bind_render("search.html"), + err_func=ErrorHandler("sources"), + get_objects="query", + ), +) # api bp_sources.add_url_rule( - '/api/search//', + "/api/search//", view_func=SearchView.as_view( - 'api_search', + "api_search", render_func=jsonify, - err_func=ErrorHandler(mode='json'), - get_objects='query')) + err_func=ErrorHandler(mode="json"), + get_objects="query", + ), +) # ChecksumView bp_sources.add_url_rule( - '/sha256/', + "/sha256/", view_func=ChecksumView.as_view( - 'checksum', - render_func=bind_render('sources/checksum.html'), - err_func=ErrorHandler('sources'), - pagination=True)) + "checksum", + render_func=bind_render("sources/checksum.html"), + err_func=ErrorHandler("sources"), + pagination=True, + ), +) # api bp_sources.add_url_rule( - '/api/sha256/', + "/api/sha256/", view_func=ChecksumView.as_view( - 'api_checksum', - render_func=jsonify, - err_func=ErrorHandler(mode='json'))) + "api_checksum", render_func=jsonify, err_func=ErrorHandler(mode="json") + ), +) # CtagView bp_sources.add_url_rule( - '/ctag/', + "/ctag/", view_func=CtagView.as_view( - 'ctag', - render_func=bind_render('sources/ctag.html'), - err_func=ErrorHandler('sources'), - pagination=True)) + "ctag", + render_func=bind_render("sources/ctag.html"), + err_func=ErrorHandler("sources"), + pagination=True, + ), +) # api bp_sources.add_url_rule( - '/api/ctag/', + "/api/ctag/", view_func=CtagView.as_view( - 'api_ctag', - render_func=jsonify, - err_func=ErrorHandler(mode='json'))) + "api_ctag", render_func=jsonify, err_func=ErrorHandler(mode="json") + ), +) # PREFIXVIEW bp_sources.add_url_rule( - '/prefix//', + "/prefix//", view_func=PrefixView.as_view( - 'prefix', - render_func=bind_render('prefix.html'), - err_func=ErrorHandler('sources'),)) + "prefix", + render_func=bind_render("prefix.html"), + err_func=ErrorHandler("sources"), + ), +) # api bp_sources.add_url_rule( - '/api/prefix//', + "/api/prefix//", view_func=PrefixView.as_view( - 'api_prefix', - render_func=jsonify, - err_func=ErrorHandler(mode='json'))) + "api_prefix", render_func=jsonify, err_func=ErrorHandler(mode="json") + ), +) # LISTPACKAGESVIEW bp_sources.add_url_rule( - '/list//', + "/list//", view_func=ListPackagesView.as_view( - 'list_packages', - render_func=bind_render('list.html'), - err_func=ErrorHandler('sources'), - pagination=True)) + "list_packages", + render_func=bind_render("list.html"), + err_func=ErrorHandler("sources"), + pagination=True, + ), +) # api bp_sources.add_url_rule( - '/api/list/', + "/api/list/", view_func=ListPackagesView.as_view( - 'api_list_packages', - render_func=jsonify, - err_func=ErrorHandler(mode='json'))) + "api_list_packages", render_func=jsonify, err_func=ErrorHandler(mode="json") + ), +) # VERSIONSVIEW bp_sources.add_url_rule( - '/src//', + "/src//", view_func=PackageVersionsView.as_view( - 'versions', - render_func=bind_render('sources/source_package.html'), - err_func=ErrorHandler('sources'))) + "versions", + render_func=bind_render("sources/source_package.html"), + err_func=ErrorHandler("sources"), + ), +) # api bp_sources.add_url_rule( - '/api/src//', + "/api/src//", view_func=PackageVersionsView.as_view( - 'api_versions', - render_func=jsonify, - err_func=ErrorHandler(mode='json'))) + "api_versions", render_func=jsonify, err_func=ErrorHandler(mode="json") + ), +) # SOURCEVIEW bp_sources.add_url_rule( - '/src//', + "/src//", view_func=SourceView.as_view( - 'source', + "source", # the render func is set by the views. - err_func=ErrorHandler('sources'), - templatename='sources/source_file.html')) + err_func=ErrorHandler("sources"), + templatename="sources/source_file.html", + ), +) # api bp_sources.add_url_rule( - '/api/src//', + "/api/src//", view_func=SourceView.as_view( - 'api_source', - err_func=ErrorHandler(mode='json'), - api=True)) + "api_source", err_func=ErrorHandler(mode="json"), api=True + ), +) # SOURCE FILE EMBEDDED ROUTING bp_sources.add_url_rule( - '/embed/file//', + "/embed/file//", view_func=SourceView.as_view( - 'embedded_source', - err_func=ErrorHandler('sources'), - templatename="sources/source_file_embedded.html")) + "embedded_source", + err_func=ErrorHandler("sources"), + templatename="sources/source_file_embedded.html", + ), +) # we redirect the old used embedded file page (/embedded/) # to the new one (/embed/file/) @bp_sources.route("/embedded//") def old_embedded_file(path_to, **kwargs): - return redirect(url_for(".embedded_source", - path_to=path_to, - **request.args)) + return redirect(url_for(".embedded_source", path_to=path_to, **request.args)) # INFO PER-VERSION bp_sources.add_url_rule( - '/info/package///', + "/info/package///", view_func=InfoPackageView.as_view( - 'info_package', - render_func=bind_render('sources/infopackage.html'), - err_func=ErrorHandler('sources'),)) + "info_package", + render_func=bind_render("sources/infopackage.html"), + err_func=ErrorHandler("sources"), + ), +) # api bp_sources.add_url_rule( - '/api/info/package///', + "/api/info/package///", view_func=InfoPackageView.as_view( - 'api_info_package', - render_func=jsonify, - err_func=ErrorHandler(mode='json'))) + "api_info_package", render_func=jsonify, err_func=ErrorHandler(mode="json") + ), +) # INFO PER-VERSION (EMBEDDED) bp_sources.add_url_rule( - '/embed/pkginfo///', + "/embed/pkginfo///", view_func=InfoPackageView.as_view( - 'embedded_info_package', - render_func=bind_render('sources/infopackage_embed.html'), - err_func=ErrorHandler('sources'))) + "embedded_info_package", + render_func=bind_render("sources/infopackage_embed.html"), + err_func=ErrorHandler("sources"), + ), +) diff --git a/lib/debsources/app/views.py b/lib/debsources/app/views.py index 34676556..92d3d2e0 100644 --- a/lib/debsources/app/views.py +++ b/lib/debsources/app/views.py @@ -98,7 +98,7 @@ def __call__(self, error, http=None): self.http = http try: method = getattr(self, "error_{}".format(self.http)) - except: + except Exception: raise Exception("Unimplemented HTTP error: {}".format(self.http)) return method(error) @@ -226,7 +226,7 @@ def dispatch_request(self): last_update = local_info.read_update_ts(update_ts_file) try: session.query(Package).first().id # database check - except: + except Exception: return jsonify(dict(status="db error", http_status_code=500)), 500 return jsonify(dict(status="ok", http_status_code=200, last_update=last_update)) @@ -386,7 +386,7 @@ def get_objects(self): """ try: page = int(request.args.get("page")) - except: + except Exception: page = 1 ctag = request.args.get("ctag") package = request.args.get("package") or None @@ -395,7 +395,7 @@ def get_objects(self): if self.d.get("pagination"): try: offset = int(current_app.config.get("LIST_OFFSET")) - except: + except Exception: offset = 60 start = (page - 1) * offset end = start + offset diff --git a/lib/debsources/debmirror.py b/lib/debsources/debmirror.py index 23daa376..14e80b5b 100644 --- a/lib/debsources/debmirror.py +++ b/lib/debsources/debmirror.py @@ -19,7 +19,6 @@ from typing import Optional from debian import deb822 -from debian.debian_support import version_compare # supported compression formats for Sources files. Order does matter: formats # appearing early in the list will be preferred to those appearing later diff --git a/lib/debsources/filetype.py b/lib/debsources/filetype.py index faaab64e..09b889a2 100644 --- a/lib/debsources/filetype.py +++ b/lib/debsources/filetype.py @@ -265,7 +265,7 @@ def get_filetype_from_filename(filename): try: if re.search(pattern, filename): return language - except: + except Exception: raise Exception("Regex error: " + str(language) + " " + str(pattern)) return None diff --git a/lib/debsources/license_helper.py b/lib/debsources/license_helper.py index afda41b7..47fd1dad 100644 --- a/lib/debsources/license_helper.py +++ b/lib/debsources/license_helper.py @@ -143,18 +143,18 @@ def parse_copyright_paragraphs_for_html_render(copyright, base_url): for files in par.files: globs.append({"files": files, "url": create_url(files, base_url)}) try: - l = { + license = { "license": parse_license_synopsis(copyright, par.license.synopsis), "text": par.license.text, } except (AttributeError, ValueError): - l = {"license": None, "text": None} + license = {"license": None, "text": None} paragraphs.append( { "globs": globs, "copyright": par.copyright, "comment": par.comment, - "license": l, + "license": license, } ) return paragraphs @@ -175,12 +175,9 @@ def parse_licenses_for_html_render(copyright): return licenses -def create_url( - glob="", - base=None, -): +def create_url(glob="", base=None): # don't create links for hidden folders/files - if base is None or not re.search("^\.", glob): + if base is None or not re.search(r"^\.", glob): if glob.count("*") > 0: # find deepest folder without * parts = glob.split("/") @@ -207,12 +204,14 @@ def parse_license_synopsis(copyright, synopsis): license = [] if any(keyword in synopsis for keyword in ["and", "or"]): licenses = re.split("(, | ?and | ?or )", synopsis) - for l in licenses: - link = match_license(l) + for license_item in licenses: + link = match_license(license_item) if not link: - license.append([l, anchor_to_license(copyright, l)]) + license.append( + [license_item, anchor_to_license(copyright, license_item)] + ) else: - license.append([l, link]) + license.append([license_item, link]) else: link = match_license(synopsis) if not link: diff --git a/lib/debsources/navigation.py b/lib/debsources/navigation.py index 06f26f6b..a3b0057f 100644 --- a/lib/debsources/navigation.py +++ b/lib/debsources/navigation.py @@ -54,7 +54,7 @@ def _get_debian_path(self, session, package, version, sources_dir) -> Path: .first() .area ) - except: + except Exception: # the package or version doesn't exist in the database # BUT: packages are stored for a longer time in the filesystem # to allow codesearch.d.n and others less up-to-date platforms diff --git a/lib/debsources/plugins/hook_sloccount.py b/lib/debsources/plugins/hook_sloccount.py index e305671e..cf3e1603 100644 --- a/lib/debsources/plugins/hook_sloccount.py +++ b/lib/debsources/plugins/hook_sloccount.py @@ -41,9 +41,9 @@ def grep(args): return rc == 0 -SLOC_TBL_HEADER = re.compile("^Totals grouped by language") -SLOC_TBL_FOOTER = re.compile("^\s*$") -SLOC_TBL_LINE = re.compile("^(?P[^:]+):\s+(?P\d+)") +SLOC_TBL_HEADER = re.compile(r"^Totals grouped by language") +SLOC_TBL_FOOTER = re.compile(r"^\s*$") +SLOC_TBL_LINE = re.compile(r"^(?P[^:]+):\s+(?P\d+)") def parse_sloccount(path): diff --git a/lib/debsources/query.py b/lib/debsources/query.py index 30c3642e..d554b71d 100644 --- a/lib/debsources/query.py +++ b/lib/debsources/query.py @@ -47,8 +47,8 @@ def pkg_names_get_packages_prefixes(cache_dir): cache_dir: the cache directory, usually comes from the app config """ try: - with (cache_dir / "pkg-prefixes").open() as f: - prefixes = [l.rstrip() for l in f] + with (cache_dir / "pkg-prefixes").open() as file: + prefixes = [line.rstrip() for line in file] except IOError: prefixes = PREFIXES_DEFAULT return prefixes @@ -279,13 +279,13 @@ def count_files_checksum(session, checksum, pkg=None, suite=None): result = session.query(sql_func.count(Checksum.id)).filter( Checksum.sha256 == checksum ) - if pkg is not None and pkg is not "": + if pkg is not None and pkg != "": result = ( result.filter(PackageName.name == pkg) .filter(Checksum.package_id == Package.id) .filter(Package.name_id == PackageName.id) ) - if suite is not None and suite is not "": + if suite is not None and suite != "": result = result.filter(Suite.suite == suite).filter( Suite.package_id == Checksum.package_id ) @@ -299,7 +299,7 @@ def get_pkg_by_name(session, pkg, suite=None): """ result = session.query(PackageName).filter_by(name=pkg) - if suite is not None and suite is not "": + if suite is not None and suite != "": result = ( result.filter(sql_func.lower(Suite.suite) == suite) .filter(Suite.package_id == Package.id) @@ -319,7 +319,7 @@ def get_pkg_by_similar_name(session, pkg, suite=None): .order_by(PackageName.name) ) - if suite is not None and suite is not "": + if suite is not None and suite != "": return filter_pkg_by_suite(session, result, suite) else: return result @@ -357,7 +357,7 @@ def get_files_by_checksum(session, checksum, package=None, suite=None): results = results.filter(PackageName.name == package) - if suite is not None and suite is not "": + if suite is not None and suite != "": results = results.filter(Suite.suite == suite).filter( Suite.package_id == Checksum.package_id ) @@ -383,7 +383,7 @@ def get_files_by_path_package(session, path, package, version=None): .filter(Checksum.file_id == File.id) ) - if version is not None and version is not "": + if version is not None and version != "": results = results.filter(Package.version == version) return results.order_by(PackageName.name, Package.version, File.path) @@ -400,7 +400,7 @@ def get_pkg_filter_prefix(session, prefix, suite=None): result = result.order_by(PackageName.name) - if suite is not None and suite is not "": + if suite is not None and suite != "": return filter_pkg_by_suite(session, result, suite) else: return result diff --git a/lib/debsources/statistics.py b/lib/debsources/statistics.py index 3b7c00c7..fafcf1f7 100644 --- a/lib/debsources/statistics.py +++ b/lib/debsources/statistics.py @@ -374,20 +374,20 @@ def stats_grouped_by(session, stat, areas=None): Reference doc/update-stats-query.bench.sql """ logging.debug("Compute %s stats for all suites" % stat) - if stat is "source_packages": + if stat == "source_packages": q = ( session.query(Suite.suite.label("suite"), sql_func.count(Package.id)) .join(Package) .group_by(Suite.suite) ) - elif stat is "source_files": + elif stat == "source_files": q = ( session.query(Suite.suite.label("suite"), sql_func.count(Checksum.id)) .join(Package) .join(Checksum) .group_by(Suite.suite) ) - elif stat is "disk_usage": + elif stat == "disk_usage": q = ( session.query(Suite.suite.label("suite"), sql_func.sum(Metric.value)) .filter(Metric.metric == "size") @@ -395,14 +395,14 @@ def stats_grouped_by(session, stat, areas=None): .join(Metric) .group_by(Suite.suite) ) - elif stat is "ctags": + elif stat == "ctags": q = ( session.query(Suite.suite.label("suite"), sql_func.count(Ctag.id)) .join(Package) .join(Ctag) .group_by(Suite.suite) ) - elif stat is "sloccount": + elif stat == "sloccount": q = ( session.query( Suite.suite.label("suite"), @@ -535,8 +535,8 @@ def licenses_summary_w_dual(results): # verify all are standard licenses = re.split(", |and |or ", result) unknown = True # verify all licenses in statement are standard - for l in licenses: - key = filter(lambda x: re.search(x, l) is not None, Licenses) + for license in licenses: + key = filter(lambda x: re.search(x, license) is not None, Licenses) if not key: unknown = False if not unknown: diff --git a/lib/debsources/tests/test_webapp.py b/lib/debsources/tests/test_webapp.py index 6ea470ea..0d649574 100644 --- a/lib/debsources/tests/test_webapp.py +++ b/lib/debsources/tests/test_webapp.py @@ -671,7 +671,10 @@ def test_non_utf8_filename(self): rv = self.app.get("/src/aspell-is/0.51-0-4/") self.assertEqual(200, rv.status_code) self.assertIn( - b'%EDslenska.alias', + ( + b'' + b"%EDslenska.alias" + ), rv.data, ) # Visit that file. diff --git a/lib/debsources/updater.py b/lib/debsources/updater.py index 8b4cdca3..bc8afebe 100644 --- a/lib/debsources/updater.py +++ b/lib/debsources/updater.py @@ -153,7 +153,7 @@ def notify_plugins( logging.info("notify (forced) %s/%s for %s" % (event, title, pkg)) if not dry: action(session, pkg, pkgdir, file_table) - except: + except Exception: logging.error("plugin hooks for {} on {} failed".format(event, pkg)) raise @@ -242,7 +242,7 @@ def _add_package(pkg, conf, session, sticky=False): exclude_files(session, pkg, pkgdir, file_table, conf["exclude"]) if not conf["dry_run"] and "hooks" in conf["backends"]: notify(conf, "add-package", session, pkg, pkgdir, file_table) - except: + except Exception: logging.exception("failed to add %s" % pkg) finally: os.chdir(workdir) @@ -272,7 +272,7 @@ def _rm_package(pkg, conf, session, db_package=None): else: with session.begin_nested(): db_storage.rm_package(session, pkg, db_package) - except: + except Exception: logging.exception("failed to remove %s" % pkg) @@ -323,7 +323,7 @@ def add_package(pkg): triggers=conf["force_triggers"], dry=conf["dry_run"], ) - except: + except Exception: logging.exception("trigger failure on %s" % pkg) # add entry for sources.txt, temporarily with no suite associated pkg_id = (pkg["package"], pkg["version"]) @@ -372,7 +372,7 @@ def garbage_collect(status, conf, session, mirror): triggers=conf["force_triggers"], dry=conf["dry_run"], ) - except: + except Exception: logging.exception("trigger failure on %s" % pkg)