flake8 compliance, black formatting for bin/ files

.gitlab-ci.yml

@@ -7,11 +7,11 @@ Build and test:
   services:
     - docker:dind
   before_script:
-    - apk add black=20.8_beta1-r0 docker-compose make
+    - apk add black=20.8_beta1-r0 py3-flake8 docker-compose make
   script:
     - cd contrib/docker
     - make build
     - make update-db
     - make test-all
     - cd ../..
-    - black --check lib/debsources
+    - make check

HACKING.md

@@ -103,7 +103,7 @@ go now?  <http://deb.li/debsrceasy>
 Running tests
 -------------
 
-See `doc/testing.txt`.
+See [testing.md](doc/testing.md].
 
 
 Coding conventions
@@ -113,20 +113,25 @@ All new Debsources code should be [PEP8][1] compliant and pass [pyflakes][2]
 validation. Before submitting patches, please make sure that the lines of code
 they touch conform to such requirements.
 
+Additionally, `black` [3] is used to format Python source files.
+
 [1]: https://www.python.org/dev/peps/pep-0008/
 [2]: https://pypi.python.org/pypi/pyflakes
+[3]: https://black.readthedocs.io/en/stable/
 
 If you develop on Debian(-based distros), a good way to check that this is the
 case is:
 
     # apt-get install python-flake8
     $ flake8 file1.py file2.py ...
 
-You can check all Debsources Python source files flake8 compliance with:
+You can check all Debsources Python source files black and flake8 compliance
+with:
 
     $ make check
 
 You can add a pre-commit hook to automatically test PEP8 compliance:
+
     $ ln -s ../../contrib/git-pre-commit .git/hooks/pre-commit
 
 If you stumble upon Debsources source files that are not flake8-compliant,

Makefile

@@ -1,5 +1,8 @@
 NOSE = nosetests3
-FLAKE = flake8
+FLAKE = flake8 --max-line-length 88 --ignore=E203,W503
+# E203 (whitespace before ':') conflicts with black formatting
+# W503 (line break before binary operator), ditto
+BLACK = black --check
 
 SRCDIR = lib/debsources
 BINDIR = bin
@@ -29,6 +32,7 @@ test-coverage:
 
 check:
 	$(FLAKE) $(SRCDIR)/ $(shell grep -H 'env python' $(BINDIR)/debsources-* | cut -f 1 -d :)
+	$(BLACK) $(SRCDIR)
 
 test-online-app:
 	contrib/test-online-app

bin/debsources-backup-db

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-# Copyright (C) 2014-2015  The Debsources developers <qa-debsources@lists.alioth.debian.org>.
+# Copyright (C) 2014-2021  The Debsources developers
+# <qa-debsources@lists.alioth.debian.org>.
 # See the AUTHORS file at the top-level directory of this distribution and at
 # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS
 #

bin/debsources-bulk-insert-checksums

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-# Copyright (C) 2014-2015  The Debsources developers <qa-debsources@lists.alioth.debian.org>.
+# Copyright (C) 2014-2021  The Debsources developers
+# <qa-debsources@lists.alioth.debian.org>.
 # See the AUTHORS file at the top-level directory of this distribution and at
 # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS
 #
@@ -23,42 +24,48 @@ from debsources import mainlib
 from debsources import fs_storage
 from debsources.plugins.hook_checksums import parse_checksums
 
-METADATA_SUFFIX = '.checksums'
+METADATA_SUFFIX = ".checksums"
 METADATA_SUFFIX_LEN = len(METADATA_SUFFIX)
 
 BINDPARAMS = [
-    bindparam('path', type_=LargeBinary),
+    bindparam("path", type_=LargeBinary),
 ]
 
-PACKAGE_ID_Q = sql_query("""
+PACKAGE_ID_Q = sql_query(
+    """
     SELECT versions.id
     FROM versions, packages
     WHERE versions.package_id = packages.id
     AND versions.version = :v
     AND packages.name = :p
-    """)
+    """
+)
 
-CHECKSUMS_EXIST_Q = sql_query("""
+CHECKSUMS_EXIST_Q = sql_query(
+    """
     SELECT 1
     FROM checksums
     WHERE package_id = :v
     LIMIT 1
-    """)
+    """
+)
 
-INSERT_SHA_Q = sql_query("""
+INSERT_SHA_Q = sql_query(
+    """
     INSERT INTO checksums (package_id, path, sha256)
     VALUES (:package_id, :path, :sha256)
     """,
-                         bindparams=BINDPARAMS)
+    bindparams=BINDPARAMS,
+)
 
 
 def insert_checksums(conn, sumsfile):
-    steps = sumsfile.split('/')
+    steps = sumsfile.split("/")
     package = steps[-2]
     version = steps[-1][:-METADATA_SUFFIX_LEN]
 
-    logging.debug('package %s/%s' % (package, version))
-    logging.debug('INSERT INTO checksums %s/%s...' % (package, version))
+    logging.debug("package %s/%s" % (package, version))
+    logging.debug("INSERT INTO checksums %s/%s..." % (package, version))
 
     package_id = conn.execute(PACKAGE_ID_Q, v=version, p=package).first()[0]
     assert package_id is not None
@@ -68,9 +75,9 @@ def insert_checksums(conn, sumsfile):
 
     checksums = []
     for (sha256, path) in parse_checksums(sumsfile):
-        checksums.append({'package_id': package_id,
-                          'path': str(path),
-                          'sha256': sha256})
+        checksums.append(
+            {"package_id": package_id, "path": str(path), "sha256": sha256}
+        )
     if checksums:
         conn.execute(INSERT_SHA_Q, checksums)
 
@@ -80,27 +87,29 @@ def is_checksum(p):
 
 
 def main(conf, sumsfile=None):
-    conn = create_engine(conf['db_uri'])
+    conn = create_engine(conf["db_uri"])
     if sumsfile:
         insert_checksums(conn, sumsfile)
     else:
-        for f in fs_storage.walk(conf['sources_dir'], test=is_checksum):
+        for f in fs_storage.walk(conf["sources_dir"], test=is_checksum):
             with conn.begin() as trans:
                 try:
                     insert_checksums(conn, f)
-                except:
-                    logging.exception('cannot inject %s' % f)
+                except Exception:
+                    logging.exception("cannot inject %s" % f)
                     trans.rollback()
 
 
-if __name__ == '__main__':
-    cmdline = argparse.ArgumentParser(
-        description='Debsources: bulk checksum insertion')
-    cmdline.add_argument('--config', '-c', dest='conffile',
-                         default=mainlib.guess_conffile(),
-                         help='alternate configuration file')
-    cmdline.add_argument('file', nargs='?',
-                         help='.checksum metadata file')
+if __name__ == "__main__":
+    cmdline = argparse.ArgumentParser(description="Debsources: bulk checksum insertion")
+    cmdline.add_argument(
+        "--config",
+        "-c",
+        dest="conffile",
+        default=mainlib.guess_conffile(),
+        help="alternate configuration file",
+    )
+    cmdline.add_argument("file", nargs="?", help=".checksum metadata file")
     args = cmdline.parse_args()
     conf = mainlib.load_configuration(args.conffile)
 

bin/debsources-bulk-insert-ctags

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-# Copyright (C) 2014-2015  The Debsources developers <qa-debsources@lists.alioth.debian.org>.
+# Copyright (C) 2014-2021  The Debsources developers
+# <qa-debsources@lists.alioth.debian.org>.
 # See the AUTHORS file at the top-level directory of this distribution and at
 # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS
 #
@@ -25,56 +26,61 @@ from sqlalchemy.sql import text as sql_query
 from debsources import fs_storage
 from debsources.plugins.hook_ctags import parse_ctags
 
-METADATA_SUFFIX = '.ctags'
+METADATA_SUFFIX = ".ctags"
 METADATA_SUFFIX_LEN = len(METADATA_SUFFIX)
 
 BINDPARAMS = [
-    bindparam('path', type_=LargeBinary),
+    bindparam("path", type_=LargeBinary),
 ]
 
-PACKAGE_ID_Q = sql_query("""
+PACKAGE_ID_Q = sql_query(
+    """
     SELECT versions.id
     FROM versions, packages
     WHERE versions.package_id = packages.id
     AND versions.version = :v
     AND packages.name = :p
-    """)
+    """
+)
 
-CTAGS_EXIST_Q = sql_query("""
+CTAGS_EXIST_Q = sql_query(
+    """
     SELECT 1
     FROM ctags
     WHERE package_id = :v
     LIMIT 1
-    """)
+    """
+)
 
-INSERT_CTAGS_Q = sql_query("""
+INSERT_CTAGS_Q = sql_query(
+    """
     INSERT INTO ctags (package_id, tag, path, line, kind, language)
     VALUES (:package_id, :tag, :path, :line, :kind, :language)
     """,
-                           bindparams=BINDPARAMS)
+    bindparams=BINDPARAMS,
+)
 
 
 def insert_ctags(conn, ctagsfile):
-    steps = ctagsfile.split('/')
+    steps = ctagsfile.split("/")
     package = steps[-2]
     version = steps[-1][:-METADATA_SUFFIX_LEN]
-    logging.debug('package %s/%s' % (package, version))
+    logging.debug("package %s/%s" % (package, version))
 
     package_id = conn.execute(PACKAGE_ID_Q, v=version, p=package).first()
     if not package_id:
-        logging.warn('skipping %s/%s (does not exist in DB)' %
-                     (package, version))
+        logging.warn("skipping %s/%s (does not exist in DB)" % (package, version))
         return
 
     has_ctags = conn.execute(CTAGS_EXIST_Q, v=package_id[0]).first()
     if has_ctags:
-        logging.warn('skipping %s/%s (already inserted)' % (package, version))
+        logging.warn("skipping %s/%s (already inserted)" % (package, version))
         return
 
-    logging.debug('INSERT INTO ctags %s/%s...' % (package, version))
+    logging.debug("INSERT INTO ctags %s/%s..." % (package, version))
     ctags = []
     for tag in parse_ctags(ctagsfile):
-        tag['package_id'] = package_id
+        tag["package_id"] = package_id
         ctags.append(tag)
     if ctags:
         conn.execute(INSERT_CTAGS_Q, ctags)
@@ -85,27 +91,29 @@ def is_ctags(p):
 
 
 def main(conf, ctagsfile=None):
-    conn = create_engine(conf['db_uri']).connect()
+    conn = create_engine(conf["db_uri"]).connect()
     if ctagsfile:
         insert_ctags(conn, ctagsfile)
     else:
-        for f in fs_storage.walk(conf['sources_dir'], test=is_ctags):
+        for f in fs_storage.walk(conf["sources_dir"], test=is_ctags):
             with conn.begin() as trans:
                 try:
                     insert_ctags(conn, f)
-                except:
-                    logging.exception('cannot inject %s' % f)
+                except Exception:
+                    logging.exception("cannot inject %s" % f)
                     trans.rollback()
 
 
-if __name__ == '__main__':
-    cmdline = argparse.ArgumentParser(
-        description='Debsources: bulk ctags insertion')
-    cmdline.add_argument('--config', '-c', dest='conffile',
-                         default=mainlib.guess_conffile(),
-                         help='alternate configuration file')
-    cmdline.add_argument('file', nargs='?',
-                         help='.ctags metadata file')
+if __name__ == "__main__":
+    cmdline = argparse.ArgumentParser(description="Debsources: bulk ctags insertion")
+    cmdline.add_argument(
+        "--config",
+        "-c",
+        dest="conffile",
+        default=mainlib.guess_conffile(),
+        help="alternate configuration file",
+    )
+    cmdline.add_argument("file", nargs="?", help=".ctags metadata file")
     args = cmdline.parse_args()
     conf = mainlib.load_configuration(args.conffile)
 

bin/debsources-dbadmin

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-# Copyright (C) 2013-2014  The Debsources developers <qa-debsources@lists.alioth.debian.org>.
+# Copyright (C) 2013-2021  The Debsources developers
+# <qa-debsources@lists.alioth.debian.org>.
 # See the AUTHORS file at the top-level directory of this distribution and at
 # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS
 #
@@ -26,19 +27,20 @@ if __name__ == "__main__":
     start_time = time.time()
 
     parser = argparse.ArgumentParser(description="Manage Debsources DB")
-    parser.add_argument("dburi",
-                        help="SQLAlchemy URI to the DB, e.g. postgres://"
-                        "username:password@localhost:5432/debsources")
-    parser.add_argument("--createdb",
-                        help="create DB schema",
-                        action="store_true")
-    parser.add_argument("--dropdb",
-                        help="destroy existing DB schema "
-                        "(WARNING: you will lose all data)",
-                        action="store_true")
-    parser.add_argument("--verbose",
-                        help="verbose logging (default: be quiet)",
-                        action="store_true")
+    parser.add_argument(
+        "dburi",
+        help="SQLAlchemy URI to the DB, e.g. postgres://"
+        "username:password@localhost:5432/debsources",
+    )
+    parser.add_argument("--createdb", help="create DB schema", action="store_true")
+    parser.add_argument(
+        "--dropdb",
+        help="destroy existing DB schema " "(WARNING: you will lose all data)",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--verbose", help="verbose logging (default: be quiet)", action="store_true"
+    )
     args = parser.parse_args()
 
     db = create_engine(args.dburi, echo=args.verbose)

bin/debsources-debian-source-archive

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-# Copyright (C) 2011-2014  The Debsources developers <qa-debsources@lists.alioth.debian.org>.
+# Copyright (C) 2011-2021  The Debsources developers
+# <qa-debsources@lists.alioth.debian.org>.
 # See the AUTHORS file at the top-level directory of this distribution and at
 # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS
 #

bin/debsources-debian-source-mirror

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-# Copyright (C) 2011-2013  The Debsources developers <qa-debsources@lists.alioth.debian.org>.
+# Copyright (C) 2011-2021  The Debsources developers
+# <qa-debsources@lists.alioth.debian.org>.
 # See the AUTHORS file at the top-level directory of this distribution and at
 # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS
 #

bin/debsources-dev-fetch-test-data

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-# Copyright (C) 2013-2014  The Debsources developers <qa-debsources@lists.alioth.debian.org>.
+# Copyright (C) 2013-2021  The Debsources developers
+# <qa-debsources@lists.alioth.debian.org>.
 # See the AUTHORS file at the top-level directory of this distribution and at
 # https://salsa.debian.org/qa/debsources/blob/master/AUTHORS
 #