Update to v1.4.0

Makefile

@@ -2,12 +2,12 @@ NAME=portstreelint
 SOURCES=src/${NAME}/__init__.py src/${NAME}/main.py src/${NAME}/library.py \
         src/${NAME}/load_config.py src/${NAME}/load_data.py src/${NAME}/check_categories.py \
         src/${NAME}/check_comment.py src/${NAME}/check_description_file.py \
-        src/${NAME}/check_installation_prefix.py src/${NAME}/check_maintainer.py \
-        src/${NAME}/check_marks.py src/${NAME}/check_plist.py src/${NAME}/check_port_path.py \
-        src/${NAME}/check_unchanging_ports.py src/${NAME}/check_vulnerabilities.py \
-        src/${NAME}/check_www_site.py src/${NAME}/show_categories.py \
-        src/${NAME}/show_maintainers.py src/${NAME}/show_notifications.py \
-        src/${NAME}/show_summary.py
+        src/${NAME}/check_installation_prefix.py src/${NAME}/check_licenses.py \
+        src/${NAME}/check_maintainer.py src/${NAME}/check_marks.py src/${NAME}/check_plist.py \
+        src/${NAME}/check_port_path.py src/${NAME}/check_unchanging_ports.py \
+        src/${NAME}/check_vulnerabilities.py src/${NAME}/check_www_site.py \
+        src/${NAME}/show_categories.py src/${NAME}/show_maintainers.py \
+        src/${NAME}/show_notifications.py src/${NAME}/show_summary.py
 
 # Default action is to show this help message:
 .help:

README.md

@@ -19,7 +19,7 @@ portstreelint - FreeBSD ports tree lint
 
 ## SYNOPSIS
 **portstreelint**
-\[--nocfg|-n\]
+\[--nocfg\]
 \[--gencfg|-g FILE\]
 \[--show-cat|-C\]
 \[--show-mnt|-M\]
@@ -98,6 +98,10 @@ The checks list includes:
 * Ports unchanged for a long time (info)
 * Makefile:PORTVERSION and Makefile:DISTVERSION used simultaneously
 * VuXML vulnerabilities for the current port versions (warning)
+* Missing Makefile:LICENSE
+* Unofficial licenses (warning)
+* Unnecessary Makefile:LICENSE_COMB=single (warning)
+* Unnecessary Makefile:LICENSE_COMB=multi (warning)
 
 It's possible to change the default values for PLIST_FILES abuse,
 BROKEN_since, DEPRECATED_since, FORBIDDEN_since and Unchanged_since
@@ -115,16 +119,17 @@ since previous run...
 
 For convenience, you can put your favourite options in a
 configuration file, which will be read before processing the
-environment and the command line, unless you use the *--nocfg|-n*
+environment and the command line, unless you use the *--nocfg*
 option. You can generate a default configuration file with the
 *--gencfg|-g* option followed by a filename. This file also offers
 full control over the checks to perform, and a way to discard
-false-positive vulnerabilities.
+false-positive vulnerabilities and unwanted licenses report
+messages.
 
 ### OPTIONS
 Options | Use
 ------- | ---
---nocfg\|-n|Don't use the configuration file
+--nocfg|Don't use the configuration file
 --gencfg\|-g FILE|Generate a default configuration file in FILE
 --show-cat\|-C|Show categories with ports count
 --show-mnt\|-M|Show maintainers with ports count
@@ -230,8 +235,8 @@ The IGNORE mark check is not reliable because this tool doesn't parse
 the ports' Makefiles, but just loads their variables without regard to
 the conditional tests that may surround them.
 
-The nonexistent plist is not very helpful because there are unaccounted
-autoplist options for some languages (Python)...
+The nonexistent plist check is not very helpful because there are still
+undocumented cases where the list is auto generated.
 
 The ports using exotic versioning schemes will be skipped from the
 vulnerability check because the library we use for version comparisons

TODO.md

@@ -8,6 +8,7 @@ Feel free to submit your own ideas!
 ## Probable evolutions
 * Checking distfiles availability
 * Improving versions comparison for versions with letters -> pnu-vuxml change needed
+* More exclusions loaded from the config file
 
 ## Possible evolutions
 * Printing the number of notifications and a congratulation message if everything is OK

man/portstreelint.8

@@ -1,12 +1,12 @@
-.Dd April 1st, 2024
+.Dd April 13, 2024
 .Dt portstreelint 8
 .Os
 .Sh NAME
 .Nm portstreelint
 .Nd FreeBSD ports tree lint
 .Sh SYNOPSIS
 .Nm
-.Op Fl \-nocfg|\-n
+.Op Fl \-nocfg
 .Op Fl \-gencfg|\-g Ar FILE
 .Op Fl \-show\-cat|\-C
 .Op Fl \-show\-mnt|\-M
@@ -130,6 +130,14 @@ Ports unchanged for a long time (info)
 Makefile:PORTVERSION and Makefile:DISTVERSION used simultaneously
 .It
 VuXML vulnerabilities for the current port versions (warning)
+.It
+Missing Makefile:LICENSE
+.It
+Unofficial licenses (warning)
+.It
+Unnecessary Makefile:LICENSE_COMB=single (warning)
+.It
+Unnecessary Makefile:LICENSE_COMB=multi (warning)
 .El
 .Pp
 It's possible to change the default values for PLIST_FILES abuse,
@@ -159,14 +167,14 @@ results and displaying only diffs since previous run...
 For convenience, you can put your favourite options in a
 configuration file, which will be read before processing the
 environment and the command line, unless you use the
-.Op Fl \-nocfg|\-n
+.Op Fl \-nocfg
 option. You can generate a default configuration file with the
 .Op Fl \-gencfg|\-g
 option followed by a filename. This file also offers full control
 over the checks to perform, and a way to discard false\-positive
-vulnerabilities.
+vulnerabilities and unwanted licenses report messages.
 .Ss OPTIONS
-.Op Fl \-nocfg|\-n
+.Op Fl \-nocfg
 Don't use the configuration file
 .Pp
 .Op Fl \-gencfg|\-g Ar FILE
@@ -337,8 +345,8 @@ The IGNORE mark check is not reliable because this tool doesn't parse
 the ports' Makefiles, but just loads their variables without regard to
 the conditional tests that may surround them.
 .Pp
-The nonexistent plist is not very helpful because there are
-unaccounted autoplist options for some languages (Python)...
+The nonexistent plist check is not very helpful because there are still
+undocumented cases where the list is auto generated.
 .Pp
 The ports using exotic versioning schemes will be skipped from the
 vulnerability check because the library we use for version comparisons

setup.cfg

@@ -3,7 +3,7 @@ name = pnu-portstreelint
 description = FreeBSD ports tree lint
 long_description = file: README.md
 long_description_content_type = text/markdown
-version = 1.3.0
+version = 1.4.0
 license = BSD 3-Clause License
 license_files = License
 author = Hubert Tournier

src/portstreelint/check_licenses.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+""" portstreelint - FreeBSD ports tree lint
+License: 3-clause BSD (see https://opensource.org/licenses/BSD-3-Clause)
+Author: Hubert Tournier
+"""
+
+import logging
+
+from .library import counters, notify_maintainer
+
+OFFICIAL_LICENSES = [
+    "AGPLv3", "AGPLv3+", "APACHE10", "APACHE11", "APACHE20", "ART10", "ART20", "ARTPERL10", "BSD",
+    "BSD0CLAUSE", "BSD2CLAUSE", "BSD3CLAUSE", "BSD4CLAUSE", "BSL", "CC-BY-1.0", "CC-BY-2.0",
+    "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5",
+    "CC-BY-NC-3.0", "CC-BY-NC-4.0", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5",
+    "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5",
+    "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0", "CC-BY-ND-1.0", "CC-BY-ND-2.0", "CC-BY-ND-2.5",
+    "CC-BY-ND-3.0", "CC-BY-ND-4.0", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0",
+    "CC-BY-SA-4.0", "CC0-1.0", "CDDL", "ClArtistic", "CPAL-1.0", "EPL", "GFDL", "GMGPL", "GPLv1",
+    "GPLv1+", "GPLv2", "GPLv2+", "GPLv3", "GPLv3+", "GPLv3RLE", "GPLv3RLE+", "ISCL", "LGPL20",
+    "LGPL20+", "LGPL21", "LGPL21+", "LGPL3", "LGPL3+", "LPPL10", "LPPL11", "LPPL12", "LPPL13",
+    "LPPL13a", "LPPL13b", "LPPL13c", "MIT", "MPL10", "MPL11", "MPL20", "NCSA", "NONE", "ODbL",
+    "OFL10", "OFL11", "OpenSSL", "OWL", "PD", "PHP202", "PHP30", "PHP301", "PostgreSQL", "PSFL",
+    "RUBY", "UNLICENSE", "WTFPL", "WTFPL1", "ZLIB", "ZPL21",
+]
+
+####################################################################################################
+def _debug_license(name, port):
+    """ """
+    print(f"===== {name} =====")
+    for key, value in port.items():
+        if key.startswith("LICENSE"):
+            print(f"{key}={value}")
+    print(f"======{'=' * len(name)}======\n")
+
+####################################################################################################
+def check_licenses(ports, excluded_licenses):
+    """ Cross-checks the licenses fields with the Makefile and compliance with rules
+    Rules at https://docs.freebsd.org/en/books/porters-handbook/makefiles/#makefile-categories-definition
+    """
+    for name, port in ports.items():
+        if "PORTNAME" in port:
+            if port["PORTNAME"] in excluded_licenses:
+                # Let's skip this port...
+                continue
+
+        if "LICENSE" not in port:
+            logging.error("Missing LICENSE in Makefile for port %s", name)
+            counters["Missing LICENSE"] += 1
+            notify_maintainer(port["maintainer"], "Missing LICENSE", name)
+        else:
+            for license_name in port["LICENSE"].split():
+                if license_name not in OFFICIAL_LICENSES and '$' not in license_name:
+                    logging.warning("Unofficial license '%s' in Makefile for port %s", license_name, name)
+                    counters["Unofficial licenses"] += 1
+                    notify_maintainer(port["maintainer"], "Unofficial license", name)
+
+            if "LICENSE_COMB" in port:
+                if port["LICENSE_COMB"] == "single":
+                    logging.warning("Unnecessary LICENSE_COMB=single in Makefile for port %s", name)
+                    counters["Unnecessary LICENSE_COMB=single"] += 1
+                    notify_maintainer(port["maintainer"], "Unnecessary LICENSE_COMB=single", name)
+                elif port["LICENSE_COMB"] not in ("multi", "dual"):
+                    logging.error("Unknown LICENSE_COMB value '%s' in Makefile for port %s (not counted)", port["LICENSE_COMB"], name)
+                elif len(port["LICENSE"].split()) == 1:
+                    for key in port:
+                        if key.startswith("LICENSE_NAME_"):
+                            # It's OK, there are additional licenses defined with LICENSE_NAME_* entries
+                            continue
+
+                    logging.warning("Unnecessary LICENSE_COMB=%s in Makefile for port %s", port["LICENSE_COMB"], name)
+                    counters["Unnecessary LICENSE_COMB=multi"] += 1
+                    notify_maintainer(port["maintainer"], "Unnecessary LICENSE_COMB=multi", name)
+
+    logging.info("Found %d ports with missing LICENSE", counters["Missing LICENSE"])
+    logging.info("Found %d ports with unofficial licenses", counters["Unofficial licenses"])
+    logging.info("Found %d ports with unnecessary LICENSE_COMB=single", counters["Unnecessary LICENSE_COMB=single"])
+    logging.info("Found %d ports with unnecessary LICENSE_COMB=multi", counters["Unnecessary LICENSE_COMB=multi"])

src/portstreelint/check_plist.py

@@ -21,6 +21,39 @@ def check_plist(ports, plist_abuse, ports_dir):
         if os.path.isdir(port_path):
             if not os.path.isfile(port_path + os.sep + "pkg-plist"):
                 if not "PLIST_FILES" in port:
+                    if "AP_GENPLIST" in port:
+                        continue
+
+                    if "USES" in port:
+                        items = port["USES"].split()
+
+                        found = False
+                        for item in items:
+                            if item in ("pear", "horde", "gem"):
+                                found = True
+                                break
+
+                            if item.startswith("pear:") \
+                            or item.startswith("horde:") \
+                            or item.startswith("gem:"):
+                                found = True
+                                break
+
+                            if item.startswith("cran:"):
+                                if "auto-plist" in item.split(':')[1].split(','):
+                                    found = True
+                                    break
+
+                        if found:
+                            continue
+
+                    if "USE_PYTHON" in port:
+                        if "autoplist" in port["USE_PYTHON"].split():
+                            continue
+
+                    if "RUBYGEM_AUTOPLIST" in port or "RUBYGEM_AUTOPLIST_ALT" in port:
+                        continue
+
                     if not "PLIST" in port and not "PLIST_SUB" in port:
                         logging.debug("Nonexistent pkg-plist/PLIST_FILES/PLIST/PLIST_SUB for port %s", name)
                         counters["Nonexistent pkg-plist"] += 1

src/portstreelint/check_www_site.py

@@ -14,8 +14,8 @@
 # Headers and timeout delay for HTTP(S) requests:
 HTTP_HEADERS = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0",
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-    "Accept-Language": "en;q=1.0, en-US;q=0.8, *;q=0.5",
+    "Accept": "*/*",
+    "Accept-Language": "en;q=1.0, *;q=0.5",
     "Accept-Encoding": "identity",
     "Connection": "keep-alive",
 }
@@ -32,7 +32,6 @@ def _resolve_hostname(hostname):
 
     return ip_address
 
-
 ####################################################################################################
 def _handle_url_errors(port_name, www_site, error, maintainer):
     """ Decides what to do with the multiple possible fetching errors """
@@ -85,7 +84,6 @@ def _handle_url_errors(port_name, www_site, error, maintainer):
 
     return is_unaccessible
 
-
 ####################################################################################################
 def check_www_site(ports, check_host, check_url):
     """ Checks the www-site field existence

src/portstreelint/library.py

@@ -41,6 +41,10 @@
     "Both PORTVERSION and DISTVERSION": 0,
     "Vulnerable port version": 0,
     "Skipped vulnerability checks": 0,
+    "Missing LICENSE": 0,
+    "Unofficial licenses": 0,
+    "Unnecessary LICENSE_COMB=single": 0,
+    "Unnecessary LICENSE_COMB=multi": 0,
 }
 
 # Global dictionary of notifications to port maintainers: