Merge from bittboy/buildroot@26c91a9

support/scripts/apply-patches.sh

@@ -119,7 +119,7 @@ function apply_patch {
         exit 1
     fi
     echo "${path}/${patch}" >> ${builddir}/.applied_patches_list
-    ${uncomp} "${path}/$patch" | patch -g0 -p1 -E -d "${builddir}" -t -N $silent
+    ${uncomp} "${path}/$patch" | patch -g0 -p1 -E --no-backup-if-mismatch -d "${builddir}" -t -N $silent
     if [ $? != 0 ] ; then
         echo "Patch failed!  Please fix ${patch}!"
         exit 1
@@ -168,6 +168,3 @@ if [ "`find $builddir/ '(' -name '*.rej' -o -name '.*.rej' ')' -print`" ] ; then
     echo "Aborting.  Reject files found."
     exit 1
 fi
-
-# Remove backup files
-find $builddir/ '(' -name '*.orig' -o -name '.*.orig' ')' -exec rm -f {} \;

support/scripts/boot-qemu-image.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+# This script expect to run from the Buildroot top directory.
+
+import os
+import pexpect
+import sys
+import time
+
+
+def main():
+    if not (len(sys.argv) == 2):
+        print("Error: incorrect number of arguments")
+        print("""Usage: boot-qemu-image.py <qemu_arch_defconfig>""")
+        sys.exit(1)
+
+    # Ignore non Qemu defconfig
+    if not sys.argv[1].startswith('qemu_'):
+        sys.exit(0)
+
+    qemu_start = os.path.join(os.getcwd(), 'output/images/start-qemu.sh')
+
+    child = pexpect.spawn(qemu_start, ['serial-only'],
+                          timeout=5, encoding='utf-8',
+                          env={"QEMU_AUDIO_DRV": "none"})
+
+    # We want only stdout into the log to avoid double echo
+    child.logfile = sys.stdout
+
+    # Let the spawn actually try to fork+exec to the wrapper, and then
+    # let the wrapper exec the qemu process.
+    time.sleep(1)
+
+    try:
+        child.expect(["buildroot login:", pexpect.TIMEOUT], timeout=60)
+    except pexpect.EOF as e:
+        # Some emulations require a fork of qemu-system, which may be
+        # missing on the system, and is not provided by Buildroot.
+        # In this case, spawn above will succeed at starting the wrapper
+        # start-qemu.sh, but that one will fail (exit with 127) in such
+        # a situation.
+        exit = [int(l.split(' ')[1])
+                for l in e.value.splitlines()
+                if l.startswith('exitstatus: ')]
+        if len(exit) and exit[0] == 127:
+            print('qemu-start.sh could not find the qemu binary')
+            sys.exit(0)
+        print("Connection problem, exiting.")
+        sys.exit(1)
+    except pexpect.TIMEOUT:
+        print("System did not boot in time, exiting.")
+        sys.exit(1)
+
+    child.sendline("root\r")
+
+    try:
+        child.expect(["# ", pexpect.TIMEOUT], timeout=60)
+    except pexpect.EOF:
+        print("Cannot connect to shell")
+        sys.exit(1)
+    except pexpect.TIMEOUT:
+        print("Timeout while waiting for shell")
+        sys.exit(1)
+
+    child.sendline("poweroff\r")
+
+    try:
+        child.expect(["System halted", pexpect.TIMEOUT], timeout=60)
+        child.expect(pexpect.EOF)
+    except pexpect.EOF:
+        pass
+    except pexpect.TIMEOUT:
+        # Qemu may not exit properly after "System halted", ignore.
+        print("Cannot halt machine")
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

support/scripts/br2-external

@@ -33,9 +33,8 @@ main() {
     # Trap any unexpected error to generate a meaningful error message
     trap "error 'unexpected error while generating ${ofile}\n'" ERR
 
-    do_validate ${@//:/ }
-
     mkdir -p "${outputdir}"
+    do_validate "${outputdir}" ${@//:/ }
     do_mk "${outputdir}"
     do_kconfig "${outputdir}"
 }
@@ -51,7 +50,9 @@ main() {
 # snippet means that there were no error.
 #
 do_validate() {
+    local outputdir="${1}"
     local br2_ext
+    shift
 
     if [ ${#} -eq 0 ]; then
         # No br2-external tree is valid
@@ -60,7 +61,7 @@ do_validate() {
 
     for br2_ext in "${@}"; do
         do_validate_one "${br2_ext}"
-    done
+    done >"${outputdir}/.br2-external.mk"
 }
 
 do_validate_one() {
@@ -74,7 +75,7 @@ do_validate_one() {
         error "'%s': permission denied\n" "${br2_ext}"
     fi
     if [ ! -f "${br2_ext}/external.desc" ]; then
-        error "'%s': does not have a name (in 'external.desc'). See %s\n" \
+        error "'%s': does not have an 'external.desc'. See %s\n" \
             "${br2_ext}" "${MANUAL_URL}"
     fi
     br2_name="$(sed -r -e '/^name: +(.*)$/!d; s//\1/' "${br2_ext}/external.desc")"

support/scripts/check-bin-arch

@@ -25,6 +25,9 @@ declare -a IGNORES=(
 	# it for a different architecture (e.g. i386 grub on x86_64).
 	"/lib/grub"
 	"/usr/lib/grub"
+
+	# Guile modules are ELF files, with a "None" machine
+	"/usr/lib/guile"
 )
 
 while getopts p:l:r:a:i: OPT ; do

support/scripts/check-dotconfig.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# This scripts check that all lines present in the defconfig are
+# still present in the .config
+
+import sys
+
+
+def main():
+    if not (len(sys.argv) == 3):
+        print("Error: incorrect number of arguments")
+        print("""Usage: check-dotconfig <configfile> <defconfig>""")
+        sys.exit(1)
+
+    configfile = sys.argv[1]
+    defconfig = sys.argv[2]
+
+    # strip() to get rid of trailing \n
+    with open(configfile) as configf:
+        configlines = [l.strip() for l in configf.readlines()]
+
+    defconfiglines = []
+    with open(defconfig) as defconfigf:
+        # strip() to get rid of trailing \n
+        for line in (line.strip() for line in defconfigf.readlines()):
+            if line.startswith("BR2_"):
+                defconfiglines.append(line)
+            elif line.startswith('# BR2_') and line.endswith(' is not set'):
+                defconfiglines.append(line)
+
+    # Check that all the defconfig lines are still present
+    missing = [line for line in defconfiglines if line not in configlines]
+
+    if missing:
+        print("WARN: defconfig {} can't be used:".format(defconfig))
+        for m in missing:
+            print("      Missing: {}".format(m))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

support/scripts/check-host-rpath

@@ -11,6 +11,7 @@ export LC_ALL=C
 main() {
     local pkg="${1}"
     local hostdir="${2}"
+    local perpackagedir="${3}"
     local file ret
 
     # Remove duplicate and trailing '/' for proper match
@@ -20,7 +21,7 @@ main() {
     while read file; do
         is_elf "${file}" || continue
         elf_needs_rpath "${file}" "${hostdir}" || continue
-        check_elf_has_rpath "${file}" "${hostdir}" && continue
+        check_elf_has_rpath "${file}" "${hostdir}" "${perpackagedir}" && continue
         if [ ${ret} -eq 0 ]; then
             ret=1
             printf "***\n"
@@ -44,6 +45,15 @@ is_elf() {
 # needs such an RPATH if at least of the libraries used by the ELF
 # executable is available in the host library directory. This function
 # returns 0 when a RPATH is needed, 1 otherwise.
+#
+# With per-package directory support, ${hostdir} will point to the
+# current package per-package host directory, and this is where this
+# function will check if the libraries needed by the executable are
+# located (or not). In practice, the ELF executable RPATH may point to
+# another package per-package host directory, but that is fine because
+# if such an executable is within the current package per-package host
+# directory, its libraries will also have been copied into the current
+# package per-package host directory.
 elf_needs_rpath() {
     local file="${1}"
     local hostdir="${2}"
@@ -62,13 +72,19 @@ elf_needs_rpath() {
 # This function checks whether at least one of the RPATH of the given
 # ELF executable (first argument) properly points to the host library
 # directory (second argument), either through an absolute RPATH or a
-# relative RPATH. Having such a RPATH will make sure the ELF
-# executable will find at runtime the shared libraries it depends
-# on. This function returns 0 when a proper RPATH was found, or 1
-# otherwise.
+# relative RPATH. In the context of per-package directory support,
+# ${hostdir} (second argument) points to the current package host
+# directory. However, it is perfectly valid for an ELF binary to have
+# a RPATH pointing to another package per-package host directory,
+# which is why such RPATH is also accepted (the per-package directory
+# gets passed as third argument). Having a RPATH pointing to the host
+# directory will make sure the ELF executable will find at runtime the
+# shared libraries it depends on. This function returns 0 when a
+# proper RPATH was found, or 1 otherwise.
 check_elf_has_rpath() {
     local file="${1}"
     local hostdir="${2}"
+    local perpackagedir="${3}"
     local rpath dir
 
     while read rpath; do
@@ -77,6 +93,12 @@ check_elf_has_rpath() {
             dir="$( sed -r -e 's:/+:/:g; s:/$::;' <<<"${dir}" )"
             [ "${dir}" = "${hostdir}/lib" ] && return 0
             [ "${dir}" = "\$ORIGIN/../lib" ] && return 0
+	    # This check is done even for builds where
+	    # BR2_PER_PACKAGE_DIRECTORIES is disabled. In this case,
+	    # PER_PACKAGE_DIR and therefore ${perpackagedir} points to
+	    # a non-existent directory, and this check will always be
+	    # false.
+            [[ ${dir} =~ ${perpackagedir}/[^/]+/host/lib ]] && return 0
         done
     done < <( readelf -d "${file}"                                              \
               |sed -r -e '/.* \(R(UN)?PATH\) +Library r(un)?path: \[(.+)\]$/!d' \

support/scripts/check-kernel-headers.sh

@@ -1,14 +1,40 @@
 #!/bin/sh
 
-SYSROOT="${1}"
+# This script (and the embedded C code) will check that the actual
+# headers version match the user told us they were:
+#
+# - if both versions are the same, all is well.
+#
+# - if the actual headers are older than the user told us, this is
+#   an error.
+#
+# - if the actual headers are more recent than the user told us, and
+#   we are doing a strict check, then this is an error.
+#
+# - if the actual headers are more recent than the user told us, and
+#   we are doing a loose check, then a warning is printed, but this is
+#   not an error.
+
+BUILDDIR="${1}"
+SYSROOT="${2}"
 # Make sure we have enough version components
-HDR_VER="${2}.0.0"
+HDR_VER="${3}.0.0"
+CHECK="${4}"  # 'strict' or 'loose'
 
 HDR_M="${HDR_VER%%.*}"
 HDR_V="${HDR_VER#*.}"
 HDR_m="${HDR_V%%.*}"
 
-EXEC="$(mktemp -t check-headers.XXXXXX)"
+# Exit on any error, so we don't try to run an unexisting program if the
+# compilation fails.
+set -e
+
+# Set the clean-up trap in advance to prevent a race condition in which we
+# create the file but get a SIGTERM before setting it. Notice that we don't
+# need to care about EXEC being empty, since 'rm -f ""' does nothing.
+trap 'rm -f "${EXEC}"' EXIT
+
+EXEC="$(mktemp -p "${BUILDDIR}" -t .check-headers.XXXXXX)"
 
 # We do not want to account for the patch-level, since headers are
 # not supposed to change for different patchlevels, so we mask it out.
@@ -18,13 +44,18 @@ ${HOSTCC} -imacros "${SYSROOT}/usr/include/linux/version.h" \
           -x c -o "${EXEC}" - <<_EOF_
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 int main(int argc __attribute__((unused)),
          char** argv __attribute__((unused)))
 {
-    if((LINUX_VERSION_CODE & ~0xFF)
-        != KERNEL_VERSION(${HDR_M},${HDR_m},0))
-    {
+    int l = LINUX_VERSION_CODE & ~0xFF;
+    int h = KERNEL_VERSION(${HDR_M},${HDR_m},0);
+
+    if ((l >= h) && !strcmp("${CHECK}", "loose"))
+        return 0;
+
+    if (l != h) {
         printf("Incorrect selection of kernel headers: ");
         printf("expected %d.%d.x, got %d.%d.x\n", ${HDR_M}, ${HDR_m},
                ((LINUX_VERSION_CODE>>16) & 0xFF),
@@ -36,6 +67,3 @@ int main(int argc __attribute__((unused)),
 _EOF_
 
 "${EXEC}"
-ret=${?}
-rm -f "${EXEC}"
-exit ${ret}

support/scripts/check-uniq-files

@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import argparse
-from collections import defaultdict
-
-warn = 'Warning: {0} file "{1}" is touched by more than one package: {2}\n'
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('packages_file_list', nargs='*',
-                        help='The packages-file-list to check from')
-    parser.add_argument('-t', '--type', metavar="TYPE",
-                        help='Report as a TYPE file (TYPE is either target, staging, or host)')
-
-    args = parser.parse_args()
-
-    if not len(args.packages_file_list) == 1:
-        sys.stderr.write('No packages-file-list was provided.\n')
-        return False
-
-    if args.type is None:
-        sys.stderr.write('No type was provided\n')
-        return False
-
-    file_to_pkg = defaultdict(list)
-    with open(args.packages_file_list[0], 'rb') as pkg_file_list:
-        for line in pkg_file_list.readlines():
-            pkg, _, file = line.rstrip(b'\n').partition(b',')
-            file_to_pkg[file].append(pkg)
-
-    for file in file_to_pkg:
-        if len(file_to_pkg[file]) > 1:
-            # If possible, try to decode the binary strings with
-            # the default user's locale
-            try:
-                sys.stderr.write(warn.format(args.type, file.decode(),
-                                             [p.decode() for p in file_to_pkg[file]]))
-            except UnicodeDecodeError:
-                # ... but fallback to just dumping them raw if they
-                # contain non-representable chars
-                sys.stderr.write(warn.format(args.type, file,
-                                             file_to_pkg[file]))
-
-
-if __name__ == "__main__":
-    sys.exit(main())

support/scripts/cve-checker

@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
+# Copyright (C) 2020 by Gregory CLEMENT <gregory.clement@bootlin.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import argparse
+import datetime
+import os
+import json
+import sys
+import cve as cvecheck
+
+
+class Package:
+    def __init__(self, name, version, ignored_cves):
+        self.name = name
+        self.version = version
+        self.cves = list()
+        self.ignored_cves = ignored_cves
+
+
+def check_package_cves(nvd_path, packages):
+    if not os.path.isdir(nvd_path):
+        os.makedirs(nvd_path)
+
+    for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
+        for pkg_name in cve.pkg_names:
+            pkg = packages.get(pkg_name, '')
+            if pkg and cve.affects(pkg.name, pkg.version, pkg.ignored_cves) == cve.CVE_AFFECTS:
+                pkg.cves.append(cve.identifier)
+
+
+html_header = """
+<head>
+<script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
+<style type=\"text/css\">
+table {
+  width: 100%;
+}
+td {
+  border: 1px solid black;
+}
+td.centered {
+  text-align: center;
+}
+td.wrong {
+  background: #ff9a69;
+}
+td.correct {
+  background: #d2ffc4;
+}
+
+</style>
+<title>CVE status for Buildroot configuration</title>
+</head>
+
+<p id=\"sortable_hint\"></p>
+"""
+
+
+html_footer = """
+</body>
+<script>
+if (typeof sorttable === \"object\") {
+  document.getElementById(\"sortable_hint\").innerHTML =
+  \"hint: the table can be sorted by clicking the column headers\"
+}
+</script>
+</html>
+"""
+
+
+def dump_html_pkg(f, pkg):
+    f.write(" <tr>\n")
+    f.write("  <td>%s</td>\n" % pkg.name)
+
+    # Current version
+    if len(pkg.version) > 20:
+        version = pkg.version[:20] + "..."
+    else:
+        version = pkg.version
+    f.write("  <td class=\"centered\">%s</td>\n" % version)
+
+    # CVEs
+    td_class = ["centered"]
+    if len(pkg.cves) == 0:
+        td_class.append("correct")
+    else:
+        td_class.append("wrong")
+    f.write("  <td class=\"%s\">\n" % " ".join(td_class))
+    for cve in pkg.cves:
+        f.write("   <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
+    f.write("  </td>\n")
+
+    f.write(" </tr>\n")
+
+
+def dump_html_all_pkgs(f, packages):
+    f.write("""
+<table class=\"sortable\">
+<tr>
+<td>Package</td>
+<td class=\"centered\">Version</td>
+<td class=\"centered\">CVEs</td>
+</tr>
+""")
+    for pkg in packages:
+        dump_html_pkg(f, pkg)
+    f.write("</table>")
+
+
+def dump_html_gen_info(f, date):
+    f.write("<p><i>Generated on %s</i></p>\n" % (str(date)))
+
+
+def dump_html(packages, date, output):
+    with open(output, 'w') as f:
+        f.write(html_header)
+        dump_html_all_pkgs(f, packages)
+        dump_html_gen_info(f, date)
+        f.write(html_footer)
+
+
+def dump_json(packages, date, output):
+    # Format packages as a dictionnary instead of a list
+    pkgs = {
+        pkg.name: {
+            "version": pkg.version,
+            "cves": pkg.cves,
+        } for pkg in packages
+    }
+    # The actual structure to dump, add date to it
+    final = {'packages': pkgs,
+             'date': str(date)}
+    with open(output, 'w') as f:
+        json.dump(final, f, indent=2, separators=(',', ': '))
+        f.write('\n')
+
+
+def resolvepath(path):
+        return os.path.abspath(os.path.expanduser(path))
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    output = parser.add_argument_group('output', 'Output file(s)')
+    output.add_argument('--html', dest='html', type=resolvepath,
+                        help='HTML output file')
+    output.add_argument('--json', dest='json', type=resolvepath,
+                        help='JSON output file')
+    parser.add_argument('--nvd-path', dest='nvd_path',
+                        help='Path to the local NVD database', type=resolvepath,
+                        required=True)
+    args = parser.parse_args()
+    if not args.html and not args.json:
+        parser.error('at least one of --html or --json (or both) is required')
+    return args
+
+
+def __main__():
+    packages = list()
+    content = json.load(sys.stdin)
+    for item in content:
+        pkg = content[item]
+        p = Package(item, pkg.get('version', ''), pkg.get('ignore_cves', ''))
+        packages.append(p)
+
+    args = parse_args()
+    date = datetime.datetime.utcnow()
+
+    print("Checking packages CVEs")
+    check_package_cves(args.nvd_path, {p.name: p for p in packages})
+
+    if args.html:
+        print("Write HTML")
+        dump_html(packages, date, args.html)
+    if args.json:
+        print("Write JSON")
+        dump_json(packages, date, args.json)
+
+
+__main__()

support/scripts/cve.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
+# Copyright (C) 2020 by Gregory CLEMENT <gregory.clement@bootlin.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import datetime
+import os
+import requests  # URL checking
+import distutils.version
+import time
+import gzip
+import sys
+import operator
+
+try:
+    import ijson
+except ImportError:
+    sys.stderr.write("You need ijson to parse NVD for CVE check\n")
+    exit(1)
+
+sys.path.append('utils/')
+
+NVD_START_YEAR = 2002
+NVD_JSON_VERSION = "1.1"
+NVD_BASE_URL = "https://nvd.nist.gov/feeds/json/cve/" + NVD_JSON_VERSION
+
+ops = {
+    '>=': operator.ge,
+    '>': operator.gt,
+    '<=': operator.le,
+    '<': operator.lt,
+    '=': operator.eq
+}
+
+
+class CVE:
+    """An accessor class for CVE Items in NVD files"""
+    CVE_AFFECTS = 1
+    CVE_DOESNT_AFFECT = 2
+    CVE_UNKNOWN = 3
+
+    def __init__(self, nvd_cve):
+        """Initialize a CVE from its NVD JSON representation"""
+        self.nvd_cve = nvd_cve
+
+    @staticmethod
+    def download_nvd_year(nvd_path, year):
+        metaf = "nvdcve-%s-%s.meta" % (NVD_JSON_VERSION, year)
+        path_metaf = os.path.join(nvd_path, metaf)
+        jsonf_gz = "nvdcve-%s-%s.json.gz" % (NVD_JSON_VERSION, year)
+        path_jsonf_gz = os.path.join(nvd_path, jsonf_gz)
+
+        # If the database file is less than a day old, we assume the NVD data
+        # locally available is recent enough.
+        if os.path.exists(path_jsonf_gz) and os.stat(path_jsonf_gz).st_mtime >= time.time() - 86400:
+            return path_jsonf_gz
+
+        # If not, we download the meta file
+        url = "%s/%s" % (NVD_BASE_URL, metaf)
+        print("Getting %s" % url)
+        page_meta = requests.get(url)
+        page_meta.raise_for_status()
+
+        # If the meta file already existed, we compare the existing
+        # one with the data newly downloaded. If they are different,
+        # we need to re-download the database.
+        # If the database does not exist locally, we need to redownload it in
+        # any case.
+        if os.path.exists(path_metaf) and os.path.exists(path_jsonf_gz):
+            meta_known = open(path_metaf, "r").read()
+            if page_meta.text == meta_known:
+                return path_jsonf_gz
+
+        # Grab the compressed JSON NVD, and write files to disk
+        url = "%s/%s" % (NVD_BASE_URL, jsonf_gz)
+        print("Getting %s" % url)
+        page_json = requests.get(url)
+        page_json.raise_for_status()
+        open(path_jsonf_gz, "wb").write(page_json.content)
+        open(path_metaf, "w").write(page_meta.text)
+        return path_jsonf_gz
+
+    @classmethod
+    def read_nvd_dir(cls, nvd_dir):
+        """
+        Iterate over all the CVEs contained in NIST Vulnerability Database
+        feeds since NVD_START_YEAR. If the files are missing or outdated in
+        nvd_dir, a fresh copy will be downloaded, and kept in .json.gz
+        """
+        for year in range(NVD_START_YEAR, datetime.datetime.now().year + 1):
+            filename = CVE.download_nvd_year(nvd_dir, year)
+            try:
+                content = ijson.items(gzip.GzipFile(filename), 'CVE_Items.item')
+            except:  # noqa: E722
+                print("ERROR: cannot read %s. Please remove the file then rerun this script" % filename)
+                raise
+            for cve in content:
+                yield cls(cve)
+
+    def each_product(self):
+        """Iterate over each product section of this cve"""
+        for vendor in self.nvd_cve['cve']['affects']['vendor']['vendor_data']:
+            for product in vendor['product']['product_data']:
+                yield product
+
+    def parse_node(self, node):
+        """
+        Parse the node inside the configurations section to extract the
+        cpe information usefull to know if a product is affected by
+        the CVE. Actually only the product name and the version
+        descriptor are needed, but we also provide the vendor name.
+        """
+
+        # The node containing the cpe entries matching the CVE can also
+        # contain sub-nodes, so we need to manage it.
+        for child in node.get('children', ()):
+            for parsed_node in self.parse_node(child):
+                yield parsed_node
+
+        for cpe in node.get('cpe_match', ()):
+            if not cpe['vulnerable']:
+                return
+            vendor, product, version = cpe['cpe23Uri'].split(':')[3:6]
+            op_start = ''
+            op_end = ''
+            v_start = ''
+            v_end = ''
+
+            if version != '*' and version != '-':
+                # Version is defined, this is a '=' match
+                op_start = '='
+                v_start = version
+            elif version == '-':
+                # no version information is available
+                op_start = '='
+                v_start = version
+            else:
+                # Parse start version, end version and operators
+                if 'versionStartIncluding' in cpe:
+                    op_start = '>='
+                    v_start = cpe['versionStartIncluding']
+
+                if 'versionStartExcluding' in cpe:
+                    op_start = '>'
+                    v_start = cpe['versionStartExcluding']
+
+                if 'versionEndIncluding' in cpe:
+                    op_end = '<='
+                    v_end = cpe['versionEndIncluding']
+
+                if 'versionEndExcluding' in cpe:
+                    op_end = '<'
+                    v_end = cpe['versionEndExcluding']
+
+            yield {
+                'vendor': vendor,
+                'product': product,
+                'v_start': v_start,
+                'op_start': op_start,
+                'v_end': v_end,
+                'op_end': op_end
+            }
+
+    def each_cpe(self):
+        for node in self.nvd_cve['configurations']['nodes']:
+            for cpe in self.parse_node(node):
+                yield cpe
+
+    @property
+    def identifier(self):
+        """The CVE unique identifier"""
+        return self.nvd_cve['cve']['CVE_data_meta']['ID']
+
+    @property
+    def pkg_names(self):
+        """The set of package names referred by this CVE definition"""
+        return set(p['product'] for p in self.each_cpe())
+
+    def affects(self, name, version, cve_ignore_list):
+        """
+        True if the Buildroot Package object passed as argument is affected
+        by this CVE.
+        """
+        if self.identifier in cve_ignore_list:
+            return self.CVE_DOESNT_AFFECT
+
+        pkg_version = distutils.version.LooseVersion(version)
+        if not hasattr(pkg_version, "version"):
+            print("Cannot parse package '%s' version '%s'" % (name, version))
+            pkg_version = None
+
+        for cpe in self.each_cpe():
+            if cpe['product'] != name:
+                continue
+            if cpe['v_start'] == '-':
+                return self.CVE_AFFECTS
+            if not cpe['v_start'] and not cpe['v_end']:
+                print("No CVE affected version")
+                continue
+            if not pkg_version:
+                continue
+
+            if cpe['v_start']:
+                try:
+                    cve_affected_version = distutils.version.LooseVersion(cpe['v_start'])
+                    inrange = ops.get(cpe['op_start'])(pkg_version, cve_affected_version)
+                except TypeError:
+                    return self.CVE_UNKNOWN
+
+                # current package version is before v_start, so we're
+                # not affected by the CVE
+                if not inrange:
+                    continue
+
+            if cpe['v_end']:
+                try:
+                    cve_affected_version = distutils.version.LooseVersion(cpe['v_end'])
+                    inrange = ops.get(cpe['op_end'])(pkg_version, cve_affected_version)
+                except TypeError:
+                    return self.CVE_UNKNOWN
+
+                # current package version is after v_end, so we're
+                # not affected by the CVE
+                if not inrange:
+                    continue
+
+            # We're in the version range affected by this CVE
+            return self.CVE_AFFECTS
+
+        return self.CVE_DOESNT_AFFECT

support/scripts/fix-rpath

@@ -127,14 +127,29 @@ main() {
 
     while read file ; do
         # check if it's an ELF file
-        if ${PATCHELF} --print-rpath "${file}" > /dev/null 2>&1; then
-            # make files writable if necessary
-            changed=$(chmod -c u+w "${file}")
-            # call patchelf to sanitize the rpath
-            ${PATCHELF} --make-rpath-relative "${rootdir}" ${sanitize_extra_args[@]} "${file}"
-            # restore the original permission
-            test "${changed}" != "" && chmod u-w "${file}"
+        rpath=$(${PATCHELF} --print-rpath "${file}" 2>&1)
+        if test $? -ne 0 ; then
+            continue
         fi
+
+        # make files writable if necessary
+        changed=$(chmod -c u+w "${file}")
+
+        # With per-package directory support, most RPATH of host
+        # binaries will point to per-package directories. This won't
+        # work with the --make-rpath-relative ${rootdir} invocation as
+        # the per-package host directory is not within ${rootdir}. So,
+        # we rewrite all RPATHs pointing to per-package directories so
+        # that they point to the global host directry.
+        changed_rpath=$(echo ${rpath} | sed "s@${PER_PACKAGE_DIR}/[^/]\+/host@${HOST_DIR}@")
+        if test "${rpath}" != "${changed_rpath}" ; then
+            ${PATCHELF} --set-rpath ${changed_rpath} "${file}"
+        fi
+
+        # call patchelf to sanitize the rpath
+        ${PATCHELF} --make-rpath-relative "${rootdir}" ${sanitize_extra_args[@]} "${file}"
+        # restore the original permission
+        test "${changed}" != "" && chmod u-w "${file}"
     done < <(find "${rootdir}" ${find_args[@]})
 
     # Restore patched patchelf utility

support/scripts/gen-bootlin-toolchains

@@ -0,0 +1,472 @@
+#!/usr/bin/env python3
+
+import os.path
+import re
+import requests
+import textwrap
+
+BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
+
+AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
+# Do not edit
+"""
+
+# In the below dict:
+
+# - 'conditions' indicate the cumulative conditions under which the
+#   toolchain will be made available. In several situations, a given
+#   toolchain is usable on several architectures variants (for
+#   example, an ARMv6 toolchain can be used on ARMv7)
+# - 'test_options' indicate one specific configuration where the
+#   toolchain can be used. It is used to create the runtime test
+#   cases. If 'test_options' does not exist, the code assumes it can
+#   be made equal to 'conditions'
+# - 'prefix' is the prefix of the cross-compilation toolchain tools
+
+arches = {
+    'aarch64': {
+        'conditions': ['BR2_aarch64'],
+        'prefix': 'aarch64',
+    },
+    'aarch64be': {
+        'conditions': ['BR2_aarch64_be'],
+        'prefix': 'aarch64_be',
+    },
+    'arcle-750d': {
+        'conditions': ['BR2_arcle', 'BR2_arc750d'],
+        'prefix': 'arc',
+    },
+    'arcle-hs38': {
+        'conditions': ['BR2_arcle', 'BR2_archs38'],
+        'prefix': 'arc',
+    },
+    'armv5-eabi': {
+        'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
+        'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
+        'prefix': 'arm',
+    },
+    'armv6-eabihf': {
+        'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
+        'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
+        'prefix': 'arm',
+    },
+    'armv7-eabihf': {
+        'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
+        'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
+        'prefix': 'arm',
+    },
+    'armv7m': {
+        'conditions': ['BR2_ARM_CPU_ARMV7M'],
+        'test_options': ['BR2_arm', 'BR2_cortex_m4'],
+        'prefix': 'arm',
+    },
+    'm68k-68xxx': {
+        'conditions': ['BR2_m68k_m68k'],
+        'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
+        'prefix': 'm68k',
+    },
+    'm68k-coldfire': {
+        'conditions': ['BR2_m68k_cf'],
+        'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
+        'prefix': 'm68k',
+    },
+    'microblazebe': {
+        'conditions': ['BR2_microblazebe'],
+        'prefix': 'microblaze',
+    },
+    'microblazeel': {
+        'conditions': ['BR2_microblazeel'],
+        'prefix': 'microblazeel',
+    },
+    'mips32': {
+        # Not sure it could be used by other mips32 variants?
+        'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips',
+    },
+    'mips32el': {
+        # Not sure it could be used by other mips32el variants?
+        'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mipsel',
+    },
+    'mips32r5el': {
+        'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mipsel',
+    },
+    'mips32r6el': {
+        'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mipsel',
+    },
+    'mips64': {
+        # Not sure it could be used by other mips64 variants?
+        'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips64',
+    },
+    'mips64-n32': {
+        # Not sure it could be used by other mips64 variants?
+        'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips64',
+    },
+    'mips64el-n32': {
+        # Not sure it could be used by other mips64el variants?
+        'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips64el',
+    },
+    'mips64r6el-n32': {
+        'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+        'prefix': 'mips64el',
+    },
+    'nios2': {
+        'conditions': ['BR2_nios2'],
+        'prefix': 'nios2',
+    },
+    'openrisc': {
+        'conditions': ['BR2_or1k'],
+        'prefix': 'or1k',
+    },
+    'powerpc-e500mc': {
+        # Not sure it could be used by other powerpc variants?
+        'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
+        'prefix': 'powerpc',
+    },
+    'powerpc64-e5500': {
+        'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
+        'prefix': 'powerpc64',
+    },
+    'powerpc64-power8': {
+        'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
+        'prefix': 'powerpc64',
+    },
+    'powerpc64le-power8': {
+        'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
+        'prefix': 'powerpc64le',
+    },
+    'riscv32-ilp32d': {
+        'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
+        'prefix': 'riscv32',
+    },
+    'riscv64': {
+        'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
+        'prefix': 'riscv64',
+    },
+    'sh-sh4': {
+        'conditions': ['BR2_sh', 'BR2_sh4'],
+        'prefix': 'sh4',
+    },
+    'sh-sh4aeb': {
+        'conditions': ['BR2_sh', 'BR2_sh4aeb'],
+        'prefix': 'sh4aeb',
+    },
+    'sparc64': {
+        'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
+        'prefix': 'sparc64',
+    },
+    'sparcv8': {
+        'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
+        'prefix': 'sparc',
+    },
+    'x86-64-core-i7': {
+        'conditions': ['BR2_x86_64',
+                       'BR2_X86_CPU_HAS_MMX',
+                       'BR2_X86_CPU_HAS_SSE',
+                       'BR2_X86_CPU_HAS_SSE2',
+                       'BR2_X86_CPU_HAS_SSE3',
+                       'BR2_X86_CPU_HAS_SSSE3',
+                       'BR2_X86_CPU_HAS_SSE4',
+                       'BR2_X86_CPU_HAS_SSE42'],
+        'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
+        'prefix': 'x86_64',
+    },
+    'x86-core2': {
+        'conditions': ['BR2_i386',
+                       'BR2_X86_CPU_HAS_MMX',
+                       'BR2_X86_CPU_HAS_SSE',
+                       'BR2_X86_CPU_HAS_SSE2',
+                       'BR2_X86_CPU_HAS_SSE3',
+                       'BR2_X86_CPU_HAS_SSSE3'],
+        'test_options': ['BR2_i386', 'BR2_x86_core2'],
+        'prefix': 'i686',
+    },
+    'x86-i686': {
+        'conditions': ['BR2_i386',
+                       '!BR2_x86_i486',
+                       '!BR2_x86_i586',
+                       '!BR2_x86_x1000'],
+        'test_options': ['BR2_i386',
+                         'BR2_x86_i686'],
+        'prefix': 'i686',
+    },
+    'xtensa-lx60': {
+        'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'],
+        'prefix': 'xtensa',
+    },
+}
+
+
+class Toolchain:
+    def __init__(self, arch, libc, variant, version):
+        self.arch = arch
+        self.libc = libc
+        self.variant = variant
+        self.version = version
+        self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
+        self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
+            (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
+        self.fragment = requests.get(self.fragment_url).text.split("\n")
+        self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
+
+    @property
+    def tarball_url(self):
+        return os.path.join(BASE_URL, self.arch, "tarballs",
+                            self.fname_prefix + ".tar.bz2")
+
+    @property
+    def hash_url(self):
+        return os.path.join(BASE_URL, self.arch, "tarballs",
+                            self.fname_prefix + ".sha256")
+
+    @property
+    def fragment_url(self):
+        return os.path.join(BASE_URL, self.arch, "fragments",
+                            self.fname_prefix + ".frag")
+
+    def gen_config_in_options(self, f):
+        f.write("config %s\n" % self.option_name)
+        f.write("\tbool \"%s %s %s %s\"\n" %
+                (self.arch, self.libc, self.variant, self.version))
+        for c in arches[self.arch]['conditions']:
+            f.write("\tdepends on %s\n" % c)
+        selects = []
+        for frag in self.fragment:
+            # libc type
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
+                selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
+            elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
+                selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
+                # all glibc toolchains have RPC support
+                selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
+            elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
+                selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
+
+            # gcc version
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
+                m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
+                assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
+                selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
+
+            # kernel headers version
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
+                m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
+                assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
+                selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
+
+            # C++
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
+                selects.append("BR2_INSTALL_LIBSTDCPP")
+
+            # SSP
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
+                selects.append("BR2_TOOLCHAIN_HAS_SSP")
+
+            # wchar
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
+                selects.append("BR2_USE_WCHAR")
+
+            # locale
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
+                # locale implies the availability of wchar
+                selects.append("BR2_USE_WCHAR")
+                selects.append("BR2_ENABLE_LOCALE")
+
+            # thread support
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
+                selects.append("BR2_TOOLCHAIN_HAS_THREADS")
+
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
+                selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
+
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
+                selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
+
+            # RPC
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
+                selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
+
+            # D language
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
+                selects.append("BR2_TOOLCHAIN_HAS_DLANG")
+
+            # fortran
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
+                selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
+
+            # OpenMP
+            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
+                selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
+
+        for select in selects:
+            f.write("\tselect %s\n" % select)
+
+        f.write("\thelp\n")
+
+        desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
+            (self.arch, self.libc)
+
+        if self.variant == "stable":
+            desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
+        else:
+            desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
+
+        f.write(textwrap.fill(desc, width=62, initial_indent="\t  ", subsequent_indent="\t  ") + "\n")
+        f.write("\n")
+        f.write("\t  https://toolchains.bootlin.com/\n")
+
+        f.write("\n")
+
+    def gen_mk(self, f):
+        f.write("ifeq ($(%s),y)\n" % self.option_name)
+        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
+        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
+                (self.arch, self.libc, self.variant))
+        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
+                os.path.join(BASE_URL, self.arch, "tarballs"))
+        f.write("endif\n\n")
+        pass
+
+    def gen_hash(self, f):
+        f.write("# From %s\n" % self.hash_url)
+        f.write("sha256  %s  %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
+
+    def gen_test(self, f):
+        if self.variant == "stable":
+            variant = "Stable"
+        else:
+            variant = "BleedingEdge"
+        testname = "TestExternalToolchainBootlin" + \
+            self.arch.replace("-", "").capitalize() + \
+            self.libc.capitalize() + variant
+        f.write("\n\n")
+        f.write("class %s(TestExternalToolchain):\n" % testname)
+        f.write("    config = \"\"\"\n")
+        if 'test_options' in arches[self.arch]:
+            test_options = arches[self.arch]['test_options']
+        else:
+            test_options = arches[self.arch]['conditions']
+        for opt in test_options:
+            if opt.startswith("!"):
+                f.write("        # %s is not set\n" % opt[1:])
+            else:
+                f.write("        %s=y\n" % opt)
+        f.write("        BR2_TOOLCHAIN_EXTERNAL=y\n")
+        f.write("        BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
+        f.write("        %s=y\n" % self.option_name)
+        f.write("        # BR2_TARGET_ROOTFS_TAR is not set\n")
+        f.write("        \"\"\"\n")
+        f.write("    toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
+        f.write("\n")
+        f.write("    def test_run(self):\n")
+        f.write("        TestExternalToolchain.common_check(self)\n")
+
+    def __repr__(self):
+        return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
+            (self.arch, self.libc, self.variant, self.version, self.option_name)
+
+
+def get_toolchains():
+    toolchains = list()
+    for arch, details in arches.items():
+        print(arch)
+        url = os.path.join(BASE_URL, arch, "available_toolchains")
+        page = requests.get(url).text
+        fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
+        # This dict will allow us to keep only the latest version for
+        # each toolchain.
+        tmp = dict()
+        for fname in fnames:
+            parts = fname.split('--')
+            assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
+            libc = parts[1]
+            if parts[2].startswith("stable-"):
+                variant = "stable"
+                version = parts[2][len("stable-"):]
+            elif parts[2].startswith("bleeding-edge-"):
+                variant = "bleeding-edge"
+                version = parts[2][len("bleeding-edge-"):]
+            tmp[(arch, libc, variant)] = version
+
+        toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
+
+    return toolchains
+
+
+def gen_config_in_options(toolchains, fpath):
+    with open(fpath, "w") as f:
+        f.write(AUTOGENERATED_COMMENT)
+
+        f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
+        f.write("\tbool\n")
+        for arch, details in arches.items():
+            f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
+        f.write("\n")
+
+        f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
+
+        f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
+        f.write("\tdefault \"$(ARCH)-linux\"\n")
+
+        f.write("\n")
+
+        f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
+        f.write("\tdefault \"toolchain-external-bootlin\"\n")
+
+        f.write("\n")
+
+        f.write("choice\n")
+        f.write("\tprompt \"Bootlin toolchain variant\"\n")
+
+        for toolchain in toolchains:
+            toolchain.gen_config_in_options(f)
+
+        f.write("endchoice\n")
+        f.write("endif\n")
+
+
+def gen_mk(toolchains, fpath):
+    with open(fpath, "w") as f:
+        f.write("#" * 80 + "\n")
+        f.write("#\n")
+        f.write("# toolchain-external-bootlin\n")
+        f.write("#\n")
+        f.write("#" * 80 + "\n")
+        f.write("\n")
+        f.write(AUTOGENERATED_COMMENT)
+        for toolchain in toolchains:
+            toolchain.gen_mk(f)
+        f.write("$(eval $(toolchain-external-package))\n")
+
+
+def gen_hash(toolchains, fpath):
+    with open(fpath, "w") as f:
+        f.write(AUTOGENERATED_COMMENT)
+        for toolchain in toolchains:
+            toolchain.gen_hash(f)
+
+
+def gen_runtime_test(toolchains, fpath):
+    with open(fpath, "w") as f:
+        f.write(AUTOGENERATED_COMMENT)
+        f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
+        for toolchain in toolchains:
+            toolchain.gen_test(f)
+
+
+def gen_toolchains(toolchains):
+    maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
+    gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
+    gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
+    gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
+    gen_runtime_test(toolchains,
+                     os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
+
+
+toolchains = get_toolchains()
+gen_toolchains(toolchains)

support/scripts/generate-gitlab-ci-yml

@@ -2,16 +2,105 @@
 set -e
 set -o pipefail
 
-input="${1}"
+main() {
+    local template="${1}"
 
-cat "${input}"
+    preamble "${template}"
+    gen_tests
+}
 
-(
-    cd configs
-    LC_ALL=C ls -1 *_defconfig
-) \
-    | sed 's/$/: { extends: .defconfig }/'
+preamble() {
+    local template="${1}"
 
-./support/testing/run-tests -l 2>&1 \
-    | sed -r -e '/^test_run \((.*)\).*/!d; s//\1: { extends: .runtime_test }/' \
-    | LC_ALL=C sort
+    cat - "${template}" <<-_EOF_
+	# This file is generated; do not edit!
+	# Builds appear on https://gitlab.com/buildroot.org/buildroot/pipelines
+
+	image: ${CI_JOB_IMAGE}
+
+_EOF_
+}
+
+gen_tests() {
+    local -a basics defconfigs runtimes
+    local do_basics do_defconfigs do_runtime
+    local defconfigs_ext cfg tst
+
+    basics=( DEVELOPERS flake8 package )
+
+    defconfigs=( $(cd configs; LC_ALL=C ls -1 *_defconfig) )
+
+    runtimes=( $(./support/testing/run-tests -l 2>&1 \
+                 | sed -r -e '/^test_run \((.*)\).*/!d; s//\1/' \
+                 | LC_ALL=C sort)
+             )
+
+    if [ -n "${CI_COMMIT_TAG}" ]; then
+        do_basics=true
+        do_defconfigs=base
+        do_runtime=true
+    elif [ "${CI_PIPELINE_SOURCE}" = "trigger" ]; then
+        case "${BR_SCHEDULE_JOBS}" in
+          (basic)
+            do_basics=true
+            do_defconfigs=check
+            defconfigs_ext=_check
+            ;;
+          (defconfig)
+            do_defconfigs=base
+            ;;
+          (runtime)
+            do_runtime=true
+            ;;
+        esac
+    else
+        case "${CI_COMMIT_REF_NAME}" in
+          (*-basics)
+            do_basics=true
+            do_defconfigs=check
+            defconfigs_ext=_check
+            ;;
+          (*-defconfigs)
+            do_defconfigs=base
+            ;;
+          (*-*_defconfig)
+            defconfigs=( "${CI_COMMIT_REF_NAME##*-}" )
+            do_defconfigs=base
+            ;;
+          (*-runtime-tests)
+            do_runtime=true
+            ;;
+          (*-tests.*)
+            runtimes=( "${CI_COMMIT_REF_NAME##*-}" )
+            do_runtime=true
+            ;;
+        esac
+    fi
+
+    # If nothing else, at least do the basics to generate a valid pipeline
+    if [    -z "${do_defconfigs}" \
+         -a -z "${do_runtime}" \
+       ]
+    then
+        do_basics=true
+    fi
+
+    if ${do_basics:-false}; then
+        for tst in "${basics[@]}"; do
+            printf 'check-%s: { extends: .check-%s_base }\n' "${tst}" "${tst}"
+        done
+    fi
+
+    if [ -n "${do_defconfigs}" ]; then
+        for cfg in "${defconfigs[@]}"; do
+            printf '%s%s: { extends: .defconfig_%s }\n' \
+                   "${cfg}" "${defconfigs_ext}" "${do_defconfigs}"
+        done
+    fi
+
+    if ${do_runtime:-false}; then
+        printf '%s: { extends: .runtime_test_base }\n' "${runtimes[@]}"
+    fi
+}
+
+main "${@}"

support/scripts/genimage.sh

@@ -30,10 +30,18 @@ done
 
 [ -n "${GENIMAGE_CFG}" ] || die "Missing argument"
 
+# Pass an empty rootpath. genimage makes a full copy of the given rootpath to
+# ${GENIMAGE_TMP}/root so passing TARGET_DIR would be a waste of time and disk
+# space. We don't rely on genimage to build the rootfs image, just to insert a
+# pre-built one in the disk image.
+
+trap 'rm -rf "${ROOTPATH_TMP}"' EXIT
+ROOTPATH_TMP="$(mktemp -d)"
+
 rm -rf "${GENIMAGE_TMP}"
 
 genimage \
-	--rootpath "${TARGET_DIR}"     \
+	--rootpath "${ROOTPATH_TMP}"     \
 	--tmppath "${GENIMAGE_TMP}"    \
 	--inputpath "${BINARIES_DIR}"  \
 	--outputpath "${BINARIES_DIR}" \

support/scripts/graph-depends

@@ -115,7 +115,7 @@ def remove_transitive_deps(pkg, deps):
 
 # List of dependencies that all/many packages have, and that we want
 # to trim when generating the dependency graph.
-MANDATORY_DEPS = ['toolchain', 'skeleton', 'host-skeleton', 'host-tar', 'host-gzip']
+MANDATORY_DEPS = ['toolchain', 'skeleton', 'host-skeleton', 'host-tar', 'host-gzip', 'host-ccache']
 
 
 # This function removes the dependency on some 'mandatory' package, like the

support/scripts/pkg-stats

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
 #
@@ -16,19 +16,22 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
+import aiohttp
 import argparse
+import asyncio
 import datetime
 import fnmatch
 import os
 from collections import defaultdict
 import re
 import subprocess
-import requests  # URL checking
 import json
-import certifi
-from urllib3 import HTTPSConnectionPool
-from urllib3.exceptions import HTTPError
-from multiprocessing import Pool
+import sys
+
+sys.path.append('utils/')
+from getdeveloperlib import parse_developers  # noqa: E402
+import cve as cvecheck  # noqa: E402
+
 
 INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)")
 URL_RE = re.compile(r"\s*https?://\S*\s*$")
@@ -38,30 +41,64 @@ RM_API_STATUS_FOUND_BY_DISTRO = 2
 RM_API_STATUS_FOUND_BY_PATTERN = 3
 RM_API_STATUS_NOT_FOUND = 4
 
-# Used to make multiple requests to the same host. It is global
-# because it's used by sub-processes.
-http_pool = None
+
+class Defconfig:
+    def __init__(self, name, path):
+        self.name = name
+        self.path = path
+        self.developers = None
+
+    def set_developers(self, developers):
+        """
+        Fills in the .developers field
+        """
+        self.developers = [
+            developer.name
+            for developer in developers
+            if developer.hasfile(self.path)
+        ]
+
+
+def get_defconfig_list():
+    """
+    Builds the list of Buildroot defconfigs, returning a list of Defconfig
+    objects.
+    """
+    return [
+        Defconfig(name[:-len('_defconfig')], os.path.join('configs', name))
+        for name in os.listdir('configs')
+        if name.endswith('_defconfig')
+    ]
 
 
 class Package:
-    all_licenses = list()
+    all_licenses = dict()
     all_license_files = list()
     all_versions = dict()
+    all_ignored_cves = dict()
+    # This is the list of all possible checks. Add new checks to this list so
+    # a tool that post-processeds the json output knows the checks before
+    # iterating over the packages.
+    status_checks = ['cve', 'developers', 'hash', 'license',
+                     'license-files', 'patches', 'pkg-check', 'url', 'version']
 
     def __init__(self, name, path):
         self.name = name
         self.path = path
+        self.pkg_path = os.path.dirname(path)
         self.infras = None
+        self.license = None
         self.has_license = False
         self.has_license_files = False
         self.has_hash = False
-        self.patch_count = 0
+        self.patch_files = []
         self.warnings = 0
         self.current_version = None
         self.url = None
-        self.url_status = None
         self.url_worker = None
-        self.latest_version = (RM_API_STATUS_ERROR, None, None)
+        self.cves = list()
+        self.latest_version = {'status': RM_API_STATUS_ERROR, 'version': None, 'id': None}
+        self.status = {}
 
     def pkgvar(self):
         return self.name.upper().replace("-", "_")
@@ -70,19 +107,32 @@ class Package:
         """
         Fills in the .url field
         """
-        self.url_status = "No Config.in"
+        self.status['url'] = ("warning", "no Config.in")
         for filename in os.listdir(os.path.dirname(self.path)):
             if fnmatch.fnmatch(filename, 'Config.*'):
                 fp = open(os.path.join(os.path.dirname(self.path), filename), "r")
                 for config_line in fp:
                     if URL_RE.match(config_line):
                         self.url = config_line.strip()
-                        self.url_status = "Found"
+                        self.status['url'] = ("ok", "found")
                         fp.close()
                         return
-                self.url_status = "Missing"
+                self.status['url'] = ("error", "missing")
                 fp.close()
 
+    @property
+    def patch_count(self):
+        return len(self.patch_files)
+
+    @property
+    def has_valid_infra(self):
+        try:
+            if self.infras[0][1] == 'virtual':
+                return False
+        except IndexError:
+            return False
+        return True
+
     def set_infra(self):
         """
         Fills in the .infras field
@@ -102,29 +152,55 @@ class Package:
 
     def set_license(self):
         """
-        Fills in the .has_license and .has_license_files fields
+        Fills in the .status['license'] and .status['license-files'] fields
         """
+        if not self.has_valid_infra:
+            self.status['license'] = ("na", "no valid package infra")
+            self.status['license-files'] = ("na", "no valid package infra")
+            return
+
         var = self.pkgvar()
+        self.status['license'] = ("error", "missing")
+        self.status['license-files'] = ("error", "missing")
         if var in self.all_licenses:
-            self.has_license = True
+            self.license = self.all_licenses[var]
+            self.status['license'] = ("ok", "found")
         if var in self.all_license_files:
-            self.has_license_files = True
+            self.status['license-files'] = ("ok", "found")
 
     def set_hash_info(self):
         """
-        Fills in the .has_hash field
+        Fills in the .status['hash'] field
         """
+        if not self.has_valid_infra:
+            self.status['hash'] = ("na", "no valid package infra")
+            self.status['hash-license'] = ("na", "no valid package infra")
+            return
+
         hashpath = self.path.replace(".mk", ".hash")
-        self.has_hash = os.path.exists(hashpath)
+        if os.path.exists(hashpath):
+            self.status['hash'] = ("ok", "found")
+        else:
+            self.status['hash'] = ("error", "missing")
 
     def set_patch_count(self):
         """
-        Fills in the .patch_count field
+        Fills in the .patch_count, .patch_files and .status['patches'] fields
         """
-        self.patch_count = 0
+        if not self.has_valid_infra:
+            self.status['patches'] = ("na", "no valid package infra")
+            return
+
         pkgdir = os.path.dirname(self.path)
         for subdir, _, _ in os.walk(pkgdir):
-            self.patch_count += len(fnmatch.filter(os.listdir(subdir), '*.patch'))
+            self.patch_files = fnmatch.filter(os.listdir(subdir), '*.patch')
+
+        if self.patch_count == 0:
+            self.status['patches'] = ("ok", "no patches")
+        elif self.patch_count < 5:
+            self.status['patches'] = ("warning", "some patches")
+        else:
+            self.status['patches'] = ("error", "lots of patches")
 
     def set_current_version(self):
         """
@@ -136,10 +212,11 @@ class Package:
 
     def set_check_package_warnings(self):
         """
-        Fills in the .warnings field
+        Fills in the .warnings and .status['pkg-check'] fields
         """
         cmd = ["./utils/check-package"]
         pkgdir = os.path.dirname(self.path)
+        self.status['pkg-check'] = ("error", "Missing")
         for root, dirs, files in os.walk(pkgdir):
             for f in files:
                 if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host":
@@ -147,11 +224,40 @@ class Package:
         o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1]
         lines = o.splitlines()
         for line in lines:
-            m = re.match("^([0-9]*) warnings generated", line)
+            m = re.match("^([0-9]*) warnings generated", line.decode())
             if m:
                 self.warnings = int(m.group(1))
+                if self.warnings == 0:
+                    self.status['pkg-check'] = ("ok", "no warnings")
+                else:
+                    self.status['pkg-check'] = ("error", "{} warnings".format(self.warnings))
                 return
 
+    @property
+    def ignored_cves(self):
+        """
+        Give the list of CVEs ignored by the package
+        """
+        return list(self.all_ignored_cves.get(self.pkgvar(), []))
+
+    def set_developers(self, developers):
+        """
+        Fills in the .developers and .status['developers'] field
+        """
+        self.developers = [
+            dev.name
+            for dev in developers
+            if dev.hasfile(self.path)
+        ]
+
+        if self.developers:
+            self.status['developers'] = ("ok", "{} developers".format(len(self.developers)))
+        else:
+            self.status['developers'] = ("warning", "no developers")
+
+    def is_status_ok(self, name):
+        return self.status[name][0] == 'ok'
+
     def __eq__(self, other):
         return self.path == other.path
 
@@ -160,7 +266,8 @@ class Package:
 
     def __str__(self):
         return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \
-            (self.name, self.path, self.has_license, self.has_license_files, self.has_hash, self.patch_count)
+            (self.name, self.path, self.is_status_ok('license'),
+             self.is_status_ok('license-files'), self.status['hash'], self.patch_count)
 
 
 def get_pkglist(npackages, package_list):
@@ -186,7 +293,6 @@ def get_pkglist(npackages, package_list):
                      "package/x11r7/x11r7.mk",
                      "package/doc-asciidoc.mk",
                      "package/pkg-.*.mk",
-                     "package/nvidia-tegra23/nvidia-tegra23.mk",
                      "toolchain/toolchain-external/pkg-toolchain-external.mk",
                      "toolchain/toolchain-external/toolchain-external.mk",
                      "toolchain/toolchain.mk",
@@ -227,8 +333,8 @@ def get_pkglist(npackages, package_list):
 def package_init_make_info():
     # Fetch all variables at once
     variables = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", "-s", "printvars",
-                                         "VARS=%_LICENSE %_LICENSE_FILES %_VERSION"])
-    variable_list = variables.splitlines()
+                                         "VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES"])
+    variable_list = variables.decode().splitlines()
 
     # We process first the host package VERSION, and then the target
     # package VERSION. This means that if a package exists in both
@@ -247,7 +353,7 @@ def package_init_make_info():
             if value == "unknown":
                 continue
             pkgvar = pkgvar[:-8]
-            Package.all_licenses.append(pkgvar)
+            Package.all_licenses[pkgvar] = value
 
         elif pkgvar.endswith("_LICENSE_FILES"):
             if pkgvar.endswith("_MANIFEST_LICENSE_FILES"):
@@ -261,81 +367,145 @@ def package_init_make_info():
             pkgvar = pkgvar[:-8]
             Package.all_versions[pkgvar] = value
 
-
-def check_url_status_worker(url, url_status):
-    if url_status != "Missing" and url_status != "No Config.in":
-        try:
-            url_status_code = requests.head(url, timeout=30).status_code
-            if url_status_code >= 400:
-                return "Invalid(%s)" % str(url_status_code)
-        except requests.exceptions.RequestException:
-            return "Invalid(Err)"
-        return "Ok"
-    return url_status
+        elif pkgvar.endswith("_IGNORE_CVES"):
+            pkgvar = pkgvar[:-12]
+            Package.all_ignored_cves[pkgvar] = value.split()
 
 
-def check_package_urls(packages):
-    Package.pool = Pool(processes=64)
-    for pkg in packages:
-        pkg.url_worker = pkg.pool.apply_async(check_url_status_worker, (pkg.url, pkg.url_status))
-    for pkg in packages:
-        pkg.url_status = pkg.url_worker.get(timeout=3600)
+check_url_count = 0
 
 
-def release_monitoring_get_latest_version_by_distro(pool, name):
+async def check_url_status(session, pkg, npkgs, retry=True):
+    global check_url_count
+
     try:
-        req = pool.request('GET', "/api/project/Buildroot/%s" % name)
-    except HTTPError:
-        return (RM_API_STATUS_ERROR, None, None)
+        async with session.get(pkg.url) as resp:
+            if resp.status >= 400:
+                pkg.status['url'] = ("error", "invalid {}".format(resp.status))
+                check_url_count += 1
+                print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name))
+                return
+    except (aiohttp.ClientError, asyncio.TimeoutError):
+        if retry:
+            return await check_url_status(session, pkg, npkgs, retry=False)
+        else:
+            pkg.status['url'] = ("error", "invalid (err)")
+            check_url_count += 1
+            print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name))
+            return
 
-    if req.status != 200:
-        return (RM_API_STATUS_NOT_FOUND, None, None)
+    pkg.status['url'] = ("ok", "valid")
+    check_url_count += 1
+    print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name))
 
-    data = json.loads(req.data)
 
-    if 'version' in data:
-        return (RM_API_STATUS_FOUND_BY_DISTRO, data['version'], data['id'])
+async def check_package_urls(packages):
+    tasks = []
+    connector = aiohttp.TCPConnector(limit_per_host=5)
+    async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess:
+        packages = [p for p in packages if p.status['url'][0] == 'ok']
+        for pkg in packages:
+            tasks.append(check_url_status(sess, pkg, len(packages)))
+        await asyncio.wait(tasks)
+
+
+def check_package_latest_version_set_status(pkg, status, version, identifier):
+    pkg.latest_version = {
+        "status": status,
+        "version": version,
+        "id": identifier,
+    }
+
+    if pkg.latest_version['status'] == RM_API_STATUS_ERROR:
+        pkg.status['version'] = ('warning', "Release Monitoring API error")
+    elif pkg.latest_version['status'] == RM_API_STATUS_NOT_FOUND:
+        pkg.status['version'] = ('warning', "Package not found on Release Monitoring")
+
+    if pkg.latest_version['version'] is None:
+        pkg.status['version'] = ('warning', "No upstream version available on Release Monitoring")
+    elif pkg.latest_version['version'] != pkg.current_version:
+        pkg.status['version'] = ('error', "The newer version {} is available upstream".format(pkg.latest_version['version']))
     else:
-        return (RM_API_STATUS_FOUND_BY_DISTRO, None, data['id'])
+        pkg.status['version'] = ('ok', 'up-to-date')
 
 
-def release_monitoring_get_latest_version_by_guess(pool, name):
+async def check_package_get_latest_version_by_distro(session, pkg, retry=True):
+    url = "https://release-monitoring.org//api/project/Buildroot/%s" % pkg.name
     try:
-        req = pool.request('GET', "/api/projects/?pattern=%s" % name)
-    except HTTPError:
-        return (RM_API_STATUS_ERROR, None, None)
+        async with session.get(url) as resp:
+            if resp.status != 200:
+                return False
 
-    if req.status != 200:
-        return (RM_API_STATUS_NOT_FOUND, None, None)
+            data = await resp.json()
+            version = data['version'] if 'version' in data else None
+            check_package_latest_version_set_status(pkg,
+                                                    RM_API_STATUS_FOUND_BY_DISTRO,
+                                                    version,
+                                                    data['id'])
+            return True
 
-    data = json.loads(req.data)
-
-    projects = data['projects']
-    projects.sort(key=lambda x: x['id'])
-
-    for p in projects:
-        if p['name'] == name and 'version' in p:
-            return (RM_API_STATUS_FOUND_BY_PATTERN, p['version'], p['id'])
-
-    return (RM_API_STATUS_NOT_FOUND, None, None)
+    except (aiohttp.ClientError, asyncio.TimeoutError):
+        if retry:
+            return await check_package_get_latest_version_by_distro(session, pkg, retry=False)
+        else:
+            return False
 
 
-def check_package_latest_version_worker(name):
-    """Wrapper to try both by name then by guess"""
-    print(name)
-    res = release_monitoring_get_latest_version_by_distro(http_pool, name)
-    if res[0] == RM_API_STATUS_NOT_FOUND:
-        res = release_monitoring_get_latest_version_by_guess(http_pool, name)
-    return res
+async def check_package_get_latest_version_by_guess(session, pkg, retry=True):
+    url = "https://release-monitoring.org/api/projects/?pattern=%s" % pkg.name
+    try:
+        async with session.get(url) as resp:
+            if resp.status != 200:
+                return False
+
+            data = await resp.json()
+            # filter projects that have the right name and a version defined
+            projects = [p for p in data['projects'] if p['name'] == pkg.name and 'version' in p]
+            projects.sort(key=lambda x: x['id'])
+
+            if len(projects) > 0:
+                check_package_latest_version_set_status(pkg,
+                                                        RM_API_STATUS_FOUND_BY_DISTRO,
+                                                        projects[0]['version'],
+                                                        projects[0]['id'])
+                return True
+
+    except (aiohttp.ClientError, asyncio.TimeoutError):
+        if retry:
+            return await check_package_get_latest_version_by_guess(session, pkg, retry=False)
+        else:
+            return False
 
 
-def check_package_latest_version(packages):
+check_latest_count = 0
+
+
+async def check_package_latest_version_get(session, pkg, npkgs):
+    global check_latest_count
+
+    if await check_package_get_latest_version_by_distro(session, pkg):
+        check_latest_count += 1
+        print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name))
+        return
+
+    if await check_package_get_latest_version_by_guess(session, pkg):
+        check_latest_count += 1
+        print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name))
+        return
+
+    check_package_latest_version_set_status(pkg,
+                                            RM_API_STATUS_NOT_FOUND,
+                                            None, None)
+    check_latest_count += 1
+    print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name))
+
+
+async def check_package_latest_version(packages):
     """
     Fills in the .latest_version field of all Package objects
 
-    This field has a special format:
-      (status, version, id)
-    with:
+    This field is a dict and has the following keys:
+
     - status: one of RM_API_STATUS_ERROR,
       RM_API_STATUS_FOUND_BY_DISTRO, RM_API_STATUS_FOUND_BY_PATTERN,
       RM_API_STATUS_NOT_FOUND
@@ -344,19 +514,34 @@ def check_package_latest_version(packages):
     - id: string containing the id of the project corresponding to this
       package, as known by release-monitoring.org
     """
-    global http_pool
-    http_pool = HTTPSConnectionPool('release-monitoring.org', port=443,
-                                    cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(),
-                                    timeout=30)
-    worker_pool = Pool(processes=64)
-    results = worker_pool.map(check_package_latest_version_worker, (pkg.name for pkg in packages))
-    for pkg, r in zip(packages, results):
-        pkg.latest_version = r
-    del http_pool
+
+    for pkg in [p for p in packages if not p.has_valid_infra]:
+        pkg.status['version'] = ("na", "no valid package infra")
+
+    tasks = []
+    connector = aiohttp.TCPConnector(limit_per_host=5)
+    async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess:
+        packages = [p for p in packages if p.has_valid_infra]
+        for pkg in packages:
+            tasks.append(check_package_latest_version_get(sess, pkg, len(packages)))
+        await asyncio.wait(tasks)
+
+
+def check_package_cves(nvd_path, packages):
+    if not os.path.isdir(nvd_path):
+        os.makedirs(nvd_path)
+
+    for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
+        for pkg_name in cve.pkg_names:
+            if pkg_name in packages:
+                pkg = packages[pkg_name]
+                if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves) == cve.CVE_AFFECTS:
+                    pkg.cves.append(cve.identifier)
 
 
 def calculate_stats(packages):
     stats = defaultdict(int)
+    stats['packages'] = len(packages)
     for pkg in packages:
         # If packages have multiple infra, take the first one. For the
         # vast majority of packages, the target and host infra are the
@@ -367,29 +552,32 @@ def calculate_stats(packages):
             stats["infra-%s" % infra] += 1
         else:
             stats["infra-unknown"] += 1
-        if pkg.has_license:
+        if pkg.is_status_ok('license'):
             stats["license"] += 1
         else:
             stats["no-license"] += 1
-        if pkg.has_license_files:
+        if pkg.is_status_ok('license-files'):
             stats["license-files"] += 1
         else:
             stats["no-license-files"] += 1
-        if pkg.has_hash:
+        if pkg.is_status_ok('hash'):
             stats["hash"] += 1
         else:
             stats["no-hash"] += 1
-        if pkg.latest_version[0] == RM_API_STATUS_FOUND_BY_DISTRO:
+        if pkg.latest_version['status'] == RM_API_STATUS_FOUND_BY_DISTRO:
             stats["rmo-mapping"] += 1
         else:
             stats["rmo-no-mapping"] += 1
-        if not pkg.latest_version[1]:
+        if not pkg.latest_version['version']:
             stats["version-unknown"] += 1
-        elif pkg.latest_version[1] == pkg.current_version:
+        elif pkg.latest_version['version'] == pkg.current_version:
             stats["version-uptodate"] += 1
         else:
             stats["version-not-uptodate"] += 1
         stats["patches"] += pkg.patch_count
+        stats["total-cves"] += len(pkg.cves)
+        if len(pkg.cves) != 0:
+            stats["pkg-cves"] += 1
     return stats
 
 
@@ -515,30 +703,30 @@ def dump_html_pkg(f, pkg):
 
     # License
     td_class = ["centered"]
-    if pkg.has_license:
+    if pkg.is_status_ok('license'):
         td_class.append("correct")
     else:
         td_class.append("wrong")
     f.write("  <td class=\"%s\">%s</td>\n" %
-            (" ".join(td_class), boolean_str(pkg.has_license)))
+            (" ".join(td_class), boolean_str(pkg.is_status_ok('license'))))
 
     # License files
     td_class = ["centered"]
-    if pkg.has_license_files:
+    if pkg.is_status_ok('license-files'):
         td_class.append("correct")
     else:
         td_class.append("wrong")
     f.write("  <td class=\"%s\">%s</td>\n" %
-            (" ".join(td_class), boolean_str(pkg.has_license_files)))
+            (" ".join(td_class), boolean_str(pkg.is_status_ok('license-files'))))
 
     # Hash
     td_class = ["centered"]
-    if pkg.has_hash:
+    if pkg.is_status_ok('hash'):
         td_class.append("correct")
     else:
         td_class.append("wrong")
     f.write("  <td class=\"%s\">%s</td>\n" %
-            (" ".join(td_class), boolean_str(pkg.has_hash)))
+            (" ".join(td_class), boolean_str(pkg.is_status_ok('hash'))))
 
     # Current version
     if len(pkg.current_version) > 20:
@@ -548,29 +736,29 @@ def dump_html_pkg(f, pkg):
     f.write("  <td class=\"centered\">%s</td>\n" % current_version)
 
     # Latest version
-    if pkg.latest_version[0] == RM_API_STATUS_ERROR:
+    if pkg.latest_version['status'] == RM_API_STATUS_ERROR:
         td_class.append("version-error")
-    if pkg.latest_version[1] is None:
+    if pkg.latest_version['version'] is None:
         td_class.append("version-unknown")
-    elif pkg.latest_version[1] != pkg.current_version:
+    elif pkg.latest_version['version'] != pkg.current_version:
         td_class.append("version-needs-update")
     else:
         td_class.append("version-good")
 
-    if pkg.latest_version[0] == RM_API_STATUS_ERROR:
+    if pkg.latest_version['status'] == RM_API_STATUS_ERROR:
         latest_version_text = "<b>Error</b>"
-    elif pkg.latest_version[0] == RM_API_STATUS_NOT_FOUND:
+    elif pkg.latest_version['status'] == RM_API_STATUS_NOT_FOUND:
         latest_version_text = "<b>Not found</b>"
     else:
-        if pkg.latest_version[1] is None:
+        if pkg.latest_version['version'] is None:
             latest_version_text = "<b>Found, but no version</b>"
         else:
             latest_version_text = "<a href=\"https://release-monitoring.org/project/%s\"><b>%s</b></a>" % \
-                (pkg.latest_version[2], str(pkg.latest_version[1]))
+                (pkg.latest_version['id'], str(pkg.latest_version['version']))
 
         latest_version_text += "<br/>"
 
-        if pkg.latest_version[0] == RM_API_STATUS_FOUND_BY_DISTRO:
+        if pkg.latest_version['status'] == RM_API_STATUS_FOUND_BY_DISTRO:
             latest_version_text += "found by <a href=\"https://release-monitoring.org/distro/Buildroot/\">distro</a>"
         else:
             latest_version_text += "found by guess"
@@ -589,18 +777,29 @@ def dump_html_pkg(f, pkg):
 
     # URL status
     td_class = ["centered"]
-    url_str = pkg.url_status
-    if pkg.url_status == "Missing" or pkg.url_status == "No Config.in":
+    url_str = pkg.status['url'][1]
+    if pkg.status['url'][0] in ("error", "warning"):
         td_class.append("missing_url")
-    elif pkg.url_status.startswith("Invalid"):
+    if pkg.status['url'][0] == "error":
         td_class.append("invalid_url")
-        url_str = "<a href=%s>%s</a>" % (pkg.url, pkg.url_status)
+        url_str = "<a href=%s>%s</a>" % (pkg.url, pkg.status['url'][1])
     else:
         td_class.append("good_url")
         url_str = "<a href=%s>Link</a>" % pkg.url
     f.write("  <td class=\"%s\">%s</td>\n" %
             (" ".join(td_class), url_str))
 
+    # CVEs
+    td_class = ["centered"]
+    if len(pkg.cves) == 0:
+        td_class.append("correct")
+    else:
+        td_class.append("wrong")
+    f.write("  <td class=\"%s\">\n" % " ".join(td_class))
+    for cve in pkg.cves:
+        f.write("   <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
+    f.write("  </td>\n")
+
     f.write(" </tr>\n")
 
 
@@ -618,6 +817,7 @@ def dump_html_all_pkgs(f, packages):
 <td class=\"centered\">Latest version</td>
 <td class=\"centered\">Warnings</td>
 <td class=\"centered\">Upstream URL</td>
+<td class=\"centered\">CVEs</td>
 </tr>
 """)
     for pkg in sorted(packages):
@@ -656,6 +856,10 @@ def dump_html_stats(f, stats):
             stats["version-not-uptodate"])
     f.write("<tr><td>Packages with no known upstream version</td><td>%s</td></tr>\n" %
             stats["version-unknown"])
+    f.write("<tr><td>Packages affected by CVEs</td><td>%s</td></tr>\n" %
+            stats["pkg-cves"])
+    f.write("<tr><td>Total number of CVEs affecting all packages</td><td>%s</td></tr>\n" %
+            stats["total-cves"])
     f.write("</table>\n")
 
 
@@ -673,7 +877,7 @@ def dump_html(packages, stats, date, commit, output):
         f.write(html_footer)
 
 
-def dump_json(packages, stats, date, commit, output):
+def dump_json(packages, defconfigs, stats, date, commit, output):
     # Format packages as a dictionnary instead of a list
     # Exclude local field that does not contains real date
     excluded_fields = ['url_worker', 'name']
@@ -684,6 +888,12 @@ def dump_json(packages, stats, date, commit, output):
             if k not in excluded_fields
         } for pkg in packages
     }
+    defconfigs = {
+        d.name: {
+            k: v
+            for k, v in d.__dict__.items()
+        } for d in defconfigs
+    }
     # Aggregate infrastructures into a single dict entry
     statistics = {
         k: v
@@ -694,6 +904,8 @@ def dump_json(packages, stats, date, commit, output):
     # The actual structure to dump, add commit and date to it
     final = {'packages': pkgs,
              'stats': statistics,
+             'defconfigs': defconfigs,
+             'package_status_checks': Package.status_checks,
              'commit': commit,
              'date': str(date)}
 
@@ -702,18 +914,24 @@ def dump_json(packages, stats, date, commit, output):
         f.write('\n')
 
 
+def resolvepath(path):
+    return os.path.abspath(os.path.expanduser(path))
+
+
 def parse_args():
     parser = argparse.ArgumentParser()
     output = parser.add_argument_group('output', 'Output file(s)')
-    output.add_argument('--html', dest='html', action='store',
+    output.add_argument('--html', dest='html', type=resolvepath,
                         help='HTML output file')
-    output.add_argument('--json', dest='json', action='store',
+    output.add_argument('--json', dest='json', type=resolvepath,
                         help='JSON output file')
     packages = parser.add_mutually_exclusive_group()
     packages.add_argument('-n', dest='npackages', type=int, action='store',
                           help='Number of packages')
     packages.add_argument('-p', dest='packages', action='store',
                           help='List of packages (comma separated)')
+    parser.add_argument('--nvd-path', dest='nvd_path',
+                        help='Path to the local NVD database', type=resolvepath)
     args = parser.parse_args()
     if not args.html and not args.json:
         parser.error('at least one of --html or --json (or both) is required')
@@ -727,10 +945,16 @@ def __main__():
     else:
         package_list = None
     date = datetime.datetime.utcnow()
-    commit = subprocess.check_output(['git', 'log', 'master', '-n', '1',
-                                      '--pretty=format:%H']).splitlines()[0]
+    commit = subprocess.check_output(['git', 'rev-parse',
+                                      'HEAD']).splitlines()[0].decode()
     print("Build package list ...")
     packages = get_pkglist(args.npackages, package_list)
+    print("Getting developers ...")
+    developers = parse_developers()
+    print("Build defconfig list ...")
+    defconfigs = get_defconfig_list()
+    for d in defconfigs:
+        d.set_developers(developers)
     print("Getting package make info ...")
     package_init_make_info()
     print("Getting package details ...")
@@ -742,10 +966,16 @@ def __main__():
         pkg.set_check_package_warnings()
         pkg.set_current_version()
         pkg.set_url()
+        pkg.set_developers(developers)
     print("Checking URL status")
-    check_package_urls(packages)
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(check_package_urls(packages))
     print("Getting latest versions ...")
-    check_package_latest_version(packages)
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(check_package_latest_version(packages))
+    if args.nvd_path:
+        print("Checking packages CVEs")
+        check_package_cves(args.nvd_path, {p.name: p for p in packages})
     print("Calculate stats")
     stats = calculate_stats(packages)
     if args.html:
@@ -753,7 +983,7 @@ def __main__():
         dump_html(packages, stats, date, commit, args.html)
     if args.json:
         print("Write JSON")
-        dump_json(packages, stats, date, commit, args.json)
+        dump_json(packages, defconfigs, stats, date, commit, args.json)
 
 
 __main__()

support/scripts/pycompile.py

@@ -1,69 +1,93 @@
 #!/usr/bin/env python
 
-'''Wrapper for python2 and python3 around compileall to raise exception
-when a python byte code generation failed.
+"""
+Byte compile all .py files from provided directories. This script is an
+alternative implementation of compileall.compile_dir written with
+cross-compilation in mind.
+"""
 
-Inspired from:
-   http://stackoverflow.com/questions/615632/how-to-detect-errors-from-compileall-compile-dir
-'''
 from __future__ import print_function
-import sys
-import py_compile
-import compileall
+
 import argparse
+import os
+import py_compile
+import re
+import sys
 
 
-def check_for_errors(comparison):
-    '''Wrap comparison operator with code checking for PyCompileError.
-    If PyCompileError was raised, re-raise it again to abort execution,
-    otherwise perform comparison as expected.
-    '''
-    def operator(self, other):
-        exc_type, value, traceback = sys.exc_info()
-        if exc_type is not None and issubclass(exc_type,
-                                               py_compile.PyCompileError):
-            print("Cannot compile %s" % value.file)
-            raise value
+def compile_one(host_path, strip_root=None, verbose=False):
+    """
+    Compile a .py file into a .pyc file located next to it.
 
-        return comparison(self, other)
+    :arg host_path:
+        Absolute path to the file to compile on the host running the build.
+    :arg strip_root:
+        Prefix to remove from the original source paths encoded in compiled
+        files.
+    :arg verbose:
+        Print compiled file paths.
+    """
+    if os.path.islink(host_path) or not os.path.isfile(host_path):
+        return  # only compile real files
 
-    return operator
+    if not re.match(r"^[_A-Za-z][_A-Za-z0-9]+\.py$",
+                    os.path.basename(host_path)):
+        return  # only compile "importable" python modules
+
+    if strip_root is not None:
+        # determine the runtime path of the file (i.e.: relative path to root
+        # dir prepended with "/").
+        runtime_path = os.path.join("/", os.path.relpath(host_path, strip_root))
+    else:
+        runtime_path = host_path
+
+    if verbose:
+        print("  PYC  {}".format(runtime_path))
+
+    # will raise an error if the file cannot be compiled
+    py_compile.compile(host_path, cfile=host_path + "c",
+                       dfile=runtime_path, doraise=True)
 
 
-class ReportProblem(int):
-    '''Class that pretends to be an int() object but implements all of its
-    comparison operators such that it'd detect being called in
-    PyCompileError handling context and abort execution
-    '''
-    VALUE = 1
-
-    def __new__(cls, *args, **kwargs):
-        return int.__new__(cls, ReportProblem.VALUE, **kwargs)
-
-    @check_for_errors
-    def __lt__(self, other):
-        return ReportProblem.VALUE < other
-
-    @check_for_errors
-    def __eq__(self, other):
-        return ReportProblem.VALUE == other
-
-    def __ge__(self, other):
-        return not self < other
-
-    def __gt__(self, other):
-        return not self < other and not self == other
-
-    def __ne__(self, other):
-        return not self == other
+def existing_dir_abs(arg):
+    """
+    argparse type callback that checks that argument is a directory and returns
+    its absolute path.
+    """
+    if not os.path.isdir(arg):
+        raise argparse.ArgumentTypeError('no such directory: {!r}'.format(arg))
+    return os.path.abspath(arg)
 
 
-parser = argparse.ArgumentParser(description='Compile Python source files in a directory tree.')
-parser.add_argument("target", metavar='DIRECTORY',
-                    help='Directory to scan')
-parser.add_argument("--force", action='store_true',
-                    help="Force compilation even if alread compiled")
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("dirs", metavar="DIR", nargs="+", type=existing_dir_abs,
+                        help="Directory to recursively scan and compile")
+    parser.add_argument("--strip-root", metavar="ROOT", type=existing_dir_abs,
+                        help="""
+                        Prefix to remove from the original source paths encoded
+                        in compiled files
+                        """)
+    parser.add_argument("--verbose", action="store_true",
+                        help="Print compiled files")
 
-args = parser.parse_args()
+    args = parser.parse_args()
 
-compileall.compile_dir(args.target, force=args.force, quiet=ReportProblem())
+    try:
+        for d in args.dirs:
+            if args.strip_root and ".." in os.path.relpath(d, args.strip_root):
+                parser.error("DIR: not inside ROOT dir: {!r}".format(d))
+            for parent, _, files in os.walk(d):
+                for f in files:
+                    compile_one(os.path.join(parent, f), args.strip_root,
+                                args.verbose)
+
+    except Exception as e:
+        print("error: {}".format(e))
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

support/scripts/setlocalversion

@@ -19,19 +19,14 @@ cd "${1:-.}" || usage
 # Check for git and a git repo.
 if head=`git rev-parse --verify --short HEAD 2>/dev/null`; then
 
-	# If we are at a tagged commit (like "v2.6.30-rc6"), we ignore it,
-	# because this version is defined in the top level Makefile.
-	if [ -z "`git describe --exact-match 2>/dev/null`" ]; then
+	atag="`git describe 2>/dev/null`"
 
-		# If we are past a tagged commit (like "v2.6.30-rc5-302-g72357d5"),
-		# we pretty print it.
-		if atag="`git describe 2>/dev/null`"; then
-			echo "$atag" | awk -F- '{printf("-%05d-%s", $(NF-1),$(NF))}'
-
-		# If we don't have a tag at all we print -g{commitish}.
-		else
-			printf '%s%s' -g $head
-		fi
+	# Show -g<commit> if we have no tag, or just the tag
+	# otherwise.
+	if [ -z "${atag}" ] ; then
+		printf "%s%s" -g ${head}
+	else
+		printf ${atag}
 	fi
 
 	# Is this git on svn?
@@ -53,13 +48,29 @@ if head=`git rev-parse --verify --short HEAD 2>/dev/null`; then
 fi
 
 # Check for mercurial and a mercurial repo.
+# In the git case, 'git describe' will show the latest tag, and unless we are
+# exactly on that tag, the number of commits since then, and last commit id.
+# Mimic something similar in the Mercurial case.
 if hgid=`HGRCPATH= hg id --id --tags 2>/dev/null`; then
 	tag=`printf '%s' "$hgid" | cut -d' ' -f2 --only-delimited`
 
 	# Do we have an untagged version?
 	if [ -z "$tag" -o "$tag" = tip ]; then
+		# current revision is not tagged, determine latest tag
+		latesttag=`HGRCPATH= hg log -r. -T '{latesttag}' 2>/dev/null`
+		# In case there is more than one tag on the latest tagged commit,
+		# 'latesttag' will separate them by colon (:). We'll retain this.
+		# In case there is no tag at all, 'null' will be returned.
+		if [ "$latesttag" = "null" ]; then
+			latesttag=''
+		fi
+
+		# add the commit id
 		id=`printf '%s' "$hgid" | sed 's/[+ ].*//'`
-		printf '%s%s' -hg "$id"
+		printf '%s%s%s' "${latesttag}" -hg "$id"
+	else
+		# current revision is tagged, just print the tag
+		printf ${tag}
 	fi
 
 	# Are there uncommitted changes?

support/scripts/size-stats

@@ -22,6 +22,7 @@ import os.path
 import argparse
 import csv
 import collections
+import math
 
 try:
     import matplotlib
@@ -32,8 +33,13 @@ except ImportError:
     sys.stderr.write("You need python-matplotlib to generate the size graph\n")
     exit(1)
 
-colors = ['#e60004', '#009836', '#2e1d86', '#ffed00',
-          '#0068b5', '#f28e00', '#940084', '#97c000']
+
+class Config:
+    biggest_first = False
+    iec = False
+    size_limit = 0.01
+    colors = ['#e60004', '#f28e00', '#ffed00', '#940084',
+              '#2e1d86', '#0068b5', '#009836', '#97c000']
 
 
 #
@@ -66,8 +72,8 @@ def add_file(filesdict, relpath, abspath, pkg):
 #
 def build_package_dict(builddir):
     filesdict = {}
-    with open(os.path.join(builddir, "build", "packages-file-list.txt")) as filelistf:
-        for l in filelistf.readlines():
+    with open(os.path.join(builddir, "build", "packages-file-list.txt")) as f:
+        for l in f.readlines():
             pkg, fpath = l.split(",", 1)
             # remove the initial './' in each file path
             fpath = fpath.strip()[2:]
@@ -127,23 +133,46 @@ def build_package_size(filesdict, builddir):
 # outputf: output file for the graph
 #
 def draw_graph(pkgsize, outputf):
+    def size2string(sz):
+        if Config.iec:
+            divider = 1024.0
+            prefixes = ['', 'Ki', 'Mi', 'Gi', 'Ti']
+        else:
+            divider = 1000.0
+            prefixes = ['', 'k', 'M', 'G', 'T']
+        while sz > divider and len(prefixes) > 1:
+            prefixes = prefixes[1:]
+            sz = sz/divider
+        # precision is made so that there are always at least three meaningful
+        # digits displayed (e.g. '3.14' and '10.4', not just '3' and '10')
+        precision = int(2-math.floor(math.log10(sz))) if sz < 1000 else 0
+        return '{:.{prec}f} {}B'.format(sz, prefixes[0], prec=precision)
+
     total = sum(pkgsize.values())
     labels = []
     values = []
     other_value = 0
-    for (p, sz) in sorted(pkgsize.items(), key=lambda x: x[1]):
-        if sz < (total * 0.01):
+    unknown_value = 0
+    for (p, sz) in sorted(pkgsize.items(), key=lambda x: x[1],
+                          reverse=Config.biggest_first):
+        if sz < (total * Config.size_limit):
             other_value += sz
+        elif p == "unknown":
+            unknown_value = sz
         else:
-            labels.append("%s (%d kB)" % (p, sz / 1000.))
+            labels.append("%s (%s)" % (p, size2string(sz)))
             values.append(sz)
-    labels.append("Other (%d kB)" % (other_value / 1000.))
-    values.append(other_value)
+    if unknown_value != 0:
+        labels.append("Unknown (%s)" % (size2string(unknown_value)))
+        values.append(unknown_value)
+    if other_value != 0:
+        labels.append("Other (%s)" % (size2string(other_value)))
+        values.append(other_value)
 
     plt.figure()
     patches, texts, autotexts = plt.pie(values, labels=labels,
                                         autopct='%1.1f%%', shadow=True,
-                                        colors=colors)
+                                        colors=Config.colors)
     # Reduce text size
     proptease = fm.FontProperties()
     proptease.set_size('xx-small')
@@ -151,7 +180,8 @@ def draw_graph(pkgsize, outputf):
     plt.setp(texts, fontproperties=proptease)
 
     plt.suptitle("Filesystem size per package", fontsize=18, y=.97)
-    plt.title("Total filesystem size: %d kB" % (total / 1000.), fontsize=10, y=.96)
+    plt.title("Total filesystem size: %s" % (size2string(total)), fontsize=10,
+              y=.96)
     plt.savefig(outputf)
 
 
@@ -209,32 +239,70 @@ def gen_packages_csv(pkgsizes, outputf):
     total = sum(pkgsizes.values())
     with open(outputf, 'w') as csvfile:
         wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
-        wr.writerow(["Package name", "Package size", "Package size in system (%)"])
+        wr.writerow(["Package name", "Package size",
+                     "Package size in system (%)"])
         for (pkg, size) in pkgsizes.items():
             wr.writerow([pkg, size, "%.1f" % (float(size) / total * 100)])
 
 
-parser = argparse.ArgumentParser(description='Draw size statistics graphs')
+#
+# Our special action for --iec, --binary, --si, --decimal
+#
+class PrefixAction(argparse.Action):
+    def __init__(self, option_strings, dest, **kwargs):
+        for key in ["type", "nargs"]:
+            if key in kwargs:
+                raise ValueError('"{}" not allowed'.format(key))
+        super(PrefixAction, self).__init__(option_strings, dest, nargs=0,
+                                           type=bool, **kwargs)
 
-parser.add_argument("--builddir", '-i', metavar="BUILDDIR", required=True,
-                    help="Buildroot output directory")
-parser.add_argument("--graph", '-g', metavar="GRAPH",
-                    help="Graph output file (.pdf or .png extension)")
-parser.add_argument("--file-size-csv", '-f', metavar="FILE_SIZE_CSV",
-                    help="CSV output file with file size statistics")
-parser.add_argument("--package-size-csv", '-p', metavar="PKG_SIZE_CSV",
-                    help="CSV output file with package size statistics")
-args = parser.parse_args()
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, option_string in ["--iec", "--binary"])
 
-# Find out which package installed what files
-pkgdict = build_package_dict(args.builddir)
 
-# Collect the size installed by each package
-pkgsize = build_package_size(pkgdict, args.builddir)
+def main():
+    parser = argparse.ArgumentParser(description='Draw size statistics graphs')
 
-if args.graph:
-    draw_graph(pkgsize, args.graph)
-if args.file_size_csv:
-    gen_files_csv(pkgdict, pkgsize, args.file_size_csv)
-if args.package_size_csv:
-    gen_packages_csv(pkgsize, args.package_size_csv)
+    parser.add_argument("--builddir", '-i', metavar="BUILDDIR", required=True,
+                        help="Buildroot output directory")
+    parser.add_argument("--graph", '-g', metavar="GRAPH",
+                        help="Graph output file (.pdf or .png extension)")
+    parser.add_argument("--file-size-csv", '-f', metavar="FILE_SIZE_CSV",
+                        help="CSV output file with file size statistics")
+    parser.add_argument("--package-size-csv", '-p', metavar="PKG_SIZE_CSV",
+                        help="CSV output file with package size statistics")
+    parser.add_argument("--biggest-first", action='store_true',
+                        help="Sort packages in decreasing size order, " +
+                             "rather than in increasing size order")
+    parser.add_argument("--iec", "--binary", "--si", "--decimal",
+                        action=PrefixAction,
+                        help="Use IEC (binary, powers of 1024) or SI (decimal, "
+                             "powers of 1000, the default) prefixes")
+    parser.add_argument("--size-limit", "-l", type=float,
+                        help='Under this size ratio, files are accounted to ' +
+                             'the generic "Other" package. Default: 0.01 (1%%)')
+    args = parser.parse_args()
+
+    Config.biggest_first = args.biggest_first
+    Config.iec = args.iec
+    if args.size_limit is not None:
+        if args.size_limit < 0.0 or args.size_limit > 1.0:
+            parser.error("--size-limit must be in [0.0..1.0]")
+        Config.size_limit = args.size_limit
+
+    # Find out which package installed what files
+    pkgdict = build_package_dict(args.builddir)
+
+    # Collect the size installed by each package
+    pkgsize = build_package_size(pkgdict, args.builddir)
+
+    if args.graph:
+        draw_graph(pkgsize, args.graph)
+    if args.file_size_csv:
+        gen_files_csv(pkgdict, pkgsize, args.file_size_csv)
+    if args.package_size_csv:
+        gen_packages_csv(pkgsize, args.package_size_csv)
+
+
+if __name__ == "__main__":
+    main()