Merge branch 'distro-config' into 'master'

Distro config reunification

See merge request security-tracker-team/security-tracker!48

5 files changed, 109 insertions, 186 deletions

diff --git a/lib/debian-releases.mk b/lib/debian-releases.mk
index c868d6edfd..ecb72a23a6 100644
--- a/lib/debian-releases.mk
+++ b/lib/debian-releases.mk
@@ -1,18 +1,22 @@
 # This file defines the variables describing all Debian repositories
 # that need to be fetched in the "update-packages" process
 
+define get_config =
+$(shell jq -r $(1) 'data/config.json')
+endef
+
 # backports suites only have Sources.xz and respective Packages.xz
 # available.
 # Cf. as well https://bugs.debian.org/664866
 #BACKPORT_RELEASES := $(OLDSTABLE) $(STABLE)
-SECURITY_RELEASES := $(OLDOLDSTABLE) $(OLDSTABLE) $(STABLE) $(TESTING)
-MAIN_RELEASES := $(SECURITY_RELEASES) sid
+MAIN_RELEASES = $(call get_config, '.distributions | to_entries[] | select(.value.release) | .key')
+SECURITY_RELEASES = $(filter-out sid, $(MAIN_RELEASES))
 
 # Define the variables for the release on the main mirror
 define add_main_release =
 $(1)_MIRROR = $$(MIRROR)
 $(1)_DIST = $(1)
-$(1)_ARCHS ?= amd64 arm64 armel armhf i386 mips64el mipsel ppc64el s390x
+$(1)_ARCHS = $(call get_config, '.distributions.$(1).architectures[]')
 $(1)_RELEASE = $(1)
 $(1)_SUBRELEASE =
 RELEASES += $(1)
diff --git a/lib/python/config.py b/lib/python/config.py
new file mode 100644
index 0000000000..c445dadb6c
--- /dev/null
+++ b/lib/python/config.py
@@ -0,0 +1,59 @@
+# config.py -- methods to read global configuration from data/config.json
+# Copyright (C) 2019 Emilio Pozuelo Monfort <pochu@debian.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+# TODO: the OrderedDict use can be dropped once we use Python 3 (>= 3.7)
+from collections import OrderedDict
+import json
+import os
+
+_config = None
+
+def get_config():
+    global _config
+    if not _config:
+        d = os.path.dirname(os.path.abspath(__file__))
+
+        with open(d + '/../../data/config.json') as f:
+            config = json.load(f, object_pairs_hook=OrderedDict)
+
+        _config = config['distributions']
+
+    return _config
+
+def get_supported_releases():
+    config = get_config()
+
+    return [d for d in config.keys() if 'release' in config[d]]
+
+def get_all_releases():
+    config = get_config()
+
+    return config.keys()
+
+def get_release_codename(release, suffix=''):
+    config = get_config()
+
+    for r in config.keys():
+        if 'release' in config[r] and config[r]['release'] == release:
+            return r + suffix
+
+    return None
+
+def get_release_alias(codename):
+    config = get_config()
+
+    return config[codename]['release']
diff --git a/lib/python/debian_support.py b/lib/python/debian_support.py
index 84f66815c1..d405440e9f 100644
--- a/lib/python/debian_support.py
+++ b/lib/python/debian_support.py
@@ -37,6 +37,8 @@ except ImportError:
 import apt_pkg
 apt_pkg.init()
 
+import config
+
 # Timeout for downloads.
 TIMEOUT = 30
 
@@ -194,8 +196,7 @@ class Release(PseudoEnum): pass
 
 def listReleases():
     releases = {}
-    rels = ("experimental", # For use in [brackets] in the list files.
-            "potato", "woody", "sarge", "etch", "lenny", "squeeze", "wheezy", "jessie", "stretch", "buster", "bullseye", "sid")
+    rels = ["experimental"] + config.get_all_releases()
     for r in range(len(rels)):
         releases[rels[r]] = Release(rels[r], r)
     Release.releases = releases
diff --git a/lib/python/dist_config.py b/lib/python/dist_config.py
deleted file mode 100644
index 107f63a088..0000000000
--- a/lib/python/dist_config.py
+++ /dev/null
@@ -1,97 +0,0 @@
-# dist_config.py -- describe how the Debian package database is assembled
-# Copyright (C) 2008 Florian Weimer <fw@deneb.enyo.de>
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-"""
-This Python moule describes how different views of the Debian package
-database are assembled from a set of on-disk files.
-
-Each view is labeled by a purpose.  Currently defined purposes are:
-
-  overview: Used to generate the release overview web page.  This
-            should not contain vulnerabilities which the security team
-            considers processed.
-
-  debsecan: Used to generate the "fix is available" data for debsecan.
-            This should reflect the recommended set of sources.list
-            entries for the release.
-"""
-
-######################################################################
-# Configuration section
-######################################################################
-
-def apply_config():
-    # Invoked at the end of the file.  Edit this to suit your needs.
-
-    common_archs = 'amd64,armel,i386,mips,mipsel,powerpc'.split(',')
-    squeeze_archs = common_archs + ['s390','ia64','kfreebsd-amd64','kfreebsd-i386','sparc' ]
-    wheezy_archs = [ 'amd64','armel','armhf','i386' ]
-    jessie_archs = [ 'amd64','armel','armhf','i386' ]
-    stretch_archs = [ 'amd64','arm64','armel','armhf','i386','mips','mips64el','mipsel','ppc64el','s390x' ]
-    buster_archs = [ 'amd64','arm64','armel','armhf','i386','mips','mips64el','mipsel','ppc64el','s390x' ]
-    bullseye_archs = [ 'amd64','arm64','armel','armhf','i386','mips64el','mipsel','ppc64el','s390x' ]
-    sid_archs = [ 'amd64','arm64','armel','armhf','i386','mips64el','mipsel','ppc64el','s390x' ]
-
-    add_release(name='squeeze',
-                architectures=squeeze_archs,
-                )
-    
-    add_release(name='wheezy',
-                architectures=wheezy_archs,
-                )
-    
-    add_release(name='jessie',
-                architectures=jessie_archs,
-                )
-
-    add_release(name='stretch',
-                architectures=stretch_archs,
-                )
-
-    add_release(name='buster',
-                architectures=buster_archs,
-                )
-
-    add_release(name='bullseye',
-                architectures=bullseye_archs,
-                )
-
-    add_release(name='sid',
-                architectures=sid_archs,
-                )
-
-######################################################################
-# Support routines
-######################################################################
-
-releases = {}
-
-def add_release(name, architectures,
-                debsecan_part=('', 'security'),
-                overview_part=('', 'security', 'proposed-updates')):
-    import debian_support
-    name = debian_support.internRelease(name)
-    if name in releases:
-        raise ValueError("duplicate release", name)
-    releases[name] = {'architectures' : architectures,
-                      'purpose' : {'debsecan' : debsecan_part,
-                                   'overview' : overview_part}}
-
-# Run the code in the configuration section
-
-apply_config()
-del apply_config
diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index 2c5a3ba58a..2404549a79 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -43,8 +43,8 @@ import sys
 import types
 import zlib
 
+import config
 import debian_support
-import dist_config
 
 class InsertError(Exception):
     """Class for capturing insert errors.
@@ -464,6 +464,7 @@ class DB:
             """)
 
     def _initViews(self, cursor):
+        testing = config.get_release_codename('testing')
         cursor.execute(
             """CREATE TEMPORARY VIEW testing_status AS
             SELECT DISTINCT sp.name AS package, st.bug_name AS bug,
@@ -479,7 +480,7 @@ class DB:
             COALESCE((SELECT NOT vulnerable
             FROM source_packages AS tsecp, source_package_status AS tsecst
             WHERE tsecp.name = sp.name
-            AND tsecp.release = 'bullseye' AND tsecp.subrelease = 'security'
+            AND tsecp.release = '%s' AND tsecp.subrelease = 'security'
             AND tsecp.archive = sp.archive
             AND tsecst.bug_name = st.bug_name
             AND tsecst.package = tsecp.rowid), 0) AS testing_security_fixed,
@@ -488,13 +489,19 @@ class DB:
             (EXISTS (SELECT * FROM package_notes_nodsa AS pnd
             WHERE pnd.bug_name = st.bug_name
             AND pnd.package = sp.name
-            AND pnd.release = 'bullseye')) AS no_dsa
+            AND pnd.release = '%s')) AS no_dsa
             FROM source_package_status AS st, source_packages AS sp
             WHERE st.vulnerable > 0 AND sp.rowid = st.package
-            AND sp.release = 'bullseye' AND sp.subrelease = ''
-            ORDER BY sp.name, st.urgency, st.bug_name""")
+            AND sp.release = '%s' AND sp.subrelease = ''
+            ORDER BY sp.name, st.urgency, st.bug_name"""
+            % (testing, testing, testing))
 
-        for (name, nickname) in (('stable', 'buster'), ('oldstable', 'stretch'), ('oldoldstable', 'jessie'),):
+        releases = config.get_supported_releases()
+        releases.remove(config.get_release_codename('testing'))
+        releases.remove('sid')
+
+        for release in releases:
+            alias = config.get_release_alias(release)
             cursor.execute(
                 """CREATE TEMPORARY VIEW %s_status AS
                 SELECT DISTINCT sp.name AS package, st.bug_name AS bug,
@@ -521,7 +528,7 @@ class DB:
                 AND secst.bug_name = st.bug_name
                 AND secst.package = secp.rowid), 0)
                 ORDER BY sp.name, urgency_to_number(urgency), st.bug_name"""
-                % (name, nickname, nickname, nickname, nickname))
+                % (alias, release, release, release, release))
 
         cursor.execute(
             """CREATE TEMPORARY VIEW debian_cve AS
@@ -582,7 +589,7 @@ class DB:
                 return -1
         self.db.createscalarfunction("subreleasepart_to_number", subreleasepart_to_number, 1)
 
-        releases = ['potato', 'woody', 'sarge', 'etch', 'lenny', 'squeeze', 'wheezy', 'jessie', 'stretch', 'buster', 'bullseye', 'sid']
+        releases = config.get_all_releases()
         def release_to_number(u):
             try:
                 return releases.index(u)
@@ -740,9 +747,6 @@ class DB:
             if unchanged:
                 continue
 
-            if release == 'squeeze-lts':
-                release = 'squeeze'
-                subrelease = 'lts'
             cursor.execute(
                 """DELETE FROM source_packages
                 WHERE release = ? AND subrelease = ? AND archive = ?""",
@@ -803,9 +807,6 @@ class DB:
                 raise ValueError("invalid file name: " + repr(filename))
 
             (release, subrelease, archive, architecture) = match.groups()
-            if release == 'squeeze-lts':
-                release = 'squeeze'
-                subrelease = 'lts'
             (unch, parsed) = self._parseFile(cursor, filename)
             unchanged = unchanged and unch
             for name in parsed.keys():
@@ -1140,7 +1141,7 @@ class DB:
         """Calculate vulnerable packages.
 
         To each package note, a release-specific vulnerability status
-        is attached.  Currently, only bullseye/testing is processed.
+        is attached.  Currently, only testing is processed.
 
         Returns a list strings describing inconsistencies.
         """
@@ -1156,17 +1157,18 @@ class DB:
         # The following does not work because stable->security ->
         # testing -> unstable propagation is no longer available.
         if False:
-            # Ignore bullseye/testing because stable issues may be
+            # Ignore testing because stable issues may be
             # fast-tracked into testing, bypassing unstable.
+            testing = config.get_release_codename('testing')
             for (bug_name, pkg_name, rel, unstable_ver, rel_ver) \
                     in list(cursor.execute(
             """SELECT a.bug_name, a.package, b.release,
             a.fixed_version, b.fixed_version
             FROM package_notes a, package_notes b
             WHERE a.bug_name = b.bug_name AND a.package = b.package
-            AND a.release = '' AND b.release NOT IN ('', 'bullseye')
+            AND a.release = '' AND b.release NOT IN ('', '%s')
             AND a.fixed_version IS NOT NULL
-            AND a.fixed_version_id < b.fixed_version_id""")):
+            AND a.fixed_version_id < b.fixed_version_id""" % (testing,))):
                 b = bugs.BugFromDB(cursor, bug_name)
                 result.append("%s:%d: inconsistent versions for package %s"
                               % (b.source_file, b.source_line, pkg_name))
@@ -1280,10 +1282,13 @@ class DB:
             "SELECT name FROM bugs WHERE NOT not_for_us"):
 
             self._calcUnstable(c, bug_name)
-            self._calcTesting(c, bug_name, 'testing', 'bullseye')
-            self._calcTesting(c, bug_name, 'stable', 'buster')
-            self._calcTesting(c, bug_name, 'oldstable', 'stretch')
-            self._calcTesting(c, bug_name, 'oldoldstable', 'jessie')
+
+            for release in config.get_supported_releases():
+                if release == 'sid':
+                    continue
+
+                alias = config.get_release_alias(release)
+                self._calcTesting(c, bug_name, alias, release)
 
         return result
 
@@ -1452,12 +1457,10 @@ class DB:
         c.execute("""INSERT INTO vulnlist
         SELECT bug_name, package, id FROM package_notes WHERE release = ''""")
 
-        if release:
+        if release != 'sid':
             c.execute("""INSERT OR REPLACE INTO vulnlist
             SELECT bug_name, package, id FROM package_notes
             WHERE release = ?""", (release,))
-        else:
-            release = 'sid'
 
         urgency_to_flag = {'low' : 'L', 'medium' : 'M', 'high' : 'H',
                            'not yet assigned' : ' '}
@@ -1732,7 +1735,7 @@ class DB:
 
             store_value('release/1/' + release, '\n'.join(result))
 
-        for release in ('sid', 'jessie', 'stretch', 'buster', 'bullseye'):
+        for release in config.get_supported_releases():
             gen_release(release)
 
         result = result_start
@@ -1745,7 +1748,7 @@ class DB:
 
     def calculateDebsecan(self):
         """Calculate all debsecan data."""
-        for release in ('', 'jessie', 'stretch', 'buster', 'bullseye'):
+        for release in config.get_supported_releases():
             self.calculateDebsecan0(release)
         self.calculateDebsecan1()
 
@@ -1778,13 +1781,16 @@ class DB:
         """A generator which returns tuples (RELEASE-LIST, VERSION),
         the available versions of the source package pkg."""
 
+        releases = config.get_supported_releases()
+        values = [pkg] + releases
+
         for (release, version) in cursor.execute(
             """SELECT release_name(release, subrelease, archive)
             AS release, version FROM source_packages
             WHERE name = ?
-            AND release IN ('jessie', 'stretch', 'buster', 'bullseye', 'sid')
+            AND release IN (""" + ",".join("?" * len(releases)) + """)
             GROUP BY release, version
-            ORDER BY release_to_number(release), subrelease_to_number(subrelease), version COLLATE version""", (pkg,)):
+            ORDER BY release_to_number(release), subrelease_to_number(subrelease), version COLLATE version""", values):
             yield release, version
 
     def getBinaryPackageVersions(self, cursor, pkg):
@@ -1830,6 +1836,9 @@ class DB:
         RELEASE-LIST, VERSION, VULNERABLE-FLAG) of source packages
         which are related to the given bug."""
 
+        releases = config.get_supported_releases()
+        values = [bug] + releases
+
         for (package, releases, version, vulnerable) in cursor.execute(
             """SELECT package, string_list(release), version, vulnerable
             FROM (SELECT p.name AS package,
@@ -1837,10 +1846,10 @@ class DB:
             p.version AS version, s.vulnerable AS vulnerable
             FROM source_package_status AS s, source_packages AS p
             WHERE s.bug_name = ? AND p.rowid = s.package
-            AND release in ('jessie', 'stretch', 'buster', 'bullseye', 'sid'))
+            AND release in (""" + ",".join("?" * len(releases)) + """))
             GROUP BY package, version, vulnerable
             ORDER BY package, releasepart_to_number(release), subreleasepart_to_number(release), version COLLATE version""",
-            (bug,)):
+            values):
             yield package, releases.split(', '), version, vulnerable
 
     def getBugsFromDebianBug(self, cursor, number):
@@ -2026,59 +2035,6 @@ class DB:
             ORDER BY n.package"""):
             yield (package, bugs.split(','), map(int, debian_bugs.split(',')))
 
-    def getEffectiveVersion(self, release, pkg, purpose, cache=None, cursor=None):
-        """Retrieve the effective version of a source package in a release.
-
-        The effective version is the version that matches the recommended
-        sources.list file for the intended purpose.  For suitable values
-        of purpose, see dist_config.
-        """
-        # The cache is structured as a (RELEASE, PACKAGE) => VAL
-        # dict, where VAL is either a dict PURPOSE => VERSION,
-        # a VERSION, or None.
-        if cache is not None:
-            sp = (release, pkg)
-            if sp in cache:
-                d = cache[sp]
-                if d.__class__ == dict:
-                    return d.get(purpose, None)
-                else:
-                    return d
-
-        if cursor is None:
-            cursor = self.cursor()
-
-        rel = dist_config.releases[release]
-        purposes = rel['purpose']
-        results = {}
-
-        Version = debian_support.Version
-        for (part, ver) in cursor.execute(
-            """SELECT DISTINCT subrelease, version FROM source_packages
-            WHERE release = ? AND name = ?""", (str(release), pkg)):
-            ver = Version(ver)
-            for (purpose, permitted) in purposes.items():
-                if part not in permitted:
-                    continue
-                if purpose in results:
-                    oldver = results[purpose]
-                    if ver <= oldver:
-                        continue
-                results[purpose] = ver
-
-        if cache is not None:
-            vers = set(map(str, results.values()))
-            l = len(vers)
-            if l == 1:
-                for r in vers:
-                    cache[sp] = Version(r)
-            elif l == 0:
-                cache[sp] = None
-            else:
-                cache[sp] = results
-
-        return results.get(purpose, None)
-
     def check(self, cursor=None):
         """Runs a simple consistency check and prints the results."""