lib/python/security_db.py (DB.initSchema):

  Add index on package_notes(package) (no schema version bump needed).
(DB.calculateDebsecan0):
  Renamed from DB.calculateDebsecan.
(DB.calculateDebsecan1):
  New method which generates version 1 format (with pinning support
  wtc.).
(DB.calculateDebsecan):
  Invokes both the version 0 and version 1 methods.

bin/update-db:
  Adjust accordingly.


git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@3129 e39458fd-73e7-0310-bf30-c45bca0a0e42

2 files changed, 204 insertions, 4 deletions

diff --git a/bin/update-db b/bin/update-db
index eadb529fcf..4bf2f312c5 100755
--- a/bin/update-db
+++ b/bin/update-db
@@ -78,8 +78,7 @@ if warnings:
 
 # debsecan data
 
-for release in ('', 'woody', 'sarge', 'etch'):
-    db.calculateDebsecan(release)
+db.calculateDebsecan()
 
 # Everything worked well.
 
diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index 3f1c5f85e3..7081e2e72e 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -233,6 +233,9 @@ class DB:
         cursor.execute(
             """CREATE UNIQUE INDEX package_notes_bug
             ON package_notes(bug_name, package, release)""")
+        cursor.execute(
+            """CREATE INDEX package_notes_package
+            ON package_notes(package)""")
 
         cursor.execute("""CREATE TABLE debian_bugs
         (bug INTEGER NOT NULL,
@@ -1229,8 +1232,8 @@ class DB:
         VALUES (?, ?, ?, ?)""",
               (bug_name, suite, status, pkgs))
 
-    def calculateDebsecan(self, release):
-        """Create data for the debsecan tool."""
+    def calculateDebsecan0(self, release):
+        """Create data for the debsecan tool (VERSION 0 format)."""
 
         c = self.cursor()
 
@@ -1344,6 +1347,204 @@ class DB:
 
         c.execute("DROP TABLE vulnlist")
 
+    def calculateDebsecan1(self):
+        """Calculates debsecan data (release-independent, VERSION 1)."""
+
+        c = self.cursor()
+
+        result_start = ['VERSION 1']
+        bug_to_index = {}
+        bug_to_remote_flag = {}
+
+        def fill_bug_to_index():
+            index = 0
+            for (bug, desc, remote) in c.execute(
+                """SELECT DISTINCT p.bug_name, b.description,
+                (SELECT range_remote FROM nvd_data
+                 WHERE cve_name = p.bug_name)
+                FROM package_notes AS p, bugs AS b
+                WHERE p.urgency <> 'unimportant'
+                AND name NOT LIKE 'FAKE-0000000-%'
+                AND b.name = p.bug_name
+                AND p.package_kind IN ('source', 'binary', 'unknown')
+                ORDER BY p.bug_name"""):
+                if remote is None:
+                    remote = '?'
+                elif remote:
+                    remote = 'R'
+                else:
+                    remote = ' '
+
+                # Normalize FAKE-* names a bit.  The line number (which
+                # makes the name unique) is completely useless for the
+                # client.
+
+                if bug[0:5] == "FAKE-":
+                    name = '-'.join(bug.split('-')[0:2])
+                else:
+                    name = bug
+
+                result_start.append("%s,,%s" % (name, desc))
+                bug_to_index[bug] = index
+                bug_to_remote_flag[bug] = remote
+                index += 1
+            result_start.append('')
+        fill_bug_to_index()
+
+        urgency_to_flag = {'low' : 'L', 'medium' : 'M', 'high' : 'H',
+                           'unknown' : ' '}
+
+        vuln_list = []
+        source_packages = {}
+        def fill_vuln_list(source_packages=source_packages):
+            for (bug, package) in list(c.execute(
+                """SELECT DISTINCT bug_name, package
+                FROM package_notes
+                WHERE bug_name NOT LIKE 'FAKE-0000000-%'
+                AND package_kind IN ('source', 'binary', 'unknown')
+                GROUP BY package, bug_name
+                ORDER BY package, bug_name""")):
+
+                unstable_fixed = '0'
+                total_urgency = ''
+                other_versions = {}
+                is_binary = False
+                is_unknown = False
+                fixed_releases = {}
+                for (release, kind, urgency, version) in list(c.execute(
+                    """SELECT release, package_kind, urgency, fixed_version
+                    FROM package_notes WHERE bug_name = ? AND package = ?""",
+                    (bug, package))):
+                    if total_urgency:
+                        if urgency == 'unknown':
+                            total_urgency = urgency
+                        elif total_urgency <> 'unknown' \
+                                 and bugs.internUrgency(urgency) \
+                                 > bugs.internUrgency(total_urgency):
+                            total_urgency = urgency
+                    else:
+                        total_urgency = urgency
+
+                    if kind == 'binary':
+                        is_binary = True
+                    elif kind == 'source':
+                        source_packages[package] = True
+                    else:
+                        is_unknown = True
+
+                    if release == '':
+                        unstable_fixed = version
+                        if version:
+                            v_ref = debian_support.Version(version)
+                            for (v,) in c.execute("""SELECT version
+                            FROM source_packages WHERE name = ?
+                            AND release = 'sid' AND subrelease = ''""",
+                                                  (package,)):
+                                if debian_support.Version(v) > v_ref:
+                                    fixed_releases['sid'] = True
+                                    break
+                    elif version is not None:
+                        fixed_releases[release] = True
+
+                        # Collect newer versions in the same release
+                        # (which are supposed to fix the same bug).
+
+                        v_ref = debian_support.Version(version)
+                        for (v,) in c.execute("""SELECT fixed_version
+                        FROM package_notes
+                        WHERE package = ? AND release = ?""",
+                                              (package, release)):
+                            if v is None:
+                                continue
+                            if debian_support.Version(v) > v_ref:
+                                other_versions[v] = True
+
+                        for (v,) in c.execute("""SELECT version
+                        FROM source_packages WHERE name = ?
+                        AND release = ? AND subrelease IN ('', 'security')""",
+                                              (package, release)):
+                            if debian_support.Version(v) > v_ref:
+                                other_versions[v] = True
+
+                # Check if the issue does not actually mark any packages
+                # as vulnerable.
+                if total_urgency == 'unimportant' \
+                       or (unstable_fixed == '0' and len(other_versions) == 0):
+                    continue
+
+                if unstable_fixed is None:
+                    unstable_fixed = ''
+                bs_flag = 'S'
+                if is_binary:
+                    assert not is_unknown
+                    bs_flag = 'B'
+                elif is_unknown:
+                    bs_flag = ' '
+
+                other_versions = other_versions.keys()
+                other_versions.sort()
+                other_versions = ' '.join(other_versions)
+
+                vuln_list.append(("%s,%d,%c%c%c"
+                                  % (package, bug_to_index[bug],
+                                     bs_flag, urgency_to_flag[total_urgency],
+                                     bug_to_remote_flag[bug]),
+                                  fixed_releases.keys(),
+                                  ",%s,%s"
+                                  % (unstable_fixed, other_versions)))
+        fill_vuln_list()
+        source_packages = source_packages.keys()
+        source_packages.sort()
+
+        def store_value(name, value):
+            value = base64.encodestring(zlib.compress(value, 9))
+            c.execute("""INSERT OR REPLACE INTO debsecan_data
+            VALUES (?, ?)""", (name, value))
+                       
+        def gen_release(release):
+            result = result_start[:]
+
+            for (prefix, releases, suffix) in vuln_list:
+                if release in releases:
+                    fixed = 'F'
+                else:
+                    fixed = ' '
+                result.append(prefix + fixed + suffix)
+            result.append('')
+
+            for sp in source_packages:
+                bp_list = []
+                for (bp,) in c.execute("""SELECT name FROM binary_packages
+                WHERE source = ? AND release = ? AND subrelease = ''
+                ORDER BY name""",
+                                       (sp, release)):
+                    bp_list.append(bp)
+                if bp_list <> [sp]:
+                    # We intentionally store the empty list, it means
+                    # that the source package is obsolete as a whole.
+                    result.append("%s,%s" % (sp, ' '.join(bp_list)))
+            result.append('')
+
+            store_value('release/1/' + release, '\n'.join(result))
+
+        for release in ('sid', 'etch', 'sarge', 'woody'):
+            gen_release(release)
+
+        result = result_start
+        for (prefix, release, suffix) in vuln_list:
+            result.append(prefix + ' ' + suffix)
+        result.append('')
+        result.append('')            
+        result.append('')
+        store_value ('release/1/GENERIC', '\n'.join(result))
+                
+    def calculateDebsecan(self):
+        """Calculate all debsecan data."""
+        for release in ('', 'woody', 'sarge', 'etch'):
+            self.calculateDebsecan0(release)
+        self.calculateDebsecan1()
+        
+
     def getDebsecan(self, name):
         """Returns the debsecan data item NAME."""
         for (data,) in self.cursor().execute(