Overhaul the source-package page

This commit addresses a long-standing bug where resolved bugs
disappear completely.  In addition, lts/security archives are no
longer shown separately, and no-dsa is marked explicitly.  The package
vulnerability state is taken from the database, so it is hopefully
quite accurate.

Remove security_db.DB.getBugsForSourcePackage() and replace it with a
global function security_db.getBugsForSourcePackage().  Add additional
named tuples BugForSourcePackage, BugForSourcePackageRelease,
BugsForSourcePackage_internal.

Add yellow CSS style.



git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@34502 e39458fd-73e7-0310-bf30-c45bca0a0e42

1 files changed, 116 insertions, 37 deletions

diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index 2844b2809f..c203125f3e 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -26,12 +26,15 @@ The data is kept in a SQLite 3 database.
 FIXME: Document the database schema once it is finished.
 """
 
+from apt_pkg import version_compare
 import apsw
 import base64
 import bugs
+from collections import namedtuple
 import cPickle
 import cStringIO
 import glob
+import itertools
 import os
 import os.path
 import re
@@ -39,8 +42,6 @@ import sys
 import types
 import zlib
 
-from collections import namedtuple
-
 import debian_support
 import dist_config
 
@@ -100,15 +101,121 @@ class SchemaMismatch(Exception):
 
     The caller is expected to remove and regenerate the database."""
 
-    def getBugsForSourcePackage(self, cursor, pkg, vulnerable, unimportant):
-        """Returns a generator for a list of (BUG, DESCRIPTION) pairs
-        which have the requested status.  Only bugs affecting supported
-        releases are returned."""
-
-# Returned by DB.getBugsForSourcePackage().
+# Returned by getBugsForSourcePackage().
+# all/open/unimportant/resolved are sequences of BugForSourcePackage.
 BugsForSourcePackage = namedtuple(
     "BugsForSourcePackage",
-    "bug description")
+    "all_releases all open unimportant resolved")
+
+# Returned by getBugsForSourcePackage().  releases is a sequence of
+# BugForSourcePackageRelease.  global_state is the aggregated state
+# across all releases (open/resolved/unimportant).
+BugForSourcePackage = namedtuple(
+    "BugForSourcePackage",
+    "bug description global_state releases")
+
+# Returned by getBugsForSourcePackage(). release, subrelease, version
+# come from the source_packages table.  vulnerable comes from
+# source_package_status.  state is open/no-dsa/resolved/unimportant
+# and inferred from vulnerable and package_notes_nodsa.
+BugForSourcePackageRelease = namedtuple(
+    "BugForSourcePackageRelease",
+    "release subrelease version vulnerable state reason")
+
+# Internally used by getBugsForSourcePackage().
+BugsForSourcePackage_internal = namedtuple(
+    "BugsForSourcePackage_internal",
+    "bug_name description release subrelease version vulnerable urgency")
+BugsForSourcePackage_query = \
+"""SELECT bugs.name AS bug_name, bugs.description AS description,
+    sp.release AS release, sp.subrelease AS subrelease, sp.version AS version,
+    st.vulnerable AS vulnerable, st.urgency AS urgency
+  FROM bugs
+  JOIN source_package_status st ON (bugs.name = st.bug_name)
+  JOIN source_packages sp ON (st.package = sp.rowid)
+  WHERE sp.name = ?
+  AND (bugs.name LIKE 'CVE-%' OR bugs.name LIKE 'TEMP-%')
+  ORDER BY bugs.name DESC, sp.release"""
+# Sort order is important for the groupby operation below.
+
+def getBugsForSourcePackage(cursor, pkg):
+    data = [BugsForSourcePackage_internal(*row) for row in
+            cursor.execute(BugsForSourcePackage_query, (pkg,))]
+    # Filter out special releases such as backports.
+    data = [row for row in data
+            if debian_support.internRelease(row.release) is not None]
+    # Obtain the set of releases actually in used, by canonical order.
+    all_releases = tuple(sorted(set(row.release for row in data),
+                                   key = debian_support.internRelease))
+    # dict from (bug_name, release) to the no-dsa reason/comment string.
+    no_dsas = {}
+    for bug_name, release, reason in cursor.execute(
+            """SELECT bug_name, release, comment FROM package_notes_nodsa
+            WHERE package = ?""", (pkg,)):
+        no_dsas[(bug_name, release)] = reason
+
+    all_bugs = []
+    # Group by bug name.
+    for bug_name, data in itertools.groupby(data,
+                                            lambda row: row.bug_name):
+        data = tuple(data)
+        description = data[0].description
+        open_seen = False
+        unimportant_seen = False
+        releases = {}
+        # Group by release.
+        for release, data1 in itertools.groupby(data, lambda row: row.release):
+            data1 = tuple(data1)
+            # The best row is the row with the highest version number.
+            # If there is a tie, the empty subrelease row wins.
+            best_row = data1[0]
+            for row in data1[1:]:
+                cmpresult = version_compare(row.version, best_row.version)
+                if cmpresult > 0 \
+                   or (cmpresult == 0 and row.subrelease == ''):
+                    best_row = row
+            reason = None
+
+            # Compute state.  Update state-seen flags for global state
+            # determination.
+            if best_row.vulnerable:
+                if best_row.urgency == 'unimportant':
+                    state = 'unimportant'
+                    unimportant_seen = True
+                else:
+                    open_seen = True
+                    reason = no_dsas.get((bug_name, best_row.release), None)
+                    if reason is not None:
+                        state = 'no-dsa'
+                    else:
+                        state = 'open'
+            else:
+                state = 'resolved'
+
+            bug = BugForSourcePackageRelease(
+                best_row.release, best_row.subrelease, best_row.version,
+                best_row.vulnerable, state, reason)
+            releases[best_row.release] = bug
+
+        # Compute global_state.
+        if open_seen:
+            global_state = 'open'
+        elif unimportant_seen:
+            global_state = 'unimportant'
+        else:
+            global_state = 'resolved'
+
+        all_bugs.append(BugForSourcePackage(bug_name, description,
+                                            global_state, releases))
+
+    # Split all_bugs into per-state sequences.
+    per_state = {'all_releases': all_releases,
+                 'all': all_bugs}
+    for state in ("open", "unimportant", "resolved"):
+        per_state[state] = tuple(bug for bug in all_bugs
+                                 if bug.global_state == state)
+
+    return BugsForSourcePackage(**per_state)
 
 # Returned by DB.getDSAsForSourcePackage().
 DSAsForSourcePackage = namedtuple(
@@ -1735,34 +1842,6 @@ class DB:
             (pkg,))
         return flag
 
-    def getBugsForSourcePackage(self, cursor, pkg, vulnerable, unimportant):
-        """Returns a generator for BugsForSourcePackage named tuples which
-        have the requested status.  Only bugs affecting supported
-        releases are returned.
-        """
-        for row in cursor.execute(
-            """SELECT DISTINCT name, description
-            FROM (SELECT bugs.name AS name, bugs.description AS description,
-            MAX(st.vulnerable
-            AND COALESCE((SELECT st2.vulnerable FROM source_packages AS sp2,
-            source_package_status AS st2
-            WHERE sp2.name = sp.name AND sp2.release = sp.release
-            AND ( sp2.subrelease = 'security' OR sp2.subrelease = 'lts' ) AND sp2.archive = sp.archive
-            AND st2.package = sp2.rowid AND st2.bug_name = st.bug_name
-            ORDER BY st2.vulnerable DESC), 1)) AS vulnerable,
-            st.urgency = 'unimportant' OR NOT vulnerable AS unimportant
-            FROM source_packages AS sp, source_package_status AS st, bugs
-            WHERE sp.name = ?
-            AND sp.release IN ('squeeze', 'wheezy', 'jessie', 'stretch', 'sid')
-            AND sp.subrelease <> 'security' AND sp.subrelease <> 'lts'
-            AND st.package = sp.rowid
-            AND bugs.name = st.bug_name
-            AND (bugs.name LIKE 'CVE-%' OR bugs.name LIKE 'TEMP-%')
-            GROUP BY bugs.name, bugs.description, sp.name)
-            WHERE vulnerable = ? AND unimportant = ?
-            ORDER BY name DESC""", (pkg, vulnerable, unimportant)):
-            yield BugsForSourcePackage(*row)
-
     def getDSAsForSourcePackage(self, cursor, package):
         for row in cursor.execute(
             """SELECT bugs.name, bugs.description