Merge branch 'fix_987283' into 'master'

Filter list for "unreported" view. Fix #987283

See merge request security-tracker-team/security-tracker!114

3 files changed, 81 insertions, 24 deletions

diff --git a/bin/tracker_service.py b/bin/tracker_service.py
index d3d27ebb89..14194321f6 100755
--- a/bin/tracker_service.py
+++ b/bin/tracker_service.py
@@ -1075,14 +1075,23 @@ checker to find out why they have not entered testing yet."""),
                         replacement="No ITP bugs are currently known.")])
 
     def page_status_unreported(self, path, params, url):
+        show_ignored = params.get('show_ignored', False)
+        show_ignored_flag = False
+        if show_ignored:
+            flags = A(url.updateParamsDict({'show_ignored' : None}),
+                      'Hide ignored issues')
+            show_ignored_flag = True
+        else:
+            flags = A(url.updateParamsDict({'show_ignored' : '1'}),
+                  'Show ignored issues')
         def gen():
-            for (bug, packages) in self.db.getUnreportedVulnerabilities():
+            for (bug, packages) in self.db.getUnreportedVulnerabilities(show_ignored=show_ignored_flag):
                 pkgs = make_list([self.make_source_package_ref(url, pkg)
                                   for pkg in packages], ", ")
                 yield self.make_xref(url, bug), pkgs
         return self.create_page(
             url, "Unfixed vulnerabilities in unstable without a filed bug",
-            [P("""The list below contains vulnerabilities for which no matching
+            [P(flags), P("""The list below contains vulnerabilities for which no matching
 Debian bug has been filed, and there is still an unfixed package in sid."""),
              make_table(gen(), caption=("Bug", "Packages"))])
 
diff --git a/data/packages/ignored-debian-bug-packages b/data/packages/ignored-debian-bug-packages
new file mode 100644
index 0000000000..8055b2f311
--- /dev/null
+++ b/data/packages/ignored-debian-bug-packages
@@ -0,0 +1,5 @@
+# This file lists packages which by default should be ignored from reporting
+# bugs for Debian unstable.
+
+linux
+gitlab
\ No newline at end of file
diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index f293e1b3e8..d02c803d56 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -420,6 +420,10 @@ class DB:
         cursor.execute(
             "CREATE TABLE removed_packages (name TEXT NOT NULL PRIMARY KEY)")
 
+        # This table is used to keep the list of source packages, for which the filing of a bug is not required.
+        cursor.execute(
+            "CREATE TABLE ignored_packages (name TEXT NOT NULL PRIMARY KEY)")
+
         cursor.execute(
             """CREATE TABLE nvd_data
             (cve_name TEXT NOT NULL PRIMARY KEY,
@@ -908,19 +912,29 @@ class DB:
         def clear_db(cleared=[False]):
             # Avoid clearing the database multiple times.
             if cleared[0]:
+                if self.verbose:
+                    print("  finished (already cleared)")
                 return
             else:
+                if self.verbose:
+                    print("  clearing database")
                 cleared[0] = True
 
-            cursor.execute("DELETE FROM debian_bugs")
-            cursor.execute("DELETE FROM bugs")
-            cursor.execute("DELETE FROM package_notes")
-            cursor.execute("DELETE FROM bugs_notes")
-            cursor.execute("DELETE FROM bugs_xref")
-            cursor.execute("DELETE FROM package_notes_nodsa")
-            cursor.execute("DELETE FROM removed_packages")
-            cursor.execute("DELETE FROM next_point_update")
+            tables = ['debian_bugs', 'bugs', 'package_notes', 'bugs_notes', 'bugs_xref', 'package_notes_nodsa', 'ignored_packages', 'removed_packages', 'next_point_update']
 
+            # clean up all tables
+            for table in tables:
+                # check first, whether the table exists
+                try:
+                    cursor.execute(f"SELECT * FROM {table} LIMIT 1")
+                except:
+                    # table does not exist
+                    if self.verbose:
+                        print(f"Table {table} does not exist")
+                    continue
+                if self.verbose:
+                    print (f"Clearing table {table}")
+                cursor.execute(f"DELETE FROM {table}")
             # The *_status tables are regenerated anyway, no need to
             # delete them here.
 
@@ -953,33 +967,43 @@ class DB:
                 "SELECT inodeprint FROM inodeprints WHERE file = ?",
                 (filename,)):
                 if old_print == current_print:
+                    if self.verbose:
+                        print("  unchanged: " + repr(filename))
                     return False
                 else:
+                    if self.verbose:
+                        print("  changed: " + repr(filename))
+                        print(f"  old: {old_print}, new: {current_print}")
                     return True
             return True
 
         source_removed_packages = '/packages/removed-packages'
+        source_ignored_unreported = '/packages/ignored-debian-bug-packages'
         sources = self.getSources()
         source_paths = [src["path"] for src in sources]
 
-        unchanged = True
         changed_source = None
-        for filename in source_paths + [source_removed_packages]:
+        for filename in source_paths + [source_removed_packages, source_ignored_unreported]:
             if has_changed(path + filename):
-                unchanged = False
+                if self.verbose:
+                    print("  changed: " + repr(path + filename))
+                    print ("  clearing database")
                 changed_source = path + filename
                 break
-        if unchanged:
+
+        if changed_source:
             if self.verbose:
-                print("  finished (no changes)")
-            return
+                print(f"  clearing database, because some files have changed (at least {changed_source})")
         else:
             if self.verbose:
-                print(f"  clearing database, because some files have changed ({changed_source})")
+                print("  finished (no changes)")
+            return
 
         clear_db()
 
         def read_one(source):
+            if self.verbose:
+                print("  reading " + repr(source.name))
             filename = source.name
             current_print = self.filePrint(filename)
 
@@ -994,9 +1018,16 @@ class DB:
             cls = getattr(bugs, cls)
             read_one(cls(path + srcpath))
 
+        # Read list of packages, which were removed from the status/unreported
         if self.verbose:
             print("  update removed packages")
-        self.readRemovedPackages(cursor, path + source_removed_packages)
+        self.readRemovedAndIgnoredPackages(cursor, path + source_removed_packages, table = "removed_packages")
+
+        # Read list of packages, which should be ignored for the status/unreported
+        if self.verbose:
+            print("  update ignored packages")
+        self.readRemovedAndIgnoredPackages(cursor, path + source_ignored_unreported, table = "ignored_packages")
+
 
         errors = []
 
@@ -1971,9 +2002,15 @@ class DB:
             ORDER BY bug""", (bug, bug, bug, bug)):
             yield bug_name
 
-    def readRemovedPackages(self, cursor, filename):
-        """Reads a file of removed packages and stores it in the database.
-        The original contents of the removed_packages table is preserved."""
+    def readRemovedAndIgnoredPackages(self, cursor, filename, table='removed_packages'):
+        """Reads a file of removed or ignored packages and stores it in the database.
+        For that the table parameter must be set to 'removed_packages'.
+        This is the default value.
+        The original contents of the removed_packages table is preserved.
+
+        This function also reads the file of packages, where filing debian bugs is being ignored
+        and stores it in the database.
+        """
 
         f = open(filename)
 
@@ -1994,8 +2031,13 @@ class DB:
                 else:
                     raise ValueError("not a package: " + repr(line))
 
+        # check, if {table} exists, otherwise create it
+        cursor.execute(
+            f"CREATE TABLE IF NOT EXISTS {table} (name TEXT NOT NULL PRIMARY KEY)")
+
+        # Add packages into the table
         cursor.executemany(
-            "INSERT OR IGNORE INTO removed_packages (name) VALUES (?)", gen())
+            f"INSERT OR IGNORE INTO {table} (name) VALUES (?)", gen())
 
 
         # Add file print to database for removed packages
@@ -2043,7 +2085,7 @@ class DB:
                 st.bug_name > 'TEMP-' AND st.bug_name LIKE 'TEMP-%'
                 ORDER BY st.bug_name""",(vulnerability,)))
 
-    def getUnreportedVulnerabilities(self, cursor=None):
+    def getUnreportedVulnerabilities(self, cursor=None, show_ignored=False):
         """Returns a list of pairs (BUG_NAME, DESCRIPTION)
         of vulnerabilities which are unfixed in unstable and lack a filed bug.
         """
@@ -2052,7 +2094,7 @@ class DB:
         last_bug = None
         result = []
         for bug, pkg in cursor.execute(
-"""SELECT DISTINCT source_package_status.bug_name, source_packages.name
+f"""SELECT DISTINCT source_package_status.bug_name, source_packages.name
   FROM source_packages
   JOIN source_package_status
     ON source_packages.rowid = source_package_status.package
@@ -2065,6 +2107,7 @@ class DB:
   AND package_notes.urgency <> 'unimportant'
   AND package_notes.rowid NOT IN (SELECT note FROM debian_bugs)
   AND source_package_status.vulnerable
+  AND ({show_ignored} OR NOT EXISTS (SELECT * FROM ignored_packages WHERE ignored_packages.name = source_packages.name))
   ORDER BY source_package_status.bug_name, source_packages.name"""):
             if last_bug is None or last_bug != bug:
                 last_bug = bug