From e8a6999e3a7897c306672716fa66afd7f1d28a13 Mon Sep 17 00:00:00 2001 From: Anton Gladky Date: Fri, 26 May 2023 13:02:11 +0000 Subject: Filter list for "unreported" view. Fix #987283 --- lib/python/security_db.py | 87 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/python/security_db.py b/lib/python/security_db.py index f293e1b3e8..d02c803d56 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -420,6 +420,10 @@ class DB: cursor.execute( "CREATE TABLE removed_packages (name TEXT NOT NULL PRIMARY KEY)") + # This table is used to keep the list of source packages, for which the filing of a bug is not required. + cursor.execute( + "CREATE TABLE ignored_packages (name TEXT NOT NULL PRIMARY KEY)") + cursor.execute( """CREATE TABLE nvd_data (cve_name TEXT NOT NULL PRIMARY KEY, @@ -908,19 +912,29 @@ class DB: def clear_db(cleared=[False]): # Avoid clearing the database multiple times. if cleared[0]: + if self.verbose: + print(" finished (already cleared)") return else: + if self.verbose: + print(" clearing database") cleared[0] = True - cursor.execute("DELETE FROM debian_bugs") - cursor.execute("DELETE FROM bugs") - cursor.execute("DELETE FROM package_notes") - cursor.execute("DELETE FROM bugs_notes") - cursor.execute("DELETE FROM bugs_xref") - cursor.execute("DELETE FROM package_notes_nodsa") - cursor.execute("DELETE FROM removed_packages") - cursor.execute("DELETE FROM next_point_update") + tables = ['debian_bugs', 'bugs', 'package_notes', 'bugs_notes', 'bugs_xref', 'package_notes_nodsa', 'ignored_packages', 'removed_packages', 'next_point_update'] + # clean up all tables + for table in tables: + # check first, whether the table exists + try: + cursor.execute(f"SELECT * FROM {table} LIMIT 1") + except: + # table does not exist + if self.verbose: + print(f"Table {table} does not exist") + continue + if self.verbose: + print (f"Clearing table {table}") + cursor.execute(f"DELETE FROM {table}") # The *_status tables are regenerated anyway, no need to # delete them here. @@ -953,33 +967,43 @@ class DB: "SELECT inodeprint FROM inodeprints WHERE file = ?", (filename,)): if old_print == current_print: + if self.verbose: + print(" unchanged: " + repr(filename)) return False else: + if self.verbose: + print(" changed: " + repr(filename)) + print(f" old: {old_print}, new: {current_print}") return True return True source_removed_packages = '/packages/removed-packages' + source_ignored_unreported = '/packages/ignored-debian-bug-packages' sources = self.getSources() source_paths = [src["path"] for src in sources] - unchanged = True changed_source = None - for filename in source_paths + [source_removed_packages]: + for filename in source_paths + [source_removed_packages, source_ignored_unreported]: if has_changed(path + filename): - unchanged = False + if self.verbose: + print(" changed: " + repr(path + filename)) + print (" clearing database") changed_source = path + filename break - if unchanged: + + if changed_source: if self.verbose: - print(" finished (no changes)") - return + print(f" clearing database, because some files have changed (at least {changed_source})") else: if self.verbose: - print(f" clearing database, because some files have changed ({changed_source})") + print(" finished (no changes)") + return clear_db() def read_one(source): + if self.verbose: + print(" reading " + repr(source.name)) filename = source.name current_print = self.filePrint(filename) @@ -994,9 +1018,16 @@ class DB: cls = getattr(bugs, cls) read_one(cls(path + srcpath)) + # Read list of packages, which were removed from the status/unreported if self.verbose: print(" update removed packages") - self.readRemovedPackages(cursor, path + source_removed_packages) + self.readRemovedAndIgnoredPackages(cursor, path + source_removed_packages, table = "removed_packages") + + # Read list of packages, which should be ignored for the status/unreported + if self.verbose: + print(" update ignored packages") + self.readRemovedAndIgnoredPackages(cursor, path + source_ignored_unreported, table = "ignored_packages") + errors = [] @@ -1971,9 +2002,15 @@ class DB: ORDER BY bug""", (bug, bug, bug, bug)): yield bug_name - def readRemovedPackages(self, cursor, filename): - """Reads a file of removed packages and stores it in the database. - The original contents of the removed_packages table is preserved.""" + def readRemovedAndIgnoredPackages(self, cursor, filename, table='removed_packages'): + """Reads a file of removed or ignored packages and stores it in the database. + For that the table parameter must be set to 'removed_packages'. + This is the default value. + The original contents of the removed_packages table is preserved. + + This function also reads the file of packages, where filing debian bugs is being ignored + and stores it in the database. + """ f = open(filename) @@ -1994,8 +2031,13 @@ class DB: else: raise ValueError("not a package: " + repr(line)) + # check, if {table} exists, otherwise create it + cursor.execute( + f"CREATE TABLE IF NOT EXISTS {table} (name TEXT NOT NULL PRIMARY KEY)") + + # Add packages into the table cursor.executemany( - "INSERT OR IGNORE INTO removed_packages (name) VALUES (?)", gen()) + f"INSERT OR IGNORE INTO {table} (name) VALUES (?)", gen()) # Add file print to database for removed packages @@ -2043,7 +2085,7 @@ class DB: st.bug_name > 'TEMP-' AND st.bug_name LIKE 'TEMP-%' ORDER BY st.bug_name""",(vulnerability,))) - def getUnreportedVulnerabilities(self, cursor=None): + def getUnreportedVulnerabilities(self, cursor=None, show_ignored=False): """Returns a list of pairs (BUG_NAME, DESCRIPTION) of vulnerabilities which are unfixed in unstable and lack a filed bug. """ @@ -2052,7 +2094,7 @@ class DB: last_bug = None result = [] for bug, pkg in cursor.execute( -"""SELECT DISTINCT source_package_status.bug_name, source_packages.name +f"""SELECT DISTINCT source_package_status.bug_name, source_packages.name FROM source_packages JOIN source_package_status ON source_packages.rowid = source_package_status.package @@ -2065,6 +2107,7 @@ class DB: AND package_notes.urgency <> 'unimportant' AND package_notes.rowid NOT IN (SELECT note FROM debian_bugs) AND source_package_status.vulnerable + AND ({show_ignored} OR NOT EXISTS (SELECT * FROM ignored_packages WHERE ignored_packages.name = source_packages.name)) ORDER BY source_package_status.bug_name, source_packages.name"""): if last_bug is None or last_bug != bug: last_bug = bug -- cgit v1.2.3