Use a separate file, data/packages/removed-packages, to list source

packages which are no longer present in the archive.

* lib/python/security_db.py
  (DB.readBugs.do_parse): Ignore duplicate packages.
  (DB.readBugs): Treat removed-packages as yet another input file.
  (DB.readRemovedPackages): Resurrect method.


git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@5668 e39458fd-73e7-0310-bf30-c45bca0a0e42

1 files changed, 36 insertions, 2 deletions

diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index 38b12646f7..c0b33c05fa 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -719,7 +719,7 @@ class DB:
                 raise InsertError(errors)
 
             cursor.executemany(
-                "INSERT OR REPLACE INTO removed_packages (name) VALUES (?)",
+                "INSERT OR IGNORE INTO removed_packages (name) VALUES (?)",
                 map(lambda x: (x,), source.removed_packages.keys()))
 
         def has_changed(filename):
@@ -733,9 +733,11 @@ class DB:
                     return True
             return True
 
+        source_removed_packages = '/packages/removed-packages'
         sources = ((bugs.CVEFile, '/CVE/list'),
                    (bugs.DSAFile, '/DSA/list'),
-                   (bugs.DTSAFile, '/DTSA/list'))
+                   (bugs.DTSAFile, '/DTSA/list'),
+                   (None, source_removed_packages))
 
         unchanged = True
         for (_, name) in sources:
@@ -759,8 +761,14 @@ class DB:
                 VALUES (?, ?)""", (current_print, filename))
 
         for (cls, name) in sources:
+            if cls is None:
+                continue
             read_one(cls(path + name))
 
+        if self.verbose:
+            print "  update removed packages"
+        self.readRemovedPackages(cursor, path + source_removed_packages)
+
         errors = []
 
         if self.verbose:
@@ -1861,6 +1869,32 @@ class DB:
             ORDER BY bug""", (bug, bug, bug, bug)):
             yield bug_name
 
+    def readRemovedPackages(self, cursor, filename):
+        """Reads a file of removed packages and stores it in the database.
+        The original contents of the removed_packages table is preserved."""
+
+        f = file(filename)
+
+        re_package = re.compile(r'^\s*([a-z0-9]\S+)\s*$')
+
+        # Not very good error reporting, but changes to that file are
+        # rare.
+
+        def gen():
+            for line in f:
+                if line == '':
+                    break
+                if line[0] == '#' or line == '\n':
+                    continue
+                match = re_package.match(line)
+                if match:
+                    yield match.groups()
+                else:
+                    raise ValueError, "not a package: " + `line`
+
+        cursor.executemany(
+            "INSERT OR IGNORE INTO removed_packages (name) VALUES (?)", gen())
+
     def getUnknownPackages(self, cursor):
         """Returns a generator for a list of unknown packages.
         Each entry has the form (PACKAGE, BUG-LIST)."""