Implement bin/update-db, to update the database with a single command.

Most processing is skipped if no input files have been modified.

lib/python/security_db.py (SchemaMismatch):
  New exception.
(DB):
  Handle schema versioning.
(DB.initSchema):
  Add subrelease column to source_packages and binary_packages.
  Set user_version.
  Remove stray commit.
(DB._parseFile):
  Return information to the caller if the file is unchanged.
(DB.readPackages):
  Move deletion code to callees.
(DB._readSourcePackages, DB._readBinaryPackages):
  Implement incremental updates.  Add subrelease.
  Need to invoke _clearVersions if any changes are made.
(DB.deleteBugs, DB.finishBugs):
  Moved into readBugs.
(DB.insertBugs):
  Rename ...
(DB.readBugs):
  ... to this one.  Implement incremental updates.
  Invoke _clearVersions if necessary.
(DB._clearVersions):
  Add.
(DB._updateVersions):
  Skip processing if _clearVersions has not been invoked.
(DB.getVersion, DB.releaseContainsPackage, DB._synthesizeReleases):
  Obsolete, remove.
(test):
  Update.

lib/python/bugs.py (CANFile, CVEFile):
  Split into two classes, which handle the differences between the two
  files.

bin/check-syntax:
  Update accordingly.

bin/update-db:
  New database update script.  Implements incremental updates.

Makefile:
  Remove references to bin/update-packages.  Simplify drastically.


git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@1994 e39458fd-73e7-0310-bf30-c45bca0a0e42

7 files changed, 299 insertions, 352 deletions

diff --git a/Makefile b/Makefile
index d5a787f043..1129bed9f5 100644
--- a/Makefile
+++ b/Makefile
@@ -14,28 +14,8 @@ SECURITY_RELEASES = sarge-security=i386,ia64 woody-security=i386
 PACKAGE_FILES = $(wildcard data/packages/*_Sources) \
 	$(wildcard data/packages/*_Packages)
 
-all: stamps/bug-lists-imported stamps/packages-imported \
-	stamps/calc-vulns
-
-stamps/bug-lists-imported: bin/update-bug-list-db \
-	$(BUG_LISTS) $(PYTHON_MODULES)
-	$(PYTHON) bin/update-bug-list-db
-	touch $@
-
-# No dependencies on the Python files.  This part of the code should
-# be quite stable.  We only run the packages import if "make
-# update-packages" has been invoked before.
-stamps/packages-imported: $(PACKAGE_FILES)
-	if test -e stamps/packages-downloaded ; then \
-		$(PYTHON) bin/update-packages import ; \
-	fi
-	touch $@
-
-stamps/calc-vulns: stamps/bug-lists-imported stamps/packages-imported
-	if test -e stamps/packages-downloaded ; then \
-		$(PYTHON) bin/update-vulnerabilities ; \
-	fi
-	touch $@
+all:
+	$(PYTHON) bin/update-db
 
 clean:
 	-rm data/security.db
@@ -64,9 +44,43 @@ stamps/DTSA-syntax: data/DTSA/list bin/check-syntax $(PYTHON_MODULES)
 	$(PYTHON) bin/check-syntax DTSA data/DTSA/list
 	touch $@
 
-.PHONY: update-packages
+.PHONY: update-packages update-etch-security
 update-packages:
-	$(PYTHON) bin/update-packages download $(MIRROR) $(RELEASES)
-	$(PYTHON) bin/update-packages download \
-		http://security.debian.org/ $(SECURITY_RELEASES)
+	set -e ; for rel in woody sarge etch sid ; do \
+		for archive in main contrib non-free ; do \
+		$(PYTHON) bin/apt-update-file \
+			$(MIRROR)/dists/$$rel/$$archive/source/Sources \
+			data/packages/$${rel}__$${archive}_Sources ; \
+		done ; \
+	        for arch in i386 ia64 ; do \
+		  for archive in main contrib non-free ; do \
+		  $(PYTHON) bin/apt-update-file \
+		    $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \
+		    data/packages/$${rel}__$${archive}_$${arch}_Packages ; \
+		  done ; \
+		done ; \
+	done
 	touch stamps/packages-downloaded
+
+ST_MIRROR = http://secure-testing.debian.net/debian-secure-testing/dists/etch/security-updates
+ST_FILE = data/packages/etch_security_
+update-testing-security:
+	$(PYTHON) bin/apt-update-file \
+	  $(ST_MIRROR)/main/source/Sources $(ST_FILE)main_Sources
+	$(PYTHON) bin/apt-update-file \
+	  $(ST_MIRROR)/main/binary-i386/Packages $(ST_FILE)main_i386_Packages
+	$(PYTHON) bin/apt-update-file \
+	  $(ST_MIRROR)/main/binary-ia64/Packages $(ST_FILE)main_ia64_Packages
+
+SEC_MIRROR = http://security.debian.org/dists
+update-security:
+	for archive in woody sarge ; do \
+	  $(PYTHON) bin/apt-update-file \
+	    $(SEC_MIRROR)/$$archive/updates/main/source/Sources \
+	    data/packages/$${archive}_security_main_Sources ; \
+	  for arch in i386 ia64 ; do \
+	    $(PYTHON) bin/apt-update-file \
+	      $(SEC_MIRROR)/$$archive/updates/main/binary-$$arch/Packages \
+	      data/packages/$${archive}_security_main_$${arch}_Packages ; \
+	  done ; \
+	done
diff --git a/bin/check-syntax b/bin/check-syntax
index d996ae3664..8e1c69c2f5 100755
--- a/bin/check-syntax
+++ b/bin/check-syntax
@@ -46,7 +46,7 @@ def do_parse(f):
     
 
 def parse_CAN(name):
-    do_parse(bugs.CVEFile(name))
+    do_parse(bugs.CANFile(name))
 
 def parse_CVE(name):
     f = bugs.CVEFile(name)
diff --git a/bin/update-bug-list-db b/bin/update-db
index 96ebd3b6a0..273d9fa7e0 100755
--- a/bin/update-bug-list-db
+++ b/bin/update-db
@@ -16,40 +16,49 @@ def setup_paths():
         if idx == -1:
             raise ImportError, "could not setup paths"
         path = path[0:idx]
-root_path = setup_paths()
+os.chdir(setup_paths())
 
 import bugs
 import debian_support
 import security_db
 
-db_file = root_path + '/data/security.db'
-new_file = not os.path.exists(db_file)
-db = security_db.DB(db_file)
-if new_file:
-    db.initSchema()
+db_file = 'data/security.db'
+try:
+    db = security_db.DB(db_file, verbose=True)
+except security_db.SchemaMismatch:
+    os.unlink(db_file)
+    db = security_db.DB(db_file, verbose=True)
+    
 cursor = db.writeTxn()
-db.deleteBugs(cursor)
+
+# Bug lists (CAN/CVE/DSA/DTSA)
+
 try:
-    db.insertBugs(cursor, bugs.CVEFile(root_path + '/data/CAN/list'))
-    db.insertBugs(cursor, bugs.CVEFile(root_path + '/data/CVE/list',
-                                       no_version_needs_note=False))
-    db.insertBugs(cursor, bugs.DSAFile(root_path + '/data/DSA/list'))
-    db.insertBugs(cursor, bugs.DTSAFile(root_path + '/data/DTSA/list'))
+    warnings = db.readBugs(cursor, 'data')
 except debian_support.ParseError, e:
-    db.rollback(cursor)
     e.printOut(sys.stderr)
     sys.exit(1)
 except security_db.InsertError, e:
-    db.rollback(cursor)
     for err in e.errors:
         print err
     sys.exit(1)
+if warnings:
+    for x in warnings:
+        print x
+    sys.exit(1)
+
+# Packages
+
+db.readPackages(cursor, 'data/packages')
 
-warnings = db.finishBugs(cursor)
+# Calculate vulnerability information.
+
+warnings = db.calculateVulnerabilities(cursor)
 if warnings:
-    db.rollback(cursor)
     for x in warnings:
         print x
     sys.exit(1)
-else:
-    db.commit(cursor)
+
+# Everything worked well.
+    
+db.commit(cursor)
diff --git a/bin/update-packages b/bin/update-packages
deleted file mode 100755
index 98b447c151..0000000000
--- a/bin/update-packages
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-
-# This script downloads and imports Debian package files.
-
-import errno
-import os
-import os.path
-import string
-import sys
-
-def setup_paths():
-    check_file = 'lib/python/debian_support.py'
-    path = os.getcwd()
-    while 1:
-        if os.path.exists("%s/%s" % (path, check_file)):
-            sys.path = [path + '/lib/python'] + sys.path
-            return path
-        idx = string.rfind(path, '/')
-        if idx == -1:
-            raise ImportError, "could not setup paths"
-        path = path[0:idx]
-root_path = setup_paths()
-
-import debian_support
-import security_db
-
-def explodeReleases(args):
-    for arg in args:
-        (release, archs) = arg.split('=')
-        # FIXME: What shall we do with these?
-        # if debian_support.internRelease(release) is None:
-        #     sys.stderr.write("error: unknown release: %s\n" % release)
-        #     sys.exit(1)
-        yield release, archs.split(',')
-
-archives = ('main', 'contrib', 'non-free')
-
-def nameSources(release, archive):
-    return '%s/data/packages/%s_%s_Sources' % (root_path, release, archive)
-
-def namePackages(release, archive, arch):
-    return '%s/data/packages/%s_%s_%s_Packages' % (root_path, release,
-                                                   archive, arch)
-
-def cmd_download(args):
-    url_base = args[0]
-    if url_base[-1] != '/':
-        url_base += '/'
-        
-    for release, archs in explodeReleases(args[1:]):
-        # Security updates are stored in a different directory.
-        if release[-9:] == '-security':
-            rrel = release[:-9] + '/updates'
-        else:
-            rrel = release
-
-        for archive in archives:
-            print "Updating source package %s/%s" % (release, archive)
-            debian_support.updateFile("%sdists/%s/%s/source/Sources"
-                                      % (url_base, rrel, archive),
-                                      nameSources(release, archive),
-                                      verbose=True)
-            for arch in archs:
-                print "Updating binary package %s/%s/%s" \
-                                      % (release, archive, arch)
-                debian_support.updateFile("%sdists/%s/%s/binary-%s/Packages"
-                                          % (url_base, rrel, archive, arch),
-                                          namePackages(release, archive, arch),
-                                          verbose=True)
-
-def cmd_import(args):
-    db_file = root_path + '/data/security.db'
-    new_file = not os.path.exists(db_file)
-    db = security_db.DB(db_file, verbose=True)
-    if new_file:
-        db.initSchema()
-    c = db.writeTxn()
-    db.readPackages(c, root_path + '/data/packages')
-    db.commit(c)
-
-cmds = {"download" : cmd_download,
-        "import" : cmd_import}
-
-if len(sys.argv) < 2 or not cmds.has_key(sys.argv[1]):
-    sys.stderr.write(\
-"""usage: update-packages download URL-BASE RELEASE=ARCH...
-       update-packages import
-""")
-    sys.exit(1)
-try:
-    cmds[sys.argv[1]](sys.argv[2:])
-except debian_support.ParseError, e:
-    e.printOut(sys.stderr)
-    sys.exit(1)
-
diff --git a/bin/update-vulnerabilities b/bin/update-vulnerabilities
deleted file mode 100755
index e2ba55554d..0000000000
--- a/bin/update-vulnerabilities
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/python
-
-# This script recalculates the vulnerability information in the
-# security database.
-
-import errno
-import os
-import os.path
-import string
-import sys
-
-def setup_paths():
-    check_file = 'lib/python/debian_support.py'
-    path = os.getcwd()
-    while 1:
-        if os.path.exists("%s/%s" % (path, check_file)):
-            sys.path = [path + '/lib/python'] + sys.path
-            return path
-        idx = string.rfind(path, '/')
-        if idx == -1:
-            raise ImportError, "could not setup paths"
-        path = path[0:idx]
-root_path = setup_paths()
-
-import security_db
-
-db_file = root_path + '/data/security.db'
-assert os.path.exists(db_file)
-db = security_db.DB(db_file, verbose=True)
-c = db.writeTxn()
-warnings = db.calculateVulnerabilities(c)
-if warnings:
-    db.rollback(c)
-    for x in warnings:
-        print x
-    sys.exit(1)
-db.commit(c)
diff --git a/lib/python/bugs.py b/lib/python/bugs.py
index 97bf4ca12c..c527cf3755 100644
--- a/lib/python/bugs.py
+++ b/lib/python/bugs.py
@@ -616,15 +616,15 @@ class FileBase(debian_support.PackageFile):
                 yield Bug(self.file.name, first_lineno, date,
                           record_name, description,
                           comments, notes=pkg_notes, xref=xref)
-            
-class CVEFile(FileBase):
-    """A CVE file, as used by the Debian testing security team."""
+
+class CANFile(FileBase):
+    """A CAN file, as used by the Debian testing security team."""
     
-    re_cve = re.compile(r'^((?:CAN|CVE)-\d{4}-(?:\d{4}|XXXX))\s+(.*?)\s*$')
+    re_cve = re.compile(r'^(CAN-\d{4}-(?:\d{4}|XXXX))\s+(.*?)\s*$')
 
-    def __init__(self, name, fileObj=None, no_version_needs_note=True):
+    def __init__(self, name, fileObj=None):
         FileBase.__init__(self, name, fileObj)
-        self.no_version_needs_note = no_version_needs_note
+        self.no_version_needs_note = True
 
     def isUniqueName(self, name):
         return BugBase.re_cve_name.match(name) is not None
@@ -632,6 +632,34 @@ class CVEFile(FileBase):
     def matchHeader(self, line):
         match = self.re_cve.match(line)
         if not match:
+            self.raiseSyntaxError("expected CAN record, got: %s" % `line`)
+            (record_name, description) = match.groups()
+        (cve, desc) = match.groups()
+        if desc:
+            if desc[0] == '(':
+                if desc[-1] <> ')':
+                    self.raiseSyntaxError("missing closing parenthesis")
+                else:
+                    desc = desc[1:-1]
+            elif desc[0] == '[':
+                if desc[-1] <> ']':
+                    self.raiseSyntaxError("missing closing bracket")
+                else:
+                    desc = desc[1:-1]
+        return (None, cve, desc)
+
+class CVEFile(FileBase):
+    """A CVE file, as used by the Debian testing security team."""
+
+    re_cve = re.compile(r'^(CVE-\d{4}-\d{4})\s+(.*?)\s*$')
+
+    def __init__(self, name, fileObj=None):
+        FileBase.__init__(self, name, fileObj)
+        self.no_version_needs_note = False
+        
+    def matchHeader(self, line):
+        match = self.re_cve.match(line)
+        if not match:
             self.raiseSyntaxError("expected CVE record, got: %s" % `line`)
             (record_name, description) = match.groups()
         (cve, desc) = match.groups()
diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index dffa846aeb..efc4f62a42 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -73,6 +73,11 @@ def mergeLists(a, b):
     result.sort()
     return result
 
+class SchemaMismatch(Exception):
+    """Raised to indicate a schema mismatch.
+
+    The caller is expected to remove and regenerate the database."""
+
 class DB:
     """Access to the security database.
 
@@ -91,6 +96,15 @@ class DB:
                           'sarge' : 'stable',
                           'woody': 'oldstable'}
 
+        c = self.cursor()
+        for (v,) in c.execute("PRAGMA user_version"):
+            if v == 0:
+                self.initSchema()
+            if v <> 1:
+                raise SchemaMismatch, `v`
+            return
+        assert False
+
     def cursor(self):
         """Creates a new database cursor.
 
@@ -140,22 +154,24 @@ class DB:
             """CREATE TABLE source_packages
             (name TEXT NOT NULL,
             release TEXT NOT NULL,
+            subrelease TEXT NOT NULL,
             archive TEXT NOT NULL,
             version TEXT NOT NULL,
             version_id INTEGER NOT NULL DEFAULT 0,
-            PRIMARY KEY (name, release, archive))""")
+            PRIMARY KEY (name, release, subrelease, archive))""")
 
         cursor.execute(
             """CREATE TABLE binary_packages
             (name TEXT NOT NULL,
             release TEXT NOT NULL,
+            subrelease TEXT NOT NULL,
             archive TEXT NOT NULL,
             version TEXT NOT NULL,
             source TEXT NOT NULL,
             source_version TEXT NOT NULL,
             archs TEXT NOT NULL,
             version_id INTEGER NOT NULL DEFAULT 0,
-            PRIMARY KEY (name, release, archive, version, source,
+            PRIMARY KEY (name, release, subrelease, archive, version, source,
             source_version))""")
         cursor.execute(
             """CREATE INDEX binary_packages_source
@@ -225,7 +241,12 @@ class DB:
             """CREATE INDEX binary_package_status_package
             ON binary_package_status(package)""")
 
-        self.commit(cursor)
+        # Put this at the end.  Any exception will leave the schema
+        # version at 0, so we automatically recreate the schema once
+        # the application is started after the underlying error has
+        # been fixed.
+
+        cursor.execute("PRAGMA user_version = 1")
 
     def filePrint(self, filename):
         """Returns a fingerprint string for filename."""
@@ -284,26 +305,23 @@ class DB:
             "SELECT inodeprint, parsed FROM inodeprints WHERE file = ?",
             (filename,)):
             if old_print == current_print:
-                return cPickle.load(cStringIO.StringIO(contents))
+                return (True, cPickle.load(cStringIO.StringIO(contents)))
             result = do_parse(debian_support.PackageFile(filename))
             cursor.execute("""UPDATE inodeprints SET inodeprint = ?, parsed = ?
             WHERE file = ?""", (current_print, toString(result), filename))
-            return result
+            return (False, result)
 
         # No inodeprints entry, load file and add one.
         result = do_parse(debian_support.PackageFile(filename))
         cursor.execute("""INSERT INTO inodeprints (file, inodeprint, parsed)
         VALUES (?, ?, ?)""", (filename, current_print, toString(result)))
-        return result
+        return (False, result)
 
     def readPackages(self, cursor, directory):
         """Reads a directory of package files."""
 
         if self.verbose:
             print "readPackages:"
-            print "  deleting old data"
-        cursor.execute("DELETE FROM source_packages")
-        cursor.execute("DELETE FROM binary_packages")
 
         self._readSourcePackages(cursor, directory)
         self._readBinaryPackages(cursor, directory)
@@ -314,7 +332,7 @@ class DB:
     def _readSourcePackages(self, cursor, directory):
         """Reads from directory with source package files."""
 
-        re_sources = re.compile(r'.*/([a-z-]+)_([a-z-]+)_Sources$')
+        re_sources = re.compile(r'.*/([a-z-]+)_([a-z-]*)_([a-z-]+)_Sources$')
 
 
         if self.verbose:
@@ -325,48 +343,71 @@ class DB:
             if match is None:
                 raise ValueError, "invalid file name: " + `filename`
 
-            (release, archive) = match.groups()
-            parsed = self._parseFile(cursor, filename)
+            (release, subrelease, archive) = match.groups()
+            (unchanged, parsed) = self._parseFile(cursor, filename)
+            if unchanged:
+                continue
+
+            cursor.execute(
+                """DELETE FROM source_packages
+                WHERE release = ? AND subrelease = ? AND archive = ?""",
+                (release, subrelease, archive))
+            self._clearVersions(cursor)
 
             def gen():
                 for (name, version, source, source_version) in parsed:
                     assert source is None
                     assert source_version is None
-                    yield name, release, archive, version
+                    yield name, release, subrelease, archive, version
             cursor.executemany(
                 """INSERT INTO source_packages
-               (name, release, archive, version) VALUES (?, ?, ?, ?)""",
+               (name, release, subrelease, archive, version)
+               VALUES (?, ?, ?, ?, ?)""",
                 gen())
         
     def _readBinaryPackages(self, cursor, directory):
         """Reads from a directory with binary package files."""
 
         re_packages \
-            = re.compile(r'.*/([a-z-]+)_([a-z-]+)_([a-z0-9]+)_Packages$')
+            = re.compile(
+            r'.*/([a-z-]+)_([a-z-]*)_([a-z-]+)_([a-z0-9]+)_Packages$')
         
         if self.verbose:
             print "  reading binary packages"
 
         packages = {}
+        unchanged = True
         for filename in glob.glob(directory + '/*_Packages'):
             match = re_packages.match(filename)
             if match is None:
                 raise ValueError, "invalid file name: " + `filename`
 
-            (release, archive, architecture) = match.groups()
-            parsed = self._parseFile(cursor, filename)
+            (release, subrelease, archive, architecture) = match.groups()
+            (unch, parsed) = self._parseFile(cursor, filename)
+            unchanged = unchanged and unch
             for (name, version, source, source_version) in parsed:
                 if source is None:
                     source = name
                 if source_version is None:
                     source_version = version
         
-                key = (name, release, archive, version, source, source_version)
+                key = (name, release, subrelease, archive, version,
+                       source, source_version)
                 if packages.has_key(key):
                     packages[key][architecture] = 1
                 else:
                     packages[key] = {architecture : 1}
 
+        if unchanged:
+            if self.verbose:
+                print "    finished (no changes)"
+            return
+
+        if self.verbose:
+            print "    deleting old data"
+        cursor.execute("DELETE FROM binary_packages")
+        self._clearVersions(cursor)
+
         l = packages.keys()
 
         if len(l) == 0:
@@ -385,41 +426,110 @@ class DB:
 
         cursor.executemany(
             """INSERT INTO binary_packages
-            (name, release, archive, version,
+            (name, release, subrelease, archive, version,
             source, source_version, archs)
-            VALUES (?, ?, ?, ?, ?, ?, ?)""",
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
             gen())
             
-    def deleteBugs(self, cursor):
-        """Deletes all record bug reports from the database."""
-        cursor.execute("DELETE FROM package_notes")
-        cursor.execute("DELETE FROM debian_bugs")
-        cursor.execute("DELETE FROM bugs")
-        cursor.execute("DELETE FROM bugs_notes")
-        cursor.execute("DELETE FROM bugs_xref")
-
-    def insertBugs(self, cursor, source):
-        """Reads the CAN/CVE/DSA/DTSA file and writes them to the database."""
-
-        errors = []
-        for bug in source:
-            try:
-                bug.writeDB(cursor)
-            except ValueError, e:
-                errors.append("%s: %d: error: %s"
-                              % (bug.source_file, bug.source_line, e))
-        if errors:
-            raise InsertError(errors)
+    def readBugs(self, cursor, path):
+        if self.verbose:
+            print "readBugs:"
 
-    def finishBugs(self, cursor):
-        """After inserting new bugs, update cross-references.
+        def clear_db(filename):
+            cursor.execute(
+                """CREATE TEMPORARY TABLE bugs_to_delete
+                (tbd TEXT NOT NULL PRIMARY KEY)""")
+            cursor.execute(
+                """INSERT INTO bugs_to_delete
+                SELECT name FROM bugs WHERE source_file = ?""",
+                (filename,))
 
-        Returns a list of warning messages."""
+            cursor.execute(
+                """DELETE FROM debian_bugs
+                WHERE EXISTS (SELECT 1
+                              FROM package_notes AS p, bugs_to_delete AS b
+                              WHERE p.id = debian_bugs.note
+                              AND p.bug_name = b.tbd)""")
+
+            cursor.execute("""DELETE FROM bugs
+                WHERE EXISTS (SELECT * FROM bugs_to_delete
+                              WHERE tbd = name)""")
+            cursor.execute("""DELETE FROM package_notes
+                WHERE EXISTS (SELECT * FROM bugs_to_delete
+                              WHERE tbd = bug_name)""")
+            cursor.execute("""DELETE FROM bugs_notes
+                WHERE EXISTS (SELECT * FROM bugs_to_delete
+                              WHERE tbd = bug_name)""")
+            cursor.execute("""DELETE FROM bugs_xref
+                WHERE EXISTS (SELECT * FROM bugs_to_delete
+                              WHERE tbd = source)""")
+
+            # The *_status tables are regenerated anyway, no need to
+            # delete them here.
+            
+            cursor.execute("""DROP TABLE bugs_to_delete""")
+            
+            self._clearVersions(cursor)
+            
+        def do_parse(source):
+            errors = []
+          
+            if self.verbose:
+                print "  reading " + `source.name`
 
-        warnings = []
+            clear_db(source.name)
+            
+            for bug in source:
+                try:
+                    bug.writeDB(cursor)
+                except ValueError, e:
+                    errors.append("%s: %d: error: %s"
+                                  % (bug.source_file, bug.source_line, e))
+            if errors:
+                raise InsertError(errors)
+
+        def read_one(source):
+            filename = source.name
+            current_print = self.filePrint(filename)
+
+            for (old_print,) in cursor.execute(
+                "SELECT inodeprint FROM inodeprints WHERE file = ?",
+                (filename,)):
+                if old_print == current_print:
+                    return False
+                do_parse(source)
+                cursor.execute(
+                    "UPDATE inodeprints SET inodeprint = ? WHERE file = ?",
+                    (current_print, filename))
+                return True
+
+            # No inodeprints entry, load file and add one.
+            do_parse(source)
+            cursor.execute(
+                "INSERT INTO inodeprints (file, inodeprint) VALUES (?, ?)",
+                (filename, current_print))
+            return True
 
-        # Check that there are no CAN/CVE collisions.
+        unchanged = True
+        if read_one(bugs.CANFile(path + '/CAN/list')):
+            unchanged = False
+        if read_one(bugs.CVEFile(path + '/CVE/list')):
+            unchanged = False
+        if read_one(bugs.DSAFile(path + '/DSA/list')):
+            unchanged = False
+        if read_one(bugs.DTSAFile(path + '/DTSA/list')):
+            unchanged = False
+
+        if unchanged:
+            if self.verbose:
+                print "  finished (no changes)"
+            return
 
+        errors = []
+        
+        if self.verbose:
+            print "  checking CAN/CVE collisions"
+        
         for b1, b2 in list(cursor.execute\
             ("""SELECT b1.name, b2.name FROM bugs AS b1, bugs AS b2
              WHERE b1.name LIKE 'CVE-%'
@@ -427,19 +537,22 @@ class DB:
             b1 = bugs.BugFromDB(cursor, b1)
             b2 = bugs.BugFromDB(cursor, b2)
 
-            warnings.append("%s:%d: duplicate CVE entries %s and %s"
-                           % (b1.source_file, b1.source_line,
-                              b1.name, b2.name))
-            warnings.append("%s:%d: location of %s"
-                            % (b1.source_file, b1.source_line, b1.name))
-            warnings.append("%s:%d: location of %s"
-                            % (b2.source_file, b2.source_line, b2.name))
+            errors.append("%s:%d: duplicate CVE entries %s and %s"
+                          % (b1.source_file, b1.source_line,
+                             b1.name, b2.name))
+            errors.append("%s:%d: location of %s"
+                          % (b1.source_file, b1.source_line, b1.name))
+            errors.append("%s:%d: location of %s"
+                          % (b2.source_file, b2.source_line, b2.name))
 
         # Normalize the CAN/CVE references to the entry which is
         # actually in the database.  After the CAN -> CVE transition,
         # this can go away (but we should check that the
         # cross-references are valid).
 
+        if self.verbose:
+            print "  normalize CAN/CVE references"
+        
         for source, target in list(cursor.execute\
             ("""SELECT source, target FROM bugs_xref
             WHERE normalized_target = ''""")):
@@ -458,11 +571,12 @@ class DB:
                     break
                 if not found:
                     b = bugs.BugFromDB(cursor, source)
-                    warnings.append\
+                    errors.append\
                         ("%s: %d: reference to unknwown CVE entry %s"
                          % (b.source_file, b.source_line, target))
 
-        # Check that the DSA/DTSA references are valid.
+        if self.verbose:
+            print "  check DSA/DTSA references"
 
         for source, target in list(cursor.execute
             ("""SELECT source, target FROM bugs_xref
@@ -473,11 +587,15 @@ class DB:
                 found = True
             if not found:
                 b = bugs.BugFromDB(cursor, source)
-                warnings.append\
+                errors.append\
                     ("%s: %d: reference to unknwown advisory %s"
                      % (b.source_file, b.source_line, target))
 
-        return warnings
+        if errors:
+            raise InsertErrors(errors)
+
+        if self.verbose:
+            print "  finished"
 
     def availableReleases(self, cursor=None):
         """Returns a list of tuples (RELEASE, ARCHIVE,
@@ -487,20 +605,21 @@ class DB:
 
         releases = {}
         for r in cursor.execute(
-            "SELECT DISTINCT release, archive FROM source_packages"):
+            """SELECT DISTINCT release, subrelease, archive
+            FROM source_packages"""):
             releases[r] = (True, [])
 
-        for (rel, archive, archs) in cursor.execute(
-            """SELECT DISTINCT release, archive, archs
+        for (rel, subrel, archive, archs) in cursor.execute(
+            """SELECT DISTINCT release, subrelease, archive, archs
             FROM binary_packages"""):
-            key = (rel, archive)
+            key = (rel, subrel, archive)
             if not releases.has_key(key):
                 releases[key] = (False, [])
             releases[key][1][:] = mergeLists(releases[key][1], archs)
 
         result = []
-        for ((rel, archive), (sources, archs)) in releases.items():
-            result.append((rel, archive, sources, archs))
+        for ((rel, subrel, archive), (sources, archs)) in releases.items():
+            result.append((rel, subrel, archive, sources, archs))
         result.sort()
 
         return result
@@ -517,48 +636,21 @@ class DB:
             WHERE name = source AND version <> source_version
             ORDER BY name, release, archive"""))
 
-    def getVersion(self, cursor, release, package):
-        """Returns the version number for package in release.
-
-        Package can be a source or binary package.  Binary package
-        versions take precedence.
-
-        Security updates etc. are not considered."""
-
-        versions = list(cursor.execute(
-            """SELECT version FROM binary_packages
-            WHERE package = ? AND release = ?""", (package, release)))
-        if versions:
-            return min(map(lambda (v,): debian_support.Version(v), versions))
-
-        versions = list(cursor.execute(
-            """SELECT version FROM source_packages
-            WHERE package = ? AND release = ?""", (package, release)))
-        if versions:
-            assert len(versions) == 1
-            return debian_support.Version(versions[0][0])
-        
-        return None
-
-    def releaseContainsPackage(self, cursor, release, package):
-        """Returns True if the source or binary package exists in release."""
-        for (c,) in cursor.execute(
-            """SELECT version FROM binary_packages
-            WHERE package = ? AND release = ?""", (package, release)):
-            return True
-        for (c,) in cursor.execute(
-            """SELECT version FROM source_packages
-            WHERE package = ? AND release = ?""", (package, release)):
-            return True
-        return False
+    def _clearVersions(self, cursor):
+        cursor.execute("DELETE FROM version_linear_order")
 
     def _updateVersions(self, cursor):
         """Updates the linear version table."""
 
-        cursor.execute("DELETE FROM version_linear_order");
-
         if self.verbose:
             print "updateVersions:"
+
+        for x in cursor.execute("SELECT * FROM version_linear_order LIMIT 1"):
+            if self.verbose:
+                print "  finished (no changes)"
+            return
+
+        if self.verbose:
             print "  reading"
 
         versions = []
@@ -606,66 +698,6 @@ class DB:
         if self.verbose:
             print "  finished"
         
-    def _synthesizeReleases(self, cursor):
-        """Creates the package lists for testing, stable and oldstable.
-
-        These package lists include security updates.
-        """
-
-        if self.verbose:
-            print "synthesizeReleases:"
-            print "  clear old data"
-            print "    source packages"
-        cursor.execute(
-            """DELETE FROM source_packages
-            WHERE release IN ('stable', 'oldstable', 'testing')""")
-        if self.verbose:
-            print "    binary packages"
-        cursor.execute(
-            """DELETE FROM binary_packages
-            WHERE release IN ('stable', 'oldstable', 'testing')""")
-
-        for (realname, nickname) in self.nicknames.items():
-            if self.verbose:
-                print "  synthesize %s to %s" % (realname, nickname)
-                print "    source packages"
-            cursor.execute(
-                """INSERT INTO source_packages 
-                SELECT name, ?, archive, '', MAX(version_id) AS vid
-                FROM source_packages WHERE release IN (?, ?)
-                GROUP BY name, archive""",
-                (nickname, realname, realname + '-security'))
-            
-            if self.verbose:
-                print "    binary packages"
-            cursor.execute(
-                """INSERT INTO binary_packages
-                SELECT DISTINCT name, ?, archive,
-                MAX (version_id) AS vid, source, source_version,
-                ''
-                FROM binary_packages WHERE release IN (?, ?)
-                GROUP BY name, archive, archs""",
-                (nickname, realname, realname + '-security'))
-
-        if self.verbose:
-            print "  patch version strings"
-            print "    source packages"
-        cursor.execute(
-            """UPDATE source_packages
-            SET version = (SELECT version FROM version_linear_order
-            WHERE id = version_id)
-            WHERE version = ''""")
-        if self.verbose:
-            print "    binary packages"
-        cursor.execute(
-            """UPDATE binary_packages
-            SET version = (SELECT version FROM version_linear_order
-            WHERE id = version_id)
-            WHERE version = ''""")
-
-        if self.verbose:
-            print "  finished"
-
     def calculateVulnerabilities(self, cursor):
         """Calculate vulnerable packages.
 
@@ -949,19 +981,15 @@ def test():
     assert mergeLists('a,c', ['b', 'de']) == ['a', 'b', 'c', 'de']
 
     import os
-    if os.path.exists('test_security.db'):
-        os.unlink('test_security.db')
-    db = DB('test_security.db')
-    db.initSchema()
+    db_file = 'test_security.db'
+    try:
+        db = DB(db_file)
+    except SchemaMismatch:
+        os.unlink(db_file)
+        db = DB(db_file)
 
     cursor = db.writeTxn()
-    db.deleteBugs(cursor)
-    db.insertBugs(cursor, bugs.CVEFile('../../data/CAN/list'))
-    db.insertBugs(cursor, bugs.CVEFile('../../data/CVE/list',
-                                              no_version_needs_note=False))
-    db.insertBugs(cursor, bugs.DSAFile('../../data/DSA/list'))
-    db.insertBugs(cursor, bugs.DTSAFile('../../data/DTSA/list'))
-    db.finishBugs(cursor)
+    db.readBugs(cursor, '../../data')
     db.commit(cursor)
 
     b = bugs.BugFromDB(cursor, 'CAN-2005-2491')