Implement bin/update-db, to update the database with a single command.

Most processing is skipped if no input files have been modified.

lib/python/security_db.py (SchemaMismatch):
  New exception.
(DB):
  Handle schema versioning.
(DB.initSchema):
  Add subrelease column to source_packages and binary_packages.
  Set user_version.
  Remove stray commit.
(DB._parseFile):
  Return information to the caller if the file is unchanged.
(DB.readPackages):
  Move deletion code to callees.
(DB._readSourcePackages, DB._readBinaryPackages):
  Implement incremental updates.  Add subrelease.
  Need to invoke _clearVersions if any changes are made.
(DB.deleteBugs, DB.finishBugs):
  Moved into readBugs.
(DB.insertBugs):
  Rename ...
(DB.readBugs):
  ... to this one.  Implement incremental updates.
  Invoke _clearVersions if necessary.
(DB._clearVersions):
  Add.
(DB._updateVersions):
  Skip processing if _clearVersions has not been invoked.
(DB.getVersion, DB.releaseContainsPackage, DB._synthesizeReleases):
  Obsolete, remove.
(test):
  Update.

lib/python/bugs.py (CANFile, CVEFile):
  Split into two classes, which handle the differences between the two
  files.

bin/check-syntax:
  Update accordingly.

bin/update-db:
  New database update script.  Implements incremental updates.

Makefile:
  Remove references to bin/update-packages.  Simplify drastically.


git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@1994 e39458fd-73e7-0310-bf30-c45bca0a0e42

4 files changed, 28 insertions, 151 deletions

diff --git a/bin/check-syntax b/bin/check-syntax
index d996ae3664..8e1c69c2f5 100755
--- a/bin/check-syntax
+++ b/bin/check-syntax
@@ -46,7 +46,7 @@ def do_parse(f):
     
 
 def parse_CAN(name):
-    do_parse(bugs.CVEFile(name))
+    do_parse(bugs.CANFile(name))
 
 def parse_CVE(name):
     f = bugs.CVEFile(name)
diff --git a/bin/update-bug-list-db b/bin/update-db
index 96ebd3b6a0..273d9fa7e0 100755
--- a/bin/update-bug-list-db
+++ b/bin/update-db
@@ -16,40 +16,49 @@ def setup_paths():
         if idx == -1:
             raise ImportError, "could not setup paths"
         path = path[0:idx]
-root_path = setup_paths()
+os.chdir(setup_paths())
 
 import bugs
 import debian_support
 import security_db
 
-db_file = root_path + '/data/security.db'
-new_file = not os.path.exists(db_file)
-db = security_db.DB(db_file)
-if new_file:
-    db.initSchema()
+db_file = 'data/security.db'
+try:
+    db = security_db.DB(db_file, verbose=True)
+except security_db.SchemaMismatch:
+    os.unlink(db_file)
+    db = security_db.DB(db_file, verbose=True)
+    
 cursor = db.writeTxn()
-db.deleteBugs(cursor)
+
+# Bug lists (CAN/CVE/DSA/DTSA)
+
 try:
-    db.insertBugs(cursor, bugs.CVEFile(root_path + '/data/CAN/list'))
-    db.insertBugs(cursor, bugs.CVEFile(root_path + '/data/CVE/list',
-                                       no_version_needs_note=False))
-    db.insertBugs(cursor, bugs.DSAFile(root_path + '/data/DSA/list'))
-    db.insertBugs(cursor, bugs.DTSAFile(root_path + '/data/DTSA/list'))
+    warnings = db.readBugs(cursor, 'data')
 except debian_support.ParseError, e:
-    db.rollback(cursor)
     e.printOut(sys.stderr)
     sys.exit(1)
 except security_db.InsertError, e:
-    db.rollback(cursor)
     for err in e.errors:
         print err
     sys.exit(1)
+if warnings:
+    for x in warnings:
+        print x
+    sys.exit(1)
+
+# Packages
+
+db.readPackages(cursor, 'data/packages')
 
-warnings = db.finishBugs(cursor)
+# Calculate vulnerability information.
+
+warnings = db.calculateVulnerabilities(cursor)
 if warnings:
-    db.rollback(cursor)
     for x in warnings:
         print x
     sys.exit(1)
-else:
-    db.commit(cursor)
+
+# Everything worked well.
+    
+db.commit(cursor)
diff --git a/bin/update-packages b/bin/update-packages
deleted file mode 100755
index 98b447c151..0000000000
--- a/bin/update-packages
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-
-# This script downloads and imports Debian package files.
-
-import errno
-import os
-import os.path
-import string
-import sys
-
-def setup_paths():
-    check_file = 'lib/python/debian_support.py'
-    path = os.getcwd()
-    while 1:
-        if os.path.exists("%s/%s" % (path, check_file)):
-            sys.path = [path + '/lib/python'] + sys.path
-            return path
-        idx = string.rfind(path, '/')
-        if idx == -1:
-            raise ImportError, "could not setup paths"
-        path = path[0:idx]
-root_path = setup_paths()
-
-import debian_support
-import security_db
-
-def explodeReleases(args):
-    for arg in args:
-        (release, archs) = arg.split('=')
-        # FIXME: What shall we do with these?
-        # if debian_support.internRelease(release) is None:
-        #     sys.stderr.write("error: unknown release: %s\n" % release)
-        #     sys.exit(1)
-        yield release, archs.split(',')
-
-archives = ('main', 'contrib', 'non-free')
-
-def nameSources(release, archive):
-    return '%s/data/packages/%s_%s_Sources' % (root_path, release, archive)
-
-def namePackages(release, archive, arch):
-    return '%s/data/packages/%s_%s_%s_Packages' % (root_path, release,
-                                                   archive, arch)
-
-def cmd_download(args):
-    url_base = args[0]
-    if url_base[-1] != '/':
-        url_base += '/'
-        
-    for release, archs in explodeReleases(args[1:]):
-        # Security updates are stored in a different directory.
-        if release[-9:] == '-security':
-            rrel = release[:-9] + '/updates'
-        else:
-            rrel = release
-
-        for archive in archives:
-            print "Updating source package %s/%s" % (release, archive)
-            debian_support.updateFile("%sdists/%s/%s/source/Sources"
-                                      % (url_base, rrel, archive),
-                                      nameSources(release, archive),
-                                      verbose=True)
-            for arch in archs:
-                print "Updating binary package %s/%s/%s" \
-                                      % (release, archive, arch)
-                debian_support.updateFile("%sdists/%s/%s/binary-%s/Packages"
-                                          % (url_base, rrel, archive, arch),
-                                          namePackages(release, archive, arch),
-                                          verbose=True)
-
-def cmd_import(args):
-    db_file = root_path + '/data/security.db'
-    new_file = not os.path.exists(db_file)
-    db = security_db.DB(db_file, verbose=True)
-    if new_file:
-        db.initSchema()
-    c = db.writeTxn()
-    db.readPackages(c, root_path + '/data/packages')
-    db.commit(c)
-
-cmds = {"download" : cmd_download,
-        "import" : cmd_import}
-
-if len(sys.argv) < 2 or not cmds.has_key(sys.argv[1]):
-    sys.stderr.write(\
-"""usage: update-packages download URL-BASE RELEASE=ARCH...
-       update-packages import
-""")
-    sys.exit(1)
-try:
-    cmds[sys.argv[1]](sys.argv[2:])
-except debian_support.ParseError, e:
-    e.printOut(sys.stderr)
-    sys.exit(1)
-
diff --git a/bin/update-vulnerabilities b/bin/update-vulnerabilities
deleted file mode 100755
index e2ba55554d..0000000000
--- a/bin/update-vulnerabilities
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/python
-
-# This script recalculates the vulnerability information in the
-# security database.
-
-import errno
-import os
-import os.path
-import string
-import sys
-
-def setup_paths():
-    check_file = 'lib/python/debian_support.py'
-    path = os.getcwd()
-    while 1:
-        if os.path.exists("%s/%s" % (path, check_file)):
-            sys.path = [path + '/lib/python'] + sys.path
-            return path
-        idx = string.rfind(path, '/')
-        if idx == -1:
-            raise ImportError, "could not setup paths"
-        path = path[0:idx]
-root_path = setup_paths()
-
-import security_db
-
-db_file = root_path + '/data/security.db'
-assert os.path.exists(db_file)
-db = security_db.DB(db_file, verbose=True)
-c = db.writeTxn()
-warnings = db.calculateVulnerabilities(c)
-if warnings:
-    db.rollback(c)
-    for x in warnings:
-        print x
-    sys.exit(1)
-db.commit(c)