Store CVE descriptions in the nvd_data table. Enable incremental

NVD updates.

lib/python/security_db.py (DB):
  Bump schema version.  Add cve_desc column to the nvd_data table.
(DB.updateNVD):
  New method.

bin/update-nvd:
  If the -i option is specified, use updateNVD instead of replaceNVD.

lib/python/nvd.py (_Parser):
  Add new member variable path.
(_Parser.characters):
  New method.
(_Parser.endElement):
  Store cve_desc variable.

bin/tracker_service.py (TrackerService.page_bug):
  Use NVD description if available.


git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@3078 e39458fd-73e7-0310-bf30-c45bca0a0e42

4 files changed, 32 insertions, 4 deletions

diff --git a/bin/tracker_service.py b/bin/tracker_service.py
index 73efd568bc..b4d79bb633 100644
--- a/bin/tracker_service.py
+++ b/bin/tracker_service.py
@@ -253,14 +253,17 @@ data source.""")],
             if source_xref:
                 yield B("Source"), source_xref
         
-            if bug.description:
+            nvd = self.db.getNVD(cursor, bug.name)
+
+            if nvd and nvd.cve_desc:
+                yield B("Description"), nvd.cve_desc
+            elif bug.description:
                 yield B("Description"), bug.description
 
             xref = list(self.db.getBugXrefs(cursor, bug.name))
             if xref:
                 yield B("References"), self.make_xref_list(url, xref)
 
-            nvd = self.db.getNVD(cursor, bug.name)
             if nvd:
                 if nvd.severity:
                     yield B("NVD severity"), nvd.severity.lower()
diff --git a/bin/update-nvd b/bin/update-nvd
index 4910845ef9..88fe39450b 100644
--- a/bin/update-nvd
+++ b/bin/update-nvd
@@ -24,12 +24,19 @@ import security_db
 db_file = 'data/security.db'
 db = security_db.DB(db_file)
 
+incremental = False
 data = []
 for name in sys.argv[1:]:
+    if name == '-i':
+        incremental = True
+        continue
     f = file(name)
     data += nvd.parse(f)
     f.close()
 
 cursor = db.writeTxn()
-db.replaceNVD(cursor, data)
+if incremental:
+    db.updateNVD(cursor, data)
+else:
+    db.replaceNVD(cursor, data)
 db.commit(cursor)
diff --git a/lib/python/nvd.py b/lib/python/nvd.py
index 0089d0fa12..77362e77a4 100644
--- a/lib/python/nvd.py
+++ b/lib/python/nvd.py
@@ -31,11 +31,13 @@ class _Parser(xml.sax.handler.ContentHandler):
         for x in ('entry', 'local', 'range', 'remote', 'user_init',
                   'avail', 'conf', 'int', 'sec_prot'):
              self.start_dispatcher[x] = getattr(self, 'TAG_' + x)
+        self.path = []
 
     def _noop(*args):
         pass
 
     def startElement(self, name, attrs):
+        self.path.append((name, attrs))
         self.start_dispatcher.get(name, self._noop)(name, attrs)
 
     def TAG_entry(self, name, attrs):
@@ -44,6 +46,7 @@ class _Parser(xml.sax.handler.ContentHandler):
         self.severity = attrs.get('severity', u'').encode('utf-8')
         self.discovered = attrs.get('discovered', u'').encode('utf-8')
 
+        self.cve_desc = ""
         self.range_local = self.range_remote = self.range_user_init = None
 
         self.loss_avail = self.loss_conf = self.loss_int \
@@ -82,6 +85,7 @@ class _Parser(xml.sax.handler.ContentHandler):
             if name[0:4] == 'CAN-':
                 name = 'CVE-' + name[4:]
             self.result.append((name,
+                                self.cve_desc,
                                 self.discovered,
                                 self.published,
                                 self.severity,
@@ -94,6 +98,12 @@ class _Parser(xml.sax.handler.ContentHandler):
                                 self.loss_sec_prot_user,
                                 self.loss_sec_prot_admin,
                                 self.loss_sec_prot_other))
+        del self.path[-1]
+
+    def characters(self, content):
+        (name, attrs) = self.path[-1]
+        if name == 'descript' and attrs['source'] == 'cve':
+            self.cve_desc = content
 
 def parse(file):
     """Parses the indicated file object.  Returns a list of tuples,
diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index 7a6b61faac..e55514aad5 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -113,7 +113,7 @@ class DB:
         self.db = apsw.Connection(name)
         self.verbose = verbose
 
-        self.schema_version = 19
+        self.schema_version = 20
         self._initFunctions()
 
         c = self.cursor()
@@ -297,6 +297,7 @@ class DB:
         cursor.execute(
             """CREATE TABLE nvd_data
             (cve_name TEXT NOT NULL PRIMARY KEY,
+            cve_desc TEXT NOT NULL,
             discovered TEXT NOT NULL,
             published TEXT NOT NULL,
             severity TEXT NOT NULL,
@@ -1362,6 +1363,13 @@ class DB:
                            + (", ?" * (len(data[0]) - 1))
                            + ")", data)
 
+    def updateNVD(self, cursor, data):
+        """Adds (and overwrites) NVD data stored in the database.
+        This can be used for incremental updates."""
+        cursor.executemany("INSERT OR REPLACE INTO nvd_data VALUES (?"
+                           + (", ?" * (len(data[0]) - 1))
+                           + ")", data)
+
     def getNVD(self, cursor, cve_name):
         """Returns a dictionary with NVD data corresponding to the CVE name,
         or None."""