bin/update-nvd: Remove duplicates before loading NVD data

git-svn-id: svn+ssh://svn.debian.org/svn/secure-testing@10437 e39458fd-73e7-0310-bf30-c45bca0a0e42

1 files changed, 8 insertions, 0 deletions

diff --git a/bin/update-nvd b/bin/update-nvd
index 88fe39450b..281c047dd9 100644
--- a/bin/update-nvd
+++ b/bin/update-nvd
@@ -34,6 +34,14 @@ for name in sys.argv[1:]:
     data += nvd.parse(f)
     f.close()
 
+# For some reason, NVD adds duplicates, so we need to get rid of them.
+# Sort afterwords to increase locality in the insert process.
+deduplicate = {}
+for x in data:
+    deduplicate[x[0]] = x
+data = deduplicate.values()
+data.sort()
+
 cursor = db.writeTxn()
 if incremental:
     db.updateNVD(cursor, data)