Aggregate data from data/wml files with JSON tracker data

CVS version numbers

english/security/oval/generate.py: INITIAL -> 1.1 
english/security/oval/parseDsa2Oval.py: 1.5 -> 1.6(DEAD) 
english/security/oval/parseJSON2Oval.py: 1.6 -> 1.7(DEAD) 
english/security/oval/oval/definition/generator.py: 1.12 -> 1.13 
english/security/oval/oval/parser/wml.py: 1.6 -> 1.7

4 files changed, 128 insertions, 184 deletions

diff --git a/english/security/oval/parseJSON2Oval.py b/english/security/oval/generate.py
index 0795035994d..25a0115c32c 100644
--- a/english/security/oval/parseJSON2Oval.py
+++ b/english/security/oval/generate.py
@@ -19,11 +19,12 @@ from oval.parser import dsa
 from oval.parser import wml
 
 
-dsaref = {}
+ovals = {}
 
 # TODO: these may need changed or reworked.
 DEBIAN_VERSION = {"wheezy" : "7.0", "jessie" : "8.2", "stretch" : "9.0",
-                  "sid" : "9.0", "etch" : "4.0", "squeeze":"6.0", "lenny":"5.0"}
+                  "sid" : "9.0", "etch" : "4.0", "squeeze":"6.0", "lenny":"5.0",
+                  "woody" : "3.0", "potato" : "2.2", "sarge" : "3.1"}
 
 def usage (prog = "parse-wml-oval.py"):
     """Print information about script flags and options"""
@@ -31,52 +32,93 @@ def usage (prog = "parse-wml-oval.py"):
     print """usage: %s [vh] [-d <directory>]\t-d\twhich directory use for
     dsa definition search\t-v\tverbose mode\t-h\tthis help""" % prog
 
-
-def printdsas(dsaref):
+def printdsas(ovals, year):
     """ Generate and print OVAL Definitions for collected DSA information """
 
-    ovalDefinitions = oval.definition.generator.createOVALDefinitions (dsaref)
+    ovalDefinitions = oval.definition.generator.createOVALDefinitions (ovals, year)
     oval.definition.generator.printOVALDefinitions (ovalDefinitions)
 
+def parsedirs (directory, postfix, depth):
+  """ Recursive search directory for DSA files contain postfix in their names.
+
+    For this files called oval.parser.dsa.parseFile() for extracting DSA information.
+  """
+
+  global ovals
+
+  if depth == 0:
+    logging.log(logging.DEBUG, "Maximum depth reached at directory " + directory)
+    return (0)
+  
+  for fileName in os.listdir (directory):
+    
+    path = "%s/%s" % (directory, fileName)
+    
+    logging.log (logging.DEBUG, "Checking %s (for %s at %s)" % (fileName, postfix, depth))
+    
+    if os.access(path, os.R_OK) and os.path.isdir (path) and not os.path.islink (path) and fileName[0] != '.':
+      logging.log(logging.DEBUG, "Entering directory " + path)
+      parsedirs (path, postfix, depth-1)
+
+    #Parse fileNames
+    if os.access(path, os.R_OK) and fileName.endswith(postfix) and fileName[0] != '.' and fileName[0] != '#':
+      result = dsa.parseFile (path)
+      if result:
+        if ovals.has_key (result[0]):
+          ovals[result[0]]["dsa"] = fileName[:-5].upper() # remove .data part
+          for (k, v) in result[1].iteritems():
+            ovals[result[0]][k] = v
+        else:
+          ovals[result[0]] = result[1]
+
+        # also parse corresponding wml file
+        wmlResult = wml.parseFile(path.replace('.data', '.wml'), DEBIAN_VERSION)
+        if wmlResult:
+          data, releases = wmlResult
+          for (k, v) in data.iteritems():
+            ovals[result[0]][k] = v
+          if not ovals[result[0]].get("release", None):
+            ovals[result[0]]['release']=releases
+
+  return 0
+
 def parseJSON(json_data, year):
     """
     Parse the JSON data and extract information needed for OVAL definitions
     :param json_data: Json_Data
     :return:
     """
+    global ovals
+
     today = date.today()
     logging.log(logging.DEBUG, "Start of JSON Parse.")
     for package in json_data:
         logging.log(logging.DEBUG, "Parsing package %s" % package)
-        for CVE in json_data[package]:
-            if CVE.find(year) < 0:
-                continue
-            logging.log(logging.DEBUG, "Getting releases for %s" % CVE)
+        for cve in json_data[package]:
+            logging.log(logging.DEBUG, "Getting releases for %s" % cve)
             release = {}
-            for rel in json_data[package][CVE]['releases']:
-                if json_data[package][CVE]['releases'][rel]['status'] != \
+            for rel in json_data[package][cve]['releases']:
+                if json_data[package][cve]['releases'][rel]['status'] != \
                         'resolved':
                     fixed_v = '0'
                     f_str = 'no'
                 else:
-                    fixed_v = json_data[package][CVE]['releases'][rel]['fixed_version']
+                    fixed_v = json_data[package][cve]['releases'][rel]['fixed_version']
                     f_str = 'yes'
                 release.update({DEBIAN_VERSION[rel]: {u'all': {
                     package: fixed_v}}})
 
-                # print json.dumps(json_data[package][CVE])
+                # print json.dumps(json_data[package][cve])
                 # sys.exit(1)
-                ovalId = oval.definition.generator.getOvalId(CVE)
-                dsaref.update({ovalId: {"packages": package,
-                                        'title': CVE,
+                ovals.update({cve: {"packages": package,
+                                        'title': cve,
                                         'vulnerable': "yes",
                                         'date': str(today.isoformat()),
                                         'fixed': f_str, 
-                                        'description': json_data[package][CVE].get("description",""),
-                                        'moreinfo': "",
-                                        'release': release, 'secrefs': CVE}})
-                logging.log(logging.DEBUG, "Created entry with ovalId %s" % ovalId)
-
+                                        'description': json_data[package][cve].get("description",""),
+                                        'secrefs': cve,
+                                        'release': release}})
+                logging.log(logging.DEBUG, "Created entry for %s" % cve)
 
 def get_json_data(json_file):
     """
@@ -105,6 +147,7 @@ def main(args):
     # unpack args
 
     json_file = args['JSONfile']
+    data_dir = args['data_directory']
     temp_file = args['tmp']
     year = args['year']
 
@@ -126,9 +169,9 @@ def main(args):
             os.remove(temp_file)
 
     parseJSON(json_data, year)
-    #parsedirs (opts['-d'], '.data', 2)
+    parsedirs(data_dir, '.data', 2)
     logging.log(logging.INFO, "Finished parsing JSON data")
-    printdsas(dsaref)
+    printdsas(ovals, year)
 
 if __name__ == "__main__":
     PARSER = argparse.ArgumentParser(description='Generates oval definitions '
@@ -141,6 +184,9 @@ if __name__ == "__main__":
                         help='Local JSON file to use. This will use a local '
                              'copy of the JSON file instead of downloading from'
                              ' it from the server. default=none', default=None)
+    PARSER.add_argument('-d', '--data-directory', type=str,
+                        help='Local directory to parse for data/wml file.'
+                        'default=.', default='.')
     PARSER.add_argument('-t', '--tmp', type=str,
                         help='Temporary file to download JSON file to. Warning:'
                              ' if this file already exists it will be removed '
diff --git a/english/security/oval/oval/definition/generator.py b/english/security/oval/oval/definition/generator.py
index 3989b4ebd64..249546628df 100644
--- a/english/security/oval/oval/definition/generator.py
+++ b/english/security/oval/oval/definition/generator.py
@@ -23,7 +23,7 @@ regex = re.compile(RE_XML_ILLEGAL)
 class OvalGeneratorException (Exception):
     pass
 
-class DSAFormatException (OvalGeneratorException):
+class CVEFormatException (OvalGeneratorException):
   code = 1
   
 def __createXMLElement (name, descr = None, attrs = {}):
@@ -299,7 +299,7 @@ def __createGeneratorHeader ():
 
   return (generator)
 
-def createPlatformDefinition (release, data, dsa):
+def createPlatformDefinition (release, data, cve):
   """ Generate OVAL definitions for current release
   
     Generate full criteria tree for specified release. Tests, states and objects 
@@ -309,13 +309,13 @@ def createPlatformDefinition (release, data, dsa):
     Argument keywords:
     release -- Debian release
     data -- dict with information about packages
-    dsa - DSA id
+    cve - CVE id
     
     return Generated XML fragment
   """
   #Raise exception if we receive too small data
   if len(data) == 0:
-    logging.log(logging.WARNING, "DSA %s: Information of affected platforms is not available." % dsa)
+    logging.log(logging.WARNING, "CVE %s: Information of affected platforms is not available." % cve)
   
   softwareCriteria = __createXMLElement ("criteria", attrs = {"comment" : "Release section", "operator" : "AND"})
   softwareCriteria.appendChild ( __createXMLElement ("criterion", attrs={"test_ref" : __createTest("release", release), "comment" : "Debian %s is installed" % release}))
@@ -397,90 +397,97 @@ def createPlatformDefinition (release, data, dsa):
   
   return (softwareCriteria)
 
-def createDefinition (dsa, dsaref):
+def createDefinition (cve, oval):
   """ Generate OVAL header of Definition tag
   
     Print general informaton about OVAL definition. Use createPlatformDefinition for generate criteria 
     sections for each affected release.
     
     Argument keywords:
-    dsa -- DSA dentificator
-    dsaref -- DSA parsed data
-  """  
-  if not dsaref.has_key("release"):
-    logging.log(logging.WARNING, "DSA %s: Release definition not well formatted. Ignoring this DSA." % dsa)
-    raise DSAFormatException
+    cve -- CVE dentificator
+    oval -- CVE parsed data
+  """
+  if not oval.has_key("release"):
+    logging.log(logging.WARNING, "CVE %s: Release definition not well formatted. Ignoring this CVE." % cve)
+    raise CVEFormatException
     
-  if not dsaref.has_key("packages"):
-    logging.log(logging.WARNING, "DSA %s: Package information missed. Ignoring this DSA." % dsa)
-    dsaref["packages"] = ""
+  if not oval.has_key("packages"):
+    logging.log(logging.WARNING, "CVE %s: Package information missed. Ignoring this CVE." % cve)
+    oval["packages"] = ""
+    return None
 
-  if not dsaref.has_key("title"):
-    logging.log(logging.WARNING, "DSA %s: title information missed." % dsa)
-    dsaref["title"] = ""
+  if not oval.has_key("title"):
+    logging.log(logging.WARNING, "CVE %s: title information missed." % cve)
+    oval["title"] = ""
 
-  if not dsaref.has_key("description"):
-    logging.log(logging.WARNING, "DSA %s: Description information missed." % dsa)
-    dsaref["description"] = ""
+  if not oval.has_key("description"):
+    logging.log(logging.WARNING, "CVE %s: Description information missed." % cve)
+    oval["description"] = ""
 
-  if not dsaref.has_key("moreinfo"):
-    logging.log(logging.WARNING, "DSA %s: Moreinfo information missed." % dsa)
-    dsaref["moreinfo"] = ""
-  
-  if not dsaref.has_key("secrefs"):
-    logging.log(logging.WARNING, "DSA %s: Secrefs information missed." % dsa)
-    dsaref["secrefs"] = ""
+  if not oval.has_key("moreinfo"):
+    logging.log(logging.WARNING, "CVE %s: Moreinfo information missed." % cve)
+    oval["moreinfo"] = ""
+
+  if not oval.has_key("dsa"):
+    logging.log(logging.WARNING, "CVE %s: DSA information missed." % cve)
+  elif oval["moreinfo"]:
+    oval["moreinfo"] = "\n%s%s" % (oval["dsa"], oval["moreinfo"])
+
+  if not oval.has_key("secrefs"):
+    logging.log(logging.WARNING, "CVE %s: Secrefs information missed." % cve)
+    oval["secrefs"] = ""
 
   ### Definition block: Metadata, Notes, Criteria
-  definition = __createXMLElement ("definition", attrs = {"id" : "oval:org.debian:def:%s" % getOvalId(dsaref["title"]), "version" : "1", "class" : "vulnerability"})
+  ovalId = getOvalId(cve)
+  definition = __createXMLElement ("definition", attrs = {"id" : "oval:org.debian:def:%s" % ovalId, "version" : "1", "class" : "vulnerability"})
 
   ### Definition : Metadata : title, affected, reference, description ###
   metadata = __createXMLElement ("metadata")
-  metadata.appendChild (__createXMLElement ("title", dsaref["title"]))
+  metadata.appendChild (__createXMLElement ("title", oval["title"]))
 
   ### Definition : Metadata : Affected : platform, product ###
   affected = __createXMLElement ("affected", attrs = {"family" : "unix"})
-  for platform in dsaref["release"]:
+  for platform in oval["release"]:
     affected.appendChild ( __createXMLElement ("platform", "Debian GNU/Linux %s" % platform))
-  affected.appendChild ( __createXMLElement ("product", dsaref.get("packages")))
+  affected.appendChild ( __createXMLElement ("product", oval.get("packages")))
     
   metadata.appendChild (affected)
   ### Definition : Metadata : Affected : END ###
 
   refpatern = re.compile (r'((CVE|CAN)-[\d-]+)')
-  for ref in dsaref.get("secrefs").split(" "):
+  for ref in oval.get("secrefs").split(" "):
     result = refpatern.search(ref)
     if result:
       (ref_id, source) = result.groups()
       metadata.appendChild ( __createXMLElement ("reference", attrs = {"source" : source, "ref_id" : ref_id, "ref_url" : "http://cve.mitre.org/cgi-bin/cvename.cgi?name=%s" % ref_id}) )
   
   #TODO: move this info to other place
-  metadata.appendChild ( __createXMLElement ("description", dsaref["description"]))
+  metadata.appendChild ( __createXMLElement ("description", oval["description"]))
   debianMetadata = __createXMLElement ("debian")
-  if dsaref.has_key("date"):
-    debianMetadata.appendChild ( __createXMLElement ("date", dsaref["date"]) )
-  debianMetadata.appendChild ( __createXMLElement ("moreinfo", dsaref["moreinfo"]) )
+  if oval.has_key("date"):
+    debianMetadata.appendChild ( __createXMLElement ("date", oval["date"]) )
+  debianMetadata.appendChild ( __createXMLElement ("moreinfo", oval["moreinfo"]))
   metadata.appendChild (debianMetadata)
   definition.appendChild ( metadata )
 
   ### Definition : Criteria ###
-  if len(dsaref["release"]) > 1:
+  if len(oval["release"]) > 1:
     #f we have more than one release - generate additional criteria section
     platformCriteria = __createXMLElement ("criteria", attrs = {"comment" : "Platform section", "operator" : "OR"})
     definition.appendChild (platformCriteria)
   else:
     platformCriteria = definition
   
-  for platform in dsaref["release"]:
-    data = dsaref["release"][platform]
-    platformCriteria.appendChild (createPlatformDefinition(platform, data, dsa))
+  for platform in oval["release"]:
+    data = oval["release"][platform]
+    platformCriteria.appendChild (createPlatformDefinition(platform, data, cve))
     
   ### Definition : Criteria END ###
 
   return (definition)
 
-def createOVALDefinitions (dsaref):
-  """ Generate XML OVAL definition tree for range of DSA
+def createOVALDefinitions (ovals, year):
+  """ Generate XML OVAL definition tree for range of CVE
   
     Generate namespace section and use other functions to generate definitions,
     tests, objects and states subsections.
@@ -506,13 +513,15 @@ def createOVALDefinitions (dsaref):
   
   definitions = doc.createElement ("definitions")
   
-  keyids = dsaref.keys()
+  keyids = ovals.keys()
   keyids.sort()
-  for dsa in keyids:
+  for cve in keyids:
     try:
-      definitions.appendChild (createDefinition(dsa, dsaref[dsa]))
-    except DSAFormatException:
-      logging.log (logging.WARNING, "DSA %s: Bad data file. Ignoring this DSA." % dsa)
+      if cve.find(year) < 0:
+        continue
+      definitions.appendChild (createDefinition(cve, ovals[cve]))
+    except CVEFormatException:
+      logging.log (logging.WARNING, "CVE %s: Bad data file. Ignoring this CVE." % cve)
       
   root.appendChild (definitions)
   
diff --git a/english/security/oval/oval/parser/wml.py b/english/security/oval/oval/parser/wml.py
index b0c4ff75eed..205a834e041 100644
--- a/english/security/oval/oval/parser/wml.py
+++ b/english/security/oval/oval/parser/wml.py
@@ -15,14 +15,10 @@ import os
 import sys
 import logging
 
-# TODO: these may need changed or reworked.
-DEBIAN_VERSION = {"wheezy" : "7.0", "jessie" : "8.2", "stretch" : "9.0",
-                  "sid" : "9.0", "etch" : "4.0", "squeeze":"6.0", "lenny":"5.0"}
-
 # Format of wml files is:
 #<define-tag description>DESCRIPTION</define-tag>
 #<define-tag moreinfo>Multiline information</define-tag>
-def parseFile (path):
+def parseFile (path, debianVersion):
   """ Parse wml file with description of Debian Security Advisories 
 	
   Keyword arguments:
@@ -81,11 +77,11 @@ def parseFile (path):
       if result:
         deb_version = result.groups()[0]
 
-      new_version_pattern = re.compile(r'version (.*?).</p>')
+      new_version_pattern = re.compile(r'version ([a-z]+).</p>')
       result = new_version_pattern.search(line)
       if result and deb_version != "":
         pack_ver = result.groups()[0]
-        releases.update({DEBIAN_VERSION[deb_version]: {u"all": {grabPackName(path) : pack_ver}}})
+        releases.update({debianVersion[deb_version]: {u"all": {grabPackName(path) : pack_ver}}})
 
   except IOError:
     logging.log (logging.ERROR, "Can't work with file %s" % path)
diff --git a/english/security/oval/parseDsa2Oval.py b/english/security/oval/parseDsa2Oval.py
deleted file mode 100755
index 5fbc9156b65..00000000000
--- a/english/security/oval/parseDsa2Oval.py
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-# Extracts the data DSA files and creates OVAL queries to
-# be used with the OVAL query interpreter (see http://oval.mitre.org)
-
-# (c) 2007 Pavel Vinogradov
-# (c) 2004 Javier Fernandez-Sanguino                                                                                                           
-# Licensed under the GNU General Public License version 2.                                                                                     
-                                                                                                                                               
-import os
-import sys
-import getopt
-import logging
-
-import oval.definition.generator
-from oval.parser import dsa
-from oval.parser import wml
-
-ovals = {}
-
-def usage (prog = "parse-wml-oval.py"):
-  """Print information about script flags and options"""
-
-  print """
-usage: %s [vh] [-d <directory>]
-\t-d\twhich directory use for dsa definition search
-\t-v\tverbose mode
-\t-h\tthis help
-  """ % prog
-   
-def printdsas (ovals):
-    """ Generate and print OVAL Definitions for collected DSA information """
-    
-    ovalDefinitions = oval.definition.generator.createOVALDefinitions (ovals)
-    oval.definition.generator.printOVALDefinitions (ovalDefinitions)
-
-def parsedirs (directory, postfix, depth):
-  """ Recursive search directory for DSA files contain postfix in their names.
-
-    For this files called oval.parser.dsa.parseFile() for extracting DSA information.
-  """
-
-  global ovals
-
-  if depth == 0:
-    logging.log(logging.DEBUG, "Maximum depth reached at directory " + directory)
-    return (0)
-  
-  for file in os.listdir (directory):
-    
-    path = "%s/%s" % (directory, file)
-    
-    logging.log (logging.DEBUG, "Checking %s (for %s at %s)" % (file, postfix, depth))
-    
-    if os.access(path, os.R_OK) and os.path.isdir (path) and not os.path.islink (path) and file[0] != '.':
-      logging.log(logging.DEBUG, "Entering directory " + path)
-      parsedirs (path, postfix, depth-1)
-
-    #Parse files
-    if os.access(path, os.R_OK) and file.endswith(postfix) and file[0] != '.' and file[0] != '#':
-      result = dsa.parseFile (path)
-      if result:
-        if ovals.has_key (result[0]):
-          for (k, v) in result[1].iteritems():
-            ovals[result[0]][k] = v
-        else:
-          ovals[result[0]] = result[1]
-
-        # also parse corresponding wml file
-        wmlResult = wml.parseFile(path.replace('.data', '.wml'))
-        if wmlResult:
-          data, releases = wmlResult
-          for (k, v) in data.iteritems():
-            ovals[result[0]][k] = v
-          if not ovals[result[0]].get("release", None):
-            ovals[result[0]]['release']=releases
-
-  return 0
-
-if __name__ == "__main__":
-    
-    # Parse cmd options with getopt
-    opts = {}
-    
-    #By default we search dsa definitions from current directory, but -d option override this
-    opts['-d'] = "./"
-    
-    try:
-        opt, args = getopt.getopt (sys.argv[1:], 'vhd:')
-    except getopt.GetoptError:
-        usage ()
-        sys.exit(1)
-    
-    for key, value in opt:
-        opts[key] = value
-    
-    if opts.has_key ('-h'):
-        usage()
-        sys.exit(0)
-        
-    if opts.has_key('-v'):
-        logging.basicConfig(level=logging.DEBUG)
-        
-    logging.basicConfig(level=logging.WARNING)
-        
-    parsedirs (opts['-d'], '.data', 2)
-    printdsas(ovals)