changed logging levels

Former-commit-id: b4b89c7785bbaafcd3e0dc280bf8f050f6c52eb0

README.txt

@@ -7,16 +7,11 @@ If found, it will generate a report linking to the associated CVE entries.
 Usage:
 $ mvn package
 $ cd target
-$ java -jar DependencyCheck-0.2.0.jar -h
-$ java -jar DependencyCheck-0.2.0.jar -a Testing -out . -scan ./test-classes/org.mortbay.jetty.jar -scan ./test-classes/struts2-core-2.1.2.jar -scan ./lib
+$ java -jar DependencyCheck-0.2.1.jar -h
+$ java -jar DependencyCheck-0.2.1.jar -a Testing -out . -scan ./test-classes/org.mortbay.jetty.jar -scan ./test-classes/struts2-core-2.1.2.jar -scan ./lib
 
 Then load the resulting 'Testing.html' into your favorite browser.
 
-Important note - DependencyCheck should be run to analyze a project at least once every week. 
-The reason for this is that it downloads data from the National Vulnerability Database hosted 
-by NIST. If more then a week goes by without DependencyCheck updating the data, a full update
-can take an 90 minutes or more (a lot of data needs to be downloaded and processed).
-
 Author: Jeremy Long (jeremy.long@gmail.com)
 
 Copyright (c) 2012 Jeremy Long. All Rights Reserved.

pom.xml

@@ -23,7 +23,7 @@ along with DependencyCheck.  If not, see <http://www.gnu.org/licenses/>.
 
     <groupId>org.codesecure</groupId>
     <artifactId>DependencyCheck</artifactId>
-    <version>0.2.0</version>
+    <version>0.2.1</version>
     <packaging>jar</packaging>
 
     <name>DependencyCheck</name>
@@ -417,5 +417,11 @@ along with DependencyCheck.  If not, see <http://www.gnu.org/licenses/>.
                 </exclusion>
             </exclusions>
         </dependency>
+        <!--
+        <dependency>
+            <groupId>org.fusesource.hawtdb</groupId>
+            <artifactId>hawtdb</artifactId>
+            <version>1.6</version>
+        </dependency>-->
     </dependencies>
 </project>

src/main/java/org/codesecure/dependencycheck/data/nvdcve/Index.java

@@ -26,8 +26,6 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import javax.xml.bind.JAXBException;
-import javax.xml.parsers.ParserConfigurationException;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.KeywordAnalyzer;
 import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
@@ -42,7 +40,6 @@ import org.codesecure.dependencycheck.data.nvdcve.xml.Importer;
 import org.codesecure.dependencycheck.utils.DownloadFailedException;
 import org.codesecure.dependencycheck.utils.Downloader;
 import org.codesecure.dependencycheck.utils.Settings;
-import org.xml.sax.SAXException;
 
 /**
  * The Index class is used to utilize and maintain the NVD CVE Index.
@@ -115,31 +112,25 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
                 }
             }
             if (maxUpdates > 3) {
-                Logger.getLogger(Index.class.getName()).log(Level.WARNING, "NVD CVE requires several updates. This could take a couple of hours. To avoid this in the future, ensure that an update is run at least every seven days.");
+                Logger.getLogger(Index.class.getName()).log(Level.WARNING, "NVD CVE requires several updates; this could take a couple of minutes.");
             }
             int count = 0;
             for (NvdCveUrl cve : update.values()) {
                 if (cve.getNeedsUpdate()) {
                     count += 1;
-                    Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Updating NVD CVE (" + count + " of " + maxUpdates + ") :" + cve.getUrl());
+                    Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Updating NVD CVE (" + count + " of " + maxUpdates + ")");
                     URL url = new URL(cve.getUrl());
                     File outputPath = null;
                     try {
+                        Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Downloading " + cve.getUrl());
                         outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
                         Downloader.fetchFile(url, outputPath, false);
+                        Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Processing " + cve.getUrl());
                         Importer.importXML(outputPath.toString());
+                        Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Completed updated " + count + " of " + maxUpdates);
                     } catch (FileNotFoundException ex) {
                         //Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
                         throw new UpdateException(ex);
-                    } catch (JAXBException ex) {
-                        //Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
-                        throw new UpdateException(ex);
-                    } catch (ParserConfigurationException ex) {
-                        //Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
-                        throw new UpdateException(ex);
-                    } catch (SAXException ex) {
-                        //Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
-                        throw new UpdateException(ex);
                     } catch (IOException ex) {
                         //Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
                         throw new UpdateException(ex);

src/main/java/org/codesecure/dependencycheck/data/nvdcve/InvalidDataException.java

@@ -24,7 +24,7 @@ package org.codesecure.dependencycheck.data.nvdcve;
  *
  * @author Jeremy
  */
-class InvalidDataException extends Exception {
+public class InvalidDataException extends Exception {
 
     /**
      * Creates an InvalidDataException

src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/Importer.java

@@ -21,13 +21,7 @@ package org.codesecure.dependencycheck.data.nvdcve.xml;
 import java.io.*;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.parsers.SAXParserFactory;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-import org.xml.sax.XMLReader;
+import org.apache.lucene.index.CorruptIndexException;
 
 /**
  * Imports a NVD CVE XML file into the Lucene NVD CVE Index.
@@ -46,57 +40,63 @@ public class Importer {
      * Imports the NVD CVE XML File into the Lucene Index.
      *
      * @param file containing the path to the NVD CVE XML file.
-     * @throws ParserConfigurationException is thrown if the parser is
-     * misconfigured.
-     * @throws FileNotFoundException is thrown when there is a
-     * FileNotFoundException.
-     * @throws IOException is thrown when there is an IOException.
-     * @throws JAXBException is thrown when there is a JAXBException.
-     * @throws SAXException is thrown when there is a SAXException.
      */
-    public static void importXML(File file) throws FileNotFoundException, IOException, JAXBException,
-            ParserConfigurationException, SAXException {
-
-        SAXParserFactory factory = SAXParserFactory.newInstance();
-        factory.setNamespaceAware(true);
-        XMLReader reader = factory.newSAXParser().getXMLReader();
-
-        JAXBContext context = JAXBContext.newInstance("org.codesecure.dependencycheck.data.nvdcve.generated");
-        NvdCveXmlFilter filter = new NvdCveXmlFilter(context);
-
-        Indexer indexer = new Indexer();
-        indexer.openIndexWriter();
-
-        filter.registerSaveDelegate(indexer);
-
-        reader.setContentHandler(filter);
-        Reader fileReader = new FileReader(file);
-        InputSource is = new InputSource(fileReader);
+    public static void importXML(File file) {
+        NvdCveParser indexer = null;
         try {
-            reader.parse(is);
+
+            indexer = new NvdCveParser();
+
+            indexer.openIndexWriter();
+
+
+            indexer.parse(file);
+
+        } catch (CorruptIndexException ex) {
+            Logger.getLogger(Importer.class.getName()).log(Level.SEVERE, null, ex);
         } catch (IOException ex) {
             Logger.getLogger(Importer.class.getName()).log(Level.SEVERE, null, ex);
-        } catch (SAXException ex) {
-            Logger.getLogger(Importer.class.getName()).log(Level.SEVERE, null, ex);
         } finally {
-            indexer.close();
+            if (indexer != null) {
+                indexer.close();
+            }
         }
     }
+//    public static void importXML(File file) throws FileNotFoundException, IOException, JAXBException,
+//            ParserConfigurationException, SAXException {
+//
+//        SAXParserFactory factory = SAXParserFactory.newInstance();
+//        factory.setNamespaceAware(true);
+//        XMLReader reader = factory.newSAXParser().getXMLReader();
+//
+//        JAXBContext context = JAXBContext.newInstance("org.codesecure.dependencycheck.data.nvdcve.generated");
+//        NvdCveXmlFilter filter = new NvdCveXmlFilter(context);
+//
+//        Indexer indexer = new Indexer();
+//        indexer.openIndexWriter();
+//
+//        filter.registerSaveDelegate(indexer);
+//
+//        reader.setContentHandler(filter);
+//        Reader fileReader = new FileReader(file);
+//        InputSource is = new InputSource(fileReader);
+//        try {
+//            reader.parse(is);
+//        } catch (IOException ex) {
+//            Logger.getLogger(Importer.class.getName()).log(Level.SEVERE, null, ex);
+//        } catch (SAXException ex) {
+//            Logger.getLogger(Importer.class.getName()).log(Level.SEVERE, null, ex);
+//        } finally {
+//            indexer.close();
+//        }
+//    }
 
     /**
      * Imports the CPE XML File into the Lucene Index.
      *
      * @param path the path to the CPE XML file.
-     * @throws ParserConfigurationException is thrown if the parser is
-     * misconfigured.
-     * @throws FileNotFoundException is thrown when there is a
-     * FileNotFoundException.
-     * @throws IOException is thrown when there is an IOException.
-     * @throws JAXBException is thrown when there is a JAXBException.
-     * @throws SAXException is thrown when there is a SAXException.
      */
-    public static void importXML(String path) throws FileNotFoundException, IOException, JAXBException,
-            ParserConfigurationException, SAXException {
+    public static void importXML(String path) {
         File f = new File(path);
         if (!f.exists()) {
             f.mkdirs();

src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/Indexer.java

@@ -106,7 +106,7 @@ public class Indexer extends Index implements EntrySaveDelegate {
         doc.add(name);
 
         Field description = new Field(Fields.DESCRIPTION, vulnerability.getSummary(), Field.Store.NO, Field.Index.ANALYZED);
-        name.setIndexOptions(IndexOptions.DOCS_ONLY);
+        description.setIndexOptions(IndexOptions.DOCS_ONLY);
         doc.add(description);
 
 

src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveParser.java

@@ -0,0 +1,173 @@
+package org.codesecure.dependencycheck.data.nvdcve.xml;
+/*
+ * This file is part of DependencyCheck.
+ *
+ * DependencyCheck is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * DependencyCheck is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * DependencyCheck. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.codesecure.dependencycheck.data.nvdcve.Fields;
+import org.codesecure.dependencycheck.data.nvdcve.Index;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@gmail.com)
+ */
+public class NvdCveParser extends Index {
+
+    /**
+     * Parses an NVD CVE xml file using a buffered readerd. This
+     * method maybe more fragile then using a partial-unmarshalling SAX
+     * Parser (aka the deprecated NvdCveXmlFilter) - but this method is
+     * orders of magnitude faster.
+     *
+     * @param file the reference to the NVD CVE file
+     */
+    public void parse(File file) {
+        FileReader fr = null;
+        BufferedReader br = null;
+        Pattern rxEntry = Pattern.compile("^\\s*<entry\\s*id\\=\\\"([^\\\"]+)\\\".*$");
+        Pattern rxEntryEnd = Pattern.compile("^\\s*</entry>.*$");
+        Pattern rxFact = Pattern.compile("^\\s*<cpe\\-lang\\:fact\\-ref name=\\\"([^\\\"]+).*$");
+        Pattern rxSummary = Pattern.compile("^\\s*<vuln:summary>([^\\<]+).*$");
+        try {
+            fr = new FileReader(file);
+            br = new BufferedReader(fr);
+            StringBuilder sb = new StringBuilder(7000);
+            String str = null;
+            String id = null;
+            Document doc = new Document();
+            boolean skipEntry = true;
+            boolean started = false;
+
+            while ((str = br.readLine()) != null) {
+                Matcher matcherEntryEnd = rxEntryEnd.matcher(str);
+
+                if (started && !matcherEntryEnd.matches()) {
+                    sb.append(str);
+                }
+                //facts occur more often, do them first.
+                Matcher matcherFact = rxFact.matcher(str);
+                if (matcherFact.matches()) {
+                    String cpe = matcherFact.group(1);
+                    if (cpe != null && cpe.startsWith("cpe:/a:")) {
+                        skipEntry = false;
+                        addVulnerableCpe(cpe, doc);
+                    }
+                    continue;
+                }
+                Matcher matcherEntry = rxEntry.matcher(str);
+                if (matcherEntry.matches()) {
+                    started = true;
+                    id = matcherEntry.group(1);
+
+                    sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>");
+                    sb.append("<vulnerabilityType ");
+                    //sb.append("xmlns=\"http://scap.nist.gov/schema/feed/vulnerability/2.0\" ");
+                    //sb.append("xmlns:vuln=\"http://scap.nist.gov/schema/vulnerability/0.4\" ");
+                    sb.append("xmlns=\"http://scap.nist.gov/schema/vulnerability/0.4\" ");
+                    sb.append("xmlns:vuln=\"http://scap.nist.gov/schema/vulnerability/0.4\" ");
+                    //sb.append("xmlns:vulnerability=\"http://scap.nist.gov/schema/feed/vulnerability/2.0\" ");
+                    sb.append("xmlns:cpe-lang=\"http://cpe.mitre.org/language/2.0\" ");
+                    sb.append("xmlns:cvss2=\"http://scap.nist.gov/schema/cvss-v2/0.2\" ");
+                    sb.append("xmlns:cvss=\"http://scap.nist.gov/schema/cvss-v2/0.2\" ");
+                    sb.append("xmlns:scap-core=\"http://scap.nist.gov/schema/scap-core/0.1\"  ");
+                    sb.append("xmlns:scap_core=\"http://scap.nist.gov/schema/scap-core/0.1\"  ");
+                    sb.append("xmlns:patch=\"http://scap.nist.gov/schema/patch/0.1\"  ");
+                    sb.append("xmlns:cve=\"http://scap.nist.gov/schema/cve/0.1\"  ");
+                    sb.append("xmlns:cce=\"http://scap.nist.gov/schema/cce/0.1\"  ");
+
+                    sb.append("id=\"").append(id).append("\">");
+                    //sb.append(str); //need to do the above to get the correct schema generated from files.
+
+                    Field name = new Field(Fields.CVE_ID, id, Field.Store.NO, Field.Index.ANALYZED);
+                    name.setIndexOptions(IndexOptions.DOCS_ONLY);
+                    doc.add(name);
+                    continue;
+                }
+                Matcher matcherSummary = rxSummary.matcher(str);
+                if (matcherSummary.matches()) {
+                    String summary = matcherSummary.group(1);
+                    Field description = new Field(Fields.DESCRIPTION, summary, Field.Store.NO, Field.Index.ANALYZED);
+                    description.setIndexOptions(IndexOptions.DOCS_ONLY);
+                    doc.add(description);
+                    continue;
+                }
+
+                if (matcherEntryEnd.matches()) {
+                    sb.append("</vulnerabilityType>");
+                    Field xml = new Field(Fields.XML, sb.toString(), Field.Store.YES, Field.Index.NO);
+                    doc.add(xml);
+
+                    if (!skipEntry) {
+                        Term name = new Term(Fields.CVE_ID, id);
+                        indexWriter.deleteDocuments(name);
+                        indexWriter.addDocument(doc);
+                        //indexWriter.updateDocument(name, doc);
+                    }
+                    //reset the document
+                    doc = new Document();
+                    sb = new StringBuilder(7000);
+                    id = null;
+                    skipEntry = true;
+                    started = false;
+                }
+            }
+
+
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
+        } catch (IOException ex) {
+            Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
+        } finally {
+            try {
+                fr.close();
+            } catch (IOException ex) {
+                Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
+            }
+            try {
+                if (br != null) {
+                    br.close();
+                }
+            } catch (IOException ex) {
+                Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
+            }
+        }
+    }
+
+    /**
+     * Adds a CPE to the Lucene Document
+     * @param cpe a string representing a CPE
+     * @param doc a lucene document
+     */
+    private void addVulnerableCpe(String cpe, Document doc) {
+        Field vulnerable = new Field(Fields.VULNERABLE_CPE, cpe, Field.Store.NO, Field.Index.ANALYZED);
+        vulnerable.setIndexOptions(IndexOptions.DOCS_ONLY);
+        doc.add(vulnerable);
+    }
+}

src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveXmlFilter.java

@@ -46,6 +46,7 @@ import org.xml.sax.helpers.XMLFilterImpl;
  *
  * @author Jeremy
  */
+@Deprecated
 public class NvdCveXmlFilter extends XMLFilterImpl {
 
     EntrySaveDelegate saveDelegate = null;
@@ -222,9 +223,9 @@ public class NvdCveXmlFilter extends XMLFilterImpl {
 
                 // then retrieve the fully unmarshalled object
                 try {
-                    JAXBElement<VulnerabilityType> result = (JAXBElement<VulnerabilityType>) unmarshallerHandler.getResult();
-                    VulnerabilityType entry = result.getValue();
                     if (saveDelegate != null) {
+                        JAXBElement<VulnerabilityType> result = (JAXBElement<VulnerabilityType>) unmarshallerHandler.getResult();
+                        VulnerabilityType entry = result.getValue();
                         saveDelegate.saveEntry(entry);
                     }
                 } catch (JAXBException je) { //we can continue with this exception.

src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/package-info.java

@@ -4,7 +4,13 @@
  * <title>org.codesecure.dependencycheck.data.nvdcve.xml</title>
  * </head>
  * <body>
- * Contains classes used to parse the NVD CVE XML file.
+ * <p>Contains classes used to parse the NVD CVE XML file.</p>
+ * <p>The basic use is that the Importer is called to import
+ * an NVD CVE file. The Importer instantiates an Indexer object
+ * (which extends Index). The Indexer creates a partial-unmarshalling
+ * SAX parser (implemented in the NvdCveXmlFilter) that extracts
+ * VulnerabilityTypes (aka Entry) from the NVD CVE data file and
+ * stores these into a Lucene Index.</p>
  * </body>
  * </html>
 */

src/main/java/org/codesecure/dependencycheck/utils/Downloader.java

@@ -31,6 +31,7 @@ import java.net.URL;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
 
 /**
  * A utility to download files from the Internet.
@@ -112,7 +113,7 @@ public class Downloader {
                 int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT);
                 conn.setConnectTimeout(timeout);
             }
-
+            conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
             conn.connect();
         } catch (IOException ex) {
             try {
@@ -124,14 +125,15 @@ public class Downloader {
             }
             throw new DownloadFailedException("Error downloading file.", ex);
         }
+        String encoding = conn.getContentEncoding();
 
         BufferedOutputStream writer = null;
         try {
-            //the following times out on some systems because the CPE is big.
-            //InputStream reader = url.openStream();
             InputStream reader;
-            if (unzip) {
+            if (unzip || (encoding != null && "gzip".equalsIgnoreCase(encoding))) {
                 reader = new GZIPInputStream(conn.getInputStream());
+            } else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
+                reader = new InflaterInputStream(conn.getInputStream());
             } else {
                 reader = conn.getInputStream();
             }

src/main/resources/configuration/log.properties

@@ -4,7 +4,9 @@ handlers=java.util.logging.ConsoleHandler, java.util.logging.FileHandler
 # FINEST, FINER, FINE, CONFIG, INFO, WARNING and SEVERE.
 
 # Configure the ConsoleHandler.
-java.util.logging.ConsoleHandler.level=SEVERE
+java.util.logging.ConsoleHandler.level=WARNING
+
+org.codesecure.dependencycheck.data.nvdcve.xml
 
 # Configure the FileHandler.
 java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter

src/main/resources/templates/HtmlReport.vsl

@@ -353,7 +353,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                             #end
                         #end
                         <h4 id="header$cnt" class="subsectionheader white">Identifiers</h4>
-                        ##:&nbsp;<a href="http://web.nvd.nist.gov/view/vuln/search-results?cpe=$esc.url($cpevalue)" target="blank">$esc.html($cpevalue)</a></h4>
+                        ##:&nbsp;<a href="http://web.nvd.nist.gov/view/vuln/search-results?cpe=$esc.url($cpevalue)" target="_blank">$esc.html($cpevalue)</a></h4>
                         <div id="content$cnt" class="subsectioncontent standardsubsection">
                         #if($cpeCount>1)
                             Several possible CPEs where identified. If one of the following are correct please update the configuration
@@ -366,7 +366,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                             <ul>
                             #foreach($id in $dependency.getIdentifiers())
                                 ##yes, we are HTML Encoding the href. this is okay. We can't URL encode as we have to trust the analyzer here...
-                                <li><b>$esc.html($id.type):</b>&nbsp;$esc.html($id.title)&nbsp;:&nbsp;<a href="$esc.html($id.url)" target="blank">$esc.html($id.value)</a>
+                                <li><b>$esc.html($id.type):</b>&nbsp;$esc.html($id.title)&nbsp;:&nbsp;<a href="$esc.html($id.url)" target="_blank">$esc.html($id.value)</a>
                                 #if( $id.descrription )
                                     <br/>$esc.html($id.description)
                                 #end
@@ -380,12 +380,12 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                     <h4 id="header$cnt" class="subsectionheader white">Published Vulnerabilities</h4>
                     <div id="content$cnt" class="subsectioncontent standardsubsection">
                         #foreach($vuln in $dependency.getVulnerabilities())
-                            <p><b><a target="blank" href="http://web.nvd.nist.gov/view/vuln/detail?vulnId=$esc.url($vuln.name)">$esc.html($vuln.name)</a></b></p>
+                            <p><b><a target="_blank" href="http://web.nvd.nist.gov/view/vuln/detail?vulnId=$esc.url($vuln.name)">$esc.html($vuln.name)</a></b></p>
                             <p>$esc.html($vuln.description)
                                 #if ($vuln.getReferences().size()>0)
                                     <ul>
                                     #foreach($ref in $vuln.getReferences())
-                                        <li>$esc.html($ref.source) - <a target="blank" href="$esc.html($ref.url)">$ref.name</a></li>
+                                        <li>$esc.html($ref.source) - <a target="_blank" href="$esc.html($ref.url)">$ref.name</a></li>
                                     #end
                                     </ul>
                                 #end

src/test/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveParserTest.java

@@ -0,0 +1,67 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.codesecure.dependencycheck.data.nvdcve.xml;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.index.CorruptIndexException;
+import org.codesecure.dependencycheck.data.nvdcve.InvalidDataException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@gmail.com)
+ */
+public class NvdCveParserTest {
+
+    public NvdCveParserTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of parse method, of class NvdCveParser.
+     */
+    @Test
+    public void testParse() throws InvalidDataException {
+        NvdCveParser instance = null;
+        try {
+            System.out.println("parse");
+            File file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").getPath());
+            instance = new NvdCveParser();
+            instance.openIndexWriter();
+            instance.parse(file);
+        } catch (CorruptIndexException ex) {
+            throw new InvalidDataException("corrupt index", ex);
+        } catch (IOException ex) {
+            throw new InvalidDataException("IO Exception", ex);
+        } finally {
+            if (instance != null) {
+                instance.close();
+            }
+        }
+    }
+}

src/test/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveXmlFilterTest.java

@@ -3,75 +3,101 @@
  * and open the template in the editor.
  */
 package org.codesecure.dependencycheck.data.nvdcve.xml;
-
-import java.io.BufferedInputStream;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Reader;
-import java.net.MalformedURLException;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.parsers.SAXParserFactory;
-import org.codesecure.dependencycheck.data.nvdcve.generated.VulnerabilityType;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import static org.junit.Assert.*;
-import org.xml.sax.Attributes;
-import org.xml.sax.InputSource;
-import org.xml.sax.Locator;
-import org.xml.sax.SAXException;
-import org.xml.sax.XMLReader;
-
-/**
- *
- * @author Jeremy
- */
-public class NvdCveXmlFilterTest {
-    
-    public NvdCveXmlFilterTest() {
-    }
-    
-    @BeforeClass
-    public static void setUpClass() {
-    }
-    
-    @AfterClass
-    public static void tearDownClass() {
-    }
-    
-    @Before
-    public void setUp() {
-    }
-    
-    @After
-    public void tearDown() {
-    }
-
-    /**
-     * Test of process method, of class NvdCveXmlFilter.
-     */
-    @Test
-    public void testFilter() throws JAXBException, SAXException, ParserConfigurationException, MalformedURLException, IOException {
-        System.out.println("filter");
-        
-        SAXParserFactory factory = SAXParserFactory.newInstance();
-        factory.setNamespaceAware(true);
-        XMLReader reader = factory.newSAXParser().getXMLReader();
-        
-        JAXBContext context = JAXBContext.newInstance("org.codesecure.dependencycheck.data.nvdcve.generated");
-        NvdCveXmlFilter filter = new NvdCveXmlFilter(context);
-
-        reader.setContentHandler(filter);
-        File file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").getPath());
-        Reader fileReader = new FileReader(file);
-        InputSource is = new InputSource(fileReader);
-        reader.parse(is);
-    }
-}
+//
+//import java.io.BufferedInputStream;
+//import java.io.DataInputStream;
+//import java.io.File;
+//import java.io.FileReader;
+//import java.io.IOException;
+//import java.io.InputStream;
+//import java.io.Reader;
+//import java.net.MalformedURLException;
+//import java.util.logging.Level;
+//import java.util.logging.Logger;
+//import javax.xml.bind.JAXBContext;
+//import javax.xml.bind.JAXBException;
+//import javax.xml.parsers.ParserConfigurationException;
+//import javax.xml.parsers.SAXParserFactory;
+//import org.apache.lucene.index.CorruptIndexException;
+//import org.codesecure.dependencycheck.data.nvdcve.InvalidDataException;
+//import org.codesecure.dependencycheck.data.nvdcve.generated.VulnerabilityType;
+//import org.junit.After;
+//import org.junit.AfterClass;
+//import org.junit.Before;
+//import org.junit.BeforeClass;
+//import org.junit.Test;
+//import static org.junit.Assert.*;
+//import org.xml.sax.Attributes;
+//import org.xml.sax.InputSource;
+//import org.xml.sax.Locator;
+//import org.xml.sax.SAXException;
+//import org.xml.sax.XMLReader;
+//
+///**
+// *
+// * @author Jeremy
+// */
+//public class NvdCveXmlFilterTest {
+//
+//    public NvdCveXmlFilterTest() {
+//    }
+//
+//    @BeforeClass
+//    public static void setUpClass() {
+//    }
+//
+//    @AfterClass
+//    public static void tearDownClass() {
+//    }
+//
+//    @Before
+//    public void setUp() {
+//    }
+//
+//    @After
+//    public void tearDown() {
+//    }
+//
+//    /**
+//     * Test of process method, of class NvdCveXmlFilter.
+//     */
+//    @Test
+//    public void testFilter() throws InvalidDataException {
+//        Indexer indexer = null;
+//        try {
+//            System.out.println("filter");
+//
+//            SAXParserFactory factory = SAXParserFactory.newInstance();
+//            factory.setNamespaceAware(true);
+//            XMLReader reader = factory.newSAXParser().getXMLReader();
+//
+//            JAXBContext context = JAXBContext.newInstance("org.codesecure.dependencycheck.data.nvdcve.generated");
+//            NvdCveXmlFilter filter = new NvdCveXmlFilter(context);
+//
+//            indexer = new Indexer();
+//            indexer.openIndexWriter();
+//
+//            filter.registerSaveDelegate(indexer);
+//
+//            reader.setContentHandler(filter);
+//            File file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").getPath());
+//            Reader fileReader = new FileReader(file);
+//            InputSource is = new InputSource(fileReader);
+//            reader.parse(is);
+//        } catch (JAXBException ex) {
+//            throw new InvalidDataException("JAXBException", ex);
+//        } catch (SAXException ex) {
+//            throw new InvalidDataException("SAXException", ex);
+//        } catch (ParserConfigurationException ex) {
+//            throw new InvalidDataException("ParserConfigurationException", ex);
+//        } catch (CorruptIndexException ex) {
+//            throw new InvalidDataException("CorruptIndexException", ex);
+//        } catch (IOException ex) {
+//            throw new InvalidDataException("IOException", ex);
+//        } finally {
+//            if (indexer != null) {
+//                indexer.close();
+//            }
+//        }
+//    }
+//}

src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java

@@ -0,0 +1,61 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.codesecure.dependencycheck.utils;
+
+import java.net.URL;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@gmail.com)
+ */
+public class DownloaderIntegrationTest {
+
+    public DownloaderIntegrationTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of fetchFile method, of class Downloader.
+     * @throws Exception thrown when an excpetion occurs.
+     */
+    @Test
+    public void testFetchFile() throws Exception {
+        System.out.println("fetchFile");
+
+//        Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, "1000");
+
+//        Settings.setString(Settings.KEYS.PROXY_PORT, "8080");
+//        Settings.setString(Settings.KEYS.PROXY_URL, "127.0.0.1");
+
+        URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
+        String outputPath = "target\\downloaded_cpe.xml";
+        Downloader.fetchFile(url, outputPath, true);
+
+        url = new URL("http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2010.xml");
+        outputPath = "target\\downloaded_cve.xml";
+        Downloader.fetchFile(url, outputPath, false);
+
+    }
+}

src/test/java/org/codesecure/dependencycheck/utils/DownloaderTest.java

@@ -1,57 +0,0 @@
-/*
- * To change this template, choose Tools | Templates
- * and open the template in the editor.
- */
-package org.codesecure.dependencycheck.utils;
-
-import java.net.URL;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- *
- * @author Jeremy Long (jeremy.long@gmail.com)
- */
-public class DownloaderTest {
-
-    public DownloaderTest() {
-    }
-
-    @BeforeClass
-    public static void setUpClass() throws Exception {
-    }
-
-    @AfterClass
-    public static void tearDownClass() throws Exception {
-    }
-
-    @Before
-    public void setUp() {
-    }
-
-    @After
-    public void tearDown() {
-    }
-
-    
-//This test is being removed because it is a bit too slow.
-//    /**
-//     * Test of fetchFile method, of class Downloader.
-//     * @throws Exception thrown when an excpetion occurs.
-//     */
-//    @Test
-//    public void testFetchFile_URL_String() throws Exception {
-//        System.out.println("fetchFile");
-//        
-////        Settings.setString(Settings.KEYS.PROXY_URL, "test");
-////        Settings.setString(Settings.KEYS.PROXY_PORT, "80");
-////        Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, "1000");
-//        
-//        URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
-//        String outputPath = "target\\downloaded_cpe.xml";
-//        Downloader.fetchFile(url, outputPath);
-//    }
-}