Updated to externalize the data properties file

Former-commit-id: 729b2181f3cfcb7aa2862c1874b8aaacb154d46b

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java

@@ -19,14 +19,10 @@
 package org.owasp.dependencycheck.data.nvdcve.xml;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
 import javax.xml.parsers.ParserConfigurationException;
+import org.xml.sax.SAXException;
 import org.owasp.dependencycheck.data.CachedWebDataSource;
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
@@ -34,10 +30,9 @@ import java.net.URL;
 import java.sql.SQLException;
 import java.util.Calendar;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
+import java.util.TreeMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.xml.parsers.SAXParser;
@@ -50,10 +45,10 @@ import org.owasp.dependencycheck.dependency.VulnerableSoftware;
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.FileUtils;
-import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
-import org.xml.sax.SAXException;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import static org.owasp.dependencycheck.data.nvdcve.xml.DataStoreMetaInfo.MODIFIED;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 
 /**
  *
@@ -62,25 +57,9 @@ import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 public class DatabaseUpdater implements CachedWebDataSource {
 
     /**
-     * The name of the properties file containing the timestamp of the last
-     * update.
+     * Utility to read and write meta-data about the data.
      */
-    private static final String UPDATE_PROPERTIES_FILE = "lastupdated.prop";
-    /**
-     * The properties file key for the last updated field - used to store the
-     * last updated time of the Modified NVD CVE xml file.
-     */
-    private static final String LAST_UPDATED_MODIFIED = "lastupdated.modified";
-    /**
-     * Stores the last updated time for each of the NVD CVE files. These
-     * timestamps should be updated if we process the modified file within 7
-     * days of the last update.
-     */
-    private static final String LAST_UPDATED_BASE = "lastupdated.";
-    /**
-     * Modified key word.
-     */
-    public static final String MODIFIED = "modified";
+    private DataStoreMetaInfo properties = null;
     /**
      * Reference to the Cve Database.
      */
@@ -89,10 +68,27 @@ public class DatabaseUpdater implements CachedWebDataSource {
      * Reference to the Cpe Index.
      */
     private CpeIndexWriter cpeIndex = null;
+    /**
+     * A flag indicating whether or not the batch update should be performed.
+     */
+    protected boolean doBatchUpdate;
 
-    public DatabaseUpdater() {
-        batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty();
-        doBatchUpdate = false;
+    /**
+     * Get the value of doBatchUpdate
+     *
+     * @return the value of doBatchUpdate
+     */
+    protected boolean isDoBatchUpdate() {
+        return doBatchUpdate;
+    }
+
+    /**
+     * Set the value of doBatchUpdate
+     *
+     * @param doBatchUpdate new value of doBatchUpdate
+     */
+    protected void setDoBatchUpdate(boolean doBatchUpdate) {
+        this.doBatchUpdate = doBatchUpdate;
     }
 
     /**
@@ -104,6 +100,8 @@ public class DatabaseUpdater implements CachedWebDataSource {
      */
     @Override
     public void update() throws UpdateException {
+        doBatchUpdate = false;
+        properties = new DataStoreMetaInfo();
         try {
             final Map<String, NvdCveUrl> update = updateNeeded();
             int maxUpdates = 0;
@@ -112,7 +110,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
                     maxUpdates += 1;
                 }
             }
-            if (maxUpdates > 3) {
+            if (maxUpdates > 3 && !properties.isBatchUpdateMode()) {
                 Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
                         "NVD CVE requires several updates; this could take a couple of minutes.");
             }
@@ -120,7 +118,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
                 openDataStores();
             }
 
-            if (isBatchUpdateMode() && isDoBatchUpdate()) {
+            if (properties.isBatchUpdateMode() && isDoBatchUpdate()) {
                 try {
                     performBatchUpdate();
                     openDataStores();
@@ -130,7 +128,6 @@ public class DatabaseUpdater implements CachedWebDataSource {
             }
 
             int count = 0;
-
             for (NvdCveUrl cve : update.values()) {
                 if (cve.getNeedsUpdate()) {
                     count += 1;
@@ -142,7 +139,6 @@ public class DatabaseUpdater implements CachedWebDataSource {
                     try {
                         Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
                                 "Downloading {0}", cve.getUrl());
-
                         outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
                         Downloader.fetchFile(url, outputPath);
 
@@ -158,7 +154,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
                         cveDB.commit();
                         cpeIndex.commit();
 
-                        writeLastUpdatedPropertyFile(cve);
+                        properties.save(cve);
 
                         Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
                                 "Completed update {0} of {1}", new Object[]{count, maxUpdates});
@@ -201,7 +197,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
                 }
             }
             if (maxUpdates >= 1) {
-                ensureModifiedIsInLastUpdatedProperties(update);
+                properties.save(update.get(MODIFIED));
                 cveDB.cleanupDatabase();
             }
         } catch (MalformedURLException ex) {
@@ -244,6 +240,55 @@ public class DatabaseUpdater implements CachedWebDataSource {
         saxParser.parse(file, cve20Handler);
     }
 
+    /**
+     * Deletes the existing data directories.
+     *
+     * @throws IOException thrown if the directory cannot be deleted
+     */
+    protected void deleteExistingData() throws IOException {
+        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database.");
+
+        final File cveDir = CveDB.getDataDirectory();
+        FileUtils.delete(cveDir);
+
+        final File cpeDir = BaseIndex.getDataDirectory();
+        FileUtils.delete(cpeDir);
+    }
+
+    private void performBatchUpdate() throws UpdateException {
+        if (properties.isBatchUpdateMode() && doBatchUpdate) {
+            final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
+            File tmp = null;
+            try {
+                deleteExistingData();
+                final File dataDirectory = CveDB.getDataDirectory().getParentFile();
+                final URL batchUrl = new URL(batchSrc);
+                if ("file".equals(batchUrl.getProtocol())) {
+                    try {
+                        tmp = new File(batchUrl.toURI());
+                    } catch (URISyntaxException ex) {
+                        final String msg = String.format("Invalid batch update URI: %s", batchSrc);
+                        throw new UpdateException(msg, ex);
+                    }
+                } else if ("http".equals(batchUrl.getProtocol())
+                        || "https".equals(batchUrl.getProtocol())) {
+                    tmp = File.createTempFile("batch_", ".zip");
+                    Downloader.fetchFile(batchUrl, tmp);
+                }
+                //TODO add FTP?
+                FileUtils.extractFiles(tmp, dataDirectory);
+
+            } catch (IOException ex) {
+                final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
+                throw new UpdateException(msg, ex);
+            } finally {
+                if (tmp != null && !tmp.delete()) {
+                    tmp.deleteOnExit();
+                }
+            }
+        }
+    }
+
     /**
      * Closes the CVE and CPE data stores.
      */
@@ -295,81 +340,9 @@ public class DatabaseUpdater implements CachedWebDataSource {
         }
     }
 
-    //<editor-fold defaultstate="collapsed" desc="Code to read/write properties files regarding the last update dates">
-    /**
-     * Writes a properties file containing the last updated date to the
-     * VULNERABLE_CPE directory.
-     *
-     * @param updatedValue the updated nvdcve entry
-     * @throws UpdateException is thrown if there is an update exception
-     */
-    private void writeLastUpdatedPropertyFile(NvdCveUrl updatedValue) throws UpdateException {
-        if (updatedValue == null) {
-            return;
-        }
-        String dir;
-        try {
-            dir = CveDB.getDataDirectory().getCanonicalPath();
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Error updating the databases propterty file.", ex);
-            throw new UpdateException("Unable to locate last updated properties file.", ex);
-        }
-        final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
-        final Properties prop = new Properties();
-        if (cveProp.exists()) {
-            FileInputStream in = null;
-            try {
-                in = new FileInputStream(cveProp);
-                prop.load(in);
-            } catch (Exception ignoreMe) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMe);
-            } finally {
-                if (in != null) {
-                    try {
-                        in.close();
-                    } catch (Exception ignoreMeToo) {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMeToo);
-                    }
-                }
-            }
-
-        }
-        prop.put("version", CveDB.DB_SCHEMA_VERSION);
-        prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
-
-        OutputStream os = null;
-        OutputStreamWriter out = null;
-        try {
-            os = new FileOutputStream(cveProp);
-            out = new OutputStreamWriter(os, "UTF-8");
-            prop.store(out, dir);
-        } catch (FileNotFoundException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-            throw new UpdateException("Unable to find last updated properties file.", ex);
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-            throw new UpdateException("Unable to update last updated properties file.", ex);
-        } finally {
-            if (out != null) {
-                try {
-                    out.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-            if (os != null) {
-                try {
-                    os.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-        }
-    }
-
     /**
      * Determines if the index needs to be updated. This is done by fetching the
-     * nvd cve meta data and checking the last update date. If the data needs to
+     * NVD CVE meta data and checking the last update date. If the data needs to
      * be refreshed this method will return the NvdCveUrl for the files that
      * need to be updated.
      *
@@ -377,27 +350,27 @@ public class DatabaseUpdater implements CachedWebDataSource {
      * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
      * data is incorrect.
      * @throws DownloadFailedException is thrown if there is an error.
-     * downloading the nvd cve download data file.
+     * downloading the NVD CVE download data file.
      * @throws UpdateException Is thrown if there is an issue with the last
      * updated properties file.
      */
-    public Map<String, NvdCveUrl> updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+    private Map<String, NvdCveUrl> updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
 
         Map<String, NvdCveUrl> currentlyPublished;
         try {
             currentlyPublished = retrieveCurrentTimestampsFromWeb();
         } catch (InvalidDataException ex) {
             final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
             throw new DownloadFailedException(msg, ex);
-
         } catch (InvalidSettingException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
             throw new DownloadFailedException("Invalid settings", ex);
         }
 
         if (currentlyPublished == null) {
-            throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page");
+            //TODO change messages once we have a new batch mode
+            throw new DownloadFailedException("Unable to retrieve valid timestamp from NVD CVE data feeds");
         }
 
         final File cpeDataDirectory;
@@ -416,97 +389,74 @@ public class DatabaseUpdater implements CachedWebDataSource {
             }
             throw new UpdateException(msg, ex);
         }
-        if (cpeDataDirectory.exists()) {
-            final File cveProp = new File(cpeDataDirectory, UPDATE_PROPERTIES_FILE);
-            if (cveProp.exists()) {
-                final Properties prop = new Properties();
-                InputStream is = null;
-                try {
-                    is = new FileInputStream(cveProp);
-                    prop.load(is);
 
-                    boolean deleteAndRecreate = false;
-                    float version;
+        if (!properties.isEmpty()) {
+            try {
+                boolean deleteAndRecreate = false;
+                float version;
 
-                    if (prop.getProperty("version") == null) {
-                        deleteAndRecreate = true;
-                    } else {
-                        try {
-                            version = Float.parseFloat(prop.getProperty("version"));
-                            final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
-                            if (currentVersion > version) {
-                                deleteAndRecreate = true;
-                            }
-                        } catch (NumberFormatException ex) {
+                if (properties.getProperty("version") == null) {
+                    deleteAndRecreate = true;
+                } else {
+                    try {
+                        version = Float.parseFloat(properties.getProperty("version"));
+                        final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+                        if (currentVersion > version) {
                             deleteAndRecreate = true;
                         }
+                    } catch (NumberFormatException ex) {
+                        deleteAndRecreate = true;
                     }
-                    if (deleteAndRecreate) {
-                        is.close();
-                        is = null;
-                        deleteExistingData();
-                        setDoBatchUpdate(isBatchUpdateMode());
-                        return currentlyPublished;
-                    }
+                }
+                if (deleteAndRecreate) {
+                    setDoBatchUpdate(properties.isBatchUpdateMode());
+                    return currentlyPublished;
+                }
 
-                    final long lastUpdated = Long.parseLong(prop.getProperty(LAST_UPDATED_MODIFIED, "0"));
-                    final Date now = new Date();
-                    final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
-                    final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
-                    final int end = Calendar.getInstance().get(Calendar.YEAR);
-                    if (lastUpdated == currentlyPublished.get(MODIFIED).timestamp) {
-                        currentlyPublished.clear(); //we don't need to update anything.
-                        setDoBatchUpdate(batchUpdateMode);
-                    } else if (withinRange(lastUpdated, now.getTime(), days)) {
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
-                        if (isBatchUpdateMode()) {
-                            setDoBatchUpdate(false);
-                        } else {
-                            for (int i = start; i <= end; i++) {
-                                currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
-                            }
-                        }
-                    } else if (isBatchUpdateMode()) {
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
-                        setDoBatchUpdate(true);
-                    } else { //we figure out which of the several XML files need to be downloaded.
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
+                final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+                final Date now = new Date();
+                final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+                final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
+                final int end = Calendar.getInstance().get(Calendar.YEAR);
+                if (lastUpdated == currentlyPublished.get(MODIFIED).getTimestamp()) {
+                    currentlyPublished.clear(); //we don't need to update anything.
+                    setDoBatchUpdate(properties.isBatchUpdateMode());
+                } else if (withinRange(lastUpdated, now.getTime(), days)) {
+                    currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
+                    if (properties.isBatchUpdateMode()) {
+                        setDoBatchUpdate(false);
+                    } else {
                         for (int i = start; i <= end; i++) {
-                            final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i));
-                            long currentTimestamp = 0;
-                            try {
-                                currentTimestamp = Long.parseLong(prop.getProperty(LAST_UPDATED_BASE + String.valueOf(i), "0"));
-                            } catch (NumberFormatException ex) {
-                                final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
-                                        LAST_UPDATED_BASE, String.valueOf(i));
-                                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
-                            }
-                            if (currentTimestamp == cve.getTimestamp()) {
-                                cve.setNeedsUpdate(false); //they default to true.
-                            }
+                            currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
                         }
                     }
-                } catch (FileNotFoundException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } catch (NumberFormatException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } finally {
-                    if (is != null) {
+                } else if (properties.isBatchUpdateMode()) {
+                    currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
+                    setDoBatchUpdate(true);
+                } else { //we figure out which of the several XML files need to be downloaded.
+                    currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
+                    for (int i = start; i <= end; i++) {
+                        final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i));
+                        long currentTimestamp = 0;
                         try {
-                            is.close();
-                        } catch (IOException ex) {
-                            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
+                            currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
+                        } catch (NumberFormatException ex) {
+                            final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+                                    DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
+                            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
+                        }
+                        if (currentTimestamp == cve.getTimestamp()) {
+                            cve.setNeedsUpdate(false); //they default to true.
                         }
                     }
                 }
-            } else {
-                //properties file does not exist - check about batch update
-                setDoBatchUpdate(isBatchUpdateMode());
+            } catch (NumberFormatException ex) {
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, "An invalid schema version or timestamp exists in the data.properties file.");
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+                setDoBatchUpdate(properties.isBatchUpdateMode());
             }
-        } else { //this condition will likely never exist - but just in case we need to handle batch updates
-            setDoBatchUpdate(isBatchUpdateMode());
+        } else {
+            setDoBatchUpdate(properties.isBatchUpdateMode());
         }
         return currentlyPublished;
     }
@@ -526,49 +476,6 @@ public class DatabaseUpdater implements CachedWebDataSource {
         final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
         return differenceInDays < range;
     }
-    /**
-     * Indicates whether or not the updates are using a batch update mode or
-     * not.
-     */
-    private boolean batchUpdateMode;
-
-    /**
-     * Get the value of batchUpdateMode.
-     *
-     * @return the value of batchUpdateMode
-     */
-    protected boolean isBatchUpdateMode() {
-        return batchUpdateMode;
-    }
-
-    /**
-     * Set the value of batchUpdateMode.
-     *
-     * @param batchUpdateMode new value of batchUpdateMode
-     */
-    protected void setBatchUpdateMode(boolean batchUpdateMode) {
-        this.batchUpdateMode = batchUpdateMode;
-    }
-    //flag indicating whether or not the batch update should be performed.
-    protected boolean doBatchUpdate;
-
-    /**
-     * Get the value of doBatchUpdate
-     *
-     * @return the value of doBatchUpdate
-     */
-    protected boolean isDoBatchUpdate() {
-        return doBatchUpdate;
-    }
-
-    /**
-     * Set the value of doBatchUpdate
-     *
-     * @param doBatchUpdate new value of doBatchUpdate
-     */
-    protected void setDoBatchUpdate(boolean doBatchUpdate) {
-        this.doBatchUpdate = doBatchUpdate;
-    }
 
     /**
      * Retrieves the timestamps from the NVD CVE meta data file.
@@ -585,7 +492,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
     protected Map<String, NvdCveUrl> retrieveCurrentTimestampsFromWeb()
             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
 
-        final Map<String, NvdCveUrl> map = new HashMap<String, NvdCveUrl>();
+        final Map<String, NvdCveUrl> map = new TreeMap<String, NvdCveUrl>();
         String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
 
         NvdCveUrl item = new NvdCveUrl();
@@ -594,11 +501,11 @@ public class DatabaseUpdater implements CachedWebDataSource {
         item.setUrl(retrieveUrl);
         item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
 
-        item.timestamp = Downloader.getLastModified(new URL(retrieveUrl));
+        item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
         map.put(MODIFIED, item);
 
         //only add these urls if we are not in batch mode
-        if (!isBatchUpdateMode()) {
+        if (!properties.isBatchUpdateMode()) {
             final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
             final int end = Calendar.getInstance().get(Calendar.YEAR);
             final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
@@ -610,191 +517,9 @@ public class DatabaseUpdater implements CachedWebDataSource {
                 item.setUrl(retrieveUrl);
                 item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
                 item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
-                map.put(item.id, item);
+                map.put(item.getId(), item);
             }
         }
         return map;
     }
-
-    /**
-     * Method to double check that the "modified" nvdcve file is listed and has
-     * a timestamp in the last updated properties file.
-     *
-     * @param update a set of updated NvdCveUrl objects
-     */
-    private void ensureModifiedIsInLastUpdatedProperties(Map<String, NvdCveUrl> update) {
-        try {
-            writeLastUpdatedPropertyFile(update.get(MODIFIED));
-        } catch (UpdateException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-        }
-    }
-
-    /**
-     * Deletes the existing data directories.
-     *
-     * @throws IOException thrown if the directory cannot be deleted
-     */
-    protected void deleteExistingData() throws IOException {
-        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database.");
-
-        final File cveDir = CveDB.getDataDirectory();
-        FileUtils.delete(cveDir);
-
-        final File cpeDir = BaseIndex.getDataDirectory();
-        FileUtils.delete(cpeDir);
-    }
-
-    private void performBatchUpdate() throws UpdateException {
-        if (batchUpdateMode && doBatchUpdate) {
-            final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
-            File tmp = null;
-            try {
-                deleteExistingData();
-                final File dataDirectory = CveDB.getDataDirectory().getParentFile();
-                final URL batchUrl = new URL(batchSrc);
-                if ("file".equals(batchUrl.getProtocol())) {
-                    try {
-                        tmp = new File(batchUrl.toURI());
-                    } catch (URISyntaxException ex) {
-                        final String msg = String.format("Invalid batch update URI: %s", batchSrc);
-                        throw new UpdateException(msg, ex);
-                    }
-                } else if ("http".equals(batchUrl.getProtocol())
-                        || "https".equals(batchUrl.getProtocol())) {
-                    tmp = File.createTempFile("batch_", ".zip");
-                    Downloader.fetchFile(batchUrl, tmp);
-                }
-                //TODO add FTP?
-                FileUtils.extractFiles(tmp, dataDirectory);
-
-            } catch (IOException ex) {
-                final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
-                throw new UpdateException(msg, ex);
-            } finally {
-                if (tmp != null && !tmp.delete()) {
-                    tmp.deleteOnExit();
-                }
-            }
-        }
-    }
-
-    /**
-     * A pojo that contains the Url and timestamp of the current NvdCve XML
-     * files.
-     */
-    protected static class NvdCveUrl {
-
-        /**
-         * an id.
-         */
-        private String id;
-
-        /**
-         * Get the value of id.
-         *
-         * @return the value of id
-         */
-        public String getId() {
-            return id;
-        }
-
-        /**
-         * Set the value of id.
-         *
-         * @param id new value of id
-         */
-        public void setId(String id) {
-            this.id = id;
-        }
-        /**
-         * a url.
-         */
-        private String url;
-
-        /**
-         * Get the value of url.
-         *
-         * @return the value of url
-         */
-        public String getUrl() {
-            return url;
-        }
-
-        /**
-         * Set the value of url.
-         *
-         * @param url new value of url
-         */
-        public void setUrl(String url) {
-            this.url = url;
-        }
-        /**
-         * The 1.2 schema URL.
-         */
-        private String oldSchemaVersionUrl;
-
-        /**
-         * Get the value of oldSchemaVersionUrl.
-         *
-         * @return the value of oldSchemaVersionUrl
-         */
-        public String getOldSchemaVersionUrl() {
-            return oldSchemaVersionUrl;
-        }
-
-        /**
-         * Set the value of oldSchemaVersionUrl.
-         *
-         * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
-         */
-        public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
-            this.oldSchemaVersionUrl = oldSchemaVersionUrl;
-        }
-        /**
-         * a timestamp - epoch time.
-         */
-        private long timestamp;
-
-        /**
-         * Get the value of timestamp - epoch time.
-         *
-         * @return the value of timestamp - epoch time
-         */
-        public long getTimestamp() {
-            return timestamp;
-        }
-
-        /**
-         * Set the value of timestamp - epoch time.
-         *
-         * @param timestamp new value of timestamp - epoch time
-         */
-        public void setTimestamp(long timestamp) {
-            this.timestamp = timestamp;
-        }
-        /**
-         * indicates whether or not this item should be updated.
-         */
-        private boolean needsUpdate = true;
-
-        /**
-         * Get the value of needsUpdate.
-         *
-         * @return the value of needsUpdate
-         */
-        public boolean getNeedsUpdate() {
-            return needsUpdate;
-        }
-
-        /**
-         * Set the value of needsUpdate.
-         *
-         * @param needsUpdate new value of needsUpdate
-         */
-        public void setNeedsUpdate(boolean needsUpdate) {
-            this.needsUpdate = needsUpdate;
-        }
-    }
-    //</editor-fold>
 }