Initial commit

.gitignore

@@ -0,0 +1,3 @@
+target/
+.idea
+*.iml

pom.xml

@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.ysoft.security</groupId>
+    <artifactId>nexus-nuget-indexer</artifactId>
+    <version>1.0</version>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>8</source>
+                    <target>8</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>3.0.2</version>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                            <mainClass>com.ysoft.security.IndexerMain</mainClass>
+                        </manifest>
+                    </archive>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.0.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <manifestEntries>
+                                        <Main-Class>com.ysoft.security.IndexerMain</Main-Class>
+                                    </manifestEntries>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.mariadb.jdbc</groupId>
+            <artifactId>mariadb-java-client</artifactId>
+            <version>1.6.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>42.2.8</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>1.4</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jfrog.artifactory.client</groupId>
+            <artifactId>artifactory-java-client-services</artifactId>
+            <version>2.6.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.18</version>
+        </dependency>
+        <dependency>
+            <groupId>javax.xml.bind</groupId>
+            <artifactId>jaxb-api</artifactId>
+            <version>2.3.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.sun.xml.bind</groupId>
+            <artifactId>jaxb-core</artifactId>
+            <version>2.3.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.sun.xml.bind</groupId>
+            <artifactId>jaxb-impl</artifactId>
+            <version>2.3.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>5.3.1</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>
+

src/main/java/com/ysoft/security/AngelaTree.java

@@ -0,0 +1,47 @@
+package com.ysoft.security;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.SortedSet;
+
+public class AngelaTree {
+    public static byte[] merkleSorted(List<String> dataList){
+        final String[] dataArray = dataList.toArray(new String[0]);
+        Arrays.sort(dataArray);
+        final byte[][] raw = new byte[dataList.size()][];
+        for (int i = 0; i < dataArray.length; i++) {
+            raw[i] = dataArray[i].getBytes(StandardCharsets.UTF_8);
+        }
+        return merkle(raw);
+    }
+    public static byte[] merkle(List<String> dataList){
+        final byte[][] raw = new byte[dataList.size()][];
+        for (int i = 0; i < dataList.size(); i++) {
+            raw[i] = dataList.get(i).getBytes(StandardCharsets.UTF_8);
+        }
+        return merkle(raw);
+    }
+    public static byte[] merkle(SortedSet<String> dataList){
+        return merkle(new ArrayList<>(dataList));
+    }
+    public static byte[] merkle(byte[]... dataArray){
+        final MessageDigest digest;
+        try {
+            digest = MessageDigest.getInstance("SHA-256");
+        } catch (NoSuchAlgorithmException e) {
+            throw new AssertionError(e);
+        }
+        for (byte[] bytes : dataArray) {
+            digest.update((byte)((bytes.length >>> 24) & 0xff));
+            digest.update((byte)((bytes.length >>> 16) & 0xff));
+            digest.update((byte)((bytes.length >>>  8) & 0xff));
+            digest.update((byte)((bytes.length       ) & 0xff));
+            digest.update(bytes);
+        }
+        return digest.digest();
+    }
+}

src/main/java/com/ysoft/security/ArtifactoryNugetSource.java

@@ -0,0 +1,99 @@
+package com.ysoft.security;
+
+import org.jfrog.artifactory.client.Artifactory;
+import org.jfrog.artifactory.client.ArtifactoryClientBuilder;
+import org.jfrog.artifactory.client.DownloadableArtifact;
+import org.jfrog.artifactory.client.model.RepoPath;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.DatatypeConverter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import static com.ysoft.security.AngelaTree.merkle;
+import static com.ysoft.security.AngelaTree.merkleSorted;
+
+public class ArtifactoryNugetSource implements NugetSource {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ArtifactoryNugetSource.class);
+
+    private final Future<Artifactory> clientFuture;
+    private final List<String> repositories;
+    private final String url;
+    private final NavigableSet<String> exclusions;
+
+    public ArtifactoryNugetSource(String url, String username, String password, List<String> repositories, NavigableSet<String> exclusions) {
+        this.repositories = repositories;
+        this.url = url;
+        this.exclusions = exclusions;
+        // This takes some time, so it should be done asynchronously. I know this way is a bit risky from deadlock PoV, but the code currently runs on a different thread…
+        this.clientFuture = CompletableFuture.supplyAsync(() -> ArtifactoryClientBuilder.create().
+                setUrl(url).
+                setUsername(username).
+                setPassword(password).
+                build()
+        );
+    }
+
+    @Override
+    public void index(long lastModifiedTime, Indexer indexer) throws IOException {
+        final List<RepoPath> repoPaths = client().searches().
+                artifactsCreatedSince(lastModifiedTime - 1). // Add -1 in order to make sure
+                repositories(repositories.toArray(new String[0])).
+                doSearch();
+        for (RepoPath repoPath : repoPaths) {
+            final String name = repoPath.getRepoKey() + "/" + repoPath.getItemPath();
+            LOGGER.info("Got file: " + name);
+            if(repoPath.getItemPath().toLowerCase().endsWith(".nupkg")) {
+                if(isBlacklisted(name)){
+                    LOGGER.info("Skipping {} because it is blacklisted", repoPath.getItemPath());
+                }else {
+                    final DownloadableArtifact downloadableArtifact = client().repository(repoPath.getRepoKey()).download(repoPath.getItemPath());
+                    try (InputStream inputStream = downloadableArtifact.doDownload()) {
+                        indexer.index(inputStream, null, null);
+                    } catch (SQLException e) {
+                        throw new IOException(e);
+                    }
+                }
+            }else{
+                LOGGER.info("Skipping {} because it does not look like a NuGet.", name);
+            }
+        }
+    }
+
+    private boolean isBlacklisted(String itemPath) {
+        // Optimization: We could try idea from https://stackoverflow.com/a/34356411 , but it seems that the code has to be fixed first
+        return exclusions.stream().anyMatch(itemPath::startsWith);
+    }
+
+    private Artifactory client() {
+        try {
+            return clientFuture.get();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        } catch (ExecutionException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    @Override
+    public String getHash() {
+        return DatatypeConverter.printHexBinary(merkle("Artifactory".getBytes(StandardCharsets.UTF_8), url.getBytes(StandardCharsets.UTF_8), merkleSorted(repositories), merkle(exclusions)));
+    }
+
+    @Override
+    public String toString() {
+        return "ArtifactoryNugetSource{" +
+                "url=" + url +
+                ", repositories=" + repositories +
+                ", exclusions=" + exclusions +
+                '}';
+    }
+}

src/main/java/com/ysoft/security/Hashing.java

@@ -0,0 +1,127 @@
+package com.ysoft.security;
+
+import javax.xml.bind.DatatypeConverter;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class Hashing {
+    private static final Map<String, String> HASHES;
+    static {
+        final HashMap<String, String> map = new HashMap<>();
+        map.put("sha1", "SHA1");
+        map.put("md5", "MD5");
+        HASHES = Collections.unmodifiableMap(map);
+    }
+
+    public static Map<String, String> hash(InputStream in) throws NoSuchAlgorithmException, IOException {
+        final MessageDigest[] digestsArray = new MessageDigest[Hashing.HASHES.size()];
+        final Map<String, MessageDigest> digestsMap = createDigestsMap(digestsArray);
+        final byte[] buffer = new byte[4096];
+        int len;
+        while ((len = in.read(buffer)) != -1) {
+            for (final MessageDigest digest: digestsArray) {
+                digest.update(buffer, 0, len);
+            }
+        }
+        final Map<String, String> hashes = finalizeHashes(digestsMap);
+        return hashes;
+    }
+
+    private static Map<String, String> finalizeHashes(Map<String, MessageDigest> digestsMap) {
+        final Map<String, String> hashes = new HashMap<>();
+        for (final Map.Entry<String, MessageDigest> md : digestsMap.entrySet()) {
+            hashes.put(md.getKey(), DatatypeConverter.printHexBinary(md.getValue().digest()));
+        }
+        return hashes;
+    }
+
+    private static Map<String, MessageDigest> createDigestsMap(MessageDigest[] digestsArray) throws NoSuchAlgorithmException {
+        final Map<String, MessageDigest> digestsMap = new HashMap<>();
+        int i = 0;
+        for (final Map.Entry<String, String> digestEntry : Hashing.HASHES.entrySet()) {
+            final MessageDigest messageDigest = MessageDigest.getInstance(digestEntry.getValue());
+            digestsArray[i] = messageDigest;
+            digestsMap.put(digestEntry.getKey(), messageDigest);
+            i++;
+        }
+        return digestsMap;
+    }
+
+    public static class HashingInputStream extends FilterInputStream {
+        private final MessageDigest[] digestsArray = new MessageDigest[Hashing.HASHES.size()];
+        private final Map<String, MessageDigest> digestsMap = createDigestsMap(digestsArray);
+
+        public HashingInputStream(InputStream in) throws NoSuchAlgorithmException {
+            super(in);
+        }
+
+        @Override
+        public int read() throws IOException {
+            final int i = super.read();
+            if(i != -1){
+                for (final MessageDigest digest: digestsArray) {
+                    digest.update((byte)i);
+                }
+            }
+            return i;
+        }
+
+        @Override
+        public int read(byte[] b) throws IOException {
+            return read(b, 0, b.length);
+        }
+
+        @Override
+        public int read(byte[] b, int off, int len) throws IOException {
+            final int count = super.read(b, off, len);
+            if(count>0) { // skip EOFs and empty reads
+                for (final MessageDigest digest : digestsArray) {
+                    digest.update(b, off, count);
+                }
+            }
+            return count;
+        }
+
+        @Override
+        public long skip(long n) throws IOException {
+            // not very efficient implementation, but I suppose this will not be used frequently.
+            long skipped = 0;
+            long toSkip = n;
+            while (toSkip > 0){ // prevents integer overflow
+                if(read() == -1){
+                    break;
+                }
+                skipped++;
+                toSkip--;
+            }
+            return skipped;
+        }
+
+        @Override
+        public synchronized void mark(int readlimit) {
+            // Not sure about the correct behavior, so I'll try to throw an unchecked exception in order to note that there is something wrong.
+            throw new RuntimeException("Mark is not supported.");
+        }
+
+        @Override
+        public synchronized void reset() throws IOException {
+            throw new IOException("Reset is not supported.");
+        }
+
+        @Override
+        public boolean markSupported() {
+            return false;
+        }
+
+        public Map<String, String> finalizeHashes(){
+            return Hashing.finalizeHashes(digestsMap);
+        }
+    }
+
+}

src/main/java/com/ysoft/security/IndexState.java

@@ -0,0 +1,19 @@
+package com.ysoft.security;
+
+public final class IndexState {
+    private final long lastModifiedTime;
+    private final String sourceHash;
+
+    public IndexState(long lastModifiedTime, String sourceHash) {
+        this.lastModifiedTime = lastModifiedTime;
+        this.sourceHash = sourceHash;
+    }
+
+    public long getLastModifiedTime() {
+        return lastModifiedTime;
+    }
+
+    public String getSourceHash() {
+        return sourceHash;
+    }
+}

src/main/java/com/ysoft/security/Indexer.java

@@ -0,0 +1,28 @@
+package com.ysoft.security;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.SQLException;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Indexer {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(Indexer.class);
+
+    private final NugetMetadataStore nugetMetadataStore;
+
+    public Indexer(NugetMetadataStore nugetMetadataStore) {
+        this.nugetMetadataStore = nugetMetadataStore;
+    }
+
+    public void index(InputStream zipIn, String expectedName, String expectedVersion) throws IOException, SQLException {
+        final NugetMetadata nugetMetadata = NugetReader.analyzeNuget(zipIn, expectedName, expectedVersion);
+        for (Map.Entry<String, Map<String, String>> file : nugetMetadata.getHashesForFiles().entrySet()) {
+            nugetMetadataStore.addHash(nugetMetadata.getNugetIdentifier().getId(), nugetMetadata.getNugetIdentifier().getVersion(), file.getKey(),
+                    file.getValue());
+        }
+    }
+}

src/main/java/com/ysoft/security/IndexerMain.java

@@ -0,0 +1,156 @@
+package com.ysoft.security;
+
+import org.apache.commons.cli.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.regex.Pattern;
+
+import static java.util.Arrays.asList;
+
+public class IndexerMain {
+    private static final Logger LOGGER = LoggerFactory.getLogger(IndexerMain.class);
+
+    private static final Options options = new Options();
+
+    private static final String OPT_SOURCE_TYPE = "source-type";
+    private static final String OPT_NEXUS_NUGET_PATH = "nexus-nuget-path";
+    private static final String OPT_NEXUS_SERVER_ID = "nexus-server-identity";
+    private static final String OPT_ARTIFACTORY_URL = "artifactory-url";
+    private static final String OPT_ARTIFACTORY_USERNAME = "artifactory-username";
+    private static final String OPT_ARTIFACTORY_PASSFILE = "artifactory-passfile";
+    private static final String OPT_ARTIFACTORY_REPOSITORY = "artifactory-repository";
+    private static final String OPT_ARTIFACTORY_EXCLUDE = "artifactory-exclude-prefix";
+    private static final String OPT_OUTPUT_DB_URL = "output-db-url";
+    private static final String OPT_OUTPUT_DB_PROPERTIES = "output-db-properties";
+
+    static {
+        options.addOption(Option.builder().longOpt(OPT_SOURCE_TYPE).required().desc("Type of source. Allowed values: “nexus” and “artifactory”").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_NEXUS_NUGET_PATH).desc("Path to nuget storage, multiple values can be separated by “" + File.pathSeparator + "”").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_NEXUS_SERVER_ID).desc("Unique identifier of indexed server, preferably URL. This is used just for distinguishing between various instances.").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_URL).desc("URL to JFrog Artifactory.").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_USERNAME).desc("Username for JFrog Artifactory.").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_PASSFILE).desc("File with password for JFrog Artifactory.").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_REPOSITORY).desc("Repositories to index. It can be used multiple times.").numberOfArgs(Option.UNLIMITED_VALUES).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_EXCLUDE).desc("Prefixes to exclude.").numberOfArgs(Option.UNLIMITED_VALUES).build());
+        options.addOption(Option.builder().longOpt(OPT_OUTPUT_DB_URL).required().desc("JDBC URL for storage DB").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_OUTPUT_DB_PROPERTIES).desc("Location of file of properties for DB connection.").numberOfArgs(1).build());
+    }
+
+    public static void main(String[] args) throws SQLException, IOException, ClassNotFoundException, InterruptedException {
+        final CommandLineParser parser = new DefaultParser();
+        final CommandLine cmd;
+        final NugetSource source;
+        final Properties dbProps;
+        try {
+            cmd = parser.parse(options, args);
+            if(!cmd.getArgList().isEmpty()){
+                throw new ParseException("Unexpected extra arguments: "+ cmd.getArgList());
+            }
+            LOGGER.info("Constructing nuget source…");
+            source = getNugetSource(cmd);
+            LOGGER.info("Constructed nuget source: {}", source);
+            dbProps = parseDbProps(cmd);
+        } catch (ParseException e) {
+            System.err.println("Bad parameters: " + e.getMessage());
+            help(System.err);
+            System.exit(1);
+            return; // satisfy compiler
+        }
+        try{
+            index(source, cmd.getOptionValue(OPT_OUTPUT_DB_URL), dbProps);
+        }catch (SQLException e){
+            System.err.println("SQL Exception(s):");
+            for(SQLException sqlException = e; sqlException != null; sqlException = sqlException.getNextException()){
+                sqlException.printStackTrace();
+            }
+            System.exit(1);
+        }
+    }
+
+    private static Properties parseDbProps(CommandLine cmd) throws ParseException {
+        final Properties dbProps = new Properties();
+        final String dbPropertiesFile = cmd.getOptionValue(OPT_OUTPUT_DB_PROPERTIES);
+        if (dbPropertiesFile != null) {
+            try (final FileInputStream inputStream = new FileInputStream(dbPropertiesFile)) {
+                dbProps.load(inputStream);
+            } catch (IOException e) {
+                throw new ParseException("Error when loading DB properties file: " + e.getMessage());
+            }
+        }
+        return dbProps;
+    }
+
+    private static NugetSource getNugetSource(CommandLine cmd) throws ParseException {
+        final String sourceType = cmd.getOptionValue(OPT_SOURCE_TYPE);
+        switch (sourceType) {
+            case "nexus":
+                return new NexusNugetSource(parsePaths(cmd.getOptionValue(OPT_NEXUS_NUGET_PATH)), cmd.getOptionValue(OPT_NEXUS_SERVER_ID));
+            case "artifactory":
+                final String password;
+                try (BufferedReader reader = new BufferedReader(new FileReader(cmd.getOptionValue(OPT_ARTIFACTORY_PASSFILE)))) {
+                    password = reader.readLine();
+                } catch (IOException e) {
+                    throw new ParseException("Error when reading password file for artifactory: "+e.getMessage());
+                }
+                final String username = cmd.getOptionValue(OPT_ARTIFACTORY_USERNAME);
+                final String[] repositories = cmd.getOptionValues(OPT_ARTIFACTORY_REPOSITORY);
+                if(repositories == null){
+                    throw new ParseException("Please specify at least one repository.");
+                }
+                final TreeSet<String> exclusions = new TreeSet<>(asList(Optional.ofNullable(cmd.getOptionValues(OPT_ARTIFACTORY_EXCLUDE)).orElseGet(() -> new String[0])));
+                return new ArtifactoryNugetSource(cmd.getOptionValue(OPT_ARTIFACTORY_URL), username, password, Arrays.asList(repositories), exclusions);
+            default:
+                throw new ParseException("Unknown source type: " + sourceType);
+        }
+    }
+
+    private static void help(PrintStream out) {
+        help(new PrintWriter(out, true));
+    }
+
+    private static void help(PrintWriter writer) {
+        new HelpFormatter().printHelp(
+                writer,
+                HelpFormatter.DEFAULT_WIDTH,
+                "java -jar nuget-indexer.jar",
+                null,
+                options,
+                HelpFormatter.DEFAULT_LEFT_PAD,
+                HelpFormatter.DEFAULT_DESC_PAD,
+                null
+        );
+    }
+
+    private static List<String> parsePaths(String pathString) {
+        return asList(pathString.split(Pattern.quote(File.pathSeparator)));
+    }
+
+    private static void index(NugetSource source, String connString, Properties dbProps) throws IOException, SQLException {
+        if(!org.postgresql.Driver.isRegistered()){
+            org.postgresql.Driver.register();
+        }
+        org.mariadb.jdbc.Driver.class.getName();
+        try (Connection dbh = DriverManager.getConnection(connString, updatedProps(dbProps))) {
+            final NugetMetadataStore nugetMetadataStore = NugetMetadataStore.open(dbh, source.getHash());
+            final long lastModifiedTime = nugetMetadataStore.getLastModifiedTime();
+            final Indexer indexer = new Indexer(nugetMetadataStore);
+            LOGGER.info("Start indexing {}…", source);
+            source.index(lastModifiedTime, indexer);
+            nugetMetadataStore.finish();
+            LOGGER.info("Finished indexing {}…", source);
+        }
+    }
+
+    private static Properties updatedProps(Properties dbProps) {
+        final Properties clone = (Properties) dbProps.clone();
+        clone.put("allowMultiQueries", "true");
+        return clone;
+    }
+
+}

src/main/java/com/ysoft/security/NexusNugetSource.java

@@ -0,0 +1,123 @@
+package com.ysoft.security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.DatatypeConverter;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import static com.ysoft.security.AngelaTree.merkle;
+import static com.ysoft.security.AngelaTree.merkleSorted;
+import static java.nio.file.Files.walkFileTree;
+
+public class NexusNugetSource implements NugetSource {
+    private static final Logger LOGGER = LoggerFactory.getLogger(NexusNugetSource.class);
+    private final List<String> paths;
+    private final String serverIdentity;
+    public NexusNugetSource(List<String> paths, String serverIdentity) {
+        this.paths = paths;
+        this.serverIdentity = serverIdentity;
+    }
+
+    @Override
+    public void index(long lastModifiedTime, Indexer indexer) throws IOException {
+        for (final String path : paths) {
+            indexDirectory(path, lastModifiedTime, indexer);
+        }
+    }
+
+    private static void indexDirectory(String searchPath, final long lastModifiedTime, final Indexer indexer) throws IOException {
+        final String prefix = searchPath + (searchPath.endsWith(File.separator) ? "" : File.separator);
+        walkFileTree(Paths.get(searchPath), new FileVisitor<Path>() {
+            public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+
+            public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
+                // We cannot use basicFileAttributes.creationTime(), because Nexus puts old (original?) timestamp for the files if mirrored
+                final FileTime fileTime;
+                final Process process = new ProcessBuilder("stat", "-c", "%Z", "--", path.toString()).redirectErrorStream(true).start();
+                try {
+                    process.getOutputStream().close();
+                    try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
+                        final String firstLine = reader.readLine();
+                        if (reader.readLine() != null) {
+                            throw new IOException("Expected EOF");
+                        }
+                        process.waitFor();
+                        final int exitValue = process.exitValue();
+                        if (exitValue != 0) {
+                            throw new IOException("Bad exit value: " + exitValue);
+                        }
+                        // The time is rounded. Add one second in order to err on the safe side.
+                        fileTime = FileTime.from(Long.parseLong(firstLine) + 1, TimeUnit.SECONDS);
+                    } catch (InterruptedException e) {
+                        throw new IOException(e);
+                    }
+                } finally {
+                    process.destroyForcibly(); // The process is not expected to do anything at this point…
+                }
+                if (fileTime.toMillis() > lastModifiedTime) {
+                    if (path.getFileName().toString().toLowerCase().endsWith(".nupkg")) {
+                        try {
+                            return process(path);
+                        } catch (SQLException e) {
+                            throw new IOException(e);
+                        }
+                    } else {
+                        LOGGER.warn("Unknown file skipped: " + path);
+                        return FileVisitResult.CONTINUE;
+                    }
+                } else {
+                    return FileVisitResult.CONTINUE;
+                }
+            }
+
+            private FileVisitResult process(Path path) throws IOException, SQLException {
+                if (path.toString().startsWith(prefix)) {
+                    final String subpath = path.toString().substring(prefix.length());
+                    if (!subpath.startsWith(".nexus" + File.separator)) {
+                        final String[] components = subpath.split(Pattern.quote(File.separator));
+                        final String name = components[0];
+                        final String version = components[1];
+                        try (InputStream in = Files.newInputStream(path)) {
+                            indexer.index(in, name, version);
+                        }
+                    }
+                    return FileVisitResult.CONTINUE;
+                } else {
+                    throw new IOException("The path does not start with the expected prefix: " + path);
+                }
+            }
+
+            public FileVisitResult visitFileFailed(Path path, IOException e) throws IOException {
+                throw e;
+            }
+
+            public FileVisitResult postVisitDirectory(Path path, IOException e) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    @Override
+    public String getHash() {
+        return DatatypeConverter.printHexBinary(merkle("Nexus".getBytes(StandardCharsets.UTF_8), serverIdentity.getBytes(StandardCharsets.UTF_8), merkleSorted(paths)));
+    }
+
+    @Override
+    public String toString() {
+        return "NexusNugetSource{" +
+                "serverIdentity='" + serverIdentity + '\'' +
+                ", paths=" + paths +
+                '}';
+    }
+}

src/main/java/com/ysoft/security/NonClosableInputStream.java

@@ -0,0 +1,16 @@
+package com.ysoft.security;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class NonClosableInputStream extends FilterInputStream {
+    public NonClosableInputStream(InputStream input) {
+        super(input);
+    }
+
+    @Override
+    public void close() throws IOException {
+        // ignore
+    }
+}

src/main/java/com/ysoft/security/NugetIdentifier.java

@@ -0,0 +1,43 @@
+package com.ysoft.security;
+
+import java.util.Objects;
+
+public final class NugetIdentifier {
+    final private String id;
+    final private String version;
+
+    public NugetIdentifier(String id, String version) {
+        this.id = id;
+        this.version = version;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getVersion() {
+        return version;
+    }
+
+    @Override
+    public String toString() {
+        return "NugetIdentifier{" +
+                "id='" + id + '\'' +
+                ", version='" + version + '\'' +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NugetIdentifier that = (NugetIdentifier) o;
+        return Objects.equals(id, that.id) &&
+                Objects.equals(version, that.version);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, version);
+    }
+}

src/main/java/com/ysoft/security/NugetMetadata.java

@@ -0,0 +1,18 @@
+package com.ysoft.security;
+
+import java.util.Map;
+
+public class NugetMetadata {
+    private final NugetIdentifier nugetIdentifier;
+    private final Map<String, Map<String, String>> hashesForFiles;
+    public NugetMetadata(NugetIdentifier nugetIdentifier, Map<String, Map<String, String>> hashesForFiles) {
+        this.nugetIdentifier = nugetIdentifier;
+        this.hashesForFiles = hashesForFiles;
+    }
+    public NugetIdentifier getNugetIdentifier() {
+        return nugetIdentifier;
+    }
+    public Map<String, Map<String, String>> getHashesForFiles() {
+        return hashesForFiles;
+    }
+}

src/main/java/com/ysoft/security/NugetMetadataStore.java

@@ -0,0 +1,168 @@
+package com.ysoft.security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.*;
+import java.util.Map;
+
+public class NugetMetadataStore {
+    private static final Logger LOGGER = LoggerFactory.getLogger(NugetMetadataStore.class);
+
+    private static final int REQUIRED_SCHEMA_VERSION = 2;
+
+    private final Connection dbh;
+
+    private final long startTime;
+
+    private final long lastModifiedTime;
+
+    private final String sourceHash;
+
+    public NugetMetadataStore(Connection dbh, long startTime, long lastModifiedTime, String sourceHash) {
+        this.dbh = dbh;
+        this.startTime = startTime;
+        this.lastModifiedTime = lastModifiedTime;
+        this.sourceHash = sourceHash;
+    }
+
+    public static NugetMetadataStore open(Connection dbh, String hash) throws SQLException, IOException {
+        LOGGER.info("Opening metadata store for {}", hash);
+        final long startTime = System.currentTimeMillis();
+        final int schemaVersion = getSchemaVersion(dbh);
+        LOGGER.info("Schema version: {}", schemaVersion);
+        updateDbStructure(dbh, schemaVersion);
+        final IndexState indexState = getIndexState(dbh, hash);
+        LOGGER.info("Index state: {}", indexState);
+        return new NugetMetadataStore(dbh, startTime, indexState.getLastModifiedTime(), indexState.getSourceHash());
+    }
+
+    private static IndexState getIndexState(Connection dbh, String sourceHash) throws SQLException {
+        try(PreparedStatement sourceStatement = dbh.prepareStatement("SELECT * FROM nuget_index_sources WHERE source_hash = ?")){
+            sourceStatement.setString(1, sourceHash);
+            try(ResultSet sourceResultSet = sourceStatement.executeQuery()){
+                if(sourceResultSet.next()) {
+                    return new IndexState(sourceResultSet.getLong("last_updated_time"), sourceHash);
+                } else {
+                    return new IndexState(-1, sourceHash);
+                }
+            }
+        }
+    }
+
+    private static int getSchemaVersion(Connection dbh) throws SQLException {
+        if(stateTableExists(dbh)){
+            try (
+                    // The name nuget_index_state is a bit misnomer, which is due to the legacy…
+                    PreparedStatement dbStatement = dbh.prepareStatement("SELECT * FROM nuget_index_state WHERE id = 1");
+                    ResultSet dbResultSet = dbStatement.executeQuery()
+            ) {
+                dbResultSet.next();
+                return dbResultSet.getInt("schema_version");
+            }
+        }else{
+            return 0;
+        }
+    }
+
+    private static boolean stateTableExists(Connection dbh) throws SQLException {
+        try(
+                ResultSet tablesResults = dbh.getMetaData().getTables(dbh.getCatalog(), null, null, null);
+        ){
+            while(tablesResults.next()){
+                final String tableName = tablesResults.getString("TABLE_NAME");
+                if(tableName.equals("nuget_index_state")){
+                    return true;
+                }
+            }
+            return false;
+        }
+    }
+
+    private static void updateDbStructure(Connection dbh, int schemaVersion) throws IOException, SQLException {
+        for(int i = schemaVersion+1; i <= REQUIRED_SCHEMA_VERSION; i++){
+            LOGGER.info("Updating schema to version "+i+"…");
+            try (final InputStream in = NugetMetadataStore.class.getResourceAsStream("/schema/" + i + ".sql")) {
+                final byte[] buffer = new byte[4096];
+                int size;
+                final ByteArrayOutputStream out = new ByteArrayOutputStream();
+                while((size = in.read(buffer)) != -1){
+                    out.write(buffer, 0, size);
+                }
+                final String sql = out.toString();
+                try (Statement statement = dbh.createStatement()) {
+                    // I know, it can catch a semicolon inside a string or comment or so, but we can live with that.
+                    // This is needed if the DB engine does not support multiple queries in a single batch.
+                    for (String sqlPart : sql.split(";")) {
+                        statement.addBatch(sqlPart);
+                    }
+                    statement.addBatch("UPDATE nuget_index_state SET schema_version = "+i);
+                    statement.executeBatch();
+                }
+            }
+
+        }
+    }
+
+    public void finish() throws SQLException {
+        updateLastUpdated(dbh, startTime, sourceHash);
+    }
+
+    private static void updateLastUpdated(Connection dbh, long lastUpdated, String hash) throws SQLException {
+        try (PreparedStatement preparedStatement = dbh.prepareStatement(getUpdateLastUpdatedStatement(dbh))) {
+            preparedStatement.setLong(1, lastUpdated);
+            preparedStatement.setString(2, hash);
+            preparedStatement.setLong(3, lastUpdated);
+            preparedStatement.execute();
+        }
+    }
+
+    private static String getUpdateLastUpdatedStatement(Connection dbh) throws SQLException {
+        String databaseProductName = dbh.getMetaData().getDatabaseProductName();
+        switch (databaseProductName) {
+            case "MySQL":
+            case "MariaDB":
+                return "INSERT INTO nuget_index_sources " +
+                        "(last_updated_time, source_hash) VALUES (?, ?) " +
+                        "ON DUPLICATE KEY UPDATE last_updated_time = ?";
+            case "PostgreSQL":
+                return "INSERT INTO nuget_index_sources " +
+                        "(last_updated_time, source_hash) VALUES (?, ?) " +
+                        "ON CONFLICT (source_hash) DO UPDATE SET last_updated_time = ?";
+            default:
+                throw new SQLException("Unexpected database: " + databaseProductName);
+        }
+    }
+
+    public void addHash(String name, String version, String fileName, Map<String, String> hashes) throws SQLException {
+        try (PreparedStatement preparedStatement = dbh.prepareStatement(getInsertCommand())) {
+            preparedStatement.setString(1, name);
+            preparedStatement.setString(2, version);
+            preparedStatement.setString(3, fileName);
+            preparedStatement.setString(4, hashes.get("sha1"));
+            preparedStatement.setString(5, hashes.get("md5"));
+            preparedStatement.execute();
+        }
+    }
+
+    private String getInsertCommand() throws SQLException {
+        String databaseProductName = dbh.getMetaData().getDatabaseProductName();
+        switch(databaseProductName){
+            case "MySQL":
+            case "MariaDB":
+                return "INSERT IGNORE INTO nuget_index_hashes (name, version, file_name, digest_hex_sha1, digest_hex_md5) VALUES(?, ?, ?, ?, ?)";
+            case "PostgreSQL":
+                return "INSERT INTO nuget_index_hashes (name, version, file_name, digest_hex_sha1, digest_hex_md5) VALUES(?, ?, ?, ?, ?)" +
+                        "ON CONFLICT (name, version, file_name, digest_hex_sha1, digest_hex_md5) DO NOTHING";
+            default:
+                throw new SQLException("Unexpected database: " + databaseProductName);
+        }
+    }
+
+    public long getLastModifiedTime() {
+        return lastModifiedTime;
+    }
+}

src/main/java/com/ysoft/security/NugetReader.java

@@ -0,0 +1,127 @@
+package com.ysoft.security;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.XMLEvent;
+
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NugetReader {
+    private static final Logger LOGGER = LoggerFactory.getLogger(NugetReader.class);
+
+    private NugetReader() {
+    }
+
+    public static NugetMetadata analyzeNuget(InputStream in, String expectedName, String expectedVersion) throws IOException {
+        NugetIdentifier nugetIdentifier = null;
+        final Map<String, Map<String, String>> hashesForFiles = new HashMap<>();
+        try (ZipArchiveInputStream zip = new ZipArchiveInputStream(new BufferedInputStream(in))) {
+            ArchiveEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                final Hashing.HashingInputStream hashIn = new Hashing.HashingInputStream(zip);
+                if (isManifest(entry)) {
+                    if (nugetIdentifier == null) {
+                        nugetIdentifier = getNugetIdentifierFromManifest(hashIn);
+                    } else {
+                        throw new IOException("Multiple NuGet manifests!");
+                    }
+                }
+                consumeStream(hashIn); // read the rest
+                if (!isBlacklistedFile(entry.getName())) {
+                    final Object previous = hashesForFiles.put(entry.getName(), hashIn.finalizeHashes());
+                    if (previous != null && entry.getName().toLowerCase().endsWith(".dll")) {
+                        throw new IOException("Multiple occurrences of file: " + entry.getName());
+                    }
+                }
+            }
+        } catch (NoSuchAlgorithmException e) {
+            throw new AssertionError(e);
+        } catch (XMLStreamException e) {
+            throw new IOException(e);
+        }
+        if (nugetIdentifier == null) {
+            throw new IOException("Missing manifest file");
+        }
+        if (expectedName != null) {
+            if (!expectedName.equalsIgnoreCase(nugetIdentifier.getId())) {
+                throw new IOException("Does not equal: " + expectedName + " and " + nugetIdentifier.getId());
+            }
+        }
+        if (expectedVersion != null) {
+            if (!expectedVersion.equals(nugetIdentifier.getVersion())) {
+                throw new IOException("Does not equal: " + expectedVersion + " and " + nugetIdentifier.getVersion());
+            }
+        }
+        final NugetMetadata nugetMetadata = new NugetMetadata(nugetIdentifier, hashesForFiles);
+        LOGGER.info("name: " + nugetIdentifier.getId() + ", version: " + nugetIdentifier.getVersion());
+        return nugetMetadata;
+    }
+
+    private static boolean isBlacklistedFile(String name) {
+        final String nn = name.toLowerCase();
+        return nn.endsWith(".xml") || nn.endsWith("/.rels");
+    }
+
+    private static void consumeStream(InputStream hashIn) throws IOException {
+        final byte[] buffer = new byte[4096];
+        //noinspection StatementWithEmptyBody
+        while (hashIn.read(buffer) > 0) {
+            // just consume in order to compute the proper hash
+        }
+    }
+
+    public static NugetIdentifier getNugetIdentifierFromManifest(InputStream input) throws XMLStreamException, IOException {
+        String id = null;
+        String version = null;
+        final XMLInputFactory xmlInputFactory = XMLInputFactory.newFactory();
+        xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); // This disables DTDs entirely for that factory
+        xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", false); // disable external entities
+        final XMLEventReader xmlEventReader = xmlInputFactory.createXMLEventReader(new NonClosableInputStream(input));
+        while (xmlEventReader.hasNext()) {
+            final XMLEvent event = xmlEventReader.nextEvent();
+            if (event.isStartElement()) {
+                switch (event.asStartElement().getName().getLocalPart()) {
+                    case "id":
+                        if (id == null) {
+                            id = xmlEventReader.nextEvent().asCharacters().getData();
+                        } else {
+                            throw new IOException("Multiple id elements.");
+                        }
+                        break;
+                    case "version":
+                        if (version == null) {
+                            version = xmlEventReader.nextEvent().asCharacters().getData();
+                        } else {
+                            throw new IOException("Multiple version elements.");
+                        }
+                        break;
+                    default:
+                        //ignore
+                }
+            }
+        }
+        if (id == null) {
+            throw new IOException("Cannot find NuGet id");
+        }
+        if (version == null) {
+            throw new IOException("Cannot find NuGet version");
+        }
+        return new NugetIdentifier(id, version);
+    }
+
+    private static boolean isManifest(ArchiveEntry zipEntry) {
+        return zipEntry.getName().toLowerCase().endsWith(".nuspec") && !zipEntry.getName().contains("/");
+    }
+
+}

src/main/java/com/ysoft/security/NugetSource.java

@@ -0,0 +1,9 @@
+package com.ysoft.security;
+
+import java.io.IOException;
+
+public interface NugetSource {
+    void index(long lastModifiedTime, Indexer indexer) throws IOException;
+
+    String getHash();
+}

src/main/resources/logback.xml

@@ -0,0 +1,18 @@
+<configuration>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%level / %d - [%thread] %logger - %message%n%xException</pattern>
+        </encoder>
+    </appender>
+
+    <!--
+      The logger name is typically the Java package name.
+      This configures the log level to log at for a package and its children packages.
+      <logger name="application" level="DEBUG" />
+    -->
+    <root level="INFO">
+        <appender-ref ref="STDOUT" />
+    </root>
+
+</configuration>

src/main/resources/schema/1.sql

@@ -0,0 +1,22 @@
+CREATE TABLE nuget_index_state (
+  id                INT NOT NULL PRIMARY KEY,
+  schema_version    INT NOT NULL,
+  last_updated_time BIGINT
+);
+
+INSERT INTO nuget_index_state (id, schema_version, last_updated_time) VALUES (
+  1,
+  0,
+  -1
+);
+
+CREATE TABLE nuget_index_hashes (
+  id SERIAL PRIMARY KEY,
+  name VARCHAR(512) NOT NULL,
+  version VARCHAR(128),
+  file_name VARCHAR(512),
+  digest_hex_sha1 CHAR(40),
+  digest_hex_md5 CHAR(32)
+);
+
+CREATE UNIQUE INDEX nuget_index_hashes_unique_combination ON nuget_index_hashes (name, version, file_name, digest_hex_sha1, digest_hex_md5);

src/main/resources/schema/2.sql

@@ -0,0 +1,9 @@
+ALTER TABLE nuget_index_state
+  DROP COLUMN last_updated_time; -- moved to another table
+
+CREATE TABLE nuget_index_sources (
+  id                SERIAL       PRIMARY KEY,
+  source_hash       VARCHAR(128) NOT NULL UNIQUE,
+  last_updated_time BIGINT       NOT NULL,
+  note              VARCHAR(256) NULL
+)

src/test/java/com/ysoft/security/NugetReaderTest.java

@@ -0,0 +1,192 @@
+package com.ysoft.security;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.junit.jupiter.api.Test;
+
+import javax.xml.stream.XMLStreamException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static com.ysoft.security.NugetReader.analyzeNuget;
+import static org.junit.jupiter.api.Assertions.*;
+
+public class NugetReaderTest {
+
+    private NugetMetadata analyzeNugetFile(String path, String expectedName, String expectedVersion) throws IOException {
+        try(InputStream in = getClass().getResourceAsStream("/"+path)){
+            if(in == null){
+                throw new FileNotFoundException(path);
+            }
+            return analyzeNuget(in, expectedName, expectedVersion);
+        }
+    }
+
+    private static final Map<String, Map<String, String>> systemGlobalizationHashes = hashesMap(
+            entry("lib/MonoAndroid10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/xamarinwatchos10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/net45/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/xamarintvos10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/xamarinmac20/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/MonoTouch10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/win8/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/xamarinios10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/wpa81/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/portable-net45%2Bwin8%2Bwp8%2Bwpa81/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("lib/wp80/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            //entry("_rels/.rels", "742fad0664278982d5e97f5515626c52d0b217d0", "98f15a6f3733cdd0c333b83918aeb0d0"),
+            //entry("[Content_Types].xml", "b5d7ea43979592a2ffeef800260fd8529f47b36d", "5a6cc58af64194af2bc55bcc867ddec3"),
+            entry("dotnet_library_license.txt", "a4cb8479639f7380ba6a632264e887f46fa7a561", "db62529d9c74388f3885fad4b435b3f7"),
+            entry("package/services/metadata/core-properties/77885db85c884affa6b80d6e5a56cf58.psmdcp", "5592a5109597abab05cab2729913709e078d73b2", "7dc5495a438658a9492c4aed7788e7c7"),
+            entry("ref/MonoAndroid10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            //entry("ref/netstandard1.3/ko/System.Globalization.xml", "632d95186fa7f021a9bce7f24c8d26c43e30c0a8", "4282b9f0103629108714f370b6e1222f"),
+            //entry("ref/netstandard1.3/System.Globalization.xml", "71564073dcf48c24a740273807d9ffa2e8a561c1", "4fe886b0090c10438c5bb543f6c5d2dc"),
+            //entry("ref/netstandard1.3/zh-hant/System.Globalization.xml", "ec06d8b60b81eb33ab84abd5bf63f4525737aefe", "11da607afc3c07782540e8fe719153c7"),
+            //entry("ref/netstandard1.3/zh-hans/System.Globalization.xml", "6ddb09a5f1c13acd2a2242c4492fdab14eea959b", "f3c1491ef616a2a563eb83e08f2092cf"),
+            //entry("ref/netstandard1.3/ja/System.Globalization.xml", "a848e00d99308336c0066b30972be0d8acb0ff5a", "e810f4e9e6028b4c432b41c05ebad6c2"),
+            //entry("ref/netstandard1.3/de/System.Globalization.xml", "22ae3ada9772bb1e27c457e068dcfaaa8bcb662e", "1ac187d1c24e59af866837ee8239a79c"),
+            //entry("ref/netstandard1.3/ru/System.Globalization.xml", "846bcd342893de6facdd183691782f4185272313", "efc817df6de191d88301ecea957ac825"),
+            //entry("ref/netstandard1.3/it/System.Globalization.xml", "2d7048ea9d7360fa92362fc7237c1b53518e79d6", "068098a1d63acbe85d0e08f228ba79be"),
+            entry("ref/netstandard1.3/System.Globalization.dll", "879325a6b71bbdea6f2d2f9d85311559653b4f11", "c481520a478dc704f80f25fd3894b563"),
+            //entry("ref/netstandard1.3/es/System.Globalization.xml", "5da83bd1fcfacf7d6e6c501b9b3648d3f86135af", "b6f6ade3994d858aca7618775aaf40d2"),
+            //entry("ref/netstandard1.3/fr/System.Globalization.xml", "e5639262d8908200a8a58f89c456256b1633a29d", "8cc404253cdc98e9450b027f6ec590a8"),
+            entry("ref/xamarinwatchos10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("ref/net45/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("ref/xamarintvos10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("ref/xamarinmac20/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("ref/MonoTouch10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            //entry("ref/netcore50/ko/System.Globalization.xml", "632d95186fa7f021a9bce7f24c8d26c43e30c0a8", "4282b9f0103629108714f370b6e1222f"),
+            //entry("ref/netcore50/System.Globalization.xml", "71564073dcf48c24a740273807d9ffa2e8a561c1", "4fe886b0090c10438c5bb543f6c5d2dc"),
+            //entry("ref/netcore50/zh-hant/System.Globalization.xml", "ec06d8b60b81eb33ab84abd5bf63f4525737aefe", "11da607afc3c07782540e8fe719153c7"),
+            //entry("ref/netcore50/zh-hans/System.Globalization.xml", "6ddb09a5f1c13acd2a2242c4492fdab14eea959b", "f3c1491ef616a2a563eb83e08f2092cf"),
+            //entry("ref/netcore50/ja/System.Globalization.xml", "a848e00d99308336c0066b30972be0d8acb0ff5a", "e810f4e9e6028b4c432b41c05ebad6c2"),
+            //entry("ref/netcore50/de/System.Globalization.xml", "22ae3ada9772bb1e27c457e068dcfaaa8bcb662e", "1ac187d1c24e59af866837ee8239a79c"),
+            //entry("ref/netcore50/ru/System.Globalization.xml", "846bcd342893de6facdd183691782f4185272313", "efc817df6de191d88301ecea957ac825"),
+            //entry("ref/netcore50/it/System.Globalization.xml", "2d7048ea9d7360fa92362fc7237c1b53518e79d6", "068098a1d63acbe85d0e08f228ba79be"),
+            entry("ref/netcore50/System.Globalization.dll", "879325a6b71bbdea6f2d2f9d85311559653b4f11", "c481520a478dc704f80f25fd3894b563"),
+            //entry("ref/netcore50/es/System.Globalization.xml", "5da83bd1fcfacf7d6e6c501b9b3648d3f86135af", "b6f6ade3994d858aca7618775aaf40d2"),
+            //entry("ref/netcore50/fr/System.Globalization.xml", "e5639262d8908200a8a58f89c456256b1633a29d", "8cc404253cdc98e9450b027f6ec590a8"),
+            //entry("ref/netstandard1.0/ko/System.Globalization.xml", "632d95186fa7f021a9bce7f24c8d26c43e30c0a8", "4282b9f0103629108714f370b6e1222f"),
+            //entry("ref/netstandard1.0/System.Globalization.xml", "71564073dcf48c24a740273807d9ffa2e8a561c1", "4fe886b0090c10438c5bb543f6c5d2dc"),
+            //entry("ref/netstandard1.0/zh-hant/System.Globalization.xml", "ec06d8b60b81eb33ab84abd5bf63f4525737aefe", "11da607afc3c07782540e8fe719153c7"),
+            //entry("ref/netstandard1.0/zh-hans/System.Globalization.xml", "6ddb09a5f1c13acd2a2242c4492fdab14eea959b", "f3c1491ef616a2a563eb83e08f2092cf"),
+            //entry("ref/netstandard1.0/ja/System.Globalization.xml", "a848e00d99308336c0066b30972be0d8acb0ff5a", "e810f4e9e6028b4c432b41c05ebad6c2"),
+            //entry("ref/netstandard1.0/de/System.Globalization.xml", "22ae3ada9772bb1e27c457e068dcfaaa8bcb662e", "1ac187d1c24e59af866837ee8239a79c"),
+            //entry("ref/netstandard1.0/ru/System.Globalization.xml", "846bcd342893de6facdd183691782f4185272313", "efc817df6de191d88301ecea957ac825"),
+            //entry("ref/netstandard1.0/it/System.Globalization.xml", "2d7048ea9d7360fa92362fc7237c1b53518e79d6", "068098a1d63acbe85d0e08f228ba79be"),
+            entry("ref/netstandard1.0/System.Globalization.dll", "fc66f3384835722177dc523e100574bd06c45725", "849f648b4f96278669f6410f8c159f94"),
+            //entry("ref/netstandard1.0/es/System.Globalization.xml", "5da83bd1fcfacf7d6e6c501b9b3648d3f86135af", "b6f6ade3994d858aca7618775aaf40d2"),
+            //entry("ref/netstandard1.0/fr/System.Globalization.xml", "e5639262d8908200a8a58f89c456256b1633a29d", "8cc404253cdc98e9450b027f6ec590a8"),
+            entry("ref/win8/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("ref/xamarinios10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("ref/wpa81/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("ref/portable-net45%2Bwin8%2Bwp8%2Bwpa81/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("ref/wp80/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"),
+            entry("System.Globalization.nuspec", "07689ff76a416c988f6af94cd7cc4b10ddb95e08", "44ff82802f7d390bfedae1e5c5be6c5e"),
+            entry("ThirdPartyNotices.txt", "58633a0b1cc282fa6ec4ca32d4b9327319ca31fe", "c967cb8266866e70f817fee256966091")
+    );
+
+    private static final Map<String, Map<String, String>> microsoftNetcoreRuntimeCoreclrArmHashes = hashesMap(
+            //entry("_rels/.rels", "fe496bc9c2a148400d4648a2c4b4c7dd2db63a9a", "80008d2f71d952183263cc6739b48c90"),
+            entry("Microsoft.NETCore.Runtime.CoreCLR-arm.nuspec", "20d364b0cce512510c0a8e5556e7d5a09171f401", "35c2b666457bdc1abbccb96153c564b1"),
+            //entry("[Content_Types].xml", "85095f137c96204b5099b3085885acfbde04108b", "9c542780fe63324caca20a23f3168c6d"),
+            entry("runtimes/win8-arm/lib/dotnet/mscorlib.ni.dll", "84b5407977ffce80854d9203ebf6d36e4c69f374", "6980c421ab76c555600d57d608abd58e"),
+            entry("runtimes/win8-arm/native/coreclr.dll", "f03b24c2be5b908d504371b351484100bfc27768", "37238f2bb0276d18c9585530104ef70b"),
+            entry("runtimes/win8-arm/native/mscordaccore.dll", "4b0af8bb4b2313682af44c4d72aec4069ebec51e", "6fae425f7e27c6ff217266b40c83f160"),
+            entry("runtimes/win8-arm/native/mscordbi.dll", "f59a9a4e1cbc1daad601e0e196987871daf755fe", "2cc25b01beee5e72cd5d0893516c2aba"),
+            entry("runtimes/win8-arm/native/dbgshim.dll", "00b7dbcf9cd520913375028e64a2a40655408e50", "bb421ab20b557d885157a035634f0e9b"),
+            entry("runtimes/win8-arm/native/mscorrc.dll", "319d8fbdba7552f7546d4e64bafc829153029f90", "dc8bfbd9ed5172ddf4c7c402118c1b4b"),
+            entry("runtimes/win8-arm/native/clretwrc.dll", "d7e56c09b3e9e40f8a4cdfee45d5aae9f8832490", "85dffa2f3dbae07c6ab0731f1f837f03"),
+            entry("runtimes/win8-arm/native/mscorrc.debug.dll", "11169d8018227b7bdc77eddf114aefa4db19f33f", "b65e2e3baf57a333019f772eb3f4eb4c"),
+            entry("package/services/metadata/core-properties/c1cbeaed81514106b6b7971ac193f132.psmdcp", "5fb4a8f7f8303e347cbf30a3e8701f2e14f7a844", "1f7935ebc3353d6c0894d00d73616372")
+            // damaged: entry("ref/dotnet/_._", "688934845f22049cb14668832efa33d45013b6b9", "598f4fe64aefab8f00bcbea4c9239abf")
+    );
+
+    private static final Map<String, Map<String, String>> microsoftAspnetRazorHashes = hashesMap(
+        entry("lib/net40/System.Web.Razor.dll", "d6b7cd752f0ff7b9f773d544add6a5ea00158599", "cd4ddc9c5b2695018d6e7c2950e7d82f"),
+        //entry("lib/net40/System.Web.Razor.xml", "4e2f4e963f640112679f70f7a8aeab746342f7b1", "2aa1d44a5b99b443f614ad6890cdb92b"),
+        //entry("_rels/.rels", "7bc360ddecbd16668555b48785e202558b67f3fc", "fec47ef79c06556ef9f3bc8f89c02075"),
+        entry("Microsoft.AspNet.Razor.nuspec", "05d426f9e8ecb4685efe1436357fd61bcc5d6df2", "2b70d4b4a93c8146db24138faa844c8c"),
+        //entry("[Content_Types].xml", "d52e5937a775c3df3eee3b5cac6bbbbb1d884009", "4a8b92ec365e327ad1c1cae004d3c088"),
+        entry("package/services/metadata/core-properties/37cf22fae31a4489a4df544d33fed45a.psmdcp", "4223c8c09f0f99751772dcb9ec0cad70af45e88b", "4892743e40333517a6aecab561e4143c"),
+        entry(".signature.p7s", "84923efb62418eedd119be96901a624d4f87cf99", "c5b87f4ac7119eb7ebbff954993e9937")
+    );
+
+    private static final Map<String, Map<String, String>> netMqHashes = hashesMap(
+        entry(".signature.p7s", "f62ab9b16f5630208353904ea7cae72784b60d5c", "e55402a31e9d8bf9a0a4ee8e1bcab495"),
+        entry("package/services/metadata/core-properties/35baa1c9e346418996e5dcf9bbc4c861.psmdcp", "e5628ff74af0a48a7f367792bf50e94301eb6d74", "98d973ededa8490ad19493996d6a47d1"),
+        entry("package/services/metadata/core-properties/25d54e2d9f1b429386de7b9853bf46f9.psmdcp", "bcdeace599c87a895ae8832177adc4bf92f5dad7", "cdfd39d415fa39bdf2052b4e65ef6f2c"),
+        //entry("lib/netstandard2.0/NetMQ.xml", "9067824d07b0b457b7846dc18ecd1f5467a0d206", "8358926b643d647167cc4e527a4a8c39"),
+        entry("lib/netstandard2.0/NetMQ.pdb", "2923e5f0088ca2a24f87613b113419e2d9e5914b", "65569b7afcbe1d46ca5820205ab5f514"),
+        entry("lib/netstandard2.0/NetMQ.dll", "8fef6ef59442061ead95457649b3d62a69775c6c", "b15546f1a77a5dda915c8bc792c2283d"),
+        //entry("lib/netstandard1.6/NetMQ.xml", "96354c3655d7e95718759b2969dd351eb20059e5", "e332c83a76c30c57e47985ca26fd029d"),
+        entry("lib/netstandard1.6/NetMQ.pdb", "d2fc620dee4fa297ae826ecb378ab63c4b358b57", "6e59b2c6a6ef2542c3fd9d6b6922c5c1"),
+        entry("lib/netstandard1.6/NetMQ.dll", "65fbf6ffdc6bb648628921d85423c85013920c0d", "0c423c8978b33dce16c6903f827109bd"),
+        //entry("lib/net40/NetMQ.xml", "cf7f6c9a2f59c121e4d6550239503c6210d9a9f0", "59bd1e8c3920f08970415fc71d1c27ce"),
+        entry("lib/net40/NetMQ.pdb", "5dd57094e2ec802554abbcf0e63fac3c3e870128", "8f70a7bc3c0508448a95cc06f81866d7"),
+        entry("lib/net40/NetMQ.dll", "6c8217c37c7f50e23d5b4948873ad73327945d74", "05c06b5716822bed55fb4cab8b9193cb"),
+        //entry("_rels/.rels", "?", "?"),
+        //entry("[Content_Types].xml", "?", "?"),
+        entry("NetMQ.nuspec", "cdd0a117da95745fcb7a86c541a0c2a6ae184238", "8fece78f2bc2ae511f0c2ac9b1386894")
+    );
+
+
+    private static Map.Entry<String, Map<String, String>> entry(String file, String sha1, String md5) {
+        final Map<String, String> hashes = new HashMap<>();
+        hashes.put("sha1", sha1.toUpperCase());
+        hashes.put("md5", md5.toUpperCase());
+        return new ImmutablePair<>(file, Collections.unmodifiableMap(hashes));
+    }
+
+    private static Map<String, Map<String, String>> hashesMap(Map.Entry<String, Map<String, String>>... entries){
+        //return Arrays.stream(entries).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        final Map<String, Map<String, String>> map = new HashMap<>();
+        for (Map.Entry<String, Map<String, String>> entry : entries) {
+            map.put(entry.getKey(), entry.getValue());
+        }
+        return Collections.unmodifiableMap(map);
+    }
+
+    @Test
+    public void testSomeRandomNuget() throws IOException {
+        assertEquals(systemGlobalizationHashes, analyzeNugetFile("System.Globalization.4.3.0.nupkg", "System.Globalization", "4.3.0").getHashesForFiles());
+    }
+
+    @Test
+    public void testNugetWithEmptyFile() throws IOException {
+        final Map<String, Map<String, String>> hashesForFiles = new HashMap<>(analyzeNugetFile("Microsoft.NETCore.Runtime.CoreCLR-arm-1.0.0.nupkg", "Microsoft.NETCore.Runtime.CoreCLR-arm", "1.0.0").getHashesForFiles());
+        hashesForFiles.remove("ref/dotnet/_._"); // remove damaged
+        assertEquals(microsoftNetcoreRuntimeCoreclrArmHashes.keySet(), hashesForFiles.keySet());
+        assertEquals(microsoftNetcoreRuntimeCoreclrArmHashes, hashesForFiles);
+    }
+
+    @Test
+    public void testSomeOtherTroublesomeNuget() throws IOException {
+        // This NuGet used to cause issues with hashing the manifest file, because of using non-zero offset when calling Hashing.HashingInputStream.read(byte[], int, int). Ideally, we would create a test for this scenario.
+        final Map<String, Map<String, String>> hashesForFiles = new HashMap<>(analyzeNugetFile("Microsoft.AspNet.Razor-2.0.20715.0.nupkg", "Microsoft.AspNet.Razor", "2.0.20715.0").getHashesForFiles());
+        assertEquals(microsoftAspnetRazorHashes, hashesForFiles);
+    }
+
+    @Test
+    public void testPackageWithDuplicateBlacklistedFiles() throws IOException {
+        // This should proceed, as the duplicate files are excluded.
+        final Map<String, Map<String, String>> hashesForFiles = new HashMap<>(analyzeNugetFile("netmq.4.0.0.207.nupkg", "NetMQ", "4.0.0.207").getHashesForFiles());
+        assertEquals(netMqHashes, hashesForFiles);
+    }
+
+    @Test
+    void testGetNugetIdentifierFromManifest() throws IOException, XMLStreamException, NoSuchAlgorithmException {
+        try (Hashing.HashingInputStream in = new Hashing.HashingInputStream(getClass().getResourceAsStream("/Microsoft.AspNet.Razor.nuspec"))) {
+            assertEquals(new NugetIdentifier("Microsoft.AspNet.Razor", "2.0.20715.0"), NugetReader.getNugetIdentifierFromManifest(in));
+            while(in.read() != -1){
+                // Eat it!
+            }
+            final Map<String, String> hashes = in.finalizeHashes();
+            assertEquals(hashes, microsoftAspnetRazorHashes.get("Microsoft.AspNet.Razor.nuspec"));
+        }
+    }
+}