Initial commit

src/main/java/com/ysoft/security/AngelaTree.java

@@ -0,0 +1,47 @@
+package com.ysoft.security;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.SortedSet;
+
+public class AngelaTree {
+    public static byte[] merkleSorted(List<String> dataList){
+        final String[] dataArray = dataList.toArray(new String[0]);
+        Arrays.sort(dataArray);
+        final byte[][] raw = new byte[dataList.size()][];
+        for (int i = 0; i < dataArray.length; i++) {
+            raw[i] = dataArray[i].getBytes(StandardCharsets.UTF_8);
+        }
+        return merkle(raw);
+    }
+    public static byte[] merkle(List<String> dataList){
+        final byte[][] raw = new byte[dataList.size()][];
+        for (int i = 0; i < dataList.size(); i++) {
+            raw[i] = dataList.get(i).getBytes(StandardCharsets.UTF_8);
+        }
+        return merkle(raw);
+    }
+    public static byte[] merkle(SortedSet<String> dataList){
+        return merkle(new ArrayList<>(dataList));
+    }
+    public static byte[] merkle(byte[]... dataArray){
+        final MessageDigest digest;
+        try {
+            digest = MessageDigest.getInstance("SHA-256");
+        } catch (NoSuchAlgorithmException e) {
+            throw new AssertionError(e);
+        }
+        for (byte[] bytes : dataArray) {
+            digest.update((byte)((bytes.length >>> 24) & 0xff));
+            digest.update((byte)((bytes.length >>> 16) & 0xff));
+            digest.update((byte)((bytes.length >>>  8) & 0xff));
+            digest.update((byte)((bytes.length       ) & 0xff));
+            digest.update(bytes);
+        }
+        return digest.digest();
+    }
+}

src/main/java/com/ysoft/security/ArtifactoryNugetSource.java

@@ -0,0 +1,99 @@
+package com.ysoft.security;
+
+import org.jfrog.artifactory.client.Artifactory;
+import org.jfrog.artifactory.client.ArtifactoryClientBuilder;
+import org.jfrog.artifactory.client.DownloadableArtifact;
+import org.jfrog.artifactory.client.model.RepoPath;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.DatatypeConverter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import static com.ysoft.security.AngelaTree.merkle;
+import static com.ysoft.security.AngelaTree.merkleSorted;
+
+public class ArtifactoryNugetSource implements NugetSource {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ArtifactoryNugetSource.class);
+
+    private final Future<Artifactory> clientFuture;
+    private final List<String> repositories;
+    private final String url;
+    private final NavigableSet<String> exclusions;
+
+    public ArtifactoryNugetSource(String url, String username, String password, List<String> repositories, NavigableSet<String> exclusions) {
+        this.repositories = repositories;
+        this.url = url;
+        this.exclusions = exclusions;
+        // This takes some time, so it should be done asynchronously. I know this way is a bit risky from deadlock PoV, but the code currently runs on a different thread…
+        this.clientFuture = CompletableFuture.supplyAsync(() -> ArtifactoryClientBuilder.create().
+                setUrl(url).
+                setUsername(username).
+                setPassword(password).
+                build()
+        );
+    }
+
+    @Override
+    public void index(long lastModifiedTime, Indexer indexer) throws IOException {
+        final List<RepoPath> repoPaths = client().searches().
+                artifactsCreatedSince(lastModifiedTime - 1). // Add -1 in order to make sure
+                repositories(repositories.toArray(new String[0])).
+                doSearch();
+        for (RepoPath repoPath : repoPaths) {
+            final String name = repoPath.getRepoKey() + "/" + repoPath.getItemPath();
+            LOGGER.info("Got file: " + name);
+            if(repoPath.getItemPath().toLowerCase().endsWith(".nupkg")) {
+                if(isBlacklisted(name)){
+                    LOGGER.info("Skipping {} because it is blacklisted", repoPath.getItemPath());
+                }else {
+                    final DownloadableArtifact downloadableArtifact = client().repository(repoPath.getRepoKey()).download(repoPath.getItemPath());
+                    try (InputStream inputStream = downloadableArtifact.doDownload()) {
+                        indexer.index(inputStream, null, null);
+                    } catch (SQLException e) {
+                        throw new IOException(e);
+                    }
+                }
+            }else{
+                LOGGER.info("Skipping {} because it does not look like a NuGet.", name);
+            }
+        }
+    }
+
+    private boolean isBlacklisted(String itemPath) {
+        // Optimization: We could try idea from https://stackoverflow.com/a/34356411 , but it seems that the code has to be fixed first
+        return exclusions.stream().anyMatch(itemPath::startsWith);
+    }
+
+    private Artifactory client() {
+        try {
+            return clientFuture.get();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        } catch (ExecutionException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    @Override
+    public String getHash() {
+        return DatatypeConverter.printHexBinary(merkle("Artifactory".getBytes(StandardCharsets.UTF_8), url.getBytes(StandardCharsets.UTF_8), merkleSorted(repositories), merkle(exclusions)));
+    }
+
+    @Override
+    public String toString() {
+        return "ArtifactoryNugetSource{" +
+                "url=" + url +
+                ", repositories=" + repositories +
+                ", exclusions=" + exclusions +
+                '}';
+    }
+}

src/main/java/com/ysoft/security/Hashing.java

@@ -0,0 +1,127 @@
+package com.ysoft.security;
+
+import javax.xml.bind.DatatypeConverter;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class Hashing {
+    private static final Map<String, String> HASHES;
+    static {
+        final HashMap<String, String> map = new HashMap<>();
+        map.put("sha1", "SHA1");
+        map.put("md5", "MD5");
+        HASHES = Collections.unmodifiableMap(map);
+    }
+
+    public static Map<String, String> hash(InputStream in) throws NoSuchAlgorithmException, IOException {
+        final MessageDigest[] digestsArray = new MessageDigest[Hashing.HASHES.size()];
+        final Map<String, MessageDigest> digestsMap = createDigestsMap(digestsArray);
+        final byte[] buffer = new byte[4096];
+        int len;
+        while ((len = in.read(buffer)) != -1) {
+            for (final MessageDigest digest: digestsArray) {
+                digest.update(buffer, 0, len);
+            }
+        }
+        final Map<String, String> hashes = finalizeHashes(digestsMap);
+        return hashes;
+    }
+
+    private static Map<String, String> finalizeHashes(Map<String, MessageDigest> digestsMap) {
+        final Map<String, String> hashes = new HashMap<>();
+        for (final Map.Entry<String, MessageDigest> md : digestsMap.entrySet()) {
+            hashes.put(md.getKey(), DatatypeConverter.printHexBinary(md.getValue().digest()));
+        }
+        return hashes;
+    }
+
+    private static Map<String, MessageDigest> createDigestsMap(MessageDigest[] digestsArray) throws NoSuchAlgorithmException {
+        final Map<String, MessageDigest> digestsMap = new HashMap<>();
+        int i = 0;
+        for (final Map.Entry<String, String> digestEntry : Hashing.HASHES.entrySet()) {
+            final MessageDigest messageDigest = MessageDigest.getInstance(digestEntry.getValue());
+            digestsArray[i] = messageDigest;
+            digestsMap.put(digestEntry.getKey(), messageDigest);
+            i++;
+        }
+        return digestsMap;
+    }
+
+    public static class HashingInputStream extends FilterInputStream {
+        private final MessageDigest[] digestsArray = new MessageDigest[Hashing.HASHES.size()];
+        private final Map<String, MessageDigest> digestsMap = createDigestsMap(digestsArray);
+
+        public HashingInputStream(InputStream in) throws NoSuchAlgorithmException {
+            super(in);
+        }
+
+        @Override
+        public int read() throws IOException {
+            final int i = super.read();
+            if(i != -1){
+                for (final MessageDigest digest: digestsArray) {
+                    digest.update((byte)i);
+                }
+            }
+            return i;
+        }
+
+        @Override
+        public int read(byte[] b) throws IOException {
+            return read(b, 0, b.length);
+        }
+
+        @Override
+        public int read(byte[] b, int off, int len) throws IOException {
+            final int count = super.read(b, off, len);
+            if(count>0) { // skip EOFs and empty reads
+                for (final MessageDigest digest : digestsArray) {
+                    digest.update(b, off, count);
+                }
+            }
+            return count;
+        }
+
+        @Override
+        public long skip(long n) throws IOException {
+            // not very efficient implementation, but I suppose this will not be used frequently.
+            long skipped = 0;
+            long toSkip = n;
+            while (toSkip > 0){ // prevents integer overflow
+                if(read() == -1){
+                    break;
+                }
+                skipped++;
+                toSkip--;
+            }
+            return skipped;
+        }
+
+        @Override
+        public synchronized void mark(int readlimit) {
+            // Not sure about the correct behavior, so I'll try to throw an unchecked exception in order to note that there is something wrong.
+            throw new RuntimeException("Mark is not supported.");
+        }
+
+        @Override
+        public synchronized void reset() throws IOException {
+            throw new IOException("Reset is not supported.");
+        }
+
+        @Override
+        public boolean markSupported() {
+            return false;
+        }
+
+        public Map<String, String> finalizeHashes(){
+            return Hashing.finalizeHashes(digestsMap);
+        }
+    }
+
+}

src/main/java/com/ysoft/security/IndexState.java

@@ -0,0 +1,19 @@
+package com.ysoft.security;
+
+public final class IndexState {
+    private final long lastModifiedTime;
+    private final String sourceHash;
+
+    public IndexState(long lastModifiedTime, String sourceHash) {
+        this.lastModifiedTime = lastModifiedTime;
+        this.sourceHash = sourceHash;
+    }
+
+    public long getLastModifiedTime() {
+        return lastModifiedTime;
+    }
+
+    public String getSourceHash() {
+        return sourceHash;
+    }
+}

src/main/java/com/ysoft/security/Indexer.java

@@ -0,0 +1,28 @@
+package com.ysoft.security;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.SQLException;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Indexer {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(Indexer.class);
+
+    private final NugetMetadataStore nugetMetadataStore;
+
+    public Indexer(NugetMetadataStore nugetMetadataStore) {
+        this.nugetMetadataStore = nugetMetadataStore;
+    }
+
+    public void index(InputStream zipIn, String expectedName, String expectedVersion) throws IOException, SQLException {
+        final NugetMetadata nugetMetadata = NugetReader.analyzeNuget(zipIn, expectedName, expectedVersion);
+        for (Map.Entry<String, Map<String, String>> file : nugetMetadata.getHashesForFiles().entrySet()) {
+            nugetMetadataStore.addHash(nugetMetadata.getNugetIdentifier().getId(), nugetMetadata.getNugetIdentifier().getVersion(), file.getKey(),
+                    file.getValue());
+        }
+    }
+}

src/main/java/com/ysoft/security/IndexerMain.java

@@ -0,0 +1,156 @@
+package com.ysoft.security;
+
+import org.apache.commons.cli.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.regex.Pattern;
+
+import static java.util.Arrays.asList;
+
+public class IndexerMain {
+    private static final Logger LOGGER = LoggerFactory.getLogger(IndexerMain.class);
+
+    private static final Options options = new Options();
+
+    private static final String OPT_SOURCE_TYPE = "source-type";
+    private static final String OPT_NEXUS_NUGET_PATH = "nexus-nuget-path";
+    private static final String OPT_NEXUS_SERVER_ID = "nexus-server-identity";
+    private static final String OPT_ARTIFACTORY_URL = "artifactory-url";
+    private static final String OPT_ARTIFACTORY_USERNAME = "artifactory-username";
+    private static final String OPT_ARTIFACTORY_PASSFILE = "artifactory-passfile";
+    private static final String OPT_ARTIFACTORY_REPOSITORY = "artifactory-repository";
+    private static final String OPT_ARTIFACTORY_EXCLUDE = "artifactory-exclude-prefix";
+    private static final String OPT_OUTPUT_DB_URL = "output-db-url";
+    private static final String OPT_OUTPUT_DB_PROPERTIES = "output-db-properties";
+
+    static {
+        options.addOption(Option.builder().longOpt(OPT_SOURCE_TYPE).required().desc("Type of source. Allowed values: “nexus” and “artifactory”").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_NEXUS_NUGET_PATH).desc("Path to nuget storage, multiple values can be separated by “" + File.pathSeparator + "”").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_NEXUS_SERVER_ID).desc("Unique identifier of indexed server, preferably URL. This is used just for distinguishing between various instances.").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_URL).desc("URL to JFrog Artifactory.").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_USERNAME).desc("Username for JFrog Artifactory.").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_PASSFILE).desc("File with password for JFrog Artifactory.").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_REPOSITORY).desc("Repositories to index. It can be used multiple times.").numberOfArgs(Option.UNLIMITED_VALUES).build());
+        options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_EXCLUDE).desc("Prefixes to exclude.").numberOfArgs(Option.UNLIMITED_VALUES).build());
+        options.addOption(Option.builder().longOpt(OPT_OUTPUT_DB_URL).required().desc("JDBC URL for storage DB").numberOfArgs(1).build());
+        options.addOption(Option.builder().longOpt(OPT_OUTPUT_DB_PROPERTIES).desc("Location of file of properties for DB connection.").numberOfArgs(1).build());
+    }
+
+    public static void main(String[] args) throws SQLException, IOException, ClassNotFoundException, InterruptedException {
+        final CommandLineParser parser = new DefaultParser();
+        final CommandLine cmd;
+        final NugetSource source;
+        final Properties dbProps;
+        try {
+            cmd = parser.parse(options, args);
+            if(!cmd.getArgList().isEmpty()){
+                throw new ParseException("Unexpected extra arguments: "+ cmd.getArgList());
+            }
+            LOGGER.info("Constructing nuget source…");
+            source = getNugetSource(cmd);
+            LOGGER.info("Constructed nuget source: {}", source);
+            dbProps = parseDbProps(cmd);
+        } catch (ParseException e) {
+            System.err.println("Bad parameters: " + e.getMessage());
+            help(System.err);
+            System.exit(1);
+            return; // satisfy compiler
+        }
+        try{
+            index(source, cmd.getOptionValue(OPT_OUTPUT_DB_URL), dbProps);
+        }catch (SQLException e){
+            System.err.println("SQL Exception(s):");
+            for(SQLException sqlException = e; sqlException != null; sqlException = sqlException.getNextException()){
+                sqlException.printStackTrace();
+            }
+            System.exit(1);
+        }
+    }
+
+    private static Properties parseDbProps(CommandLine cmd) throws ParseException {
+        final Properties dbProps = new Properties();
+        final String dbPropertiesFile = cmd.getOptionValue(OPT_OUTPUT_DB_PROPERTIES);
+        if (dbPropertiesFile != null) {
+            try (final FileInputStream inputStream = new FileInputStream(dbPropertiesFile)) {
+                dbProps.load(inputStream);
+            } catch (IOException e) {
+                throw new ParseException("Error when loading DB properties file: " + e.getMessage());
+            }
+        }
+        return dbProps;
+    }
+
+    private static NugetSource getNugetSource(CommandLine cmd) throws ParseException {
+        final String sourceType = cmd.getOptionValue(OPT_SOURCE_TYPE);
+        switch (sourceType) {
+            case "nexus":
+                return new NexusNugetSource(parsePaths(cmd.getOptionValue(OPT_NEXUS_NUGET_PATH)), cmd.getOptionValue(OPT_NEXUS_SERVER_ID));
+            case "artifactory":
+                final String password;
+                try (BufferedReader reader = new BufferedReader(new FileReader(cmd.getOptionValue(OPT_ARTIFACTORY_PASSFILE)))) {
+                    password = reader.readLine();
+                } catch (IOException e) {
+                    throw new ParseException("Error when reading password file for artifactory: "+e.getMessage());
+                }
+                final String username = cmd.getOptionValue(OPT_ARTIFACTORY_USERNAME);
+                final String[] repositories = cmd.getOptionValues(OPT_ARTIFACTORY_REPOSITORY);
+                if(repositories == null){
+                    throw new ParseException("Please specify at least one repository.");
+                }
+                final TreeSet<String> exclusions = new TreeSet<>(asList(Optional.ofNullable(cmd.getOptionValues(OPT_ARTIFACTORY_EXCLUDE)).orElseGet(() -> new String[0])));
+                return new ArtifactoryNugetSource(cmd.getOptionValue(OPT_ARTIFACTORY_URL), username, password, Arrays.asList(repositories), exclusions);
+            default:
+                throw new ParseException("Unknown source type: " + sourceType);
+        }
+    }
+
+    private static void help(PrintStream out) {
+        help(new PrintWriter(out, true));
+    }
+
+    private static void help(PrintWriter writer) {
+        new HelpFormatter().printHelp(
+                writer,
+                HelpFormatter.DEFAULT_WIDTH,
+                "java -jar nuget-indexer.jar",
+                null,
+                options,
+                HelpFormatter.DEFAULT_LEFT_PAD,
+                HelpFormatter.DEFAULT_DESC_PAD,
+                null
+        );
+    }
+
+    private static List<String> parsePaths(String pathString) {
+        return asList(pathString.split(Pattern.quote(File.pathSeparator)));
+    }
+
+    private static void index(NugetSource source, String connString, Properties dbProps) throws IOException, SQLException {
+        if(!org.postgresql.Driver.isRegistered()){
+            org.postgresql.Driver.register();
+        }
+        org.mariadb.jdbc.Driver.class.getName();
+        try (Connection dbh = DriverManager.getConnection(connString, updatedProps(dbProps))) {
+            final NugetMetadataStore nugetMetadataStore = NugetMetadataStore.open(dbh, source.getHash());
+            final long lastModifiedTime = nugetMetadataStore.getLastModifiedTime();
+            final Indexer indexer = new Indexer(nugetMetadataStore);
+            LOGGER.info("Start indexing {}…", source);
+            source.index(lastModifiedTime, indexer);
+            nugetMetadataStore.finish();
+            LOGGER.info("Finished indexing {}…", source);
+        }
+    }
+
+    private static Properties updatedProps(Properties dbProps) {
+        final Properties clone = (Properties) dbProps.clone();
+        clone.put("allowMultiQueries", "true");
+        return clone;
+    }
+
+}

src/main/java/com/ysoft/security/NexusNugetSource.java

@@ -0,0 +1,123 @@
+package com.ysoft.security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.DatatypeConverter;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import static com.ysoft.security.AngelaTree.merkle;
+import static com.ysoft.security.AngelaTree.merkleSorted;
+import static java.nio.file.Files.walkFileTree;
+
+public class NexusNugetSource implements NugetSource {
+    private static final Logger LOGGER = LoggerFactory.getLogger(NexusNugetSource.class);
+    private final List<String> paths;
+    private final String serverIdentity;
+    public NexusNugetSource(List<String> paths, String serverIdentity) {
+        this.paths = paths;
+        this.serverIdentity = serverIdentity;
+    }
+
+    @Override
+    public void index(long lastModifiedTime, Indexer indexer) throws IOException {
+        for (final String path : paths) {
+            indexDirectory(path, lastModifiedTime, indexer);
+        }
+    }
+
+    private static void indexDirectory(String searchPath, final long lastModifiedTime, final Indexer indexer) throws IOException {
+        final String prefix = searchPath + (searchPath.endsWith(File.separator) ? "" : File.separator);
+        walkFileTree(Paths.get(searchPath), new FileVisitor<Path>() {
+            public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+
+            public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
+                // We cannot use basicFileAttributes.creationTime(), because Nexus puts old (original?) timestamp for the files if mirrored
+                final FileTime fileTime;
+                final Process process = new ProcessBuilder("stat", "-c", "%Z", "--", path.toString()).redirectErrorStream(true).start();
+                try {
+                    process.getOutputStream().close();
+                    try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
+                        final String firstLine = reader.readLine();
+                        if (reader.readLine() != null) {
+                            throw new IOException("Expected EOF");
+                        }
+                        process.waitFor();
+                        final int exitValue = process.exitValue();
+                        if (exitValue != 0) {
+                            throw new IOException("Bad exit value: " + exitValue);
+                        }
+                        // The time is rounded. Add one second in order to err on the safe side.
+                        fileTime = FileTime.from(Long.parseLong(firstLine) + 1, TimeUnit.SECONDS);
+                    } catch (InterruptedException e) {
+                        throw new IOException(e);
+                    }
+                } finally {
+                    process.destroyForcibly(); // The process is not expected to do anything at this point…
+                }
+                if (fileTime.toMillis() > lastModifiedTime) {
+                    if (path.getFileName().toString().toLowerCase().endsWith(".nupkg")) {
+                        try {
+                            return process(path);
+                        } catch (SQLException e) {
+                            throw new IOException(e);
+                        }
+                    } else {
+                        LOGGER.warn("Unknown file skipped: " + path);
+                        return FileVisitResult.CONTINUE;
+                    }
+                } else {
+                    return FileVisitResult.CONTINUE;
+                }
+            }
+
+            private FileVisitResult process(Path path) throws IOException, SQLException {
+                if (path.toString().startsWith(prefix)) {
+                    final String subpath = path.toString().substring(prefix.length());
+                    if (!subpath.startsWith(".nexus" + File.separator)) {
+                        final String[] components = subpath.split(Pattern.quote(File.separator));
+                        final String name = components[0];
+                        final String version = components[1];
+                        try (InputStream in = Files.newInputStream(path)) {
+                            indexer.index(in, name, version);
+                        }
+                    }
+                    return FileVisitResult.CONTINUE;
+                } else {
+                    throw new IOException("The path does not start with the expected prefix: " + path);
+                }
+            }
+
+            public FileVisitResult visitFileFailed(Path path, IOException e) throws IOException {
+                throw e;
+            }
+
+            public FileVisitResult postVisitDirectory(Path path, IOException e) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    @Override
+    public String getHash() {
+        return DatatypeConverter.printHexBinary(merkle("Nexus".getBytes(StandardCharsets.UTF_8), serverIdentity.getBytes(StandardCharsets.UTF_8), merkleSorted(paths)));
+    }
+
+    @Override
+    public String toString() {
+        return "NexusNugetSource{" +
+                "serverIdentity='" + serverIdentity + '\'' +
+                ", paths=" + paths +
+                '}';
+    }
+}

src/main/java/com/ysoft/security/NonClosableInputStream.java

@@ -0,0 +1,16 @@
+package com.ysoft.security;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class NonClosableInputStream extends FilterInputStream {
+    public NonClosableInputStream(InputStream input) {
+        super(input);
+    }
+
+    @Override
+    public void close() throws IOException {
+        // ignore
+    }
+}

src/main/java/com/ysoft/security/NugetIdentifier.java

@@ -0,0 +1,43 @@
+package com.ysoft.security;
+
+import java.util.Objects;
+
+public final class NugetIdentifier {
+    final private String id;
+    final private String version;
+
+    public NugetIdentifier(String id, String version) {
+        this.id = id;
+        this.version = version;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getVersion() {
+        return version;
+    }
+
+    @Override
+    public String toString() {
+        return "NugetIdentifier{" +
+                "id='" + id + '\'' +
+                ", version='" + version + '\'' +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NugetIdentifier that = (NugetIdentifier) o;
+        return Objects.equals(id, that.id) &&
+                Objects.equals(version, that.version);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, version);
+    }
+}

src/main/java/com/ysoft/security/NugetMetadata.java

@@ -0,0 +1,18 @@
+package com.ysoft.security;
+
+import java.util.Map;
+
+public class NugetMetadata {
+    private final NugetIdentifier nugetIdentifier;
+    private final Map<String, Map<String, String>> hashesForFiles;
+    public NugetMetadata(NugetIdentifier nugetIdentifier, Map<String, Map<String, String>> hashesForFiles) {
+        this.nugetIdentifier = nugetIdentifier;
+        this.hashesForFiles = hashesForFiles;
+    }
+    public NugetIdentifier getNugetIdentifier() {
+        return nugetIdentifier;
+    }
+    public Map<String, Map<String, String>> getHashesForFiles() {
+        return hashesForFiles;
+    }
+}

src/main/java/com/ysoft/security/NugetMetadataStore.java

@@ -0,0 +1,168 @@
+package com.ysoft.security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.*;
+import java.util.Map;
+
+public class NugetMetadataStore {
+    private static final Logger LOGGER = LoggerFactory.getLogger(NugetMetadataStore.class);
+
+    private static final int REQUIRED_SCHEMA_VERSION = 2;
+
+    private final Connection dbh;
+
+    private final long startTime;
+
+    private final long lastModifiedTime;
+
+    private final String sourceHash;
+
+    public NugetMetadataStore(Connection dbh, long startTime, long lastModifiedTime, String sourceHash) {
+        this.dbh = dbh;
+        this.startTime = startTime;
+        this.lastModifiedTime = lastModifiedTime;
+        this.sourceHash = sourceHash;
+    }
+
+    public static NugetMetadataStore open(Connection dbh, String hash) throws SQLException, IOException {
+        LOGGER.info("Opening metadata store for {}", hash);
+        final long startTime = System.currentTimeMillis();
+        final int schemaVersion = getSchemaVersion(dbh);
+        LOGGER.info("Schema version: {}", schemaVersion);
+        updateDbStructure(dbh, schemaVersion);
+        final IndexState indexState = getIndexState(dbh, hash);
+        LOGGER.info("Index state: {}", indexState);
+        return new NugetMetadataStore(dbh, startTime, indexState.getLastModifiedTime(), indexState.getSourceHash());
+    }
+
+    private static IndexState getIndexState(Connection dbh, String sourceHash) throws SQLException {
+        try(PreparedStatement sourceStatement = dbh.prepareStatement("SELECT * FROM nuget_index_sources WHERE source_hash = ?")){
+            sourceStatement.setString(1, sourceHash);
+            try(ResultSet sourceResultSet = sourceStatement.executeQuery()){
+                if(sourceResultSet.next()) {
+                    return new IndexState(sourceResultSet.getLong("last_updated_time"), sourceHash);
+                } else {
+                    return new IndexState(-1, sourceHash);
+                }
+            }
+        }
+    }
+
+    private static int getSchemaVersion(Connection dbh) throws SQLException {
+        if(stateTableExists(dbh)){
+            try (
+                    // The name nuget_index_state is a bit misnomer, which is due to the legacy…
+                    PreparedStatement dbStatement = dbh.prepareStatement("SELECT * FROM nuget_index_state WHERE id = 1");
+                    ResultSet dbResultSet = dbStatement.executeQuery()
+            ) {
+                dbResultSet.next();
+                return dbResultSet.getInt("schema_version");
+            }
+        }else{
+            return 0;
+        }
+    }
+
+    private static boolean stateTableExists(Connection dbh) throws SQLException {
+        try(
+                ResultSet tablesResults = dbh.getMetaData().getTables(dbh.getCatalog(), null, null, null);
+        ){
+            while(tablesResults.next()){
+                final String tableName = tablesResults.getString("TABLE_NAME");
+                if(tableName.equals("nuget_index_state")){
+                    return true;
+                }
+            }
+            return false;
+        }
+    }
+
+    private static void updateDbStructure(Connection dbh, int schemaVersion) throws IOException, SQLException {
+        for(int i = schemaVersion+1; i <= REQUIRED_SCHEMA_VERSION; i++){
+            LOGGER.info("Updating schema to version "+i+"…");
+            try (final InputStream in = NugetMetadataStore.class.getResourceAsStream("/schema/" + i + ".sql")) {
+                final byte[] buffer = new byte[4096];
+                int size;
+                final ByteArrayOutputStream out = new ByteArrayOutputStream();
+                while((size = in.read(buffer)) != -1){
+                    out.write(buffer, 0, size);
+                }
+                final String sql = out.toString();
+                try (Statement statement = dbh.createStatement()) {
+                    // I know, it can catch a semicolon inside a string or comment or so, but we can live with that.
+                    // This is needed if the DB engine does not support multiple queries in a single batch.
+                    for (String sqlPart : sql.split(";")) {
+                        statement.addBatch(sqlPart);
+                    }
+                    statement.addBatch("UPDATE nuget_index_state SET schema_version = "+i);
+                    statement.executeBatch();
+                }
+            }
+
+        }
+    }
+
+    public void finish() throws SQLException {
+        updateLastUpdated(dbh, startTime, sourceHash);
+    }
+
+    private static void updateLastUpdated(Connection dbh, long lastUpdated, String hash) throws SQLException {
+        try (PreparedStatement preparedStatement = dbh.prepareStatement(getUpdateLastUpdatedStatement(dbh))) {
+            preparedStatement.setLong(1, lastUpdated);
+            preparedStatement.setString(2, hash);
+            preparedStatement.setLong(3, lastUpdated);
+            preparedStatement.execute();
+        }
+    }
+
+    private static String getUpdateLastUpdatedStatement(Connection dbh) throws SQLException {
+        String databaseProductName = dbh.getMetaData().getDatabaseProductName();
+        switch (databaseProductName) {
+            case "MySQL":
+            case "MariaDB":
+                return "INSERT INTO nuget_index_sources " +
+                        "(last_updated_time, source_hash) VALUES (?, ?) " +
+                        "ON DUPLICATE KEY UPDATE last_updated_time = ?";
+            case "PostgreSQL":
+                return "INSERT INTO nuget_index_sources " +
+                        "(last_updated_time, source_hash) VALUES (?, ?) " +
+                        "ON CONFLICT (source_hash) DO UPDATE SET last_updated_time = ?";
+            default:
+                throw new SQLException("Unexpected database: " + databaseProductName);
+        }
+    }
+
+    public void addHash(String name, String version, String fileName, Map<String, String> hashes) throws SQLException {
+        try (PreparedStatement preparedStatement = dbh.prepareStatement(getInsertCommand())) {
+            preparedStatement.setString(1, name);
+            preparedStatement.setString(2, version);
+            preparedStatement.setString(3, fileName);
+            preparedStatement.setString(4, hashes.get("sha1"));
+            preparedStatement.setString(5, hashes.get("md5"));
+            preparedStatement.execute();
+        }
+    }
+
+    private String getInsertCommand() throws SQLException {
+        String databaseProductName = dbh.getMetaData().getDatabaseProductName();
+        switch(databaseProductName){
+            case "MySQL":
+            case "MariaDB":
+                return "INSERT IGNORE INTO nuget_index_hashes (name, version, file_name, digest_hex_sha1, digest_hex_md5) VALUES(?, ?, ?, ?, ?)";
+            case "PostgreSQL":
+                return "INSERT INTO nuget_index_hashes (name, version, file_name, digest_hex_sha1, digest_hex_md5) VALUES(?, ?, ?, ?, ?)" +
+                        "ON CONFLICT (name, version, file_name, digest_hex_sha1, digest_hex_md5) DO NOTHING";
+            default:
+                throw new SQLException("Unexpected database: " + databaseProductName);
+        }
+    }
+
+    public long getLastModifiedTime() {
+        return lastModifiedTime;
+    }
+}

src/main/java/com/ysoft/security/NugetReader.java

@@ -0,0 +1,127 @@
+package com.ysoft.security;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.XMLEvent;
+
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NugetReader {
+    private static final Logger LOGGER = LoggerFactory.getLogger(NugetReader.class);
+
+    private NugetReader() {
+    }
+
+    public static NugetMetadata analyzeNuget(InputStream in, String expectedName, String expectedVersion) throws IOException {
+        NugetIdentifier nugetIdentifier = null;
+        final Map<String, Map<String, String>> hashesForFiles = new HashMap<>();
+        try (ZipArchiveInputStream zip = new ZipArchiveInputStream(new BufferedInputStream(in))) {
+            ArchiveEntry entry;
+            while ((entry = zip.getNextEntry()) != null) {
+                final Hashing.HashingInputStream hashIn = new Hashing.HashingInputStream(zip);
+                if (isManifest(entry)) {
+                    if (nugetIdentifier == null) {
+                        nugetIdentifier = getNugetIdentifierFromManifest(hashIn);
+                    } else {
+                        throw new IOException("Multiple NuGet manifests!");
+                    }
+                }
+                consumeStream(hashIn); // read the rest
+                if (!isBlacklistedFile(entry.getName())) {
+                    final Object previous = hashesForFiles.put(entry.getName(), hashIn.finalizeHashes());
+                    if (previous != null && entry.getName().toLowerCase().endsWith(".dll")) {
+                        throw new IOException("Multiple occurrences of file: " + entry.getName());
+                    }
+                }
+            }
+        } catch (NoSuchAlgorithmException e) {
+            throw new AssertionError(e);
+        } catch (XMLStreamException e) {
+            throw new IOException(e);
+        }
+        if (nugetIdentifier == null) {
+            throw new IOException("Missing manifest file");
+        }
+        if (expectedName != null) {
+            if (!expectedName.equalsIgnoreCase(nugetIdentifier.getId())) {
+                throw new IOException("Does not equal: " + expectedName + " and " + nugetIdentifier.getId());
+            }
+        }
+        if (expectedVersion != null) {
+            if (!expectedVersion.equals(nugetIdentifier.getVersion())) {
+                throw new IOException("Does not equal: " + expectedVersion + " and " + nugetIdentifier.getVersion());
+            }
+        }
+        final NugetMetadata nugetMetadata = new NugetMetadata(nugetIdentifier, hashesForFiles);
+        LOGGER.info("name: " + nugetIdentifier.getId() + ", version: " + nugetIdentifier.getVersion());
+        return nugetMetadata;
+    }
+
+    private static boolean isBlacklistedFile(String name) {
+        final String nn = name.toLowerCase();
+        return nn.endsWith(".xml") || nn.endsWith("/.rels");
+    }
+
+    private static void consumeStream(InputStream hashIn) throws IOException {
+        final byte[] buffer = new byte[4096];
+        //noinspection StatementWithEmptyBody
+        while (hashIn.read(buffer) > 0) {
+            // just consume in order to compute the proper hash
+        }
+    }
+
+    public static NugetIdentifier getNugetIdentifierFromManifest(InputStream input) throws XMLStreamException, IOException {
+        String id = null;
+        String version = null;
+        final XMLInputFactory xmlInputFactory = XMLInputFactory.newFactory();
+        xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); // This disables DTDs entirely for that factory
+        xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", false); // disable external entities
+        final XMLEventReader xmlEventReader = xmlInputFactory.createXMLEventReader(new NonClosableInputStream(input));
+        while (xmlEventReader.hasNext()) {
+            final XMLEvent event = xmlEventReader.nextEvent();
+            if (event.isStartElement()) {
+                switch (event.asStartElement().getName().getLocalPart()) {
+                    case "id":
+                        if (id == null) {
+                            id = xmlEventReader.nextEvent().asCharacters().getData();
+                        } else {
+                            throw new IOException("Multiple id elements.");
+                        }
+                        break;
+                    case "version":
+                        if (version == null) {
+                            version = xmlEventReader.nextEvent().asCharacters().getData();
+                        } else {
+                            throw new IOException("Multiple version elements.");
+                        }
+                        break;
+                    default:
+                        //ignore
+                }
+            }
+        }
+        if (id == null) {
+            throw new IOException("Cannot find NuGet id");
+        }
+        if (version == null) {
+            throw new IOException("Cannot find NuGet version");
+        }
+        return new NugetIdentifier(id, version);
+    }
+
+    private static boolean isManifest(ArchiveEntry zipEntry) {
+        return zipEntry.getName().toLowerCase().endsWith(".nuspec") && !zipEntry.getName().contains("/");
+    }
+
+}

src/main/java/com/ysoft/security/NugetSource.java

@@ -0,0 +1,9 @@
+package com.ysoft.security;
+
+import java.io.IOException;
+
+public interface NugetSource {
+    void index(long lastModifiedTime, Indexer indexer) throws IOException;
+
+    String getHash();
+}