commit a48c57ef092bd1945956db6e2965347edc8ec6ad Author: Šesták Vít Date: Fri Jan 31 14:48:19 2020 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e02456b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +target/ +.idea +*.iml diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..67bedf8 --- /dev/null +++ b/pom.xml @@ -0,0 +1,113 @@ + + + 4.0.0 + + com.ysoft.security + nexus-nuget-indexer + 1.0 + + UTF-8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + + true + com.ysoft.security.IndexerMain + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + package + + shade + + + + + + com.ysoft.security.IndexerMain + + + + + + + + + + + + + org.mariadb.jdbc + mariadb-java-client + 1.6.3 + + + org.postgresql + postgresql + 42.2.8 + + + commons-cli + commons-cli + 1.4 + + + org.jfrog.artifactory.client + artifactory-java-client-services + 2.6.2 + + + org.apache.commons + commons-compress + 1.18 + + + javax.xml.bind + jaxb-api + 2.3.0 + + + com.sun.xml.bind + jaxb-core + 2.3.0 + + + com.sun.xml.bind + jaxb-impl + 2.3.0 + + + org.junit.jupiter + junit-jupiter-api + 5.3.1 + test + + + + + diff --git a/src/main/java/com/ysoft/security/AngelaTree.java b/src/main/java/com/ysoft/security/AngelaTree.java new file mode 100644 index 0000000..2430f7f --- /dev/null +++ b/src/main/java/com/ysoft/security/AngelaTree.java @@ -0,0 +1,47 @@ +package com.ysoft.security; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.SortedSet; + +public class AngelaTree { + public static byte[] merkleSorted(List dataList){ + final String[] dataArray = dataList.toArray(new String[0]); + Arrays.sort(dataArray); + final byte[][] raw = new byte[dataList.size()][]; + for (int i = 0; i < dataArray.length; i++) { + raw[i] = dataArray[i].getBytes(StandardCharsets.UTF_8); + } + return merkle(raw); + } + public static byte[] merkle(List dataList){ + final byte[][] raw = new byte[dataList.size()][]; + for (int i = 0; i < dataList.size(); i++) { + raw[i] = dataList.get(i).getBytes(StandardCharsets.UTF_8); + } + return merkle(raw); + } + public static byte[] merkle(SortedSet dataList){ + return merkle(new ArrayList<>(dataList)); + } + public static byte[] merkle(byte[]... dataArray){ + final MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + for (byte[] bytes : dataArray) { + digest.update((byte)((bytes.length >>> 24) & 0xff)); + digest.update((byte)((bytes.length >>> 16) & 0xff)); + digest.update((byte)((bytes.length >>> 8) & 0xff)); + digest.update((byte)((bytes.length ) & 0xff)); + digest.update(bytes); + } + return digest.digest(); + } +} diff --git a/src/main/java/com/ysoft/security/ArtifactoryNugetSource.java b/src/main/java/com/ysoft/security/ArtifactoryNugetSource.java new file mode 100644 index 0000000..c2e1878 --- /dev/null +++ b/src/main/java/com/ysoft/security/ArtifactoryNugetSource.java @@ -0,0 +1,99 @@ +package com.ysoft.security; + +import org.jfrog.artifactory.client.Artifactory; +import org.jfrog.artifactory.client.ArtifactoryClientBuilder; +import org.jfrog.artifactory.client.DownloadableArtifact; +import org.jfrog.artifactory.client.model.RepoPath; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.DatatypeConverter; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.List; +import java.util.NavigableSet; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static com.ysoft.security.AngelaTree.merkle; +import static com.ysoft.security.AngelaTree.merkleSorted; + +public class ArtifactoryNugetSource implements NugetSource { + private static final Logger LOGGER = LoggerFactory.getLogger(ArtifactoryNugetSource.class); + + private final Future clientFuture; + private final List repositories; + private final String url; + private final NavigableSet exclusions; + + public ArtifactoryNugetSource(String url, String username, String password, List repositories, NavigableSet exclusions) { + this.repositories = repositories; + this.url = url; + this.exclusions = exclusions; + // This takes some time, so it should be done asynchronously. I know this way is a bit risky from deadlock PoV, but the code currently runs on a different thread… + this.clientFuture = CompletableFuture.supplyAsync(() -> ArtifactoryClientBuilder.create(). + setUrl(url). + setUsername(username). + setPassword(password). + build() + ); + } + + @Override + public void index(long lastModifiedTime, Indexer indexer) throws IOException { + final List repoPaths = client().searches(). + artifactsCreatedSince(lastModifiedTime - 1). // Add -1 in order to make sure + repositories(repositories.toArray(new String[0])). + doSearch(); + for (RepoPath repoPath : repoPaths) { + final String name = repoPath.getRepoKey() + "/" + repoPath.getItemPath(); + LOGGER.info("Got file: " + name); + if(repoPath.getItemPath().toLowerCase().endsWith(".nupkg")) { + if(isBlacklisted(name)){ + LOGGER.info("Skipping {} because it is blacklisted", repoPath.getItemPath()); + }else { + final DownloadableArtifact downloadableArtifact = client().repository(repoPath.getRepoKey()).download(repoPath.getItemPath()); + try (InputStream inputStream = downloadableArtifact.doDownload()) { + indexer.index(inputStream, null, null); + } catch (SQLException e) { + throw new IOException(e); + } + } + }else{ + LOGGER.info("Skipping {} because it does not look like a NuGet.", name); + } + } + } + + private boolean isBlacklisted(String itemPath) { + // Optimization: We could try idea from https://stackoverflow.com/a/34356411 , but it seems that the code has to be fixed first + return exclusions.stream().anyMatch(itemPath::startsWith); + } + + private Artifactory client() { + try { + return clientFuture.get(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + throw new AssertionError(e); + } + } + + @Override + public String getHash() { + return DatatypeConverter.printHexBinary(merkle("Artifactory".getBytes(StandardCharsets.UTF_8), url.getBytes(StandardCharsets.UTF_8), merkleSorted(repositories), merkle(exclusions))); + } + + @Override + public String toString() { + return "ArtifactoryNugetSource{" + + "url=" + url + + ", repositories=" + repositories + + ", exclusions=" + exclusions + + '}'; + } +} diff --git a/src/main/java/com/ysoft/security/Hashing.java b/src/main/java/com/ysoft/security/Hashing.java new file mode 100644 index 0000000..92b57ae --- /dev/null +++ b/src/main/java/com/ysoft/security/Hashing.java @@ -0,0 +1,127 @@ +package com.ysoft.security; + +import javax.xml.bind.DatatypeConverter; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class Hashing { + private static final Map HASHES; + static { + final HashMap map = new HashMap<>(); + map.put("sha1", "SHA1"); + map.put("md5", "MD5"); + HASHES = Collections.unmodifiableMap(map); + } + + public static Map hash(InputStream in) throws NoSuchAlgorithmException, IOException { + final MessageDigest[] digestsArray = new MessageDigest[Hashing.HASHES.size()]; + final Map digestsMap = createDigestsMap(digestsArray); + final byte[] buffer = new byte[4096]; + int len; + while ((len = in.read(buffer)) != -1) { + for (final MessageDigest digest: digestsArray) { + digest.update(buffer, 0, len); + } + } + final Map hashes = finalizeHashes(digestsMap); + return hashes; + } + + private static Map finalizeHashes(Map digestsMap) { + final Map hashes = new HashMap<>(); + for (final Map.Entry md : digestsMap.entrySet()) { + hashes.put(md.getKey(), DatatypeConverter.printHexBinary(md.getValue().digest())); + } + return hashes; + } + + private static Map createDigestsMap(MessageDigest[] digestsArray) throws NoSuchAlgorithmException { + final Map digestsMap = new HashMap<>(); + int i = 0; + for (final Map.Entry digestEntry : Hashing.HASHES.entrySet()) { + final MessageDigest messageDigest = MessageDigest.getInstance(digestEntry.getValue()); + digestsArray[i] = messageDigest; + digestsMap.put(digestEntry.getKey(), messageDigest); + i++; + } + return digestsMap; + } + + public static class HashingInputStream extends FilterInputStream { + private final MessageDigest[] digestsArray = new MessageDigest[Hashing.HASHES.size()]; + private final Map digestsMap = createDigestsMap(digestsArray); + + public HashingInputStream(InputStream in) throws NoSuchAlgorithmException { + super(in); + } + + @Override + public int read() throws IOException { + final int i = super.read(); + if(i != -1){ + for (final MessageDigest digest: digestsArray) { + digest.update((byte)i); + } + } + return i; + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + final int count = super.read(b, off, len); + if(count>0) { // skip EOFs and empty reads + for (final MessageDigest digest : digestsArray) { + digest.update(b, off, count); + } + } + return count; + } + + @Override + public long skip(long n) throws IOException { + // not very efficient implementation, but I suppose this will not be used frequently. + long skipped = 0; + long toSkip = n; + while (toSkip > 0){ // prevents integer overflow + if(read() == -1){ + break; + } + skipped++; + toSkip--; + } + return skipped; + } + + @Override + public synchronized void mark(int readlimit) { + // Not sure about the correct behavior, so I'll try to throw an unchecked exception in order to note that there is something wrong. + throw new RuntimeException("Mark is not supported."); + } + + @Override + public synchronized void reset() throws IOException { + throw new IOException("Reset is not supported."); + } + + @Override + public boolean markSupported() { + return false; + } + + public Map finalizeHashes(){ + return Hashing.finalizeHashes(digestsMap); + } + } + +} diff --git a/src/main/java/com/ysoft/security/IndexState.java b/src/main/java/com/ysoft/security/IndexState.java new file mode 100644 index 0000000..b6dc7ef --- /dev/null +++ b/src/main/java/com/ysoft/security/IndexState.java @@ -0,0 +1,19 @@ +package com.ysoft.security; + +public final class IndexState { + private final long lastModifiedTime; + private final String sourceHash; + + public IndexState(long lastModifiedTime, String sourceHash) { + this.lastModifiedTime = lastModifiedTime; + this.sourceHash = sourceHash; + } + + public long getLastModifiedTime() { + return lastModifiedTime; + } + + public String getSourceHash() { + return sourceHash; + } +} diff --git a/src/main/java/com/ysoft/security/Indexer.java b/src/main/java/com/ysoft/security/Indexer.java new file mode 100644 index 0000000..684ed5d --- /dev/null +++ b/src/main/java/com/ysoft/security/Indexer.java @@ -0,0 +1,28 @@ +package com.ysoft.security; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Indexer { + + private static final Logger LOGGER = LoggerFactory.getLogger(Indexer.class); + + private final NugetMetadataStore nugetMetadataStore; + + public Indexer(NugetMetadataStore nugetMetadataStore) { + this.nugetMetadataStore = nugetMetadataStore; + } + + public void index(InputStream zipIn, String expectedName, String expectedVersion) throws IOException, SQLException { + final NugetMetadata nugetMetadata = NugetReader.analyzeNuget(zipIn, expectedName, expectedVersion); + for (Map.Entry> file : nugetMetadata.getHashesForFiles().entrySet()) { + nugetMetadataStore.addHash(nugetMetadata.getNugetIdentifier().getId(), nugetMetadata.getNugetIdentifier().getVersion(), file.getKey(), + file.getValue()); + } + } +} diff --git a/src/main/java/com/ysoft/security/IndexerMain.java b/src/main/java/com/ysoft/security/IndexerMain.java new file mode 100644 index 0000000..94fb6fe --- /dev/null +++ b/src/main/java/com/ysoft/security/IndexerMain.java @@ -0,0 +1,156 @@ +package com.ysoft.security; + +import org.apache.commons.cli.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.*; +import java.util.regex.Pattern; + +import static java.util.Arrays.asList; + +public class IndexerMain { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexerMain.class); + + private static final Options options = new Options(); + + private static final String OPT_SOURCE_TYPE = "source-type"; + private static final String OPT_NEXUS_NUGET_PATH = "nexus-nuget-path"; + private static final String OPT_NEXUS_SERVER_ID = "nexus-server-identity"; + private static final String OPT_ARTIFACTORY_URL = "artifactory-url"; + private static final String OPT_ARTIFACTORY_USERNAME = "artifactory-username"; + private static final String OPT_ARTIFACTORY_PASSFILE = "artifactory-passfile"; + private static final String OPT_ARTIFACTORY_REPOSITORY = "artifactory-repository"; + private static final String OPT_ARTIFACTORY_EXCLUDE = "artifactory-exclude-prefix"; + private static final String OPT_OUTPUT_DB_URL = "output-db-url"; + private static final String OPT_OUTPUT_DB_PROPERTIES = "output-db-properties"; + + static { + options.addOption(Option.builder().longOpt(OPT_SOURCE_TYPE).required().desc("Type of source. Allowed values: “nexus” and “artifactory”").numberOfArgs(1).build()); + options.addOption(Option.builder().longOpt(OPT_NEXUS_NUGET_PATH).desc("Path to nuget storage, multiple values can be separated by “" + File.pathSeparator + "”").numberOfArgs(1).build()); + options.addOption(Option.builder().longOpt(OPT_NEXUS_SERVER_ID).desc("Unique identifier of indexed server, preferably URL. This is used just for distinguishing between various instances.").numberOfArgs(1).build()); + options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_URL).desc("URL to JFrog Artifactory.").numberOfArgs(1).build()); + options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_USERNAME).desc("Username for JFrog Artifactory.").numberOfArgs(1).build()); + options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_PASSFILE).desc("File with password for JFrog Artifactory.").numberOfArgs(1).build()); + options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_REPOSITORY).desc("Repositories to index. It can be used multiple times.").numberOfArgs(Option.UNLIMITED_VALUES).build()); + options.addOption(Option.builder().longOpt(OPT_ARTIFACTORY_EXCLUDE).desc("Prefixes to exclude.").numberOfArgs(Option.UNLIMITED_VALUES).build()); + options.addOption(Option.builder().longOpt(OPT_OUTPUT_DB_URL).required().desc("JDBC URL for storage DB").numberOfArgs(1).build()); + options.addOption(Option.builder().longOpt(OPT_OUTPUT_DB_PROPERTIES).desc("Location of file of properties for DB connection.").numberOfArgs(1).build()); + } + + public static void main(String[] args) throws SQLException, IOException, ClassNotFoundException, InterruptedException { + final CommandLineParser parser = new DefaultParser(); + final CommandLine cmd; + final NugetSource source; + final Properties dbProps; + try { + cmd = parser.parse(options, args); + if(!cmd.getArgList().isEmpty()){ + throw new ParseException("Unexpected extra arguments: "+ cmd.getArgList()); + } + LOGGER.info("Constructing nuget source…"); + source = getNugetSource(cmd); + LOGGER.info("Constructed nuget source: {}", source); + dbProps = parseDbProps(cmd); + } catch (ParseException e) { + System.err.println("Bad parameters: " + e.getMessage()); + help(System.err); + System.exit(1); + return; // satisfy compiler + } + try{ + index(source, cmd.getOptionValue(OPT_OUTPUT_DB_URL), dbProps); + }catch (SQLException e){ + System.err.println("SQL Exception(s):"); + for(SQLException sqlException = e; sqlException != null; sqlException = sqlException.getNextException()){ + sqlException.printStackTrace(); + } + System.exit(1); + } + } + + private static Properties parseDbProps(CommandLine cmd) throws ParseException { + final Properties dbProps = new Properties(); + final String dbPropertiesFile = cmd.getOptionValue(OPT_OUTPUT_DB_PROPERTIES); + if (dbPropertiesFile != null) { + try (final FileInputStream inputStream = new FileInputStream(dbPropertiesFile)) { + dbProps.load(inputStream); + } catch (IOException e) { + throw new ParseException("Error when loading DB properties file: " + e.getMessage()); + } + } + return dbProps; + } + + private static NugetSource getNugetSource(CommandLine cmd) throws ParseException { + final String sourceType = cmd.getOptionValue(OPT_SOURCE_TYPE); + switch (sourceType) { + case "nexus": + return new NexusNugetSource(parsePaths(cmd.getOptionValue(OPT_NEXUS_NUGET_PATH)), cmd.getOptionValue(OPT_NEXUS_SERVER_ID)); + case "artifactory": + final String password; + try (BufferedReader reader = new BufferedReader(new FileReader(cmd.getOptionValue(OPT_ARTIFACTORY_PASSFILE)))) { + password = reader.readLine(); + } catch (IOException e) { + throw new ParseException("Error when reading password file for artifactory: "+e.getMessage()); + } + final String username = cmd.getOptionValue(OPT_ARTIFACTORY_USERNAME); + final String[] repositories = cmd.getOptionValues(OPT_ARTIFACTORY_REPOSITORY); + if(repositories == null){ + throw new ParseException("Please specify at least one repository."); + } + final TreeSet exclusions = new TreeSet<>(asList(Optional.ofNullable(cmd.getOptionValues(OPT_ARTIFACTORY_EXCLUDE)).orElseGet(() -> new String[0]))); + return new ArtifactoryNugetSource(cmd.getOptionValue(OPT_ARTIFACTORY_URL), username, password, Arrays.asList(repositories), exclusions); + default: + throw new ParseException("Unknown source type: " + sourceType); + } + } + + private static void help(PrintStream out) { + help(new PrintWriter(out, true)); + } + + private static void help(PrintWriter writer) { + new HelpFormatter().printHelp( + writer, + HelpFormatter.DEFAULT_WIDTH, + "java -jar nuget-indexer.jar", + null, + options, + HelpFormatter.DEFAULT_LEFT_PAD, + HelpFormatter.DEFAULT_DESC_PAD, + null + ); + } + + private static List parsePaths(String pathString) { + return asList(pathString.split(Pattern.quote(File.pathSeparator))); + } + + private static void index(NugetSource source, String connString, Properties dbProps) throws IOException, SQLException { + if(!org.postgresql.Driver.isRegistered()){ + org.postgresql.Driver.register(); + } + org.mariadb.jdbc.Driver.class.getName(); + try (Connection dbh = DriverManager.getConnection(connString, updatedProps(dbProps))) { + final NugetMetadataStore nugetMetadataStore = NugetMetadataStore.open(dbh, source.getHash()); + final long lastModifiedTime = nugetMetadataStore.getLastModifiedTime(); + final Indexer indexer = new Indexer(nugetMetadataStore); + LOGGER.info("Start indexing {}…", source); + source.index(lastModifiedTime, indexer); + nugetMetadataStore.finish(); + LOGGER.info("Finished indexing {}…", source); + } + } + + private static Properties updatedProps(Properties dbProps) { + final Properties clone = (Properties) dbProps.clone(); + clone.put("allowMultiQueries", "true"); + return clone; + } + +} diff --git a/src/main/java/com/ysoft/security/NexusNugetSource.java b/src/main/java/com/ysoft/security/NexusNugetSource.java new file mode 100644 index 0000000..2a655e2 --- /dev/null +++ b/src/main/java/com/ysoft/security/NexusNugetSource.java @@ -0,0 +1,123 @@ +package com.ysoft.security; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.DatatypeConverter; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.sql.SQLException; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import static com.ysoft.security.AngelaTree.merkle; +import static com.ysoft.security.AngelaTree.merkleSorted; +import static java.nio.file.Files.walkFileTree; + +public class NexusNugetSource implements NugetSource { + private static final Logger LOGGER = LoggerFactory.getLogger(NexusNugetSource.class); + private final List paths; + private final String serverIdentity; + public NexusNugetSource(List paths, String serverIdentity) { + this.paths = paths; + this.serverIdentity = serverIdentity; + } + + @Override + public void index(long lastModifiedTime, Indexer indexer) throws IOException { + for (final String path : paths) { + indexDirectory(path, lastModifiedTime, indexer); + } + } + + private static void indexDirectory(String searchPath, final long lastModifiedTime, final Indexer indexer) throws IOException { + final String prefix = searchPath + (searchPath.endsWith(File.separator) ? "" : File.separator); + walkFileTree(Paths.get(searchPath), new FileVisitor() { + public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFileAttributes) throws IOException { + return FileVisitResult.CONTINUE; + } + + public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) throws IOException { + // We cannot use basicFileAttributes.creationTime(), because Nexus puts old (original?) timestamp for the files if mirrored + final FileTime fileTime; + final Process process = new ProcessBuilder("stat", "-c", "%Z", "--", path.toString()).redirectErrorStream(true).start(); + try { + process.getOutputStream().close(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + final String firstLine = reader.readLine(); + if (reader.readLine() != null) { + throw new IOException("Expected EOF"); + } + process.waitFor(); + final int exitValue = process.exitValue(); + if (exitValue != 0) { + throw new IOException("Bad exit value: " + exitValue); + } + // The time is rounded. Add one second in order to err on the safe side. + fileTime = FileTime.from(Long.parseLong(firstLine) + 1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new IOException(e); + } + } finally { + process.destroyForcibly(); // The process is not expected to do anything at this point… + } + if (fileTime.toMillis() > lastModifiedTime) { + if (path.getFileName().toString().toLowerCase().endsWith(".nupkg")) { + try { + return process(path); + } catch (SQLException e) { + throw new IOException(e); + } + } else { + LOGGER.warn("Unknown file skipped: " + path); + return FileVisitResult.CONTINUE; + } + } else { + return FileVisitResult.CONTINUE; + } + } + + private FileVisitResult process(Path path) throws IOException, SQLException { + if (path.toString().startsWith(prefix)) { + final String subpath = path.toString().substring(prefix.length()); + if (!subpath.startsWith(".nexus" + File.separator)) { + final String[] components = subpath.split(Pattern.quote(File.separator)); + final String name = components[0]; + final String version = components[1]; + try (InputStream in = Files.newInputStream(path)) { + indexer.index(in, name, version); + } + } + return FileVisitResult.CONTINUE; + } else { + throw new IOException("The path does not start with the expected prefix: " + path); + } + } + + public FileVisitResult visitFileFailed(Path path, IOException e) throws IOException { + throw e; + } + + public FileVisitResult postVisitDirectory(Path path, IOException e) throws IOException { + return FileVisitResult.CONTINUE; + } + }); + } + + @Override + public String getHash() { + return DatatypeConverter.printHexBinary(merkle("Nexus".getBytes(StandardCharsets.UTF_8), serverIdentity.getBytes(StandardCharsets.UTF_8), merkleSorted(paths))); + } + + @Override + public String toString() { + return "NexusNugetSource{" + + "serverIdentity='" + serverIdentity + '\'' + + ", paths=" + paths + + '}'; + } +} diff --git a/src/main/java/com/ysoft/security/NonClosableInputStream.java b/src/main/java/com/ysoft/security/NonClosableInputStream.java new file mode 100644 index 0000000..7b44e4c --- /dev/null +++ b/src/main/java/com/ysoft/security/NonClosableInputStream.java @@ -0,0 +1,16 @@ +package com.ysoft.security; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class NonClosableInputStream extends FilterInputStream { + public NonClosableInputStream(InputStream input) { + super(input); + } + + @Override + public void close() throws IOException { + // ignore + } +} diff --git a/src/main/java/com/ysoft/security/NugetIdentifier.java b/src/main/java/com/ysoft/security/NugetIdentifier.java new file mode 100644 index 0000000..b65f6d7 --- /dev/null +++ b/src/main/java/com/ysoft/security/NugetIdentifier.java @@ -0,0 +1,43 @@ +package com.ysoft.security; + +import java.util.Objects; + +public final class NugetIdentifier { + final private String id; + final private String version; + + public NugetIdentifier(String id, String version) { + this.id = id; + this.version = version; + } + + public String getId() { + return id; + } + + public String getVersion() { + return version; + } + + @Override + public String toString() { + return "NugetIdentifier{" + + "id='" + id + '\'' + + ", version='" + version + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NugetIdentifier that = (NugetIdentifier) o; + return Objects.equals(id, that.id) && + Objects.equals(version, that.version); + } + + @Override + public int hashCode() { + return Objects.hash(id, version); + } +} diff --git a/src/main/java/com/ysoft/security/NugetMetadata.java b/src/main/java/com/ysoft/security/NugetMetadata.java new file mode 100644 index 0000000..6247088 --- /dev/null +++ b/src/main/java/com/ysoft/security/NugetMetadata.java @@ -0,0 +1,18 @@ +package com.ysoft.security; + +import java.util.Map; + +public class NugetMetadata { + private final NugetIdentifier nugetIdentifier; + private final Map> hashesForFiles; + public NugetMetadata(NugetIdentifier nugetIdentifier, Map> hashesForFiles) { + this.nugetIdentifier = nugetIdentifier; + this.hashesForFiles = hashesForFiles; + } + public NugetIdentifier getNugetIdentifier() { + return nugetIdentifier; + } + public Map> getHashesForFiles() { + return hashesForFiles; + } +} diff --git a/src/main/java/com/ysoft/security/NugetMetadataStore.java b/src/main/java/com/ysoft/security/NugetMetadataStore.java new file mode 100644 index 0000000..869fc0d --- /dev/null +++ b/src/main/java/com/ysoft/security/NugetMetadataStore.java @@ -0,0 +1,168 @@ +package com.ysoft.security; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.sql.*; +import java.util.Map; + +public class NugetMetadataStore { + private static final Logger LOGGER = LoggerFactory.getLogger(NugetMetadataStore.class); + + private static final int REQUIRED_SCHEMA_VERSION = 2; + + private final Connection dbh; + + private final long startTime; + + private final long lastModifiedTime; + + private final String sourceHash; + + public NugetMetadataStore(Connection dbh, long startTime, long lastModifiedTime, String sourceHash) { + this.dbh = dbh; + this.startTime = startTime; + this.lastModifiedTime = lastModifiedTime; + this.sourceHash = sourceHash; + } + + public static NugetMetadataStore open(Connection dbh, String hash) throws SQLException, IOException { + LOGGER.info("Opening metadata store for {}", hash); + final long startTime = System.currentTimeMillis(); + final int schemaVersion = getSchemaVersion(dbh); + LOGGER.info("Schema version: {}", schemaVersion); + updateDbStructure(dbh, schemaVersion); + final IndexState indexState = getIndexState(dbh, hash); + LOGGER.info("Index state: {}", indexState); + return new NugetMetadataStore(dbh, startTime, indexState.getLastModifiedTime(), indexState.getSourceHash()); + } + + private static IndexState getIndexState(Connection dbh, String sourceHash) throws SQLException { + try(PreparedStatement sourceStatement = dbh.prepareStatement("SELECT * FROM nuget_index_sources WHERE source_hash = ?")){ + sourceStatement.setString(1, sourceHash); + try(ResultSet sourceResultSet = sourceStatement.executeQuery()){ + if(sourceResultSet.next()) { + return new IndexState(sourceResultSet.getLong("last_updated_time"), sourceHash); + } else { + return new IndexState(-1, sourceHash); + } + } + } + } + + private static int getSchemaVersion(Connection dbh) throws SQLException { + if(stateTableExists(dbh)){ + try ( + // The name nuget_index_state is a bit misnomer, which is due to the legacy… + PreparedStatement dbStatement = dbh.prepareStatement("SELECT * FROM nuget_index_state WHERE id = 1"); + ResultSet dbResultSet = dbStatement.executeQuery() + ) { + dbResultSet.next(); + return dbResultSet.getInt("schema_version"); + } + }else{ + return 0; + } + } + + private static boolean stateTableExists(Connection dbh) throws SQLException { + try( + ResultSet tablesResults = dbh.getMetaData().getTables(dbh.getCatalog(), null, null, null); + ){ + while(tablesResults.next()){ + final String tableName = tablesResults.getString("TABLE_NAME"); + if(tableName.equals("nuget_index_state")){ + return true; + } + } + return false; + } + } + + private static void updateDbStructure(Connection dbh, int schemaVersion) throws IOException, SQLException { + for(int i = schemaVersion+1; i <= REQUIRED_SCHEMA_VERSION; i++){ + LOGGER.info("Updating schema to version "+i+"…"); + try (final InputStream in = NugetMetadataStore.class.getResourceAsStream("/schema/" + i + ".sql")) { + final byte[] buffer = new byte[4096]; + int size; + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + while((size = in.read(buffer)) != -1){ + out.write(buffer, 0, size); + } + final String sql = out.toString(); + try (Statement statement = dbh.createStatement()) { + // I know, it can catch a semicolon inside a string or comment or so, but we can live with that. + // This is needed if the DB engine does not support multiple queries in a single batch. + for (String sqlPart : sql.split(";")) { + statement.addBatch(sqlPart); + } + statement.addBatch("UPDATE nuget_index_state SET schema_version = "+i); + statement.executeBatch(); + } + } + + } + } + + public void finish() throws SQLException { + updateLastUpdated(dbh, startTime, sourceHash); + } + + private static void updateLastUpdated(Connection dbh, long lastUpdated, String hash) throws SQLException { + try (PreparedStatement preparedStatement = dbh.prepareStatement(getUpdateLastUpdatedStatement(dbh))) { + preparedStatement.setLong(1, lastUpdated); + preparedStatement.setString(2, hash); + preparedStatement.setLong(3, lastUpdated); + preparedStatement.execute(); + } + } + + private static String getUpdateLastUpdatedStatement(Connection dbh) throws SQLException { + String databaseProductName = dbh.getMetaData().getDatabaseProductName(); + switch (databaseProductName) { + case "MySQL": + case "MariaDB": + return "INSERT INTO nuget_index_sources " + + "(last_updated_time, source_hash) VALUES (?, ?) " + + "ON DUPLICATE KEY UPDATE last_updated_time = ?"; + case "PostgreSQL": + return "INSERT INTO nuget_index_sources " + + "(last_updated_time, source_hash) VALUES (?, ?) " + + "ON CONFLICT (source_hash) DO UPDATE SET last_updated_time = ?"; + default: + throw new SQLException("Unexpected database: " + databaseProductName); + } + } + + public void addHash(String name, String version, String fileName, Map hashes) throws SQLException { + try (PreparedStatement preparedStatement = dbh.prepareStatement(getInsertCommand())) { + preparedStatement.setString(1, name); + preparedStatement.setString(2, version); + preparedStatement.setString(3, fileName); + preparedStatement.setString(4, hashes.get("sha1")); + preparedStatement.setString(5, hashes.get("md5")); + preparedStatement.execute(); + } + } + + private String getInsertCommand() throws SQLException { + String databaseProductName = dbh.getMetaData().getDatabaseProductName(); + switch(databaseProductName){ + case "MySQL": + case "MariaDB": + return "INSERT IGNORE INTO nuget_index_hashes (name, version, file_name, digest_hex_sha1, digest_hex_md5) VALUES(?, ?, ?, ?, ?)"; + case "PostgreSQL": + return "INSERT INTO nuget_index_hashes (name, version, file_name, digest_hex_sha1, digest_hex_md5) VALUES(?, ?, ?, ?, ?)" + + "ON CONFLICT (name, version, file_name, digest_hex_sha1, digest_hex_md5) DO NOTHING"; + default: + throw new SQLException("Unexpected database: " + databaseProductName); + } + } + + public long getLastModifiedTime() { + return lastModifiedTime; + } +} diff --git a/src/main/java/com/ysoft/security/NugetReader.java b/src/main/java/com/ysoft/security/NugetReader.java new file mode 100644 index 0000000..1160ab0 --- /dev/null +++ b/src/main/java/com/ysoft/security/NugetReader.java @@ -0,0 +1,127 @@ +package com.ysoft.security; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; + +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.events.XMLEvent; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class NugetReader { + private static final Logger LOGGER = LoggerFactory.getLogger(NugetReader.class); + + private NugetReader() { + } + + public static NugetMetadata analyzeNuget(InputStream in, String expectedName, String expectedVersion) throws IOException { + NugetIdentifier nugetIdentifier = null; + final Map> hashesForFiles = new HashMap<>(); + try (ZipArchiveInputStream zip = new ZipArchiveInputStream(new BufferedInputStream(in))) { + ArchiveEntry entry; + while ((entry = zip.getNextEntry()) != null) { + final Hashing.HashingInputStream hashIn = new Hashing.HashingInputStream(zip); + if (isManifest(entry)) { + if (nugetIdentifier == null) { + nugetIdentifier = getNugetIdentifierFromManifest(hashIn); + } else { + throw new IOException("Multiple NuGet manifests!"); + } + } + consumeStream(hashIn); // read the rest + if (!isBlacklistedFile(entry.getName())) { + final Object previous = hashesForFiles.put(entry.getName(), hashIn.finalizeHashes()); + if (previous != null && entry.getName().toLowerCase().endsWith(".dll")) { + throw new IOException("Multiple occurrences of file: " + entry.getName()); + } + } + } + } catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } catch (XMLStreamException e) { + throw new IOException(e); + } + if (nugetIdentifier == null) { + throw new IOException("Missing manifest file"); + } + if (expectedName != null) { + if (!expectedName.equalsIgnoreCase(nugetIdentifier.getId())) { + throw new IOException("Does not equal: " + expectedName + " and " + nugetIdentifier.getId()); + } + } + if (expectedVersion != null) { + if (!expectedVersion.equals(nugetIdentifier.getVersion())) { + throw new IOException("Does not equal: " + expectedVersion + " and " + nugetIdentifier.getVersion()); + } + } + final NugetMetadata nugetMetadata = new NugetMetadata(nugetIdentifier, hashesForFiles); + LOGGER.info("name: " + nugetIdentifier.getId() + ", version: " + nugetIdentifier.getVersion()); + return nugetMetadata; + } + + private static boolean isBlacklistedFile(String name) { + final String nn = name.toLowerCase(); + return nn.endsWith(".xml") || nn.endsWith("/.rels"); + } + + private static void consumeStream(InputStream hashIn) throws IOException { + final byte[] buffer = new byte[4096]; + //noinspection StatementWithEmptyBody + while (hashIn.read(buffer) > 0) { + // just consume in order to compute the proper hash + } + } + + public static NugetIdentifier getNugetIdentifierFromManifest(InputStream input) throws XMLStreamException, IOException { + String id = null; + String version = null; + final XMLInputFactory xmlInputFactory = XMLInputFactory.newFactory(); + xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); // This disables DTDs entirely for that factory + xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", false); // disable external entities + final XMLEventReader xmlEventReader = xmlInputFactory.createXMLEventReader(new NonClosableInputStream(input)); + while (xmlEventReader.hasNext()) { + final XMLEvent event = xmlEventReader.nextEvent(); + if (event.isStartElement()) { + switch (event.asStartElement().getName().getLocalPart()) { + case "id": + if (id == null) { + id = xmlEventReader.nextEvent().asCharacters().getData(); + } else { + throw new IOException("Multiple id elements."); + } + break; + case "version": + if (version == null) { + version = xmlEventReader.nextEvent().asCharacters().getData(); + } else { + throw new IOException("Multiple version elements."); + } + break; + default: + //ignore + } + } + } + if (id == null) { + throw new IOException("Cannot find NuGet id"); + } + if (version == null) { + throw new IOException("Cannot find NuGet version"); + } + return new NugetIdentifier(id, version); + } + + private static boolean isManifest(ArchiveEntry zipEntry) { + return zipEntry.getName().toLowerCase().endsWith(".nuspec") && !zipEntry.getName().contains("/"); + } + +} diff --git a/src/main/java/com/ysoft/security/NugetSource.java b/src/main/java/com/ysoft/security/NugetSource.java new file mode 100644 index 0000000..e3ec9ea --- /dev/null +++ b/src/main/java/com/ysoft/security/NugetSource.java @@ -0,0 +1,9 @@ +package com.ysoft.security; + +import java.io.IOException; + +public interface NugetSource { + void index(long lastModifiedTime, Indexer indexer) throws IOException; + + String getHash(); +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..c0ade16 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,18 @@ + + + + + %level / %d - [%thread] %logger - %message%n%xException + + + + + + + + + diff --git a/src/main/resources/schema/1.sql b/src/main/resources/schema/1.sql new file mode 100644 index 0000000..6657bb6 --- /dev/null +++ b/src/main/resources/schema/1.sql @@ -0,0 +1,22 @@ +CREATE TABLE nuget_index_state ( + id INT NOT NULL PRIMARY KEY, + schema_version INT NOT NULL, + last_updated_time BIGINT +); + +INSERT INTO nuget_index_state (id, schema_version, last_updated_time) VALUES ( + 1, + 0, + -1 +); + +CREATE TABLE nuget_index_hashes ( + id SERIAL PRIMARY KEY, + name VARCHAR(512) NOT NULL, + version VARCHAR(128), + file_name VARCHAR(512), + digest_hex_sha1 CHAR(40), + digest_hex_md5 CHAR(32) +); + +CREATE UNIQUE INDEX nuget_index_hashes_unique_combination ON nuget_index_hashes (name, version, file_name, digest_hex_sha1, digest_hex_md5); \ No newline at end of file diff --git a/src/main/resources/schema/2.sql b/src/main/resources/schema/2.sql new file mode 100644 index 0000000..c77a608 --- /dev/null +++ b/src/main/resources/schema/2.sql @@ -0,0 +1,9 @@ +ALTER TABLE nuget_index_state + DROP COLUMN last_updated_time; -- moved to another table + +CREATE TABLE nuget_index_sources ( + id SERIAL PRIMARY KEY, + source_hash VARCHAR(128) NOT NULL UNIQUE, + last_updated_time BIGINT NOT NULL, + note VARCHAR(256) NULL +) \ No newline at end of file diff --git a/src/test/java/com/ysoft/security/NugetReaderTest.java b/src/test/java/com/ysoft/security/NugetReaderTest.java new file mode 100644 index 0000000..2b13ee3 --- /dev/null +++ b/src/test/java/com/ysoft/security/NugetReaderTest.java @@ -0,0 +1,192 @@ +package com.ysoft.security; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.junit.jupiter.api.Test; + +import javax.xml.stream.XMLStreamException; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.security.NoSuchAlgorithmException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.ysoft.security.NugetReader.analyzeNuget; +import static org.junit.jupiter.api.Assertions.*; + +public class NugetReaderTest { + + private NugetMetadata analyzeNugetFile(String path, String expectedName, String expectedVersion) throws IOException { + try(InputStream in = getClass().getResourceAsStream("/"+path)){ + if(in == null){ + throw new FileNotFoundException(path); + } + return analyzeNuget(in, expectedName, expectedVersion); + } + } + + private static final Map> systemGlobalizationHashes = hashesMap( + entry("lib/MonoAndroid10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/xamarinwatchos10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/net45/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/xamarintvos10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/xamarinmac20/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/MonoTouch10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/win8/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/xamarinios10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/wpa81/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/portable-net45%2Bwin8%2Bwp8%2Bwpa81/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("lib/wp80/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + //entry("_rels/.rels", "742fad0664278982d5e97f5515626c52d0b217d0", "98f15a6f3733cdd0c333b83918aeb0d0"), + //entry("[Content_Types].xml", "b5d7ea43979592a2ffeef800260fd8529f47b36d", "5a6cc58af64194af2bc55bcc867ddec3"), + entry("dotnet_library_license.txt", "a4cb8479639f7380ba6a632264e887f46fa7a561", "db62529d9c74388f3885fad4b435b3f7"), + entry("package/services/metadata/core-properties/77885db85c884affa6b80d6e5a56cf58.psmdcp", "5592a5109597abab05cab2729913709e078d73b2", "7dc5495a438658a9492c4aed7788e7c7"), + entry("ref/MonoAndroid10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + //entry("ref/netstandard1.3/ko/System.Globalization.xml", "632d95186fa7f021a9bce7f24c8d26c43e30c0a8", "4282b9f0103629108714f370b6e1222f"), + //entry("ref/netstandard1.3/System.Globalization.xml", "71564073dcf48c24a740273807d9ffa2e8a561c1", "4fe886b0090c10438c5bb543f6c5d2dc"), + //entry("ref/netstandard1.3/zh-hant/System.Globalization.xml", "ec06d8b60b81eb33ab84abd5bf63f4525737aefe", "11da607afc3c07782540e8fe719153c7"), + //entry("ref/netstandard1.3/zh-hans/System.Globalization.xml", "6ddb09a5f1c13acd2a2242c4492fdab14eea959b", "f3c1491ef616a2a563eb83e08f2092cf"), + //entry("ref/netstandard1.3/ja/System.Globalization.xml", "a848e00d99308336c0066b30972be0d8acb0ff5a", "e810f4e9e6028b4c432b41c05ebad6c2"), + //entry("ref/netstandard1.3/de/System.Globalization.xml", "22ae3ada9772bb1e27c457e068dcfaaa8bcb662e", "1ac187d1c24e59af866837ee8239a79c"), + //entry("ref/netstandard1.3/ru/System.Globalization.xml", "846bcd342893de6facdd183691782f4185272313", "efc817df6de191d88301ecea957ac825"), + //entry("ref/netstandard1.3/it/System.Globalization.xml", "2d7048ea9d7360fa92362fc7237c1b53518e79d6", "068098a1d63acbe85d0e08f228ba79be"), + entry("ref/netstandard1.3/System.Globalization.dll", "879325a6b71bbdea6f2d2f9d85311559653b4f11", "c481520a478dc704f80f25fd3894b563"), + //entry("ref/netstandard1.3/es/System.Globalization.xml", "5da83bd1fcfacf7d6e6c501b9b3648d3f86135af", "b6f6ade3994d858aca7618775aaf40d2"), + //entry("ref/netstandard1.3/fr/System.Globalization.xml", "e5639262d8908200a8a58f89c456256b1633a29d", "8cc404253cdc98e9450b027f6ec590a8"), + entry("ref/xamarinwatchos10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("ref/net45/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("ref/xamarintvos10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("ref/xamarinmac20/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("ref/MonoTouch10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + //entry("ref/netcore50/ko/System.Globalization.xml", "632d95186fa7f021a9bce7f24c8d26c43e30c0a8", "4282b9f0103629108714f370b6e1222f"), + //entry("ref/netcore50/System.Globalization.xml", "71564073dcf48c24a740273807d9ffa2e8a561c1", "4fe886b0090c10438c5bb543f6c5d2dc"), + //entry("ref/netcore50/zh-hant/System.Globalization.xml", "ec06d8b60b81eb33ab84abd5bf63f4525737aefe", "11da607afc3c07782540e8fe719153c7"), + //entry("ref/netcore50/zh-hans/System.Globalization.xml", "6ddb09a5f1c13acd2a2242c4492fdab14eea959b", "f3c1491ef616a2a563eb83e08f2092cf"), + //entry("ref/netcore50/ja/System.Globalization.xml", "a848e00d99308336c0066b30972be0d8acb0ff5a", "e810f4e9e6028b4c432b41c05ebad6c2"), + //entry("ref/netcore50/de/System.Globalization.xml", "22ae3ada9772bb1e27c457e068dcfaaa8bcb662e", "1ac187d1c24e59af866837ee8239a79c"), + //entry("ref/netcore50/ru/System.Globalization.xml", "846bcd342893de6facdd183691782f4185272313", "efc817df6de191d88301ecea957ac825"), + //entry("ref/netcore50/it/System.Globalization.xml", "2d7048ea9d7360fa92362fc7237c1b53518e79d6", "068098a1d63acbe85d0e08f228ba79be"), + entry("ref/netcore50/System.Globalization.dll", "879325a6b71bbdea6f2d2f9d85311559653b4f11", "c481520a478dc704f80f25fd3894b563"), + //entry("ref/netcore50/es/System.Globalization.xml", "5da83bd1fcfacf7d6e6c501b9b3648d3f86135af", "b6f6ade3994d858aca7618775aaf40d2"), + //entry("ref/netcore50/fr/System.Globalization.xml", "e5639262d8908200a8a58f89c456256b1633a29d", "8cc404253cdc98e9450b027f6ec590a8"), + //entry("ref/netstandard1.0/ko/System.Globalization.xml", "632d95186fa7f021a9bce7f24c8d26c43e30c0a8", "4282b9f0103629108714f370b6e1222f"), + //entry("ref/netstandard1.0/System.Globalization.xml", "71564073dcf48c24a740273807d9ffa2e8a561c1", "4fe886b0090c10438c5bb543f6c5d2dc"), + //entry("ref/netstandard1.0/zh-hant/System.Globalization.xml", "ec06d8b60b81eb33ab84abd5bf63f4525737aefe", "11da607afc3c07782540e8fe719153c7"), + //entry("ref/netstandard1.0/zh-hans/System.Globalization.xml", "6ddb09a5f1c13acd2a2242c4492fdab14eea959b", "f3c1491ef616a2a563eb83e08f2092cf"), + //entry("ref/netstandard1.0/ja/System.Globalization.xml", "a848e00d99308336c0066b30972be0d8acb0ff5a", "e810f4e9e6028b4c432b41c05ebad6c2"), + //entry("ref/netstandard1.0/de/System.Globalization.xml", "22ae3ada9772bb1e27c457e068dcfaaa8bcb662e", "1ac187d1c24e59af866837ee8239a79c"), + //entry("ref/netstandard1.0/ru/System.Globalization.xml", "846bcd342893de6facdd183691782f4185272313", "efc817df6de191d88301ecea957ac825"), + //entry("ref/netstandard1.0/it/System.Globalization.xml", "2d7048ea9d7360fa92362fc7237c1b53518e79d6", "068098a1d63acbe85d0e08f228ba79be"), + entry("ref/netstandard1.0/System.Globalization.dll", "fc66f3384835722177dc523e100574bd06c45725", "849f648b4f96278669f6410f8c159f94"), + //entry("ref/netstandard1.0/es/System.Globalization.xml", "5da83bd1fcfacf7d6e6c501b9b3648d3f86135af", "b6f6ade3994d858aca7618775aaf40d2"), + //entry("ref/netstandard1.0/fr/System.Globalization.xml", "e5639262d8908200a8a58f89c456256b1633a29d", "8cc404253cdc98e9450b027f6ec590a8"), + entry("ref/win8/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("ref/xamarinios10/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("ref/wpa81/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("ref/portable-net45%2Bwin8%2Bwp8%2Bwpa81/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("ref/wp80/_._", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "d41d8cd98f00b204e9800998ecf8427e"), + entry("System.Globalization.nuspec", "07689ff76a416c988f6af94cd7cc4b10ddb95e08", "44ff82802f7d390bfedae1e5c5be6c5e"), + entry("ThirdPartyNotices.txt", "58633a0b1cc282fa6ec4ca32d4b9327319ca31fe", "c967cb8266866e70f817fee256966091") + ); + + private static final Map> microsoftNetcoreRuntimeCoreclrArmHashes = hashesMap( + //entry("_rels/.rels", "fe496bc9c2a148400d4648a2c4b4c7dd2db63a9a", "80008d2f71d952183263cc6739b48c90"), + entry("Microsoft.NETCore.Runtime.CoreCLR-arm.nuspec", "20d364b0cce512510c0a8e5556e7d5a09171f401", "35c2b666457bdc1abbccb96153c564b1"), + //entry("[Content_Types].xml", "85095f137c96204b5099b3085885acfbde04108b", "9c542780fe63324caca20a23f3168c6d"), + entry("runtimes/win8-arm/lib/dotnet/mscorlib.ni.dll", "84b5407977ffce80854d9203ebf6d36e4c69f374", "6980c421ab76c555600d57d608abd58e"), + entry("runtimes/win8-arm/native/coreclr.dll", "f03b24c2be5b908d504371b351484100bfc27768", "37238f2bb0276d18c9585530104ef70b"), + entry("runtimes/win8-arm/native/mscordaccore.dll", "4b0af8bb4b2313682af44c4d72aec4069ebec51e", "6fae425f7e27c6ff217266b40c83f160"), + entry("runtimes/win8-arm/native/mscordbi.dll", "f59a9a4e1cbc1daad601e0e196987871daf755fe", "2cc25b01beee5e72cd5d0893516c2aba"), + entry("runtimes/win8-arm/native/dbgshim.dll", "00b7dbcf9cd520913375028e64a2a40655408e50", "bb421ab20b557d885157a035634f0e9b"), + entry("runtimes/win8-arm/native/mscorrc.dll", "319d8fbdba7552f7546d4e64bafc829153029f90", "dc8bfbd9ed5172ddf4c7c402118c1b4b"), + entry("runtimes/win8-arm/native/clretwrc.dll", "d7e56c09b3e9e40f8a4cdfee45d5aae9f8832490", "85dffa2f3dbae07c6ab0731f1f837f03"), + entry("runtimes/win8-arm/native/mscorrc.debug.dll", "11169d8018227b7bdc77eddf114aefa4db19f33f", "b65e2e3baf57a333019f772eb3f4eb4c"), + entry("package/services/metadata/core-properties/c1cbeaed81514106b6b7971ac193f132.psmdcp", "5fb4a8f7f8303e347cbf30a3e8701f2e14f7a844", "1f7935ebc3353d6c0894d00d73616372") + // damaged: entry("ref/dotnet/_._", "688934845f22049cb14668832efa33d45013b6b9", "598f4fe64aefab8f00bcbea4c9239abf") + ); + + private static final Map> microsoftAspnetRazorHashes = hashesMap( + entry("lib/net40/System.Web.Razor.dll", "d6b7cd752f0ff7b9f773d544add6a5ea00158599", "cd4ddc9c5b2695018d6e7c2950e7d82f"), + //entry("lib/net40/System.Web.Razor.xml", "4e2f4e963f640112679f70f7a8aeab746342f7b1", "2aa1d44a5b99b443f614ad6890cdb92b"), + //entry("_rels/.rels", "7bc360ddecbd16668555b48785e202558b67f3fc", "fec47ef79c06556ef9f3bc8f89c02075"), + entry("Microsoft.AspNet.Razor.nuspec", "05d426f9e8ecb4685efe1436357fd61bcc5d6df2", "2b70d4b4a93c8146db24138faa844c8c"), + //entry("[Content_Types].xml", "d52e5937a775c3df3eee3b5cac6bbbbb1d884009", "4a8b92ec365e327ad1c1cae004d3c088"), + entry("package/services/metadata/core-properties/37cf22fae31a4489a4df544d33fed45a.psmdcp", "4223c8c09f0f99751772dcb9ec0cad70af45e88b", "4892743e40333517a6aecab561e4143c"), + entry(".signature.p7s", "84923efb62418eedd119be96901a624d4f87cf99", "c5b87f4ac7119eb7ebbff954993e9937") + ); + + private static final Map> netMqHashes = hashesMap( + entry(".signature.p7s", "f62ab9b16f5630208353904ea7cae72784b60d5c", "e55402a31e9d8bf9a0a4ee8e1bcab495"), + entry("package/services/metadata/core-properties/35baa1c9e346418996e5dcf9bbc4c861.psmdcp", "e5628ff74af0a48a7f367792bf50e94301eb6d74", "98d973ededa8490ad19493996d6a47d1"), + entry("package/services/metadata/core-properties/25d54e2d9f1b429386de7b9853bf46f9.psmdcp", "bcdeace599c87a895ae8832177adc4bf92f5dad7", "cdfd39d415fa39bdf2052b4e65ef6f2c"), + //entry("lib/netstandard2.0/NetMQ.xml", "9067824d07b0b457b7846dc18ecd1f5467a0d206", "8358926b643d647167cc4e527a4a8c39"), + entry("lib/netstandard2.0/NetMQ.pdb", "2923e5f0088ca2a24f87613b113419e2d9e5914b", "65569b7afcbe1d46ca5820205ab5f514"), + entry("lib/netstandard2.0/NetMQ.dll", "8fef6ef59442061ead95457649b3d62a69775c6c", "b15546f1a77a5dda915c8bc792c2283d"), + //entry("lib/netstandard1.6/NetMQ.xml", "96354c3655d7e95718759b2969dd351eb20059e5", "e332c83a76c30c57e47985ca26fd029d"), + entry("lib/netstandard1.6/NetMQ.pdb", "d2fc620dee4fa297ae826ecb378ab63c4b358b57", "6e59b2c6a6ef2542c3fd9d6b6922c5c1"), + entry("lib/netstandard1.6/NetMQ.dll", "65fbf6ffdc6bb648628921d85423c85013920c0d", "0c423c8978b33dce16c6903f827109bd"), + //entry("lib/net40/NetMQ.xml", "cf7f6c9a2f59c121e4d6550239503c6210d9a9f0", "59bd1e8c3920f08970415fc71d1c27ce"), + entry("lib/net40/NetMQ.pdb", "5dd57094e2ec802554abbcf0e63fac3c3e870128", "8f70a7bc3c0508448a95cc06f81866d7"), + entry("lib/net40/NetMQ.dll", "6c8217c37c7f50e23d5b4948873ad73327945d74", "05c06b5716822bed55fb4cab8b9193cb"), + //entry("_rels/.rels", "?", "?"), + //entry("[Content_Types].xml", "?", "?"), + entry("NetMQ.nuspec", "cdd0a117da95745fcb7a86c541a0c2a6ae184238", "8fece78f2bc2ae511f0c2ac9b1386894") + ); + + + private static Map.Entry> entry(String file, String sha1, String md5) { + final Map hashes = new HashMap<>(); + hashes.put("sha1", sha1.toUpperCase()); + hashes.put("md5", md5.toUpperCase()); + return new ImmutablePair<>(file, Collections.unmodifiableMap(hashes)); + } + + private static Map> hashesMap(Map.Entry>... entries){ + //return Arrays.stream(entries).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + final Map> map = new HashMap<>(); + for (Map.Entry> entry : entries) { + map.put(entry.getKey(), entry.getValue()); + } + return Collections.unmodifiableMap(map); + } + + @Test + public void testSomeRandomNuget() throws IOException { + assertEquals(systemGlobalizationHashes, analyzeNugetFile("System.Globalization.4.3.0.nupkg", "System.Globalization", "4.3.0").getHashesForFiles()); + } + + @Test + public void testNugetWithEmptyFile() throws IOException { + final Map> hashesForFiles = new HashMap<>(analyzeNugetFile("Microsoft.NETCore.Runtime.CoreCLR-arm-1.0.0.nupkg", "Microsoft.NETCore.Runtime.CoreCLR-arm", "1.0.0").getHashesForFiles()); + hashesForFiles.remove("ref/dotnet/_._"); // remove damaged + assertEquals(microsoftNetcoreRuntimeCoreclrArmHashes.keySet(), hashesForFiles.keySet()); + assertEquals(microsoftNetcoreRuntimeCoreclrArmHashes, hashesForFiles); + } + + @Test + public void testSomeOtherTroublesomeNuget() throws IOException { + // This NuGet used to cause issues with hashing the manifest file, because of using non-zero offset when calling Hashing.HashingInputStream.read(byte[], int, int). Ideally, we would create a test for this scenario. + final Map> hashesForFiles = new HashMap<>(analyzeNugetFile("Microsoft.AspNet.Razor-2.0.20715.0.nupkg", "Microsoft.AspNet.Razor", "2.0.20715.0").getHashesForFiles()); + assertEquals(microsoftAspnetRazorHashes, hashesForFiles); + } + + @Test + public void testPackageWithDuplicateBlacklistedFiles() throws IOException { + // This should proceed, as the duplicate files are excluded. + final Map> hashesForFiles = new HashMap<>(analyzeNugetFile("netmq.4.0.0.207.nupkg", "NetMQ", "4.0.0.207").getHashesForFiles()); + assertEquals(netMqHashes, hashesForFiles); + } + + @Test + void testGetNugetIdentifierFromManifest() throws IOException, XMLStreamException, NoSuchAlgorithmException { + try (Hashing.HashingInputStream in = new Hashing.HashingInputStream(getClass().getResourceAsStream("/Microsoft.AspNet.Razor.nuspec"))) { + assertEquals(new NugetIdentifier("Microsoft.AspNet.Razor", "2.0.20715.0"), NugetReader.getNugetIdentifierFromManifest(in)); + while(in.read() != -1){ + // Eat it! + } + final Map hashes = in.finalizeHashes(); + assertEquals(hashes, microsoftAspnetRazorHashes.get("Microsoft.AspNet.Razor.nuspec")); + } + } +} \ No newline at end of file diff --git a/src/test/resources/Microsoft.AspNet.Razor-2.0.20715.0.nupkg b/src/test/resources/Microsoft.AspNet.Razor-2.0.20715.0.nupkg new file mode 100644 index 0000000..70dc1f4 Binary files /dev/null and b/src/test/resources/Microsoft.AspNet.Razor-2.0.20715.0.nupkg differ diff --git a/src/test/resources/Microsoft.AspNet.Razor.nuspec b/src/test/resources/Microsoft.AspNet.Razor.nuspec new file mode 100644 index 0000000..3aa3c24 Binary files /dev/null and b/src/test/resources/Microsoft.AspNet.Razor.nuspec differ diff --git a/src/test/resources/Microsoft.NETCore.Runtime.CoreCLR-arm-1.0.0.nupkg b/src/test/resources/Microsoft.NETCore.Runtime.CoreCLR-arm-1.0.0.nupkg new file mode 100644 index 0000000..f391761 Binary files /dev/null and b/src/test/resources/Microsoft.NETCore.Runtime.CoreCLR-arm-1.0.0.nupkg differ diff --git a/src/test/resources/System.Globalization.4.3.0.nupkg b/src/test/resources/System.Globalization.4.3.0.nupkg new file mode 100644 index 0000000..5d440d5 Binary files /dev/null and b/src/test/resources/System.Globalization.4.3.0.nupkg differ diff --git a/src/test/resources/netmq.4.0.0.207.nupkg b/src/test/resources/netmq.4.0.0.207.nupkg new file mode 100644 index 0000000..82cc16b Binary files /dev/null and b/src/test/resources/netmq.4.0.0.207.nupkg differ