ID-1570 - Fix dice-where and update dce-id to use the latest version

dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/Download.java

@@ -1,5 +1,7 @@
 package technology.dice.dicewhere.downloader;
 
+import java.io.File;
+import java.nio.file.Files;
 import java.util.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -8,6 +10,7 @@
 import technology.dice.dicewhere.downloader.exception.DownloaderException;
 import technology.dice.dicewhere.downloader.md5.MD5Checksum;
 import technology.dice.dicewhere.downloader.source.FileSource;
+import technology.dice.dicewhere.downloader.stream.StreamWithMD5Decorator;
 
 public abstract class Download {
 

dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java

@@ -8,8 +8,8 @@
 
 public interface FileAcceptor<T> {
 
-  StreamConsumer<T> getStreamConsumer(MD5Checksum originalFileMd5, Instant originalFileTimestamp,
-      boolean noMd5Check);
+  StreamConsumer<T> getStreamConsumer(
+      MD5Checksum originalFileMd5, Instant originalFileTimestamp, boolean noMd5Check);
 
   boolean destinationExists();
 

dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java

@@ -73,13 +73,11 @@ public Optional<MD5Checksum> existingFileMd5() {
           BufferedInputStream bis = new BufferedInputStream(is);
           StreamWithMD5Decorator md5Is = StreamWithMD5Decorator.of(bis)) {
         byte[] buffer = new byte[BUFFER];
-        while ((md5Is.read(buffer)) != -1) {
-        }
+        while ((md5Is.read(buffer)) != -1) {}
         return Optional.of(md5Is.md5());
       } catch (IOException | NoSuchAlgorithmException e) {
         throw new RuntimeException(
-            "Could not obtain md5 of the file existing at the target: " + destination,
-            e);
+            "Could not obtain md5 of the file existing at the target: " + destination, e);
       }
     }
     return Optional.empty();

dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java

@@ -5,13 +5,15 @@
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.Optional;
 import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
 import technology.dice.dicewhere.downloader.md5.MD5Checksum;
 
 public class StreamWithMD5Decorator extends InputStream {
 
   private final MessageDigest md5;
   DigestInputStream inputStream;
+  private Optional<MD5Checksum> checksum = Optional.empty();
 
   private StreamWithMD5Decorator(DigestInputStream inputStream, MessageDigest md5) {
     this.inputStream = inputStream;
@@ -25,17 +27,26 @@
   }
 
   public MD5Checksum md5() {
-    String hex = (new HexBinaryAdapter()).marshal(this.md5.digest());
-    return MD5Checksum.of(hex);
+    return checksum.orElseGet(
+        () -> {
+          String hex = (new HexBinaryAdapter()).marshal(this.md5.digest());
+          checksum = Optional.of(MD5Checksum.of(hex));
+          return checksum.get();
+        });
   }
 
   @Override
   public int read() throws IOException {
-    return this.inputStream.read();
+    return inputStream.read();
+  }
+
+  @Override
+  public int read(byte[] b, int off, int len) throws IOException {
+    return inputStream.read(b, off, len);
   }
 
   @Override
   public void close() throws IOException {
-    this.inputStream.close();
+    inputStream.close();
   }
 }

dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java

@@ -33,8 +33,7 @@
 public class LocalFileAcceptorTest extends TestCase {
 
   private static final int TEST_FILE_SIZE = 1024 * 1024;
-  @ClassRule
-  static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
+  @ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
 
   @BeforeClass
   public static void beforeClass() {
@@ -45,16 +44,22 @@ public static void beforeClass() {
   public void corruptedFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException {
     Pair<Path, String> tempFile = generateTempFile();
     Path destinationDir = Files.createTempDirectory("dice-where");
-    IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
-        new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
-    wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
-        aResponse().withStatus(HttpStatus.SC_OK)
-            .withHeader("Etag", "aaa")
-            .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
-            .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
-    wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
-        .willReturn(aResponse().withBody(
-            IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+    IpInfoSiteSource ipInfoSiteSource =
+        new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
+    wireMockRule.stubFor(
+        WireMock.head(UrlPattern.ANY)
+            .willReturn(
+                aResponse()
+                    .withStatus(HttpStatus.SC_OK)
+                    .withHeader("Etag", "aaa")
+                    .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
+                    .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
+    wireMockRule.stubFor(
+        WireMock.get(UrlPattern.ANY)
+            .willReturn(
+                aResponse()
+                    .withBody(
+                        IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
 
     FileInfo fileInfo = ipInfoSiteSource.fileInfo();
     ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
@@ -68,72 +73,100 @@ public void corruptedFilePreexistingSet() throws IOException, NoSuchAlgorithmExc
     Pair<Path, String> existingFile = generateTempFile();
     Path destinationDir = Files.createTempDirectory("dice-where");
     Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb"));
-    IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
-        new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
-    wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
-        aResponse().withStatus(HttpStatus.SC_OK)
-            .withHeader("Etag", "aaa")
-            .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
-            .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
-    wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
-        .willReturn(aResponse().withBody(
-            IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+    IpInfoSiteSource ipInfoSiteSource =
+        new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
+    wireMockRule.stubFor(
+        WireMock.head(UrlPattern.ANY)
+            .willReturn(
+                aResponse()
+                    .withStatus(HttpStatus.SC_OK)
+                    .withHeader("Etag", "aaa")
+                    .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
+                    .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
+    wireMockRule.stubFor(
+        WireMock.get(UrlPattern.ANY)
+            .willReturn(
+                aResponse()
+                    .withBody(
+                        IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
 
     FileInfo fileInfo = ipInfoSiteSource.fileInfo();
     ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
     assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
     assertEquals(Files.list(destinationDir).count(), 1);
     assertFalse(Files.exists(destinationDir.resolve("file.mdb")));
-    assertTrue(Arrays.equals(Files.readAllBytes(existingFile.getLeft()),
-        Files.readAllBytes(Files.list(destinationDir).findFirst().get())));
+    assertTrue(
+        Arrays.equals(
+            Files.readAllBytes(existingFile.getLeft()),
+            Files.readAllBytes(Files.list(destinationDir).findFirst().get())));
   }
 
   @Test
   public void goodFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException {
     Pair<Path, String> tempFile = generateTempFile();
     Path destinationDir = Files.createTempDirectory("dice-where");
-    IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
-        new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
-    wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
-        aResponse().withStatus(HttpStatus.SC_OK)
-            .withHeader("Etag", tempFile.getRight())
-            .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
-            .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
-    wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
-        .willReturn(aResponse().withBody(
-            IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+    IpInfoSiteSource ipInfoSiteSource =
+        new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
+    wireMockRule.stubFor(
+        WireMock.head(UrlPattern.ANY)
+            .willReturn(
+                aResponse()
+                    .withStatus(HttpStatus.SC_OK)
+                    .withHeader("Etag", tempFile.getRight())
+                    .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
+                    .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
+    wireMockRule.stubFor(
+        WireMock.get(UrlPattern.ANY)
+            .willReturn(
+                aResponse()
+                    .withBody(
+                        IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
 
     FileInfo fileInfo = ipInfoSiteSource.fileInfo();
     ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
     assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
     assertEquals(1, Files.list(destinationDir).count());
-    assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()),
-        Files.readAllBytes(destinationDir.resolve("file.mdb"))));
+    assertTrue(
+        Arrays.equals(
+            Files.readAllBytes(tempFile.getLeft()),
+            Files.readAllBytes(destinationDir.resolve("file.mdb"))));
   }
 
   @Test
   public void goodFilePreexistingSet() throws IOException, NoSuchAlgorithmException {
     Pair<Path, String> tempFile = generateTempFile();
     Pair<Path, String> existingFile = generateTempFile();
+
     Path destinationDir = Files.createTempDirectory("dice-where");
     Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb"));
-    IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
-        new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
-    wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
-        aResponse().withStatus(HttpStatus.SC_OK)
-            .withHeader("Etag", tempFile.getRight())
-            .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
-            .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
-    wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
-        .willReturn(aResponse().withBody(
-            IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
 
+    wireMockRule.stubFor(
+        WireMock.head(UrlPattern.ANY)
+            .willReturn(
+                aResponse()
+                    .withStatus(HttpStatus.SC_OK)
+                    .withHeader("Etag", tempFile.getRight())
+                    .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
+                    .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
+
+    wireMockRule.stubFor(
+        WireMock.get(UrlPattern.ANY)
+            .willReturn(
+                aResponse()
+                    .withBody(
+                        IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+
+    IpInfoSiteSource ipInfoSiteSource =
+        new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
     FileInfo fileInfo = ipInfoSiteSource.fileInfo();
+
     ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
     assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
     assertEquals(2, Files.list(destinationDir).count());
-    assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()),
-        Files.readAllBytes(destinationDir.resolve("file.mdb"))));
+    assertTrue(
+        Arrays.equals(
+            Files.readAllBytes(tempFile.getLeft()),
+            Files.readAllBytes(destinationDir.resolve("file.mdb"))));
   }
 
   private Pair<Path, String> generateTempFile() throws IOException, NoSuchAlgorithmException {
@@ -145,4 +178,4 @@ private Pair<Path, String> generateTempFile() throws IOException, NoSuchAlgorith
     String hex = (new HexBinaryAdapter()).marshal(md.digest(contents));
     return Pair.of(tempFile, hex);
   }
-}
+}

dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java

@@ -0,0 +1,81 @@
+package technology.dice.dicewhere.downloader.stream;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.github.tomakehurst.wiremock.matching.UrlPattern;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.security.NoSuchAlgorithmException;
+import junit.framework.TestCase;
+import org.apache.commons.io.IOUtils;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.internal.runners.JUnit4ClassRunner;
+import org.junit.runner.RunWith;
+
+@RunWith(JUnit4ClassRunner.class)
+public class StreamWithMD5DecoratorTest extends TestCase {
+
+  private static final String PATH = "/maxmind/maxmind-city-1.zip";
+
+  @ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
+
+  @BeforeClass
+  public static void beforeClass() {
+    wireMockRule.start();
+  }
+
+  @Test
+  public void shouldSuccessfullyReadAndCalculateDigestOfStream()
+      throws IOException, NoSuchAlgorithmException, URISyntaxException {
+    Path path = Path.of(getClass().getResource(PATH).toURI());
+    StreamWithMD5Decorator is = StreamWithMD5Decorator.of(new FileInputStream(path.toFile()));
+
+    // Exhaust stream for the complete hash digest
+    byte[] buffer = new byte[8192];
+    while ((is.read(buffer)) != -1) {}
+
+    String first = is.md5().stringFormat();
+    IOUtils.toString(is, Charset.defaultCharset());
+
+    // Assert the Stream Hash before and after
+    assertEquals(first, is.md5().stringFormat());
+    assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06");
+  }
+
+  @Test
+  public void shouldSuccessfullyReadAndCalculateDigestOfStreamFromHttp()
+      throws IOException, NoSuchAlgorithmException, URISyntaxException {
+    Path path = Path.of(getClass().getResource(PATH).toURI());
+
+    wireMockRule.stubFor(
+        WireMock.get(UrlPattern.ANY)
+            .willReturn(
+                aResponse().withBody(IOUtils.toByteArray(new FileInputStream(path.toFile())))));
+
+    URL url = new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb");
+    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+
+    StreamWithMD5Decorator is = StreamWithMD5Decorator.of(connection.getInputStream());
+    // Exhaust stream for the complete hash digest
+    byte[] buffer = new byte[8192];
+    while ((is.read(buffer)) != -1) {}
+
+    String first = is.md5().stringFormat();
+    // Read from the stream
+    IOUtils.toString(is, Charset.defaultCharset());
+
+    // Assert the Stream Hash before and after
+    assertEquals(first, is.md5().stringFormat());
+    assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06");
+  }
+}