diff --git a/dice-where-downloader-lib/pom.xml b/dice-where-downloader-lib/pom.xml index 5a2966a..55386a1 100644 --- a/dice-where-downloader-lib/pom.xml +++ b/dice-where-downloader-lib/pom.xml @@ -15,55 +15,8 @@ 1.7.30 2.14.2 2.3.1 - 1.19.8 - 3.0.1 - - - - org.jacoco - jacoco-maven-plugin - ${jacoco.version} - - - pre-unit-tests - - prepare-agent - - - - post-unit-tests - test - - report - - - - default-check - - check - - - - - BUNDLE - - - COMPLEXITY - COVEREDRATIO - 0.1 - - - - - - - - - - - software.amazon.awssdk @@ -107,37 +60,6 @@ ${javax-bind.version} - - org.testcontainers - testcontainers - ${test-containers.version} - test - - - org.testcontainers - junit-jupiter - ${test-containers.version} - test - - - org.testcontainers - localstack - ${test-containers.version} - test - - - org.testcontainers - nginx - ${test-containers.version} - test - - - org.wiremock - wiremock - ${wiremock.version} - test - - diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/Download.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/Download.java index 598e557..6a671a9 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/Download.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/Download.java @@ -10,7 +10,6 @@ import technology.dice.dicewhere.downloader.source.FileSource; public abstract class Download { - private static final Logger LOG = LoggerFactory.getLogger(Download.class); protected final boolean noCheckMd5; @@ -39,7 +38,6 @@ protected DownloadExecutionResult process(FileAcceptor acceptor, FileSource f result = processFileDoesNotExist(acceptor, fileSource, pathWritable); } LOG.info("A new file was" + (result.isNewFileDownloaded() ? "" : " not") + " downloaded"); - LOG.info("Download is " + (!result.isSuccessful() ? "un" : "") + "successful"); return result; } @@ -47,21 +45,18 @@ private DownloadExecutionResult processFileDoesNotExist( FileAcceptor acceptor, FileSource fileSource, boolean pathWritable) { if (pathWritable) { - final MD5Checksum md5Checksum = fileSource.produce(acceptor, noCheckMd5); - LOG.info("File transferred"); + final MD5Checksum md5Checksum = fileSource.produce(acceptor); + LOG.info("File successfully transferred"); if (!noCheckMd5) { boolean checksumMatches = md5Checksum.matches(fileSource.fileInfo().getMd5Checksum()); if (!checksumMatches) { - LOG.error( + LOG.warn( "Local and remote files' MD5 do not match: " + md5Checksum.stringFormat() + " Vs. " + fileSource.fileInfo().getMd5Checksum().stringFormat()); } else { - LOG.info("MD5 matches that of the remote file: " - + md5Checksum.stringFormat() - + " Vs. " - + fileSource.fileInfo().getMd5Checksum().stringFormat()); + LOG.info("MD5 matches that of the remote file"); } return new DownloadExecutionResult( true, checksumMatches, md5Checksum, acceptor.getUri(), checksumMatches); @@ -92,10 +87,7 @@ private DownloadExecutionResult processFileExists( + " Vs. " + fileSource.fileInfo().getMd5Checksum().stringFormat()); } else { - LOG.info("MD5 matches that of the remote file: " - + existingMd5.map(md5 -> md5.stringFormat()).orElse("?") - + " Vs. " - + fileSource.fileInfo().getMd5Checksum().stringFormat()); + LOG.info("MD5 matches that of the remote file"); } return new DownloadExecutionResult( false, @@ -114,8 +106,7 @@ private DownloadExecutionResult processFileExists( protected abstract DownloadExecutionResult execute(); - protected void checkNecessaryEnvironmentVariables() { - } + protected void checkNecessaryEnvironmentVariables() {} public boolean isVerbose() { return verbose; diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/actions/ipinfo/IpInfoBaseDownload.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/actions/ipinfo/IpInfoBaseDownload.java index c8fee5d..0d49402 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/actions/ipinfo/IpInfoBaseDownload.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/actions/ipinfo/IpInfoBaseDownload.java @@ -1,6 +1,8 @@ package technology.dice.dicewhere.downloader.actions.ipinfo; +import java.net.URI; import technology.dice.dicewhere.downloader.Download; +import technology.dice.dicewhere.downloader.PathUtils; public abstract class IpInfoBaseDownload extends Download { diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java index e741f25..0b70281 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java @@ -7,9 +7,7 @@ import technology.dice.dicewhere.downloader.stream.StreamConsumer; public interface FileAcceptor { - - StreamConsumer getStreamConsumer(MD5Checksum originalFileMd5, Instant originalFileTimestamp, - boolean noMd5Check); + StreamConsumer getStreamConsumer(MD5Checksum originalFileMd5, Instant originalFileTimestamp); boolean destinationExists(); diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java index 71adab3..bfc5be1 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java @@ -19,7 +19,6 @@ import technology.dice.dicewhere.downloader.stream.StreamWithMD5Decorator; public class LocalFileAcceptor implements FileAcceptor { - private static final Logger LOG = LoggerFactory.getLogger(LocalFileAcceptor.class); public static final int BUFFER = 8192; @@ -31,7 +30,7 @@ public LocalFileAcceptor(Path destination) { @Override public StreamConsumer getStreamConsumer( - MD5Checksum originalFileMd5, Instant originalFileTimestamp, boolean noMd5Check) { + MD5Checksum originalFileMd5, Instant originalFileTimestamp) { return (stream, size) -> { try { Files.createDirectories(destination); @@ -40,10 +39,6 @@ public StreamConsumer getStreamConsumer( LOG.debug("Destination directory already exists"); } Files.copy(stream, destination, StandardCopyOption.REPLACE_EXISTING); - if ((!noMd5Check) && (!originalFileMd5.matches(stream.md5()))) { - LOG.error("MD5 mismatch. Deleting destination file"); - Files.delete(destination); - } return null; }; } @@ -73,12 +68,11 @@ public Optional existingFileMd5() { BufferedInputStream bis = new BufferedInputStream(is); StreamWithMD5Decorator md5Is = StreamWithMD5Decorator.of(bis)) { byte[] buffer = new byte[BUFFER]; - while ((md5Is.read(buffer)) != -1) { - } + while ((md5Is.read(buffer)) != -1) {} return Optional.of(md5Is.md5()); } catch (IOException | NoSuchAlgorithmException e) { throw new RuntimeException( - "Could not obtain md5 of the file existing at the target: " + destination, + "Could not obtain md5 of the file existing at the target: " + destination.toString(), e); } } diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/s3/S3FileAcceptor.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/s3/S3FileAcceptor.java index d145646..9f68d1a 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/s3/S3FileAcceptor.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/s3/S3FileAcceptor.java @@ -8,11 +8,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; @@ -25,8 +22,8 @@ public class S3FileAcceptor implements FileAcceptor { - private static final Logger LOG = LoggerFactory.getLogger(S3FileAcceptor.class); private static final String LATEST_KEY = "latest"; + public static final String MD5_METADATA_KEY = "md5"; public static final String TIMESTAMP_METADATA_KEY = "ts"; private final S3Client client; private final String bucket; @@ -45,10 +42,11 @@ public S3FileAcceptor( @Override public StreamConsumer getStreamConsumer( - MD5Checksum originalFileMd5, Instant originalFileTimestamp, boolean noMd5Check) { + MD5Checksum originalFileMd5, Instant originalFileTimestamp) { return (StreamConsumer) (stream, size) -> { Map objectMetadata = new HashMap<>(); + objectMetadata.put(MD5_METADATA_KEY, originalFileMd5.stringFormat()); objectMetadata.put( TIMESTAMP_METADATA_KEY, String.valueOf(originalFileTimestamp.toEpochMilli())); PutObjectRequest putObjectRequest = @@ -60,29 +58,22 @@ public StreamConsumer getStreamConsumer( .storageClass(StorageClass.INTELLIGENT_TIERING) .build(); client.putObject(putObjectRequest, RequestBody.fromInputStream(stream, size)); + Latest latest = new Latest(clock.instant(), key); String latestContent = mapper.writeValueAsString(latest); - if ((!noMd5Check) && (!originalFileMd5.matches(stream.md5()))) { - LOG.error("MD5 mismatch. Deleting destination file"); - client.deleteObject(DeleteObjectRequest.builder() - .bucket(bucket) - .key(key) - .build()); - } else { + PutObjectRequest putLatest = + PutObjectRequest.builder() + .key(Paths.get(key).getParent().toString() + "/" + LATEST_KEY) + .bucket(bucket) + .contentLength((long) latestContent.length()) + .storageClass(StorageClass.INTELLIGENT_TIERING) + .build(); + client.putObject( + putLatest, + RequestBody.fromInputStream( + new StringInputStream(latestContent), latestContent.length())); - PutObjectRequest putLatest = - PutObjectRequest.builder() - .key(Paths.get(key).getParent().toString() + "/" + LATEST_KEY) - .bucket(bucket) - .contentLength((long) latestContent.length()) - .storageClass(StorageClass.INTELLIGENT_TIERING) - .build(); - client.putObject( - putLatest, - RequestBody.fromInputStream( - new StringInputStream(latestContent), latestContent.length())); - } return null; }; } @@ -112,8 +103,8 @@ public Optional existingFileMd5() { try { final HeadObjectResponse headObjectResponse = client.headObject(headObjectRequest); - return Optional.ofNullable(headObjectResponse.eTag()) - .map(m -> MD5Checksum.of(m.replaceAll("\"", ""))); + final Map metadata = headObjectResponse.metadata(); + return Optional.ofNullable(metadata.get(MD5_METADATA_KEY)).map(m -> MD5Checksum.of(m)); } catch (NoSuchKeyException e) { return Optional.empty(); } diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/BaseUrlSource.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/BaseUrlSource.java index 95cfea9..c06f895 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/BaseUrlSource.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/BaseUrlSource.java @@ -11,7 +11,6 @@ import technology.dice.dicewhere.downloader.stream.StreamWithMD5Decorator; public abstract class BaseUrlSource implements FileSource { - protected FileInfo fileInfo; protected final URL dataFileLocation; @@ -20,14 +19,14 @@ protected BaseUrlSource(URL dataFileLocation) { } @Override - public MD5Checksum produce(FileAcceptor acceptor, boolean noMd5Check) { + public MD5Checksum produce(FileAcceptor acceptor) { try { HttpURLConnection httpConnection = (HttpURLConnection) this.dataFileLocation.openConnection(); httpConnection.setRequestMethod("GET"); try (StreamWithMD5Decorator is = StreamWithMD5Decorator.of(httpConnection.getInputStream())) { acceptor - .getStreamConsumer(fileInfo.getMd5Checksum(), fileInfo.getTimestamp(), noMd5Check) + .getStreamConsumer(fileInfo.getMd5Checksum(), fileInfo.getTimestamp()) .consume(is, fileInfo.getSize()); return is.md5(); } diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/FileSource.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/FileSource.java index 85bd265..2181252 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/FileSource.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/FileSource.java @@ -5,8 +5,7 @@ import technology.dice.dicewhere.downloader.destination.FileAcceptor; public interface FileSource { - FileInfo fileInfo(); - MD5Checksum produce(FileAcceptor consumer, boolean noMd5Check); + MD5Checksum produce(FileAcceptor consumer); } diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/s3/S3Source.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/s3/S3Source.java index 92dca72..a687f2f 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/s3/S3Source.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/source/s3/S3Source.java @@ -22,8 +22,8 @@ import technology.dice.dicewhere.downloader.stream.StreamWithMD5Decorator; public class S3Source implements FileSource { - private static Logger LOG = LoggerFactory.getLogger(S3Source.class); + public static final String MD5_METADATA_KEY = "md5"; public static final String TIMESTAMP_METADATA_KEY = "ts"; private final S3Client client; private final String bucket; @@ -43,11 +43,11 @@ public FileInfo fileInfo() { HeadObjectRequest.builder().key(key).bucket(bucket).build(); final HeadObjectResponse headObjectResponse = client.headObject(headObjectRequest); - if (headObjectResponse.eTag() == null) { + final Map metadata = headObjectResponse.metadata(); + if (!metadata.containsKey(MD5_METADATA_KEY)) { throw new DownloaderException( "Remote file does not have md5 information. Please delete the file and re-upload"); } - final Map metadata = headObjectResponse.metadata(); if (!metadata.containsKey(TIMESTAMP_METADATA_KEY)) { LOG.warn("Timestamp not available at source. Using now as timestamp."); } @@ -59,7 +59,7 @@ public FileInfo fileInfo() { Optional.ofNullable(metadata.get(TIMESTAMP_METADATA_KEY)) .map(m -> Instant.ofEpochMilli(Long.valueOf(m))) .orElse(Instant.now()), - MD5Checksum.of(headObjectResponse.eTag().replaceAll("\"", "")), + MD5Checksum.of(metadata.get(MD5_METADATA_KEY)), size); } @@ -67,12 +67,12 @@ public FileInfo fileInfo() { } @Override - public MD5Checksum produce(FileAcceptor consumer, boolean noMd5Check) { + public MD5Checksum produce(FileAcceptor consumer) { GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); try (final ResponseInputStream object = client.getObject(getObjectRequest); StreamWithMD5Decorator is = StreamWithMD5Decorator.of(object)) { consumer - .getStreamConsumer(fileInfo.getMd5Checksum(), fileInfo.getTimestamp(), noMd5Check) + .getStreamConsumer(fileInfo.getMd5Checksum(), fileInfo.getTimestamp()) .consume(is, fileInfo.getSize()); return is.md5(); } catch (IOException | NoSuchAlgorithmException e) { diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamConsumer.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamConsumer.java index 88de87e..babd9b5 100644 --- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamConsumer.java +++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamConsumer.java @@ -5,5 +5,5 @@ @FunctionalInterface public interface StreamConsumer { - T consume(StreamWithMD5Decorator stream, long size) throws IOException; + T consume(InputStream stream, long size) throws IOException; } diff --git a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java deleted file mode 100644 index 121d2f7..0000000 --- a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java +++ /dev/null @@ -1,148 +0,0 @@ -package technology.dice.dicewhere.downloader.destination.local; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; -import static org.junit.Assert.assertNotEquals; - -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.github.tomakehurst.wiremock.matching.UrlPattern; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import java.util.Random; -import javax.xml.bind.annotation.adapters.HexBinaryAdapter; -import junit.framework.TestCase; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpStatus; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.internal.runners.JUnit4ClassRunner; -import org.junit.runner.RunWith; -import org.testcontainers.shaded.org.apache.commons.lang3.tuple.Pair; -import technology.dice.dicewhere.downloader.files.FileInfo; -import technology.dice.dicewhere.downloader.source.ipinfosite.IpInfoSiteSource; - -@RunWith(JUnit4ClassRunner.class) -public class LocalFileAcceptorTest extends TestCase { - - private static final int TEST_FILE_SIZE = 1024 * 1024; - @ClassRule - static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort()); - - @BeforeClass - public static void beforeClass() { - wireMockRule.start(); - } - - @Test - public void corruptedFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException { - Pair tempFile = generateTempFile(); - Path destinationDir = Files.createTempDirectory("dice-where"); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", "aaa") - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); - - FileInfo fileInfo = ipInfoSiteSource.fileInfo(); - ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false); - assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); - assertFalse(Files.list(destinationDir).findAny().isPresent()); - } - - @Test - public void corruptedFilePreexistingSet() throws IOException, NoSuchAlgorithmException { - Pair tempFile = generateTempFile(); - Pair existingFile = generateTempFile(); - Path destinationDir = Files.createTempDirectory("dice-where"); - Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb")); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", "aaa") - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); - - FileInfo fileInfo = ipInfoSiteSource.fileInfo(); - ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false); - assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); - assertEquals(Files.list(destinationDir).count(), 1); - assertFalse(Files.exists(destinationDir.resolve("file.mdb"))); - assertTrue(Arrays.equals(Files.readAllBytes(existingFile.getLeft()), - Files.readAllBytes(Files.list(destinationDir).findFirst().get()))); - } - - @Test - public void goodFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException { - Pair tempFile = generateTempFile(); - Path destinationDir = Files.createTempDirectory("dice-where"); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", tempFile.getRight()) - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); - - FileInfo fileInfo = ipInfoSiteSource.fileInfo(); - ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false); - assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); - assertEquals(1, Files.list(destinationDir).count()); - assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()), - Files.readAllBytes(destinationDir.resolve("file.mdb")))); - } - - @Test - public void goodFilePreexistingSet() throws IOException, NoSuchAlgorithmException { - Pair tempFile = generateTempFile(); - Pair existingFile = generateTempFile(); - Path destinationDir = Files.createTempDirectory("dice-where"); - Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb")); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", tempFile.getRight()) - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); - - FileInfo fileInfo = ipInfoSiteSource.fileInfo(); - ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false); - assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); - assertEquals(2, Files.list(destinationDir).count()); - assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()), - Files.readAllBytes(destinationDir.resolve("file.mdb")))); - } - - private Pair generateTempFile() throws IOException, NoSuchAlgorithmException { - byte[] contents = new byte[TEST_FILE_SIZE]; - new Random().nextBytes(contents); - Path tempFile = Files.createTempFile("dice-where", "downloader"); - Files.write(tempFile, contents); - MessageDigest md = MessageDigest.getInstance("MD5"); - String hex = (new HexBinaryAdapter()).marshal(md.digest(contents)); - return Pair.of(tempFile, hex); - } -} \ No newline at end of file diff --git a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/s3/S3FileAcceptorTest.java b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/s3/S3FileAcceptorTest.java deleted file mode 100644 index bfaaf0a..0000000 --- a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/s3/S3FileAcceptorTest.java +++ /dev/null @@ -1,309 +0,0 @@ -package technology.dice.dicewhere.downloader.destination.s3; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertThrows; - -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.github.tomakehurst.wiremock.matching.UrlPattern; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.time.Clock; -import java.time.Instant; -import java.util.Map; -import java.util.Random; -import java.util.stream.Collectors; -import javax.xml.bind.annotation.adapters.HexBinaryAdapter; -import junit.framework.TestCase; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpStatus; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.internal.runners.JUnit4ClassRunner; -import org.junit.runner.RunWith; -import org.testcontainers.containers.localstack.LocalStackContainer; -import org.testcontainers.containers.localstack.LocalStackContainer.Service; -import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; -import org.testcontainers.shaded.org.apache.commons.lang3.tuple.Pair; -import org.testcontainers.utility.DockerImageName; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.Delete; -import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; -import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsResponse; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.ObjectIdentifier; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.utils.StringInputStream; -import technology.dice.dicewhere.downloader.ObjectMapperInstance; -import technology.dice.dicewhere.downloader.files.FileInfo; -import technology.dice.dicewhere.downloader.source.ipinfosite.IpInfoSiteSource; - -@RunWith(JUnit4ClassRunner.class) -public class S3FileAcceptorTest extends TestCase { - - private static final int TEST_FILE_SIZE = 1024 * 1024; - public static final String TEST_BUCKET = "test-bucket"; - public static final String TEST_KEY = "downloads/test-file"; - - @ClassRule - static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort()); - @ClassRule - static LocalStackContainer localStack = new LocalStackContainer( - DockerImageName.parse("localstack/localstack:3.0")).withServices(Service.S3) - .waitingFor(new HttpWaitStrategy()); - - private static S3Client S3_CLIENT; - - @BeforeClass - public static void beforeClass() { - localStack.start(); - wireMockRule.start(); - - S3_CLIENT = S3Client.builder().endpointOverride(localStack.getEndpoint()).credentialsProvider( - StaticCredentialsProvider.create( - AwsBasicCredentials.create(localStack.getAccessKey(), localStack.getSecretKey()))) - .region(Region.of(localStack.getRegion())).build(); - S3_CLIENT.createBucket(CreateBucketRequest.builder().bucket(TEST_BUCKET).build()); - - } - - @Before - public void before() { - S3_CLIENT.createBucket(CreateBucketRequest.builder().bucket(TEST_BUCKET).build()); - } - - @After - public void after() { - emptyBucket(TEST_BUCKET); - S3_CLIENT.deleteBucket(DeleteBucketRequest.builder().bucket(TEST_BUCKET).build()); - } - - @Test - public void corruptedFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException { - Pair tempFile = generateTempFile(); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", "aaa") - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); - - FileInfo fileInfo = ipInfoSiteSource.fileInfo(); - ipInfoSiteSource.produce( - new S3FileAcceptor(S3_CLIENT, TEST_BUCKET, TEST_KEY, ObjectMapperInstance.INSTANCE, - Clock.systemUTC()), false); - assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); - assertThrows(NoSuchKeyException.class, () -> S3_CLIENT.getObject( - GetObjectRequest.builder() - .bucket(TEST_BUCKET) - .key(TEST_KEY) - .build())); - assertThrows(NoSuchKeyException.class, () -> S3_CLIENT.getObject( - GetObjectRequest.builder() - .bucket(TEST_BUCKET) - .key("downloads/latest") - .build())); - } - - @Test - public void corruptedFilePreexistingSet() throws IOException, NoSuchAlgorithmException { - Pair preexistingFile = generateTempFile(); - S3_CLIENT.putObject(PutObjectRequest.builder() - .bucket(TEST_BUCKET) - .key("downloads/existingFile.zip") - .build(), preexistingFile.getLeft()); - final Instant originalFileUploadedAt = Instant.now(); - ImmutableMap existingLatest = ImmutableMap.of("key", - "downloads/existingFile.zip", "uploadedAt", originalFileUploadedAt); - String existingLatestString = ObjectMapperInstance.INSTANCE.writeValueAsString(existingLatest); - S3_CLIENT.putObject(PutObjectRequest.builder() - .bucket(TEST_BUCKET) - .key("downloads/latest") - .build(), RequestBody.fromInputStream(new StringInputStream(existingLatestString), - existingLatestString.length())); - - Pair tempFile = generateTempFile(); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", "bbb") - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); - - FileInfo fileInfo = ipInfoSiteSource.fileInfo(); - ipInfoSiteSource.produce( - new S3FileAcceptor(S3_CLIENT, TEST_BUCKET, TEST_KEY, ObjectMapperInstance.INSTANCE, - Clock.systemUTC()), false); - assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat()); - assertThrows(NoSuchKeyException.class, () -> S3_CLIENT.getObject( - GetObjectRequest.builder() - .bucket(TEST_BUCKET) - .key(TEST_KEY) - .build())); - ResponseInputStream latest = S3_CLIENT.getObject( - GetObjectRequest.builder() - .bucket(TEST_BUCKET) - .key("downloads/latest") - .build()); - Map latestInformation = ObjectMapperInstance.INSTANCE.readValue(latest, - Map.class); - - assertEquals(latestInformation.get("key"), "downloads/existingFile.zip"); - assertEquals(Instant.parse(latestInformation.get("uploadedAt")), originalFileUploadedAt); - } - - @Test - public void goodFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException { - Pair tempFile = generateTempFile(); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", tempFile.getRight()) - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); - - FileInfo fileInfo = ipInfoSiteSource.fileInfo(); - ipInfoSiteSource.produce( - new S3FileAcceptor(S3_CLIENT, TEST_BUCKET, TEST_KEY, ObjectMapperInstance.INSTANCE, - Clock.systemUTC()), false); - assertEquals(fileInfo.getMd5Checksum().stringFormat().toLowerCase(), - tempFile.getRight().toLowerCase()); - ResponseInputStream uploadedFile = S3_CLIENT.getObject( - GetObjectRequest.builder() - .bucket(TEST_BUCKET) - .key(TEST_KEY) - .build()); - ResponseInputStream latest = S3_CLIENT.getObject( - GetObjectRequest.builder() - .bucket(TEST_BUCKET) - .key("downloads/latest") - .build()); - Map latestInformation = ObjectMapperInstance.INSTANCE.readValue(latest, - Map.class); - - assertEquals(TEST_FILE_SIZE, uploadedFile.response().contentLength().intValue()); - assertEquals(tempFile.getRight().toLowerCase(), - uploadedFile.response().eTag().toLowerCase().replace("\"", "")); - assertEquals(latestInformation.get("key"), "downloads/test-file"); - } - - @Test - public void goodFilePreexistingSet() throws IOException, NoSuchAlgorithmException { - Pair preexistingFile = generateTempFile(); - S3_CLIENT.putObject(PutObjectRequest.builder() - .bucket(TEST_BUCKET) - .key("downloads/existingFile.zip") - .build(), preexistingFile.getLeft()); - final Instant originalFileUploadedAt = Instant.now(); - ImmutableMap existingLatest = ImmutableMap.of("key", - "downloads/existingFile.zip", "uploadedAt", originalFileUploadedAt); - String existingLatestString = ObjectMapperInstance.INSTANCE.writeValueAsString(existingLatest); - S3_CLIENT.putObject(PutObjectRequest.builder() - .bucket(TEST_BUCKET) - .key("downloads/latest") - .build(), RequestBody.fromInputStream(new StringInputStream(existingLatestString), - existingLatestString.length())); - - Pair tempFile = generateTempFile(); - IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource( - new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb")); - wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn( - aResponse().withStatus(HttpStatus.SC_OK) - .withHeader("Etag", tempFile.getRight()) - .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE)) - .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT"))); - wireMockRule.stubFor(WireMock.get(UrlPattern.ANY) - .willReturn(aResponse().withBody( - IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile()))))); - - FileInfo fileInfo = ipInfoSiteSource.fileInfo(); - ipInfoSiteSource.produce( - new S3FileAcceptor(S3_CLIENT, TEST_BUCKET, TEST_KEY, ObjectMapperInstance.INSTANCE, - Clock.systemUTC()), false); - assertEquals(fileInfo.getMd5Checksum().stringFormat().toLowerCase(), - tempFile.getRight().toLowerCase()); - ResponseInputStream uploadedFile = S3_CLIENT.getObject( - GetObjectRequest.builder() - .bucket(TEST_BUCKET) - .key(TEST_KEY) - .build()); - ResponseInputStream latest = S3_CLIENT.getObject( - GetObjectRequest.builder() - .bucket(TEST_BUCKET) - .key("downloads/latest") - .build()); - Map latestInformation = ObjectMapperInstance.INSTANCE.readValue(latest, - Map.class); - - assertEquals(TEST_FILE_SIZE, uploadedFile.response().contentLength().intValue()); - assertEquals(tempFile.getRight().toLowerCase(), - uploadedFile.response().eTag().toLowerCase().replace("\"", "")); - assertEquals(latestInformation.get("key"), "downloads/test-file"); - assertNotEquals(latestInformation.get("uploadedAt"), originalFileUploadedAt); - } - - private Pair generateTempFile() throws IOException, NoSuchAlgorithmException { - byte[] contents = new byte[TEST_FILE_SIZE]; - new Random().nextBytes(contents); - Path tempFile = Files.createTempFile("dice-where", "downloader"); - Files.write(tempFile, contents); - MessageDigest md = MessageDigest.getInstance("MD5"); - String hex = (new HexBinaryAdapter()).marshal(md.digest(contents)); - return Pair.of(tempFile, hex); - } - - public static void emptyBucket(String bucketName) { - ListObjectsResponse listObjectsResponse = S3_CLIENT.listObjects( - ListObjectsRequest.builder() - .bucket(bucketName) - .build()); - - if (listObjectsResponse.contents().size() > 0) { - Delete del = Delete.builder() - .objects(listObjectsResponse.contents().stream() - .map(o -> ObjectIdentifier.builder().key(o.key()).build()) - .collect(Collectors.toList())) - .build(); - - DeleteObjectsRequest multiObjectDeleteRequest = DeleteObjectsRequest.builder() - .bucket(bucketName) - .delete(del) - .build(); - - S3_CLIENT.deleteObjects(multiObjectDeleteRequest); - } - } -} \ No newline at end of file