diff --git a/pom.xml b/pom.xml index 89c7c9d2..de354946 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ io.jenkins.plugins azure-sdk - 30.vf3165534d6e8 + 55.v31806ceca163 org.jenkins-ci.plugins @@ -95,10 +95,6 @@ org.jenkins-ci.plugins credentials-binding - - org.jenkins-ci.plugins - jackson2-api - io.jenkins.blueocean blueocean-rest diff --git a/src/main/java/com/microsoftopentechnologies/windowsazurestorage/helper/AzureUtils.java b/src/main/java/com/microsoftopentechnologies/windowsazurestorage/helper/AzureUtils.java index 77c7d8da..ebd9cc16 100644 --- a/src/main/java/com/microsoftopentechnologies/windowsazurestorage/helper/AzureUtils.java +++ b/src/main/java/com/microsoftopentechnologies/windowsazurestorage/helper/AzureUtils.java @@ -76,8 +76,7 @@ public static boolean validateStorageAccount( return true; } - public static BlobServiceClient getCloudStorageAccount(final StorageAccountInfo storageAccount) - throws MalformedURLException, URISyntaxException { + public static BlobServiceClient getCloudStorageAccount(final StorageAccountInfo storageAccount) { return getCloudStorageAccount(storageAccount, new RequestRetryOptions()); } @@ -140,7 +139,7 @@ public static String generateBlobSASURL( StorageAccountInfo storageAccount, String containerName, String blobName, - BlobSasPermission permissions) throws Exception { + BlobSasPermission permissions) { BlobServiceClient cloudStorageAccount = getCloudStorageAccount(storageAccount); @@ -149,7 +148,7 @@ public static String generateBlobSASURL( // At this point need to throw an error back since container itself did not exist. if (!container.exists()) { - throw new Exception("WAStorageClient: generateBlobSASURL: Container " + containerName + throw new IllegalStateException("WAStorageClient: generateBlobSASURL: Container " + containerName + " does not exist in storage account " + storageAccount.getStorageAccName()); } @@ -173,12 +172,12 @@ public static String generateFileSASURL( StorageAccountInfo storageAccount, String shareName, String fileName, - ShareFileSasPermission permissions) throws Exception { + ShareFileSasPermission permissions) throws MalformedURLException, URISyntaxException { ShareServiceClient shareServiceClient = getShareClient(storageAccount); ShareClient fileShare = shareServiceClient.getShareClient((shareName)); if (!fileShare.exists()) { - throw new Exception("WAStorageClient: generateFileSASURL: Share " + shareName + throw new IllegalStateException("WAStorageClient: generateFileSASURL: Share " + shareName + " does not exist in storage account " + storageAccount.getStorageAccName()); } diff --git a/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadService.java b/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadService.java index 395b2f5d..7f4d75e8 100644 --- a/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadService.java +++ b/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadService.java @@ -56,7 +56,9 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.Serializable; +import java.net.MalformedURLException; import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; @@ -149,6 +151,7 @@ protected static class UploadObject implements Serializable { private String sas; private String storageType; private String storageAccount; + private String containerOrShareName; private PartialBlobProperties blobProperties; private Map metadata; @@ -164,16 +167,28 @@ protected static class UploadObject implements Serializable { */ public UploadObject(String name, FilePath src, String url, String sas, String storageType, String storageAccount, PartialBlobProperties blobProperties, Map metadata) { + this(name, src, url, sas, storageType, storageAccount, null, blobProperties, metadata); + } + + public UploadObject(String name, FilePath src, String url, String sas, String storageType, + String storageAccount, String containerOrShareName, + PartialBlobProperties blobProperties, Map metadata + ) { this.name = name; this.src = src; this.url = url; this.sas = sas; this.storageType = storageType; this.storageAccount = storageAccount; + this.containerOrShareName = containerOrShareName; this.blobProperties = blobProperties; this.metadata = metadata; } + public String getContainerOrShareName() { + return containerOrShareName; + } + public String getName() { return name; } @@ -362,7 +377,8 @@ protected void updateAzureBlobs(List results, } protected String generateWriteSASURL(StorageAccountInfo storageAccountInfo, String fileName, - String storageType, String name) throws Exception { + String storageType, String name) + throws MalformedURLException, URISyntaxException { if (storageType.equalsIgnoreCase(Constants.BLOB_STORAGE)) { return AzureUtils.generateBlobSASURL(storageAccountInfo, name, fileName, @@ -371,7 +387,7 @@ protected String generateWriteSASURL(StorageAccountInfo storageAccountInfo, Stri return AzureUtils.generateFileSASURL(storageAccountInfo, name, fileName, new ShareFileSasPermission().setWritePermission(true)); } - throw new Exception("Unknown storage type. Please re-configure your job and build again."); + throw new IllegalStateException("Unknown storage type. Please re-configure your job and build again."); } /** diff --git a/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToBlobService.java b/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToBlobService.java index 7ceb752b..83618a94 100644 --- a/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToBlobService.java +++ b/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToBlobService.java @@ -95,7 +95,6 @@ protected void uploadArchive(String archiveIncludes) } } - @SuppressWarnings("HttpUrlsUsage") private UploadObject generateUploadObject(FilePath path, StorageAccountInfo accountInfo, BlockBlobClient blob, String containerName, PartialBlobProperties blobProperties, diff --git a/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToFileService.java b/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToFileService.java index 378a84f1..7e061918 100644 --- a/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToFileService.java +++ b/src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToFileService.java @@ -15,26 +15,52 @@ package com.microsoftopentechnologies.windowsazurestorage.service; +import com.azure.core.credential.AzureSasCredential; import com.azure.core.http.rest.PagedIterable; +import com.azure.core.http.rest.Response; import com.azure.storage.file.share.ShareClient; import com.azure.storage.file.share.ShareDirectoryClient; import com.azure.storage.file.share.ShareFileClient; import com.azure.storage.file.share.ShareServiceClient; +import com.azure.storage.file.share.ShareServiceClientBuilder; import com.azure.storage.file.share.models.ShareFileItem; +import com.azure.storage.file.share.models.ShareFileUploadInfo; +import com.azure.storage.file.share.models.ShareFileUploadOptions; +import com.microsoftopentechnologies.windowsazurestorage.beans.StorageAccountInfo; import com.microsoftopentechnologies.windowsazurestorage.exceptions.WAStorageException; import com.microsoftopentechnologies.windowsazurestorage.helper.AzureUtils; +import com.microsoftopentechnologies.windowsazurestorage.helper.Constants; +import com.microsoftopentechnologies.windowsazurestorage.service.model.PartialBlobProperties; import com.microsoftopentechnologies.windowsazurestorage.service.model.UploadServiceData; import com.microsoftopentechnologies.windowsazurestorage.service.model.UploadType; import hudson.FilePath; +import hudson.ProxyConfiguration; +import hudson.remoting.VirtualChannel; import hudson.util.DirScanner; +import io.jenkins.plugins.azuresdk.HttpClientRetriever; +import jenkins.MasterToSlaveFileCallable; +import jenkins.model.Jenkins; import org.apache.commons.lang.StringUtils; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; public class UploadToFileService extends UploadService { public UploadToFileService(UploadServiceData serviceData) { @@ -50,6 +76,7 @@ protected void uploadIndividuals(String embeddedVP, FilePath[] paths, FilePath w @Override protected void uploadIndividuals(String embeddedVP, FilePath[] paths) throws WAStorageException { final UploadServiceData serviceData = getServiceData(); + FilePath workspace = getServiceData().getRemoteWorkspace(); try { final ShareClient fileShare = getCloudFileShare(); UploadType uploadType = serviceData.getUploadType(); @@ -57,18 +84,112 @@ protected void uploadIndividuals(String embeddedVP, FilePath[] paths) throws WAS cleanupFileShare(fileShare); } + List uploadObjects = new ArrayList<>(); + for (FilePath src : paths) { final String filePath = getItemPath(src, embeddedVP, serviceData); ShareDirectoryClient rootDirectoryClient = fileShare.getRootDirectoryClient(); final ShareFileClient cloudFile = rootDirectoryClient.getFileClient(filePath); ensureDirExist(fileShare, filePath); - getExecutorService().submit(new FileUploadThread(cloudFile, src, serviceData.getIndividualBlobs())); + + UploadObject uploadObject = generateUploadObject(src, serviceData.getStorageAccountInfo(), + cloudFile, fileShare.getShareName(), null, updateMetadata(new HashMap<>())); + uploadObjects.add(uploadObject); } + + List results = workspace + .act(new UploadOnAgent(Jenkins.get().getProxy(), uploadObjects)); + + updateAzureBlobs(results, serviceData.getIndividualBlobs()); } catch (URISyntaxException | IOException | InterruptedException e) { throw new WAStorageException("fail to upload individual files to azure file storage", e); } } + private UploadObject generateUploadObject(FilePath path, StorageAccountInfo accountInfo, + ShareFileClient client, String shareName, + PartialBlobProperties properties, + Map metadata) + throws MalformedURLException, URISyntaxException { + String sas = generateWriteSASURL(accountInfo, client.getFilePath(), Constants.FILE_STORAGE, shareName); + + return new UploadObject(client.getFilePath(), path, client.getFileUrl(), sas, Constants.BLOB_STORAGE, + client.getAccountName(), shareName, properties, metadata); + } + + static final class UploadOnAgent extends MasterToSlaveFileCallable> { + private static final Logger LOGGER = Logger.getLogger(UploadOnAgent.class.getName()); + private static final int TIMEOUT = 30; + private static final int ERROR_ON_UPLOAD = 500; + + private final ProxyConfiguration proxy; + private final List uploadObjects; + + UploadOnAgent(ProxyConfiguration proxy, List uploadObjects) { + this.proxy = proxy; + this.uploadObjects = uploadObjects; + } + + private ShareServiceClient getFileShareClient(UploadObject uploadObject) { + return new ShareServiceClientBuilder() + .credential(new AzureSasCredential(uploadObject.getSas())) + .httpClient(HttpClientRetriever.get(proxy)) + .endpoint(uploadObject.getUrl()) + .buildClient(); + } + + @Override + public List invoke(File f, VirtualChannel channel) { + return uploadObjects.parallelStream() + .map(uploadObject -> { + ShareServiceClient fileShareClient = getFileShareClient(uploadObject); + + ShareClient shareClient = fileShareClient + .getShareClient(uploadObject.getContainerOrShareName()); + ShareFileClient fileClient = shareClient.getFileClient(uploadObject.getName()); + return uploadCloudFile(fileClient, uploadObject); + + }) + .collect(Collectors.toList()); + } + + private UploadResult uploadCloudFile(ShareFileClient fileClient, UploadObject uploadObject) { + long startTime = System.currentTimeMillis(); + File file = new File(uploadObject.getSrc().getRemote()); + try (FileInputStream fis = new FileInputStream(file); + BufferedInputStream bis = new BufferedInputStream(fis)) { + long bytes = Files.size(file.toPath()); + fileClient.create(bytes); + + + ShareFileUploadOptions fileUploadOptions = new ShareFileUploadOptions(bis); + Response response = fileClient + .uploadWithResponse(fileUploadOptions, Duration.ofSeconds(TIMEOUT), null); + + long endTime = System.currentTimeMillis(); + + String fileHash = null; + byte[] md5 = response.getValue().getContentMd5(); + if (md5 != null) { + fileHash = new String(md5, StandardCharsets.UTF_8); + } + + return new UploadResult(response.getStatusCode(), null, + fileHash, + file.getName(), + uploadObject.getUrl(), file.length(), uploadObject.getStorageType(), + startTime, endTime); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, "Failed uploading file", e); + return new UploadResult(ERROR_ON_UPLOAD, null, + null, + file.getName(), + uploadObject.getUrl(), file.length(), uploadObject.getStorageType(), + startTime, 0); + } + } + } + @Override protected void uploadArchive(String archiveIncludes) throws WAStorageException { final UploadServiceData serviceData = getServiceData(); diff --git a/src/test/java/IntegrationTests/IntegrationTest.java b/src/test/java/IntegrationTests/IntegrationTest.java index 63347058..e723c534 100644 --- a/src/test/java/IntegrationTests/IntegrationTest.java +++ b/src/test/java/IntegrationTests/IntegrationTest.java @@ -48,7 +48,8 @@ protected static class TestEnvironment { azureStorageAccountKey1 = TestEnvironment.loadFromEnv("AZURE_STORAGE_TEST_STORAGE_ACCOUNT_KEY1"); azureStorageAccountKey2 = TestEnvironment.loadFromEnv("AZURE_STORAGE_TEST_STORAGE_ACCOUNT_KEY2"); - blobURL = Utils.DEF_BLOB_URL; + blobURL = Utils.DEF_BLOB_URL.replace("https://", + String.format("https://%s.", azureStorageAccountName)); AzureStorageAccount.StorageAccountCredential u = new AzureStorageAccount.StorageAccountCredential(azureStorageAccountName, azureStorageAccountKey1, blobURL); sampleStorageAccount = new StorageAccountInfo(azureStorageAccountName,azureStorageAccountKey1, blobURL); containerName = name;