Upload to file share on agent (#200)

pom.xml

@@ -46,7 +46,7 @@
         <dependency>
             <groupId>io.jenkins.plugins</groupId>
             <artifactId>azure-sdk</artifactId>
-            <version>30.vf3165534d6e8</version>
+            <version>55.v31806ceca163</version>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.plugins</groupId>
@@ -95,10 +95,6 @@
             <groupId>org.jenkins-ci.plugins</groupId>
             <artifactId>credentials-binding</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.jenkins-ci.plugins</groupId>
-            <artifactId>jackson2-api</artifactId>
-        </dependency>
         <dependency>
             <groupId>io.jenkins.blueocean</groupId>
             <artifactId>blueocean-rest</artifactId>

src/main/java/com/microsoftopentechnologies/windowsazurestorage/helper/AzureUtils.java

@@ -76,8 +76,7 @@ public static boolean validateStorageAccount(
         return true;
     }
 
-    public static BlobServiceClient getCloudStorageAccount(final StorageAccountInfo storageAccount)
-            throws MalformedURLException, URISyntaxException {
+    public static BlobServiceClient getCloudStorageAccount(final StorageAccountInfo storageAccount) {
         return getCloudStorageAccount(storageAccount, new RequestRetryOptions());
     }
 
@@ -140,7 +139,7 @@ public static String generateBlobSASURL(
             StorageAccountInfo storageAccount,
             String containerName,
             String blobName,
-            BlobSasPermission permissions) throws Exception {
+            BlobSasPermission permissions) {
 
         BlobServiceClient cloudStorageAccount = getCloudStorageAccount(storageAccount);
 
@@ -149,7 +148,7 @@ public static String generateBlobSASURL(
 
         // At this point need to throw an error back since container itself did not exist.
         if (!container.exists()) {
-            throw new Exception("WAStorageClient: generateBlobSASURL: Container " + containerName
+            throw new IllegalStateException("WAStorageClient: generateBlobSASURL: Container " + containerName
                     + " does not exist in storage account " + storageAccount.getStorageAccName());
         }
 
@@ -173,12 +172,12 @@ public static String generateFileSASURL(
             StorageAccountInfo storageAccount,
             String shareName,
             String fileName,
-            ShareFileSasPermission permissions) throws Exception {
+            ShareFileSasPermission permissions) throws MalformedURLException, URISyntaxException {
         ShareServiceClient shareServiceClient = getShareClient(storageAccount);
 
         ShareClient fileShare = shareServiceClient.getShareClient((shareName));
         if (!fileShare.exists()) {
-            throw new Exception("WAStorageClient: generateFileSASURL: Share " + shareName
+            throw new IllegalStateException("WAStorageClient: generateFileSASURL: Share " + shareName
                     + " does not exist in storage account " + storageAccount.getStorageAccName());
         }
 

src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadService.java

@@ -56,7 +56,9 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.Serializable;
+import java.net.MalformedURLException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.ArrayList;
@@ -149,6 +151,7 @@ protected static class UploadObject implements Serializable {
         private String sas;
         private String storageType;
         private String storageAccount;
+        private String containerOrShareName;
         private PartialBlobProperties blobProperties;
         private Map<String, String> metadata;
 
@@ -164,16 +167,28 @@ protected static class UploadObject implements Serializable {
          */
         public UploadObject(String name, FilePath src, String url, String sas, String storageType,
                             String storageAccount, PartialBlobProperties blobProperties, Map<String, String> metadata) {
+            this(name, src, url, sas, storageType, storageAccount, null, blobProperties, metadata);
+        }
+
+        public UploadObject(String name, FilePath src, String url, String sas, String storageType,
+                            String storageAccount, String containerOrShareName,
+                            PartialBlobProperties blobProperties, Map<String, String> metadata
+        ) {
             this.name = name;
             this.src = src;
             this.url = url;
             this.sas = sas;
             this.storageType = storageType;
             this.storageAccount = storageAccount;
+            this.containerOrShareName = containerOrShareName;
             this.blobProperties = blobProperties;
             this.metadata = metadata;
         }
 
+        public String getContainerOrShareName() {
+            return containerOrShareName;
+        }
+
         public String getName() {
             return name;
         }
@@ -362,7 +377,8 @@ protected void updateAzureBlobs(List<UploadResult> results,
     }
 
     protected String generateWriteSASURL(StorageAccountInfo storageAccountInfo, String fileName,
-                                         String storageType, String name) throws Exception {
+                                         String storageType, String name)
+            throws MalformedURLException, URISyntaxException {
         if (storageType.equalsIgnoreCase(Constants.BLOB_STORAGE)) {
 
             return AzureUtils.generateBlobSASURL(storageAccountInfo, name, fileName,
@@ -371,7 +387,7 @@ protected String generateWriteSASURL(StorageAccountInfo storageAccountInfo, Stri
             return AzureUtils.generateFileSASURL(storageAccountInfo, name, fileName,
                     new ShareFileSasPermission().setWritePermission(true));
         }
-        throw new Exception("Unknown storage type. Please re-configure your job and build again.");
+        throw new IllegalStateException("Unknown storage type. Please re-configure your job and build again.");
     }
 
     /**

src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToBlobService.java

@@ -95,7 +95,6 @@ protected void uploadArchive(String archiveIncludes)
         }
     }
 
-    @SuppressWarnings("HttpUrlsUsage")
     private UploadObject generateUploadObject(FilePath path, StorageAccountInfo accountInfo,
                                               BlockBlobClient blob, String containerName,
                                               PartialBlobProperties blobProperties,

src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToFileService.java

@@ -15,26 +15,52 @@
 
 package com.microsoftopentechnologies.windowsazurestorage.service;
 
+import com.azure.core.credential.AzureSasCredential;
 import com.azure.core.http.rest.PagedIterable;
+import com.azure.core.http.rest.Response;
 import com.azure.storage.file.share.ShareClient;
 import com.azure.storage.file.share.ShareDirectoryClient;
 import com.azure.storage.file.share.ShareFileClient;
 import com.azure.storage.file.share.ShareServiceClient;
+import com.azure.storage.file.share.ShareServiceClientBuilder;
 import com.azure.storage.file.share.models.ShareFileItem;
+import com.azure.storage.file.share.models.ShareFileUploadInfo;
+import com.azure.storage.file.share.models.ShareFileUploadOptions;
+import com.microsoftopentechnologies.windowsazurestorage.beans.StorageAccountInfo;
 import com.microsoftopentechnologies.windowsazurestorage.exceptions.WAStorageException;
 import com.microsoftopentechnologies.windowsazurestorage.helper.AzureUtils;
+import com.microsoftopentechnologies.windowsazurestorage.helper.Constants;
+import com.microsoftopentechnologies.windowsazurestorage.service.model.PartialBlobProperties;
 import com.microsoftopentechnologies.windowsazurestorage.service.model.UploadServiceData;
 import com.microsoftopentechnologies.windowsazurestorage.service.model.UploadType;
 import hudson.FilePath;
+import hudson.ProxyConfiguration;
+import hudson.remoting.VirtualChannel;
 import hudson.util.DirScanner;
+import io.jenkins.plugins.azuresdk.HttpClientRetriever;
+import jenkins.MasterToSlaveFileCallable;
+import jenkins.model.Jenkins;
 import org.apache.commons.lang.StringUtils;
 
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 public class UploadToFileService extends UploadService {
     public UploadToFileService(UploadServiceData serviceData) {
@@ -50,25 +76,120 @@ protected void uploadIndividuals(String embeddedVP, FilePath[] paths, FilePath w
     @Override
     protected void uploadIndividuals(String embeddedVP, FilePath[] paths) throws WAStorageException {
         final UploadServiceData serviceData = getServiceData();
+        FilePath workspace = getServiceData().getRemoteWorkspace();
         try {
             final ShareClient fileShare = getCloudFileShare();
             UploadType uploadType = serviceData.getUploadType();
             if (uploadType == UploadType.INDIVIDUAL || uploadType == UploadType.BOTH) {
                 cleanupFileShare(fileShare);
             }
 
+            List<UploadObject> uploadObjects = new ArrayList<>();
+
             for (FilePath src : paths) {
                 final String filePath = getItemPath(src, embeddedVP, serviceData);
                 ShareDirectoryClient rootDirectoryClient = fileShare.getRootDirectoryClient();
                 final ShareFileClient cloudFile = rootDirectoryClient.getFileClient(filePath);
                 ensureDirExist(fileShare, filePath);
-                getExecutorService().submit(new FileUploadThread(cloudFile, src, serviceData.getIndividualBlobs()));
+
+                UploadObject uploadObject = generateUploadObject(src, serviceData.getStorageAccountInfo(),
+                        cloudFile, fileShare.getShareName(), null, updateMetadata(new HashMap<>()));
+                uploadObjects.add(uploadObject);
             }
+
+            List<UploadResult> results = workspace
+                    .act(new UploadOnAgent(Jenkins.get().getProxy(), uploadObjects));
+
+            updateAzureBlobs(results, serviceData.getIndividualBlobs());
         } catch (URISyntaxException | IOException | InterruptedException e) {
             throw new WAStorageException("fail to upload individual files to azure file storage", e);
         }
     }
 
+    private UploadObject generateUploadObject(FilePath path, StorageAccountInfo accountInfo,
+                                              ShareFileClient client, String shareName,
+                                              PartialBlobProperties properties,
+                                              Map<String, String> metadata)
+            throws MalformedURLException, URISyntaxException {
+        String sas = generateWriteSASURL(accountInfo, client.getFilePath(), Constants.FILE_STORAGE, shareName);
+
+        return new UploadObject(client.getFilePath(), path, client.getFileUrl(), sas, Constants.BLOB_STORAGE,
+                client.getAccountName(), shareName, properties, metadata);
+    }
+
+    static final class UploadOnAgent extends MasterToSlaveFileCallable<List<UploadResult>> {
+        private static final Logger LOGGER = Logger.getLogger(UploadOnAgent.class.getName());
+        private static final int TIMEOUT = 30;
+        private static final int ERROR_ON_UPLOAD = 500;
+
+        private final ProxyConfiguration proxy;
+        private final List<UploadObject> uploadObjects;
+
+        UploadOnAgent(ProxyConfiguration proxy, List<UploadObject> uploadObjects) {
+            this.proxy = proxy;
+            this.uploadObjects = uploadObjects;
+        }
+
+        private ShareServiceClient getFileShareClient(UploadObject uploadObject) {
+            return new ShareServiceClientBuilder()
+                    .credential(new AzureSasCredential(uploadObject.getSas()))
+                    .httpClient(HttpClientRetriever.get(proxy))
+                    .endpoint(uploadObject.getUrl())
+                    .buildClient();
+        }
+
+        @Override
+        public List<UploadResult> invoke(File f, VirtualChannel channel) {
+            return uploadObjects.parallelStream()
+                    .map(uploadObject -> {
+                        ShareServiceClient fileShareClient = getFileShareClient(uploadObject);
+
+                        ShareClient shareClient = fileShareClient
+                                .getShareClient(uploadObject.getContainerOrShareName());
+                        ShareFileClient fileClient = shareClient.getFileClient(uploadObject.getName());
+                        return uploadCloudFile(fileClient, uploadObject);
+
+                    })
+                    .collect(Collectors.toList());
+        }
+
+        private UploadResult uploadCloudFile(ShareFileClient fileClient, UploadObject uploadObject) {
+            long startTime = System.currentTimeMillis();
+            File file = new File(uploadObject.getSrc().getRemote());
+            try (FileInputStream fis = new FileInputStream(file);
+                 BufferedInputStream bis = new BufferedInputStream(fis)) {
+                long bytes = Files.size(file.toPath());
+                fileClient.create(bytes);
+
+
+                ShareFileUploadOptions fileUploadOptions = new ShareFileUploadOptions(bis);
+                Response<ShareFileUploadInfo> response = fileClient
+                        .uploadWithResponse(fileUploadOptions, Duration.ofSeconds(TIMEOUT), null);
+
+                long endTime = System.currentTimeMillis();
+
+                String fileHash = null;
+                byte[] md5 = response.getValue().getContentMd5();
+                if (md5 != null) {
+                    fileHash = new String(md5, StandardCharsets.UTF_8);
+                }
+
+                return new UploadResult(response.getStatusCode(), null,
+                        fileHash,
+                        file.getName(),
+                        uploadObject.getUrl(), file.length(), uploadObject.getStorageType(),
+                        startTime, endTime);
+            } catch (Exception e) {
+                LOGGER.log(Level.SEVERE, "Failed uploading file", e);
+                return new UploadResult(ERROR_ON_UPLOAD, null,
+                        null,
+                        file.getName(),
+                        uploadObject.getUrl(), file.length(), uploadObject.getStorageType(),
+                        startTime, 0);
+            }
+        }
+    }
+
     @Override
     protected void uploadArchive(String archiveIncludes) throws WAStorageException {
         final UploadServiceData serviceData = getServiceData();

src/test/java/IntegrationTests/IntegrationTest.java

@@ -48,7 +48,8 @@ protected static class TestEnvironment {
             azureStorageAccountKey1 = TestEnvironment.loadFromEnv("AZURE_STORAGE_TEST_STORAGE_ACCOUNT_KEY1");
             azureStorageAccountKey2 = TestEnvironment.loadFromEnv("AZURE_STORAGE_TEST_STORAGE_ACCOUNT_KEY2");
 
-            blobURL = Utils.DEF_BLOB_URL;
+            blobURL = Utils.DEF_BLOB_URL.replace("https://",
+                    String.format("https://%s.", azureStorageAccountName));
             AzureStorageAccount.StorageAccountCredential u = new AzureStorageAccount.StorageAccountCredential(azureStorageAccountName, azureStorageAccountKey1, blobURL);
             sampleStorageAccount = new StorageAccountInfo(azureStorageAccountName,azureStorageAccountKey1, blobURL);
             containerName = name;