feat(storage transfer): Added samples for storage transfer (#2059)

storagetransfer/composer.json

@@ -1,9 +1,10 @@
 {
     "require": {
-        "google/cloud-storage-transfer": "^1.4",
+        "google/cloud-storage-transfer": "^2.0",
         "paragonie/random_compat": "^9.0.0"
     },
     "require-dev": {
-        "google/cloud-storage": "^1.20.1"
+        "google/cloud-storage": "^1.20.1",
+        "google/cloud-pubsub": "^2.0"
     }
 }

storagetransfer/src/check_latest_transfer_operation.php

@@ -0,0 +1,58 @@
+<?php
+/**
+ * Copyright 2024 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Google\Cloud\Samples\StorageTransfer;
+
+# [START storagetransfer_get_latest_transfer_operation]
+use Google\Cloud\StorageTransfer\V1\Client\StorageTransferServiceClient;
+use Google\Cloud\StorageTransfer\V1\GetTransferJobRequest;
+
+/**
+ * Checks the latest transfer operation for a given transfer job.
+ *
+ * @param string $projectId Your Google Cloud project ID.
+ * @param string $jobName Storage Transfer Service job name.
+ */
+function check_latest_transfer_operation(
+    string $projectId,
+    string $jobName
+): void {
+    // $project = 'my-project-id';
+    // $jobName = 'myJob/1234567890';
+    $transferJob = new GetTransferJobRequest([
+        'project_id' => $projectId,
+        'job_name' => $jobName
+    ]);
+
+    $client = new StorageTransferServiceClient();
+    $request = $client->getTransferJob($transferJob);
+    $latestOperationName = $request->getLatestOperationName();
+
+    if ($latestOperationName) {
+        $transferOperation = $client->resumeOperation($latestOperationName);
+        $operation = $transferOperation->getLastProtoResponse();
+
+        printf('Latest transfer operation for %s is: %s ' . PHP_EOL, $jobName, $operation->serializeToJsonString());
+    } else {
+        printf('Transfer job %s has not ran yet.' . PHP_EOL, $jobName);
+    }
+}
+# [END storagetransfer_get_latest_transfer_operation]
+
+// The following 2 lines are only needed to run the samples
+require_once __DIR__ . '/../../testing/sample_helpers.php';
+\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv);

storagetransfer/src/event_driven_gcs_transfer.php

@@ -0,0 +1,71 @@
+<?php
+
+/**
+ * Copyright 2024 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Google\Cloud\Samples\StorageTransfer;
+
+# [START storagetransfer_create_event_driven_gcs_transfer]
+
+use Google\Cloud\StorageTransfer\V1\Client\StorageTransferServiceClient;
+use Google\Cloud\StorageTransfer\V1\CreateTransferJobRequest;
+use Google\Cloud\StorageTransfer\V1\EventStream;
+use Google\Cloud\StorageTransfer\V1\GcsData;
+use Google\Cloud\StorageTransfer\V1\TransferJob;
+use Google\Cloud\StorageTransfer\V1\TransferJob\Status;
+use Google\Cloud\StorageTransfer\V1\TransferSpec;
+
+/**
+ * Creates an event driven transfer that tracks a Pubsub subscription.
+ *
+ * @param string $projectId Your Google Cloud project ID.
+ * @param string $sourceGcsBucketName The name of the GCS bucket to transfer objects from.
+ * @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to.
+ * @param string $pubsubId The subscription ID to a Pubsub queue to track.
+ */
+function event_driven_gcs_transfer(
+    string $projectId,
+    string $sourceGcsBucketName,
+    string $sinkGcsBucketName,
+    string $pubsubId
+): void {
+    // $project = 'my-project-id';
+    // $sourceGcsBucketName = 'my-source-bucket';
+    // $sinkGcsBucketName = 'my-sink-bucket';
+    // $pubsubId = 'projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID';
+
+    $transferJob = new TransferJob([
+        'project_id' => $projectId,
+        'transfer_spec' => new TransferSpec([
+            'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]),
+            'gcs_data_source' => new GcsData(['bucket_name' => $sourceGcsBucketName])
+        ]),
+        'event_stream' => new EventStream(['name' => $pubsubId]),
+        'status' => Status::ENABLED
+    ]);
+
+    $client = new StorageTransferServiceClient();
+    $createRequest = (new CreateTransferJobRequest())
+        ->setTransferJob($transferJob);
+    $response = $client->createTransferJob($createRequest);
+
+    printf('Created an event driven transfer from %s to %s with name %s .' . PHP_EOL, $sourceGcsBucketName, $sinkGcsBucketName, $response->getName());
+}
+# [END storagetransfer_create_event_driven_gcs_transfer]
+
+// The following 2 lines are only needed to run the samples
+require_once __DIR__ . '/../../testing/sample_helpers.php';
+\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv);

storagetransfer/src/manifest_request.php

@@ -0,0 +1,79 @@
+<?php
+/**
+ * Copyright 2024 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Google\Cloud\Samples\StorageTransfer;
+
+# [START storagetransfer_manifest_request]
+
+use Google\Cloud\StorageTransfer\V1\Client\StorageTransferServiceClient;
+use Google\Cloud\StorageTransfer\V1\CreateTransferJobRequest;
+use Google\Cloud\StorageTransfer\V1\GcsData;
+use Google\Cloud\StorageTransfer\V1\PosixFilesystem;
+use Google\Cloud\StorageTransfer\V1\RunTransferJobRequest;
+use Google\Cloud\StorageTransfer\V1\TransferJob;
+use Google\Cloud\StorageTransfer\V1\TransferJob\Status;
+use Google\Cloud\StorageTransfer\V1\TransferManifest;
+use Google\Cloud\StorageTransfer\V1\TransferSpec;
+
+/**
+ * Creates and runs a transfer from the local file system to the sink bucket
+ *
+ * @param string $projectId Your Google Cloud project ID.
+ * @param string $sourceAgentPoolName The agent pool associated with the POSIX data source.
+ * @param string $rootDirectory The root directory path on the source filesystem.
+ * @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to.
+ * @param string $manifestLocation Transfer manifest location. Must be a `gs:` URL.
+ */
+function manifest_request(
+    string $projectId,
+    string $sourceAgentPoolName,
+    string $rootDirectory,
+    string $sinkGcsBucketName,
+    string $manifestLocation
+): void {
+    // $project = 'my-project-id';
+    // $sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default';
+    // $rootDirectory = '/directory/to/transfer/source';
+    // $sinkGcsBucketName = 'my-sink-bucket';
+    // $manifestLocation = 'gs://my-bucket/sample_manifest.csv';
+    $transferJob = new TransferJob([
+        'project_id' => $projectId,
+        'transfer_spec' => new TransferSpec([
+            'source_agent_pool_name' => $sourceAgentPoolName,
+            'posix_data_source' => new PosixFilesystem(['root_directory' => $rootDirectory]),
+            'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]),
+            'transfer_manifest' => new TransferManifest(['location' => $manifestLocation])
+        ]),
+        'status' => Status::ENABLED
+    ]);
+
+    $client = new StorageTransferServiceClient();
+    $createRequest = (new CreateTransferJobRequest())
+        ->setTransferJob($transferJob);
+    $response = $client->createTransferJob($createRequest);
+    $runRequest = (new RunTransferJobRequest())
+        ->setJobName($response->getName())
+        ->setProjectId($projectId);
+    $client->runTransferJob($runRequest);
+
+    printf('Created and ran transfer job from %s to %s using manifest %s with name %s ' . PHP_EOL, $rootDirectory, $sinkGcsBucketName, $manifestLocation, $response->getName());
+}
+# [END storagetransfer_manifest_request]
+
+// The following 2 lines are only needed to run the samples
+require_once __DIR__ . '/../../testing/sample_helpers.php';
+\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv);

storagetransfer/src/nearline_request.php

@@ -0,0 +1,107 @@
+<?php
+/**
+ * Copyright 2024 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Google\Cloud\Samples\StorageTransfer;
+
+# [START storagetransfer_transfer_to_nearline]
+
+use DateTime;
+use Google\Cloud\StorageTransfer\V1\Client\StorageTransferServiceClient;
+use Google\Cloud\StorageTransfer\V1\CreateTransferJobRequest;
+use Google\Cloud\StorageTransfer\V1\GcsData;
+use Google\Cloud\StorageTransfer\V1\ObjectConditions;
+use Google\Cloud\StorageTransfer\V1\RunTransferJobRequest;
+use Google\Cloud\StorageTransfer\V1\Schedule;
+use Google\Cloud\StorageTransfer\V1\TransferJob;
+use Google\Cloud\StorageTransfer\V1\TransferJob\Status;
+use Google\Cloud\StorageTransfer\V1\TransferOptions;
+use Google\Cloud\StorageTransfer\V1\TransferSpec;
+use Google\Protobuf\Duration as ProtobufDuration;
+use Google\Type\Date;
+use Google\Type\TimeOfDay;
+
+/**
+ * Create a daily migration from a GCS bucket to another GCS bucket for objects untouched for 30+ days.
+ *
+ * @param string $projectId Your Google Cloud project ID.
+ * @param string $description A useful description for your transfer job.
+ * @param string $sourceGcsBucketName The name of the GCS bucket to transfer objects from.
+ * @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to.
+ * @param string $startDate Date to start daily migration.
+ */
+function nearline_request(
+    string $projectId,
+    string $description,
+    string $sourceGcsBucketName,
+    string $sinkGcsBucketName,
+    string $startDate
+): void {
+    // $project = 'my-project-id';
+    // $description = 'My transfer job';
+    // $sourceGcsBucketName = 'my-source-bucket';
+    // $sinkGcsBucketName = 'my-sink-bucket';
+    // $startDate = new DateTime();
+
+    $dateTime = new DateTime($startDate);
+    $date = new Date([
+        'year' => $dateTime->format('Y'),
+        'month' => $dateTime->format('m'),
+        'day' => $dateTime->format('d'),
+    ]);
+
+    $time = new TimeOfDay([
+        'hours' => $dateTime->format('H'),
+        'minutes' => $dateTime->format('i'),
+        'seconds' => $dateTime->format('s'),
+    ]);
+
+    $transferJob = new TransferJob([
+        'project_id' => $projectId,
+        'description' => $description,
+        'schedule' => new Schedule([
+            'schedule_start_date' => $date,
+            'start_time_of_day' => $time
+        ]),
+        'transfer_spec' => new TransferSpec([
+            'gcs_data_source' => new GcsData(['bucket_name' => $sourceGcsBucketName]),
+            'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]),
+            'object_conditions' => new ObjectConditions([
+                'min_time_elapsed_since_last_modification' => new ProtobufDuration([
+                    'seconds' => 2592000
+                ])
+            ]),
+            'transfer_options' => new TransferOptions(['delete_objects_from_source_after_transfer' => true])
+        ]),
+        'status' => Status::ENABLED
+    ]);
+
+    $client = new StorageTransferServiceClient();
+    $createRequest = (new CreateTransferJobRequest())
+        ->setTransferJob($transferJob);
+    $response = $client->createTransferJob($createRequest);
+    $runRequest = (new RunTransferJobRequest())
+        ->setJobName($response->getName())
+        ->setProjectId($projectId);
+    $client->runTransferJob($runRequest);
+
+    printf('Created and ran transfer job : %s' . PHP_EOL, $response->getName());
+}
+# [END storagetransfer_transfer_to_nearline]
+
+// The following 2 lines are only needed to run the samples
+require_once __DIR__ . '/../../testing/sample_helpers.php';
+\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv);