Skip to content

Commit c49fd08

Browse files
BenWhiteheadBenWhitehead
authored
feat: add new Storage#moveBlob method to atomically rename an object (#2882)
1 parent 34e5903 commit c49fd08

13 files changed

+436
-2
lines changed

‎google-cloud-storage/clirr-ignored-differences.xml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,5 +108,16 @@
108108
<method>io.opentelemetry.api.OpenTelemetry getOpenTelemetry()</method>
109109
</difference>
110110

111+
<!-- Move Object -->
112+
<difference>
113+
<differenceType>7012</differenceType>
114+
<className>com/google/cloud/storage/Storage</className>
115+
<method>* moveBlob(*)</method>
116+
</difference>
117+
<difference>
118+
<differenceType>7012</differenceType>
119+
<className>com/google/cloud/storage/spi/v1/StorageRpc</className>
120+
<method>* moveObject(*)</method>
121+
</difference>
111122

112123
</differences>

‎google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcRetryAlgorithmManager.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.google.storage.v2.ListBucketsRequest;
3333
import com.google.storage.v2.ListObjectsRequest;
3434
import com.google.storage.v2.LockBucketRetentionPolicyRequest;
35+
import com.google.storage.v2.MoveObjectRequest;
3536
import com.google.storage.v2.QueryWriteStatusRequest;
3637
import com.google.storage.v2.ReadObjectRequest;
3738
import com.google.storage.v2.RestoreObjectRequest;
@@ -124,6 +125,12 @@ public ResultRetryAlgorithm<?> getFor(RewriteObjectRequest req) {
124125
: retryStrategy.getNonidempotentHandler();
125126
}
126127

128+
public ResultRetryAlgorithm<?> getFor(MoveObjectRequest req) {
129+
return req.hasIfGenerationMatch()
130+
? retryStrategy.getIdempotentHandler()
131+
: retryStrategy.getNonidempotentHandler();
132+
}
133+
127134
public ResultRetryAlgorithm<?> getFor(SetIamPolicyRequest req) {
128135
if (req.getPolicy().getEtag().equals(ByteString.empty())) {
129136
return retryStrategy.getNonidempotentHandler();

‎google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@
9090
import com.google.storage.v2.ListObjectsRequest;
9191
import com.google.storage.v2.ListObjectsResponse;
9292
import com.google.storage.v2.LockBucketRetentionPolicyRequest;
93+
import com.google.storage.v2.MoveObjectRequest;
9394
import com.google.storage.v2.Object;
9495
import com.google.storage.v2.ObjectAccessControl;
9596
import com.google.storage.v2.ReadObjectRequest;
@@ -1418,6 +1419,31 @@ public BlobWriteSession blobWriteSession(BlobInfo info, BlobWriteOption... optio
14181419
return BlobWriteSessions.of(writableByteChannelSession);
14191420
}
14201421

1422+
@Override
1423+
public Blob moveBlob(MoveBlobRequest request) {
1424+
Object srcObj = codecs.blobId().encode(request.getSource());
1425+
Object dstObj = codecs.blobId().encode(request.getTarget());
1426+
Opts<ObjectSourceOpt> srcOpts =
1427+
Opts.unwrap(request.getSourceOptions()).resolveFrom(request.getSource()).projectAsSource();
1428+
Opts<ObjectTargetOpt> dstOpts =
1429+
Opts.unwrap(request.getTargetOptions()).resolveFrom(request.getTarget());
1430+
MoveObjectRequest.Builder b =
1431+
MoveObjectRequest.newBuilder()
1432+
.setBucket(srcObj.getBucket())
1433+
.setSourceObject(srcObj.getName())
1434+
.setDestinationObject(dstObj.getName());
1435+
1436+
srcOpts.moveObjectsRequest().apply(b);
1437+
dstOpts.moveObjectsRequest().apply(b);
1438+
1439+
MoveObjectRequest req = b.build();
1440+
return Retrying.run(
1441+
getOptions(),
1442+
retryAlgorithmManager.getFor(req),
1443+
() -> storageClient.moveObjectCallable().call(req),
1444+
syntaxDecoders.blob);
1445+
}
1446+
14211447
@Override
14221448
public GrpcStorageOptions getOptions() {
14231449
return (GrpcStorageOptions) super.getOptions();

‎google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRetryAlgorithmManager.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.google.cloud.storage.spi.v1.StorageRpc;
2828
import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest;
2929
import com.google.common.base.MoreObjects;
30+
import com.google.common.collect.ImmutableMap;
3031
import java.io.Serializable;
3132
import java.util.List;
3233
import java.util.Map;
@@ -236,6 +237,14 @@ public ResultRetryAlgorithm<?> getForObjectsRewrite(RewriteRequest pb) {
236237
: retryStrategy.getNonidempotentHandler();
237238
}
238239

240+
public ResultRetryAlgorithm<?> getForObjectsMove(
241+
ImmutableMap<StorageRpc.Option, ?> sourceOptions,
242+
ImmutableMap<StorageRpc.Option, ?> targetOptions) {
243+
return targetOptions.containsKey(StorageRpc.Option.IF_GENERATION_MATCH)
244+
? retryStrategy.getIdempotentHandler()
245+
: retryStrategy.getNonidempotentHandler();
246+
}
247+
239248
public ResultRetryAlgorithm<?> getForObjectsCompose(
240249
List<StorageObject> sources, StorageObject target, Map<StorageRpc.Option, ?> optionsMap) {
241250
return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH)

‎google-cloud-storage/src/main/java/com/google/cloud/storage/OtelStorageDecorator.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1434,6 +1434,25 @@ public BlobWriteSession blobWriteSession(BlobInfo blobInfo, BlobWriteOption... o
14341434
}
14351435
}
14361436

1437+
@Override
1438+
public Blob moveBlob(MoveBlobRequest request) {
1439+
Span span =
1440+
tracer
1441+
.spanBuilder("moveBlob")
1442+
.setAttribute("gsutil.uri.source", request.getSource().toGsUtilUriWithGeneration())
1443+
.setAttribute("gsutil.uri.target", request.getTarget().toGsUtilUriWithGeneration())
1444+
.startSpan();
1445+
try (Scope ignore = span.makeCurrent()) {
1446+
return delegate.moveBlob(request);
1447+
} catch (Throwable t) {
1448+
span.recordException(t);
1449+
span.setStatus(StatusCode.ERROR, t.getClass().getSimpleName());
1450+
throw t;
1451+
} finally {
1452+
span.end();
1453+
}
1454+
}
1455+
14371456
@Override
14381457
public StorageOptions getOptions() {
14391458
return delegate.getOptions();

‎google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import com.google.cloud.storage.UnifiedOpts.ObjectSourceOpt;
5151
import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt;
5252
import com.google.cloud.storage.UnifiedOpts.Opts;
53+
import com.google.common.base.MoreObjects;
5354
import com.google.common.collect.ImmutableList;
5455
import com.google.common.collect.ImmutableMap;
5556
import com.google.common.collect.ImmutableSet;
@@ -73,6 +74,7 @@
7374
import java.util.LinkedList;
7475
import java.util.List;
7576
import java.util.Map;
77+
import java.util.Objects;
7678
import java.util.Set;
7779
import java.util.concurrent.TimeUnit;
7880
import java.util.stream.Stream;
@@ -2778,6 +2780,150 @@ public static Builder newBuilder() {
27782780
}
27792781
}
27802782

2783+
/**
2784+
* A class to contain all information needed for a Google Cloud Storage Object Move.
2785+
*
2786+
* @since 2.48.0
2787+
* @see Storage#moveBlob(MoveBlobRequest)
2788+
*/
2789+
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
2790+
final class MoveBlobRequest {
2791+
private final BlobId source;
2792+
private final BlobId target;
2793+
private final ImmutableList<BlobSourceOption> sourceOptions;
2794+
private final ImmutableList<BlobTargetOption> targetOptions;
2795+
2796+
MoveBlobRequest(
2797+
BlobId source,
2798+
BlobId target,
2799+
ImmutableList<BlobSourceOption> sourceOptions,
2800+
ImmutableList<BlobTargetOption> targetOptions) {
2801+
this.source = source;
2802+
this.target = target;
2803+
this.sourceOptions = sourceOptions;
2804+
this.targetOptions = targetOptions;
2805+
}
2806+
2807+
public BlobId getSource() {
2808+
return source;
2809+
}
2810+
2811+
public BlobId getTarget() {
2812+
return target;
2813+
}
2814+
2815+
public List<BlobSourceOption> getSourceOptions() {
2816+
return sourceOptions;
2817+
}
2818+
2819+
public List<BlobTargetOption> getTargetOptions() {
2820+
return targetOptions;
2821+
}
2822+
2823+
public Builder toBuilder() {
2824+
return new Builder(source, target, sourceOptions, targetOptions);
2825+
}
2826+
2827+
public static Builder newBuilder() {
2828+
return new Builder();
2829+
}
2830+
2831+
@Override
2832+
public boolean equals(Object o) {
2833+
if (this == o) {
2834+
return true;
2835+
}
2836+
if (!(o instanceof MoveBlobRequest)) {
2837+
return false;
2838+
}
2839+
MoveBlobRequest that = (MoveBlobRequest) o;
2840+
return Objects.equals(source, that.source)
2841+
&& Objects.equals(target, that.target)
2842+
&& Objects.equals(sourceOptions, that.sourceOptions)
2843+
&& Objects.equals(targetOptions, that.targetOptions);
2844+
}
2845+
2846+
@Override
2847+
public int hashCode() {
2848+
return Objects.hash(source, target, sourceOptions, targetOptions);
2849+
}
2850+
2851+
@Override
2852+
public String toString() {
2853+
return MoreObjects.toStringHelper(this)
2854+
.add("source", source)
2855+
.add("target", target)
2856+
.add("sourceOptions", sourceOptions)
2857+
.add("targetOptions", targetOptions)
2858+
.toString();
2859+
}
2860+
2861+
public static final class Builder {
2862+
2863+
private BlobId source;
2864+
private BlobId target;
2865+
private ImmutableList<BlobSourceOption> sourceOptions;
2866+
private ImmutableList<BlobTargetOption> targetOptions;
2867+
2868+
private Builder() {
2869+
this(null, null, ImmutableList.of(), ImmutableList.of());
2870+
}
2871+
2872+
private Builder(
2873+
BlobId source,
2874+
BlobId target,
2875+
ImmutableList<BlobSourceOption> sourceOptions,
2876+
ImmutableList<BlobTargetOption> targetOptions) {
2877+
this.source = source;
2878+
this.target = target;
2879+
this.sourceOptions = sourceOptions;
2880+
this.targetOptions = targetOptions;
2881+
}
2882+
2883+
public Builder setSource(BlobId source) {
2884+
this.source = requireNonNull(source, "source must be non null");
2885+
return this;
2886+
}
2887+
2888+
public Builder setTarget(BlobId target) {
2889+
this.target = requireNonNull(target, "target must be non null");
2890+
return this;
2891+
}
2892+
2893+
public Builder setSourceOptions(Iterable<BlobSourceOption> sourceOptions) {
2894+
this.sourceOptions =
2895+
ImmutableList.copyOf(requireNonNull(sourceOptions, "sourceOptions must be non null"));
2896+
return this;
2897+
}
2898+
2899+
public Builder setTargetOptions(Iterable<BlobTargetOption> targetOptions) {
2900+
this.targetOptions =
2901+
ImmutableList.copyOf(requireNonNull(targetOptions, "targetOptions must be non null"));
2902+
return this;
2903+
}
2904+
2905+
public Builder setSourceOptions(BlobSourceOption... sourceOptions) {
2906+
this.sourceOptions =
2907+
ImmutableList.copyOf(requireNonNull(sourceOptions, "sourceOptions must be non null"));
2908+
return this;
2909+
}
2910+
2911+
public Builder setTargetOptions(BlobTargetOption... targetOptions) {
2912+
this.targetOptions =
2913+
ImmutableList.copyOf(requireNonNull(targetOptions, "targetOptions must be non null"));
2914+
return this;
2915+
}
2916+
2917+
public MoveBlobRequest build() {
2918+
return new MoveBlobRequest(
2919+
requireNonNull(source, "source must be non null"),
2920+
requireNonNull(target, "target must be non null"),
2921+
sourceOptions,
2922+
targetOptions);
2923+
}
2924+
}
2925+
}
2926+
27812927
/**
27822928
* Creates a new bucket.
27832929
*
@@ -4882,4 +5028,16 @@ default void close() throws Exception {}
48825028
default BlobWriteSession blobWriteSession(BlobInfo blobInfo, BlobWriteOption... options) {
48835029
return throwGrpcOnly(fmtMethodName("blobWriteSession", BlobInfo.class, BlobWriteOption.class));
48845030
}
5031+
5032+
/**
5033+
* Atomically move an object from one name to another.
5034+
*
5035+
* <p>This new method is an atomic equivalent of the existing {@link Storage#copy(CopyRequest)} +
5036+
* {@link Storage#delete(BlobId)}, however without the ability to change metadata fields for the
5037+
* target object.
5038+
*
5039+
* @since 2.48.0
5040+
*/
5041+
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
5042+
Blob moveBlob(MoveBlobRequest request);
48855043
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1696,6 +1696,27 @@ public BlobWriteSession blobWriteSession(BlobInfo blobInfo, BlobWriteOption... o
16961696
return BlobWriteSessions.of(writableByteChannelSession);
16971697
}
16981698

1699+
@Override
1700+
public Blob moveBlob(MoveBlobRequest request) {
1701+
Opts<ObjectSourceOpt> srcOpts =
1702+
Opts.unwrap(request.getSourceOptions()).resolveFrom(request.getSource()).projectAsSource();
1703+
Opts<ObjectTargetOpt> dstOpts =
1704+
Opts.unwrap(request.getTargetOptions()).resolveFrom(request.getTarget());
1705+
ImmutableMap<StorageRpc.Option, ?> sourceOptions = srcOpts.getRpcOptions();
1706+
ImmutableMap<StorageRpc.Option, ?> targetOptions = dstOpts.getRpcOptions();
1707+
1708+
return run(
1709+
retryAlgorithmManager.getForObjectsMove(sourceOptions, targetOptions),
1710+
() ->
1711+
storageRpc.moveObject(
1712+
request.getSource().getBucket(),
1713+
request.getSource().getName(),
1714+
request.getTarget().getName(),
1715+
sourceOptions,
1716+
targetOptions),
1717+
o -> codecs.blobInfo().decode(o).asBlob(this));
1718+
}
1719+
16991720
@Override
17001721
public BlobInfo internalCreateFrom(Path path, BlobInfo info, Opts<ObjectTargetOpt> opts)
17011722
throws IOException {

0 commit comments

Comments
 (0)