Skip to content

Commit e0191b5

Browse files
BenWhiteheadBenWhitehead
authored
feat: introduce new BlobWriteSession (#2123)
When writing a new Blob to GCS, there are secondary session related state and actions which can't be represented by WriteChannel. BlobWriteSession provides a new construct to allow retrieving the resultant object which is created after the WritableByteChannel is closed. Along with the new session, configuration for this is now performed at the StorageOptions level where cross session considerations can influence the implementation of the returned session. The configurable option present for this new StorageWriterConfig is chunkSize. In the future new configurations will be added with their corresponding options. For example, in a future release it will be possible to change from in memory buffering to instead buffer to disk thereby reducing heap usage.
1 parent 8cce2e0 commit e0191b5

18 files changed

+740
-53
lines changed

‎google-cloud-storage/clirr-ignored-differences.xml

+3-21
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,11 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<!-- see https://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
33
<differences>
4-
<!-- Not breaking, internal only interface and the new methods have default implementations -->
4+
<!-- Not breaking, new method has a default implementation -->
55
<difference>
66
<differenceType>7012</differenceType>
7-
<className>com/google/cloud/storage/UnbufferedWritableByteChannelSession$UnbufferedWritableByteChannel</className>
8-
<method>* write(*)</method>
9-
</difference>
10-
<!-- Allow accessing the underlying Apiary instance -->
11-
<difference>
12-
<differenceType>7012</differenceType>
13-
<className>com/google/cloud/storage/spi/v1/StorageRpc</className>
14-
<method>* getStorage()</method>
15-
</difference>
16-
17-
<difference>
18-
<differenceType>8001</differenceType>
19-
<className>com/google/cloud/storage/Hasher$ConstantConcatValueHasher</className>
20-
</difference>
21-
22-
<!-- Not breaking, internal only class -->
23-
<difference>
24-
<differenceType>7002</differenceType>
25-
<className>com/google/cloud/storage/HttpDownloadSessionBuilder$ReadableByteChannelSessionBuilder</className>
26-
<method>com.google.cloud.storage.HttpDownloadSessionBuilder$ReadableByteChannelSessionBuilder setCallback(java.util.function.Consumer)</method>
7+
<className>com/google/cloud/storage/Storage</className>
8+
<method>com.google.cloud.storage.BlobWriteSession blobWriteSession(com.google.cloud.storage.BlobInfo, com.google.cloud.storage.Storage$BlobWriteOption[])</method>
279
</difference>
2810

2911
</differences>

‎google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteSession.java

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.core.ApiFuture;
20+
import com.google.api.core.BetaApi;
21+
import java.io.IOException;
22+
import java.nio.channels.WritableByteChannel;
23+
24+
/**
25+
* A session to write an object to Google Cloud Storage.
26+
*
27+
* <p>A session can only write a single version of an object. If writing multiple versions of an
28+
* object a new session must be created each time.
29+
*
30+
* <p>Provides an api that allows writing to and retrieving the resulting {@link BlobInfo} after
31+
* write finalization.
32+
*
33+
* <p>The underlying implementation is dictated based upon the specified {@link
34+
* BlobWriteSessionConfig} provided at {@link StorageOptions} creation time.
35+
*
36+
* @see GrpcStorageOptions.Builder#setBlobWriteSessionConfig(BlobWriteSessionConfig)
37+
* @see BlobWriteSessionConfig
38+
* @see BlobWriteSessionConfigs
39+
* @since 2.26.0 This new api is in preview and is subject to breaking changes.
40+
*/
41+
@BetaApi
42+
public interface BlobWriteSession {
43+
44+
/**
45+
* Open the {@link WritableByteChannel} for this session.
46+
*
47+
* <p>A session may only be {@code open}ed once. If multiple calls to open are made, an illegal
48+
* state exception will be thrown
49+
*
50+
* <p>Upon calling {@link WritableByteChannel#close()} the object creation will be finalized, and
51+
* {@link #getResult()}s future should resolve.
52+
*
53+
* @throws IOException When creating the {@link WritableByteChannel} if an unrecoverable
54+
* underlying IOException occurs it can be rethrown
55+
* @throws IllegalStateException if open is called more than once
56+
* @since 2.26.0 This new api is in preview and is subject to breaking changes.
57+
*/
58+
@BetaApi
59+
WritableByteChannel open() throws IOException;
60+
61+
/**
62+
* Return an {@link ApiFuture}{@code <BlobInfo>} which will represent the state of the object upon
63+
* finalization and success response from Google Cloud Storage.
64+
*
65+
* <p>This future will not resolve until: 1. The object is successfully finalized and created in
66+
* Google Cloud Storage 2. A terminal failure occurs, the terminal failure will become the
67+
* exception result
68+
*
69+
* @since 2.26.0 This new api is in preview and is subject to breaking changes.
70+
*/
71+
@BetaApi
72+
ApiFuture<BlobInfo> getResult();
73+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteSessionConfig.java

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.core.InternalApi;
20+
import com.google.cloud.storage.Conversions.Decoder;
21+
import com.google.cloud.storage.Storage.BlobWriteOption;
22+
import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt;
23+
import com.google.cloud.storage.UnifiedOpts.Opts;
24+
import com.google.storage.v2.WriteObjectResponse;
25+
import java.io.IOException;
26+
import java.time.Clock;
27+
28+
/**
29+
* A sealed internal implementation only class which provides the means of configuring a {@link
30+
* BlobWriteSession}.
31+
*
32+
* <p>A {@code BlobWriteSessionConfig} will be used to configure all {@link BlobWriteSession}s
33+
* produced by an instance of {@link Storage}.
34+
*
35+
* @see BlobWriteSessionConfigs
36+
* @see GrpcStorageOptions.Builder#setBlobWriteSessionConfig(BlobWriteSessionConfig)
37+
* @see Storage#blobWriteSession(BlobInfo, BlobWriteOption...)
38+
* @since 2.26.0 This new api is in preview and is subject to breaking changes.
39+
*/
40+
// When we have java modules, actually seal this to internal extension only
41+
@InternalApi
42+
public abstract class BlobWriteSessionConfig {
43+
44+
@InternalApi
45+
BlobWriteSessionConfig() {}
46+
47+
@InternalApi
48+
abstract WriterFactory createFactory(Clock clock) throws IOException;
49+
50+
@InternalApi
51+
interface WriterFactory {
52+
@InternalApi
53+
WritableByteChannelSession<?, BlobInfo> writeSession(
54+
StorageInternal s,
55+
BlobInfo info,
56+
Opts<ObjectTargetOpt> opts,
57+
Decoder<WriteObjectResponse, BlobInfo> d);
58+
}
59+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteSessionConfigs.java

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.core.BetaApi;
20+
import com.google.cloud.storage.GrpcStorageOptions.GrpcStorageDefaults;
21+
import com.google.cloud.storage.Storage.BlobWriteOption;
22+
23+
/**
24+
* Factory class to select and construct {@link BlobWriteSessionConfig}s.
25+
*
26+
* @see BlobWriteSessionConfig
27+
* @see GrpcStorageOptions.Builder#setBlobWriteSessionConfig(BlobWriteSessionConfig)
28+
* @see Storage#blobWriteSession(BlobInfo, BlobWriteOption...)
29+
* @since 2.26.0 This new api is in preview and is subject to breaking changes.
30+
*/
31+
@BetaApi
32+
public final class BlobWriteSessionConfigs {
33+
34+
private BlobWriteSessionConfigs() {}
35+
36+
/**
37+
* Factory to produce the default configuration for uploading an object to Cloud Storage.
38+
*
39+
* <p>Configuration of the chunk size can be performed via {@link
40+
* DefaultBlobWriteSessionConfig#withChunkSize(int)}.
41+
*
42+
* @see GrpcStorageDefaults#getDefaultStorageWriterConfig()
43+
* @since 2.26.0 This new api is in preview and is subject to breaking changes.
44+
*/
45+
@BetaApi
46+
public static DefaultBlobWriteSessionConfig getDefault() {
47+
return new DefaultBlobWriteSessionConfig(ByteSizeConstants._16MiB);
48+
}
49+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteSessions.java

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.core.ApiFuture;
20+
import java.io.IOException;
21+
import java.nio.channels.WritableByteChannel;
22+
23+
final class BlobWriteSessions {
24+
25+
private BlobWriteSessions() {}
26+
27+
static BlobWriteSession of(WritableByteChannelSession<?, BlobInfo> s) {
28+
return new WritableByteChannelSessionAdapter(s);
29+
}
30+
31+
static final class WritableByteChannelSessionAdapter implements BlobWriteSession {
32+
private final WritableByteChannelSession<?, BlobInfo> delegate;
33+
34+
private WritableByteChannelSessionAdapter(WritableByteChannelSession<?, BlobInfo> delegate) {
35+
this.delegate = delegate;
36+
}
37+
38+
@Override
39+
public WritableByteChannel open() throws IOException {
40+
return delegate.open();
41+
}
42+
43+
@Override
44+
public ApiFuture<BlobInfo> getResult() {
45+
return delegate.getResult();
46+
}
47+
}
48+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/CrossTransportUtils.java

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.cloud.storage.TransportCompatibility.Transport;
20+
import java.util.Arrays;
21+
import java.util.stream.Collectors;
22+
23+
final class CrossTransportUtils {
24+
25+
static <T> T throwHttpJsonOnly(String methodName) {
26+
return throwHttpJsonOnly(Storage.class, methodName);
27+
}
28+
29+
static <T> T throwHttpJsonOnly(Class<?> clazz, String methodName) {
30+
return throwTransportOnly(clazz, methodName, Transport.HTTP);
31+
}
32+
33+
static <T> T throwGrpcOnly(String methodName) {
34+
return throwGrpcOnly(Storage.class, methodName);
35+
}
36+
37+
static <T> T throwGrpcOnly(Class<?> clazz, String methodName) {
38+
return throwTransportOnly(clazz, methodName, Transport.GRPC);
39+
}
40+
41+
static <T> T throwTransportOnly(Class<?> clazz, String methodName, Transport transport) {
42+
String builder;
43+
switch (transport) {
44+
case HTTP:
45+
builder = "StorageOptions.http()";
46+
break;
47+
case GRPC:
48+
builder = "StorageOptions.grpc()";
49+
break;
50+
default:
51+
throw new IllegalStateException(
52+
String.format("Broken Java Enum: %s received value: '%s'", Transport.class, transport));
53+
}
54+
String message =
55+
String.format(
56+
"%s#%s is only supported for %s transport. Please use %s to construct a compatible instance.",
57+
clazz.getName(), methodName, transport, builder);
58+
throw new UnsupportedOperationException(message);
59+
}
60+
61+
static String fmtMethodName(String name, Class<?>... args) {
62+
return name
63+
+ "("
64+
+ Arrays.stream(args).map(Class::getName).collect(Collectors.joining(", "))
65+
+ ")";
66+
}
67+
}

0 commit comments

Comments
 (0)