Skip to content

Commit d91e47f

Browse files
cpriti-oscpriti-os
authored
feat(storage): add support for partial success in ListBuckets (#13320)
This change adds support for the returnPartialSuccess parameter in ListBuckets calls for both JSON and gRPC client. When this option is enabled by setting ReturnPartialSuccess = true on the BucketIterator, the GCS API will return a list of reachable buckets even if some are temporarily unavailable. The resource names of any unreachable buckets are returned in a new Unreachable field in the ListBucketsResponse Integration and unit tests added.
1 parent 466d309 commit d91e47f

File tree

7 files changed

+252
-71
lines changed

7 files changed

+252
-71
lines changed

‎storage/bucket.go‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2331,6 +2331,11 @@ func (it *ObjectIterator) Next() (*ObjectAttrs, error) {
23312331
// whose names begin with the prefix. By default, all buckets in the project
23322332
// are returned.
23332333
//
2334+
// To receive a partial list of buckets when some are unavailable, set the
2335+
// iterator's ReturnPartialSuccess field to true. You can then call the
2336+
// iterator's Unreachable method to retrieve the names of the unreachable
2337+
// buckets.
2338+
//
23342339
// Note: The returned iterator is not safe for concurrent operations without explicit synchronization.
23352340
func (c *Client) Buckets(ctx context.Context, projectID string) *BucketIterator {
23362341
o := makeStorageOpts(true, c.retry, "")
@@ -2343,12 +2348,24 @@ func (c *Client) Buckets(ctx context.Context, projectID string) *BucketIterator
23432348
type BucketIterator struct {
23442349
// Prefix restricts the iterator to buckets whose names begin with it.
23452350
Prefix string
2346-
2347-
ctx context.Context
2348-
projectID string
2349-
buckets []*BucketAttrs
2350-
pageInfo *iterator.PageInfo
2351-
nextFunc func() error
2351+
// If true, the iterator will return a partial result of buckets even if
2352+
// some buckets are unreachable. Call the Unreachable() method to retrieve the
2353+
// list of unreachable buckets. By default (false), the iterator will return
2354+
// an error if any buckets are unreachable.
2355+
ReturnPartialSuccess bool
2356+
2357+
ctx context.Context
2358+
projectID string
2359+
buckets []*BucketAttrs
2360+
unreachable []string
2361+
pageInfo *iterator.PageInfo
2362+
nextFunc func() error
2363+
}
2364+
2365+
// Unreachable returns a list of bucket names that could not be reached
2366+
// during the iteration if ReturnPartialSuccess was set to true.
2367+
func (it *BucketIterator) Unreachable() []string {
2368+
return it.unreachable
23522369
}
23532370

23542371
// Next returns the next result. Its second return value is iterator.Done if

‎storage/bucket_test.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"github.com/google/go-cmp/cmp"
2929
gax "github.com/googleapis/gax-go/v2"
3030
"google.golang.org/api/googleapi"
31+
"google.golang.org/api/iterator"
3132
"google.golang.org/api/option"
3233
raw "google.golang.org/api/storage/v1"
3334
"google.golang.org/protobuf/proto"
@@ -1674,3 +1675,107 @@ func TestDefaultSignBlobRetry(t *testing.T) {
16741675
t.Fatalf("BucketHandle.SignedURL: %v", err)
16751676
}
16761677
}
1678+
1679+
func TestBucketIterator_PartialSuccess(t *testing.T) {
1680+
ctx := context.Background()
1681+
const projectID = "test-project"
1682+
1683+
// Mocked JSON responses from the GCS API.
1684+
partialSuccessListBucketsResponse := `{
1685+
"kind": "storage#buckets",
1686+
"nextPageToken": "",
1687+
"items": [
1688+
{
1689+
"kind": "storage#bucket",
1690+
"id": "bucket-1",
1691+
"name": "bucket-1"
1692+
},
1693+
{
1694+
"kind": "storage#bucket",
1695+
"id": "bucket-2",
1696+
"name": "bucket-2"
1697+
}
1698+
],
1699+
"unreachable": [
1700+
"projects/_/buckets/unreachable-1",
1701+
"projects/_/buckets/unreachable-2"
1702+
]
1703+
}`
1704+
1705+
listBucketsResponse := `{
1706+
"kind": "storage#buckets",
1707+
"nextPageToken": "",
1708+
"items": [
1709+
{
1710+
"kind": "storage#bucket",
1711+
"id": "bucket-1",
1712+
"name": "bucket-1"
1713+
},
1714+
{
1715+
"kind": "storage#bucket",
1716+
"id": "bucket-2",
1717+
"name": "bucket-2"
1718+
}
1719+
]
1720+
}`
1721+
1722+
testCases := []struct {
1723+
name string
1724+
returnPartialSuccess bool
1725+
jsonResponse string
1726+
wantUnreachable []string
1727+
}{
1728+
{
1729+
name: "ReturnPartialSuccess is true",
1730+
returnPartialSuccess: true,
1731+
jsonResponse: partialSuccessListBucketsResponse,
1732+
wantUnreachable: []string{"projects/_/buckets/unreachable-1", "projects/_/buckets/unreachable-2"},
1733+
},
1734+
{
1735+
name: "ReturnPartialSuccess is false",
1736+
returnPartialSuccess: false,
1737+
jsonResponse: listBucketsResponse,
1738+
wantUnreachable: nil,
1739+
},
1740+
}
1741+
1742+
for _, tc := range testCases {
1743+
t.Run(tc.name, func(t *testing.T) {
1744+
mt := mockTransport{}
1745+
mt.addResult(&http.Response{
1746+
StatusCode: 200,
1747+
Body: bodyReader(tc.jsonResponse),
1748+
Header: http.Header{"Content-Type": []string{"application/json"}},
1749+
}, nil)
1750+
1751+
client, err := NewClient(ctx, option.WithHTTPClient(&http.Client{Transport: &mt}))
1752+
if err != nil {
1753+
t.Fatalf("NewClient: %v", err)
1754+
}
1755+
1756+
it := client.Buckets(ctx, projectID)
1757+
it.ReturnPartialSuccess = tc.returnPartialSuccess
1758+
1759+
var buckets []*BucketAttrs
1760+
for {
1761+
b, err := it.Next()
1762+
if err == iterator.Done {
1763+
break
1764+
}
1765+
if err != nil {
1766+
t.Fatalf("it.Next() got err: %v", err)
1767+
}
1768+
buckets = append(buckets, b)
1769+
}
1770+
1771+
if got, want := len(buckets), 2; got != want {
1772+
t.Errorf("got %d buckets, want %d", got, want)
1773+
}
1774+
1775+
unreachable := it.Unreachable()
1776+
if diff := cmp.Diff(unreachable, tc.wantUnreachable); diff != "" {
1777+
t.Errorf("Unreachable() mismatch (-want +got):\n%s", diff)
1778+
}
1779+
})
1780+
}
1781+
}

‎storage/go.mod‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,57 +6,57 @@ retract [v1.25.0, v1.27.0] // due to https://github.com/googleapis/google-cloud-
66

77
require (
88
cloud.google.com/go v0.121.6
9-
cloud.google.com/go/auth v0.16.5
10-
cloud.google.com/go/compute/metadata v0.8.0
9+
cloud.google.com/go/auth v0.17.0
10+
cloud.google.com/go/compute/metadata v0.9.0
1111
cloud.google.com/go/iam v1.5.2
1212
cloud.google.com/go/longrunning v0.7.0
1313
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0
1414
github.com/google/go-cmp v0.7.0
1515
github.com/google/uuid v1.6.0
1616
github.com/googleapis/gax-go/v2 v2.15.0
1717
go.opentelemetry.io/contrib/detectors/gcp v1.36.0
18-
go.opentelemetry.io/otel v1.36.0
18+
go.opentelemetry.io/otel v1.37.0
1919
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0
20-
go.opentelemetry.io/otel/sdk v1.36.0
21-
go.opentelemetry.io/otel/sdk/metric v1.36.0
22-
go.opentelemetry.io/otel/trace v1.36.0
23-
golang.org/x/oauth2 v0.30.0
24-
golang.org/x/sync v0.16.0
25-
google.golang.org/api v0.247.0
20+
go.opentelemetry.io/otel/sdk v1.37.0
21+
go.opentelemetry.io/otel/sdk/metric v1.37.0
22+
go.opentelemetry.io/otel/trace v1.37.0
23+
golang.org/x/oauth2 v0.33.0
24+
golang.org/x/sync v0.18.0
25+
google.golang.org/api v0.256.0
2626
google.golang.org/genproto v0.0.0-20250603155806-513f23925822
2727
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c
28-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c
29-
google.golang.org/grpc v1.74.3
30-
google.golang.org/protobuf v1.36.7
28+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101
29+
google.golang.org/grpc v1.76.0
30+
google.golang.org/protobuf v1.36.10
3131
)
3232

3333
require (
3434
cel.dev/expr v0.24.0 // indirect
3535
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
3636
cloud.google.com/go/monitoring v1.24.2 // indirect
37-
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
37+
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
3838
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
3939
github.com/cespare/xxhash/v2 v2.3.0 // indirect
4040
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
4141
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
4242
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
4343
github.com/felixge/httpsnoop v1.0.4 // indirect
44-
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
44+
github.com/go-jose/go-jose/v4 v4.1.2 // indirect
4545
github.com/go-logr/logr v1.4.3 // indirect
4646
github.com/go-logr/stdr v1.2.2 // indirect
4747
github.com/google/martian/v3 v3.3.3 // indirect
4848
github.com/google/s2a-go v0.1.9 // indirect
49-
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
49+
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
5050
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
5151
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
5252
github.com/zeebo/errs v1.4.0 // indirect
5353
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
5454
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
5555
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
56-
go.opentelemetry.io/otel/metric v1.36.0 // indirect
57-
golang.org/x/crypto v0.41.0 // indirect
58-
golang.org/x/net v0.43.0 // indirect
59-
golang.org/x/sys v0.35.0 // indirect
60-
golang.org/x/text v0.28.0 // indirect
61-
golang.org/x/time v0.12.0 // indirect
56+
go.opentelemetry.io/otel/metric v1.37.0 // indirect
57+
golang.org/x/crypto v0.43.0 // indirect
58+
golang.org/x/net v0.46.0 // indirect
59+
golang.org/x/sys v0.37.0 // indirect
60+
golang.org/x/text v0.30.0 // indirect
61+
golang.org/x/time v0.14.0 // indirect
6262
)

0 commit comments

Comments
 (0)