diff --git a/storage/contexts.go b/storage/contexts.go new file mode 100644 index 000000000000..06ace64a55be --- /dev/null +++ b/storage/contexts.go @@ -0,0 +1,115 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "time" + + "cloud.google.com/go/storage/internal/apiv2/storagepb" + raw "google.golang.org/api/storage/v1" +) + +// ObjectContexts is a container for custom object contexts. +type ObjectContexts struct { + Custom map[string]ObjectCustomContextPayload +} + +// ObjectCustomContextPayload holds the value of a user-defined object context and +// other metadata. To delete a key from Custom object contexts, set Delete as true. +type ObjectCustomContextPayload struct { + Value string + Delete bool + // Read-only fields. Any updates to CreateTime and UpdateTime will be ignored. + // These fields are handled by the server. + CreateTime time.Time + UpdateTime time.Time +} + +// toContexts converts the raw library's ObjectContexts type to the object contexts. +func toObjectContexts(c *raw.ObjectContexts) *ObjectContexts { + if c == nil { + return nil + } + customContexts := make(map[string]ObjectCustomContextPayload, len(c.Custom)) + for k, v := range c.Custom { + customContexts[k] = ObjectCustomContextPayload{ + Value: v.Value, + CreateTime: convertTime(v.CreateTime), + UpdateTime: convertTime(v.UpdateTime), + } + } + return &ObjectContexts{ + Custom: customContexts, + } +} + +// toRawObjectContexts converts the object contexts to the raw library's ObjectContexts type. +func toRawObjectContexts(c *ObjectContexts) *raw.ObjectContexts { + if c == nil { + return nil + } + customContexts := make(map[string]raw.ObjectCustomContextPayload) + for k, v := range c.Custom { + if v.Delete { + // If Delete is true, populate null fields to signify deletion. + customContexts[k] = raw.ObjectCustomContextPayload{NullFields: []string{k}} + } else { + customContexts[k] = raw.ObjectCustomContextPayload{ + Value: v.Value, + ForceSendFields: []string{k}, + } + } + } + return &raw.ObjectContexts{ + Custom: customContexts, + } +} + +func toObjectContextsFromProto(c *storagepb.ObjectContexts) *ObjectContexts { + if c == nil { + return nil + } + customContexts := make(map[string]ObjectCustomContextPayload, len(c.GetCustom())) + for k, v := range c.GetCustom() { + customContexts[k] = ObjectCustomContextPayload{ + Value: v.GetValue(), + CreateTime: v.GetCreateTime().AsTime(), + UpdateTime: v.GetUpdateTime().AsTime(), + } + } + return &ObjectContexts{ + Custom: customContexts, + } +} + +func toProtoObjectContexts(c *ObjectContexts) *storagepb.ObjectContexts { + if c == nil { + return nil + } + customContexts := make(map[string]*storagepb.ObjectCustomContextPayload) + for k, v := range c.Custom { + // To delete a key, it is added to gRPC fieldMask and with an empty value + // in gRPC request body. Hence, the key is skipped here in customContexts map. + // See grpcStorageClient.UpdateObject method for more details. + if !v.Delete { + customContexts[k] = &storagepb.ObjectCustomContextPayload{ + Value: v.Value, + } + } + } + return &storagepb.ObjectContexts{ + Custom: customContexts, + } +} diff --git a/storage/contexts_test.go b/storage/contexts_test.go new file mode 100644 index 000000000000..d65ed96904a1 --- /dev/null +++ b/storage/contexts_test.go @@ -0,0 +1,236 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "testing" + "time" + + "cloud.google.com/go/storage/internal/apiv2/storagepb" + "github.com/google/go-cmp/cmp" + raw "google.golang.org/api/storage/v1" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestToObjectContexts(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + testCases := []struct { + name string + raw *raw.ObjectContexts + want *ObjectContexts + }{ + { + name: "nil raw object contexts", + raw: nil, + want: nil, + }, + { + name: "empty custom contexts", + raw: &raw.ObjectContexts{ + Custom: map[string]raw.ObjectCustomContextPayload{}, + }, + want: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{}, + }, + }, + { + name: "with custom contexts", + raw: &raw.ObjectContexts{ + Custom: map[string]raw.ObjectCustomContextPayload{ + "key1": {Value: "value1", CreateTime: now.Format(time.RFC3339Nano), UpdateTime: now.Format(time.RFC3339Nano)}, + "key2": {Value: "value2", CreateTime: now.Format(time.RFC3339Nano), UpdateTime: now.Format(time.RFC3339Nano)}, + }, + }, + want: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "key1": {Value: "value1", CreateTime: now, UpdateTime: now}, + "key2": {Value: "value2", CreateTime: now, UpdateTime: now}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := toObjectContexts(tc.raw) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("toObjectContexts() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestToRawObjectContexts(t *testing.T) { + testCases := []struct { + name string + obj *ObjectContexts + want *raw.ObjectContexts + }{ + { + name: "nil object contexts", + obj: nil, + want: nil, + }, + { + name: "empty custom contexts", + obj: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{}, + }, + want: &raw.ObjectContexts{ + Custom: map[string]raw.ObjectCustomContextPayload{}, + }, + }, + { + name: "with custom contexts", + obj: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "key1": {Value: "value1"}, + "key2": {Value: "value2", Delete: true}, // Should have NullFields + }, + }, + want: &raw.ObjectContexts{ + Custom: map[string]raw.ObjectCustomContextPayload{ + "key1": {Value: "value1", ForceSendFields: []string{"key1"}}, + "key2": {NullFields: []string{"key2"}}, + }, + }, + }, + { + name: "with custom contexts, no delete", + obj: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "key1": {Value: "value1"}, + "key2": {Value: "value2"}, + }, + }, + want: &raw.ObjectContexts{ + Custom: map[string]raw.ObjectCustomContextPayload{ + "key1": {Value: "value1", ForceSendFields: []string{"key1"}}, + "key2": {Value: "value2", ForceSendFields: []string{"key2"}}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := toRawObjectContexts(tc.obj) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("toRawObjectContexts() mismatch (-want +got): %s", diff) + } + }) + } +} + +func TestToObjectContextsFromProto(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + testCases := []struct { + name string + proto *storagepb.ObjectContexts + want *ObjectContexts + }{ + { + name: "nil proto object contexts", + proto: nil, + want: nil, + }, + { + name: "empty custom contexts", + proto: &storagepb.ObjectContexts{ + Custom: map[string]*storagepb.ObjectCustomContextPayload{}, + }, + want: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{}, + }, + }, + { + name: "with custom contexts", + proto: &storagepb.ObjectContexts{ + Custom: map[string]*storagepb.ObjectCustomContextPayload{ + "key1": {Value: "value1", CreateTime: timestamppb.New(now), UpdateTime: timestamppb.New(now)}, + "key2": {Value: "value2", CreateTime: timestamppb.New(now), UpdateTime: timestamppb.New(now)}, + }, + }, + want: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "key1": {Value: "value1", CreateTime: now, UpdateTime: now}, + "key2": {Value: "value2", CreateTime: now, UpdateTime: now}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := toObjectContextsFromProto(tc.proto) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("toObjectContextsFromProto() mismatch (-want +got): %s", diff) + } + }) + } +} + +func TestToProtoObjectContexts(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + testCases := []struct { + name string + obj *ObjectContexts + want *storagepb.ObjectContexts + }{ + { + name: "nil object contexts", + obj: nil, + want: nil, + }, + { + name: "empty custom contexts", + obj: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{}, + }, + want: &storagepb.ObjectContexts{ + Custom: map[string]*storagepb.ObjectCustomContextPayload{}, + }, + }, + { + name: "with custom contexts", + obj: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "key1": {Value: "value1", CreateTime: now, UpdateTime: now}, + "key2": {Value: "value2", Delete: true}, // Should be skipped in proto conversion + "key3": {Value: "value3"}, + }, + }, + want: &storagepb.ObjectContexts{ + Custom: map[string]*storagepb.ObjectCustomContextPayload{ + "key1": {Value: "value1"}, + "key3": {Value: "value3"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := toProtoObjectContexts(tc.obj) + if diff := cmp.Diff(tc.want, got, protocmp.Transform()); diff != "" { + t.Errorf("toProtoObjectContexts() mismatch (-want +got): %s", diff) + } + }) + } +} diff --git a/storage/grpc_client.go b/storage/grpc_client.go index f6b2e5e694b6..7dc3d0a189c8 100644 --- a/storage/grpc_client.go +++ b/storage/grpc_client.go @@ -460,6 +460,7 @@ func (c *grpcStorageClient) ListObjects(ctx context.Context, bucket string, q *Q ReadMask: q.toFieldMask(), // a nil Query still results in a "*" FieldMask SoftDeleted: it.query.SoftDeleted, IncludeFoldersAsPrefixes: it.query.IncludeFoldersAsPrefixes, + Filter: it.query.Filter, } if s.userProject != "" { ctx = setUserProjectMetadata(ctx, s.userProject) @@ -630,6 +631,18 @@ func (c *grpcStorageClient) UpdateObject(ctx context.Context, params *updateObje } } + if uattrs.Contexts != nil && uattrs.Contexts.Custom != nil { + if len(uattrs.Contexts.Custom) == 0 { + // pass fieldMask with no key value and empty map to delete all keys + fieldMask.Paths = append(fieldMask.Paths, "contexts.custom") + } else { + for key := range uattrs.Contexts.Custom { + // pass fieldMask with key value with empty value in map to delete key + fieldMask.Paths = append(fieldMask.Paths, fmt.Sprintf("contexts.custom.%s", key)) + } + } + } + req.UpdateMask = fieldMask if len(fieldMask.Paths) < 1 { diff --git a/storage/http_client.go b/storage/http_client.go index 4c651544f9c0..26b2ec491d5a 100644 --- a/storage/http_client.go +++ b/storage/http_client.go @@ -365,6 +365,8 @@ func (c *httpStorageClient) ListObjects(ctx context.Context, bucket string, q *Q req.IncludeTrailingDelimiter(it.query.IncludeTrailingDelimiter) req.MatchGlob(it.query.MatchGlob) req.IncludeFoldersAsPrefixes(it.query.IncludeFoldersAsPrefixes) + req.Filter(it.query.Filter) + if selection := it.query.toFieldSelection(); selection != "" { req.Fields("nextPageToken", googleapi.Field(selection)) } @@ -518,6 +520,19 @@ func (c *httpStorageClient) UpdateObject(ctx context.Context, params *updateObje forceSendFields = append(forceSendFields, "Retention") } } + + if uattrs.Contexts != nil && uattrs.Contexts.Custom != nil { + if len(uattrs.Contexts.Custom) == 0 { + // To delete all contexts, "Contexts" must be added to nullFields. + // Sending empty Custom map in the request body is a no-op without this. + nullFields = append(nullFields, "Contexts") + } else { + attrs.Contexts = uattrs.Contexts + // This is to ensure any new values or deletions are updated + forceSendFields = append(forceSendFields, "Contexts") + } + } + rawObj := attrs.toRawObject(params.bucket) rawObj.ForceSendFields = forceSendFields rawObj.NullFields = nullFields diff --git a/storage/integration_test.go b/storage/integration_test.go index c4b29cae8a2b..39403b307359 100644 --- a/storage/integration_test.go +++ b/storage/integration_test.go @@ -2328,11 +2328,17 @@ func TestIntegration_ObjectCompose(t *testing.T) { b.Object("obj/with/slashes" + uidSpaceObjects.New()), b.Object("obj/" + uidSpaceObjects.New()), } + wantCustomContexts := map[string]ObjectCustomContextPayload{ + "key_0": {Value: "val_0"}, + "key_1": {Value: "val_1"}, + "key_2": {Value: "val_2"}, + "key_3": {Value: "val_3"}, + } var compSrcs []*ObjectHandle wantContents := make([]byte, 0) // Write objects to compose - for _, obj := range objects { + for i, obj := range objects { c := randomContents() if err := writeObject(ctx, obj, "text/plain", c); err != nil { t.Errorf("Write for %v failed with %v", obj, err) @@ -2340,9 +2346,15 @@ func TestIntegration_ObjectCompose(t *testing.T) { compSrcs = append(compSrcs, obj) wantContents = append(wantContents, c...) defer obj.Delete(ctx) + initialContexts := &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{fmt.Sprintf("key_%v", i): {Value: wantCustomContexts[fmt.Sprintf("key_%v", i)].Value}}, + } + if _, err := obj.Update(ctx, ObjectAttrsToUpdate{Contexts: initialContexts}); err != nil { + t.Fatalf("obj.Update: %v", err) + } } - checkCompose := func(obj *ObjectHandle, contentTypeSet *string) { + checkCompose := func(obj *ObjectHandle, contentTypeSet *string, wantCustomContexts map[string]ObjectCustomContextPayload) { r, err := obj.NewReader(ctx) if err != nil { t.Fatalf("new reader: %v", err) @@ -2362,6 +2374,17 @@ func TestIntegration_ObjectCompose(t *testing.T) { if !(contentTypeSet == nil && (got == "" || got == "application/octet-stream")) && got != *contentTypeSet { t.Errorf("Composed object content-type = %q, want %q", got, *contentTypeSet) } + attrs, err := obj.Attrs(ctx) + if err != nil { + t.Fatalf("obj.Attrs: %v", err) + } + var gotCustomContexts map[string]ObjectCustomContextPayload + if attrs.Contexts != nil { + gotCustomContexts = attrs.Contexts.Custom + } + if diff := cmp.Diff(wantCustomContexts, gotCustomContexts, cmpopts.IgnoreFields(ObjectCustomContextPayload{}, "UpdateTime", "CreateTime")); diff != "" { + t.Errorf("custom contexts mismatch (-want +got):\n%s", diff) + } } // Compose should work even if the user sets no destination attributes. @@ -2374,7 +2397,7 @@ func TestIntegration_ObjectCompose(t *testing.T) { if attrs.ComponentCount != int64(len(objects)) { t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects))) } - checkCompose(compDst, nil) + checkCompose(compDst, nil, wantCustomContexts) // It should also work if we do. contentType := "text/json" @@ -2388,7 +2411,23 @@ func TestIntegration_ObjectCompose(t *testing.T) { if attrs.ComponentCount != int64(len(objects)) { t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects))) } - checkCompose(compDst, &contentType) + checkCompose(compDst, &contentType, wantCustomContexts) + + // overriding contexts. + customContexts := &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{"some-key": {Value: "some-value"}}, + } + compDst = b.Object("composed3") + c = compDst.ComposerFrom(compSrcs...) + c.Contexts = customContexts + attrs, err = c.Run(ctx) + if err != nil { + t.Fatalf("ComposeFrom with contexts error: %v", err) + } + if attrs.ComponentCount != int64(len(objects)) { + t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects))) + } + checkCompose(compDst, nil, customContexts.Custom) }) } @@ -2438,10 +2477,16 @@ func TestIntegration_Copy(t *testing.T) { h.mustDeleteObject(obj) }) - // Set metadata on the source object to check if it's copied. + // Set metadata and custom contexts on the source object to check if it's copied. + initialContexts := &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "sourceKey": {Value: "sourceValue"}, + }, + } if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ ContentLanguage: "en", ContentDisposition: "inline", + Contexts: initialContexts, }); err != nil { t.Fatalf("obj.Update: %v", err) } @@ -2456,6 +2501,7 @@ func TestIntegration_Copy(t *testing.T) { type copierAttrs struct { contentEncoding string maxBytesPerCall int64 + contexts *ObjectContexts } for _, test := range []struct { @@ -2493,6 +2539,19 @@ func TestIntegration_Copy(t *testing.T) { copierAttrs: &copierAttrs{maxBytesPerCall: 1048576}, numExpectedRewriteCalls: 3, }, + { + desc: "copy with custom contexts", + toObj: "copy-with-custom-contexts", + toBucket: bucketInSameRegion, + copierAttrs: &copierAttrs{ + contexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "newKey": {Value: "newValue"}, + }, + }, + }, + numExpectedRewriteCalls: 1, + }, } { t.Run(test.desc, func(t *testing.T) { copyObj := test.toBucket.Object(test.toObj) @@ -2505,6 +2564,9 @@ func TestIntegration_Copy(t *testing.T) { if attrs.maxBytesPerCall != 0 { copier.maxBytesRewrittenPerCall = attrs.maxBytesPerCall } + if attrs.contexts != nil { + copier.Contexts = attrs.contexts + } } rewriteCallsCount := 0 @@ -2530,6 +2592,15 @@ func TestIntegration_Copy(t *testing.T) { if attrs.ContentEncoding != test.copierAttrs.contentEncoding { t.Errorf("unexpected ContentEncoding; got: %s, want: %s", attrs.ContentEncoding, test.copierAttrs.contentEncoding) } + want := initialContexts.Custom + if test.copierAttrs.contexts != nil { + want = test.copierAttrs.contexts.Custom + } + if attrs.Contexts != nil { + if diff := cmp.Diff(attrs.Contexts.Custom, want, cmpopts.IgnoreFields(ObjectCustomContextPayload{}, "UpdateTime", "CreateTime")); diff != "" { + t.Errorf("custom contexts mismatch (-got +want):\n%s", diff) + } + } } else { // Check that metadata is copied when no destination attributes are provided. if attrs.ContentLanguage != "en" { @@ -2538,6 +2609,9 @@ func TestIntegration_Copy(t *testing.T) { if attrs.ContentDisposition != "inline" { t.Errorf("unexpected ContentDisposition; got: %s, want: inline", attrs.ContentDisposition) } + if diff := cmp.Diff(attrs.Contexts.Custom, initialContexts.Custom, cmpopts.IgnoreFields(ObjectCustomContextPayload{}, "UpdateTime", "CreateTime")); diff != "" { + t.Errorf("inherited custom contexts mismatch (-got +want):\n%s", diff) + } } // Check the copied contents @@ -7032,6 +7106,233 @@ func (te *openTelemetryTestExporter) Unregister(ctx context.Context) { te.tp.Shutdown(ctx) } +func TestIntegration_ObjectPatchCustomContexts(t *testing.T) { + multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { + h := testHelper{t} + + testCases := []struct { + name string + initialContexts *ObjectContexts + patchContexts *ObjectContexts + expectedContexts map[string]ObjectCustomContextPayload + }{ + { + name: "add individual contexts", + initialContexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "basekey-unicode-å": {Value: "baseval-unicode-é"}, + }, + }, + patchContexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "newKey": {Value: "newValue"}, + }, + }, + expectedContexts: map[string]ObjectCustomContextPayload{ + "basekey-unicode-å": {Value: "baseval-unicode-é"}, + "newKey": {Value: "newValue"}, + }, + }, + { + name: "modify individual contexts", + initialContexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "keyToModify": {Value: "oldValue"}, + "otherKey": {Value: "otherValue"}, + }, + }, + patchContexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "keyToModify": {Value: "newValue"}, + }, + }, + expectedContexts: map[string]ObjectCustomContextPayload{ + "keyToModify": {Value: "newValue"}, + "otherKey": {Value: "otherValue"}, + }, + }, + { + name: "remove individual contexts", + initialContexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "keyToRemove": {Value: "valueToRemove"}, + "anotherKey": {Value: "anotherValue"}, + }, + }, + patchContexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "keyToRemove": {Delete: true}, + }, + }, + expectedContexts: map[string]ObjectCustomContextPayload{ + "anotherKey": {Value: "anotherValue"}, + }, + }, + { + name: "remove all contexts", + initialContexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "keyToRemove": {Value: "valueToRemove"}, + "anotherKey": {Value: "anotherValue"}, + }, + }, + patchContexts: &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{}, + }, + expectedContexts: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + obj := client.Bucket(bucket).Object(uidSpaceObjects.New()) + + // Create object with initial contexts. + wc := obj.NewWriter(ctx) + wc.Contexts = tc.initialContexts + h.mustWrite(wc, []byte("hello")) + defer h.mustDeleteObject(obj) + + attrs, err := obj.Attrs(ctx) + if err != nil { + t.Fatalf("obj.Attrs: %v", err) + } + if len(attrs.Contexts.Custom) != len(tc.initialContexts.Custom) { + t.Fatalf("got initial contexts %v, want %v", attrs.Contexts.Custom, tc.initialContexts.Custom) + } + + // Patch the object with new contexts. + ua := ObjectAttrsToUpdate{Contexts: tc.patchContexts} + updatedAttrs := h.mustUpdateObject(obj, ua, attrs.Metageneration) + + // Verify contexts are updated as expected. + if tc.expectedContexts == nil { + if updatedAttrs.Contexts != nil && updatedAttrs.Contexts.Custom != nil { + t.Fatalf("expected nil contexts but got: %v", updatedAttrs.Contexts.Custom) + } + return + } + if len(updatedAttrs.Contexts.Custom) != len(tc.expectedContexts) { + t.Errorf("got %d custom contexts, want %d. got: %v, want: %v", len(updatedAttrs.Contexts.Custom), len(tc.expectedContexts), updatedAttrs.Contexts.Custom, tc.expectedContexts) + } + + if diff := cmp.Diff(updatedAttrs.Contexts.Custom, tc.expectedContexts, cmpopts.IgnoreFields(ObjectCustomContextPayload{}, "UpdateTime", "CreateTime")); diff != "" { + t.Errorf("%s: name mismatch (-got +want):\n%s", tc.name, diff) + } + }) + } + }) +} + +func TestIntegration_ObjectGetListCustomContexts(t *testing.T) { + multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, prefix string, client *Client) { + h := testHelper{t} + + // Create objects with and without custom contexts + object1Name := prefix + uidSpaceObjects.New() + "-ctx1" + obj1 := client.Bucket(bucket).Object(object1Name) + contexts1 := &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "keyA": {Value: "valueA"}, + "keyB": {Value: "valueB"}, + "key-unicode-á": {Value: "value-unicode-é"}, + }, + } + wc1 := obj1.NewWriter(ctx) + wc1.Contexts = contexts1 + h.mustWrite(wc1, []byte("content1")) + defer h.mustDeleteObject(obj1) + + object2Name := prefix + uidSpaceObjects.New() + "-ctx2" + obj2 := client.Bucket(bucket).Object(object2Name) + contexts2 := &ObjectContexts{ + Custom: map[string]ObjectCustomContextPayload{ + "keyA": {Value: "valueX"}, // Different value for keyA + "keyC": {Value: "valueC"}, + }, + } + wc2 := obj2.NewWriter(ctx) + wc2.Contexts = contexts2 + h.mustWrite(wc2, []byte("content2")) + defer h.mustDeleteObject(obj2) + + object3Name := prefix + uidSpaceObjects.New() + "-no-ctx" + obj3 := client.Bucket(bucket).Object(object3Name) + h.mustWrite(obj3.NewWriter(ctx), []byte("content3")) + defer h.mustDeleteObject(obj3) + + filterTests := []struct { + name string + query *Query + expectedNames []string + }{ + { + name: "FilterByKeyValue", + query: &Query{ + Prefix: prefix, + Filter: "contexts.\"keyA\"=\"valueA\"", + }, + expectedNames: []string{object1Name}, + }, + { + name: "FilterByAbsenceOfKeyValue", + query: &Query{ + Prefix: prefix, + Filter: "-contexts.\"keyB\"=\"valueB\"", + }, + expectedNames: []string{object2Name, object3Name}, + }, + { + name: "FilterByKeyPresence", + query: &Query{ + Prefix: prefix, + Filter: "contexts.\"keyA\":*", + }, + expectedNames: []string{object1Name, object2Name}, + }, + { + name: "FilterByKeyAbsence", + query: &Query{ + Prefix: prefix, + Filter: "-contexts.\"keyD\":*", + }, + expectedNames: []string{object1Name, object2Name, object3Name}, + }, + { + name: "FilterByUnicodeKeyValue", + query: &Query{ + Prefix: prefix, + Filter: "contexts.\"key-unicode-á\"=\"value-unicode-é\"", + }, + expectedNames: []string{object1Name}, + }, + } + + for _, tc := range filterTests { + t.Run(tc.name, func(t *testing.T) { + it := client.Bucket(bucket).Objects(ctx, tc.query) + var foundNames []string + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + t.Fatalf("%s iterator.Next failed: %v", tc.name, err) + } + foundNames = append(foundNames, attrs.Name) + } + + sort.Strings(foundNames) + sort.Strings(tc.expectedNames) + if diff := cmp.Diff(foundNames, tc.expectedNames); diff != "" { + t.Errorf("%s: name mismatch (-got +want):\n%s", tc.name, diff) + } + }) + } + }) +} + func TestIntegration_UniverseDomains(t *testing.T) { // Direct connectivity is not supported yet for this feature. const disableDP = "GOOGLE_CLOUD_DISABLE_DIRECT_PATH" diff --git a/storage/storage.go b/storage/storage.go index 32a145b8d5d0..55a81fab79cb 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1119,6 +1119,13 @@ type ObjectAttrsToUpdate struct { // extending the RetainUntil time on the object retention must be done // on an ObjectHandle with OverrideUnlockedRetention set to true. Retention *ObjectRetention + + // Contexts allows adding, modifying, or deleting individual object contexts. + // To add or modify a context, set the value field in ObjectCustomContextPayload. + // To delete a context, set the Delete field in ObjectCustomContextPayload to true. + // To remove all contexts, pass Custom as an empty map in Contexts. Passing nil Custom + // map will be no-op. + Contexts *ObjectContexts } // Delete deletes the single specified object. @@ -1411,6 +1418,7 @@ func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { Metadata: o.Metadata, CustomTime: ct, Retention: o.Retention.toRawObjectRetention(), + Contexts: toRawObjectContexts(o.Contexts), } } @@ -1445,6 +1453,7 @@ func (o *ObjectAttrs) toProtoObject(b string) *storagepb.Object { KmsKey: o.KMSKeyName, Generation: o.Generation, Size: o.Size, + Contexts: toProtoObjectContexts(o.Contexts), } } @@ -1488,6 +1497,10 @@ func (uattrs *ObjectAttrsToUpdate) toProtoObject(bucket, object string) *storage o.Metadata = uattrs.Metadata + if uattrs.Contexts != nil { + o.Contexts = toProtoObjectContexts(uattrs.Contexts) + } + return o } @@ -1670,6 +1683,12 @@ type ObjectAttrs struct { // ObjectHandle.Attrs will return ErrObjectNotExist if the object is soft-deleted. // This field is read-only. HardDeleteTime time.Time + + // Contexts store custom key-value metadata that the user could + // annotate object with. These key-value pairs can be used to filter objects + // during list calls. See https://cloud.google.com/storage/docs/object-contexts + // for more details. + Contexts *ObjectContexts } // isZero reports whether the ObjectAttrs struct is empty (i.e. all the @@ -1783,6 +1802,7 @@ func newObject(o *raw.Object) *ObjectAttrs { Retention: toObjectRetention(o.Retention), SoftDeleteTime: convertTime(o.SoftDeleteTime), HardDeleteTime: convertTime(o.HardDeleteTime), + Contexts: toObjectContexts(o.Contexts), } } @@ -1821,6 +1841,7 @@ func newObjectFromProto(o *storagepb.Object) *ObjectAttrs { ComponentCount: int64(o.ComponentCount), SoftDeleteTime: convertProtoTime(o.GetSoftDeleteTime()), HardDeleteTime: convertProtoTime(o.GetHardDeleteTime()), + Contexts: toObjectContextsFromProto(o.GetContexts()), } } @@ -1933,6 +1954,11 @@ type Query struct { // If true, only objects that have been soft-deleted will be listed. // By default, soft-deleted objects are not listed. SoftDeleted bool + + // Filters objects based on object attributes like custom contexts. + // See https://docs.cloud.google.com/storage/docs/listing-objects#filter-by-object-contexts + // for more details. + Filter string } // attrToFieldMap maps the field names of ObjectAttrs to the underlying field @@ -1971,6 +1997,7 @@ var attrToFieldMap = map[string]string{ "Retention": "retention", "HardDeleteTime": "hardDeleteTime", "SoftDeleteTime": "softDeleteTime", + "Contexts": "contexts", } // attrToProtoFieldMap maps the field names of ObjectAttrs to the underlying field @@ -2006,6 +2033,7 @@ var attrToProtoFieldMap = map[string]string{ "ComponentCount": "component_count", "HardDeleteTime": "hard_delete_time", "SoftDeleteTime": "soft_delete_time", + "Contexts": "contexts", // MediaLink was explicitly excluded from the proto as it is an HTTP-ism. // "MediaLink": "mediaLink", // TODO: add object retention - b/308194853