From bce454106f0db8b7ade828d5979b8b63742c8ac1 Mon Sep 17 00:00:00 2001 From: Luke Date: Wed, 30 Jan 2019 20:14:54 +0000 Subject: [PATCH 01/35] Adding foward compatibility for key encoding --- datastore/key.go | 64 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/datastore/key.go b/datastore/key.go index 6ab83eaf..49a0dee0 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -18,8 +18,59 @@ import ( "google.golang.org/appengine/internal" pb "google.golang.org/appengine/internal/datastore" + + // Adding Support the new key encoding format + // in cloud.google.com/go/datastore + nds "cloud.google.com/go/datastore" ) +var errKeyConversion string = "Key conversions must be enabled in the application.\n See for more details." +var convKey *KeyConverter + +// EnableKeyConversion enables forward key conversation +// abilities. Calling this function in a single handler +// will enable the feature for all handlers. This function can be called in +// the /_ah/start handler. +// Support for key converstion. Variable holds the appid +// so that key conversion can retrieve it without a context. +func EnableKeyConversion(ctx context) *KeyConverter { + convKey = *KeyConverter{ + appid: internal.FullyQualifiedAppID(ctx), + } + return +} + +// KeyConverter is the struct used to hold the appid +// for the process of key conversion +type KeyConverter struct { + appid string +} + +// convertKey takes at new datastore key type and +// returns a old key type +func convertKey(key *nds.Key) (*Key, error) { + // if key conerstion is not enabled return right away + if convKey == nil { + return nil, errors.New(errKeyConversion) + } + var pKey *Key + var err error + if key.Parent != nil { + pKey, err = convertKey(key.Parent) + if err != nil { + return nil, err + } + } + return &Key{ + intID: key.ID, + kind: key.Kind, + namespace: key.Namespace, + parent: pKey, + stringID: key.Name, + appID: convKey.appid, + }, nil +} + type KeyRangeCollisionError struct { start int64 end int64 @@ -254,7 +305,18 @@ func DecodeKey(encoded string) (*Key, error) { ref := new(pb.Reference) if err := proto.Unmarshal(b, ref); err != nil { - return nil, err + // if there was an err on the key lets try + // to decode the new key type by default + nKey, nerr := nds.DecodeKey(encoded) + if nerr != nil { + // returning the orginal error on purpose + return nil, err + } + oKey, err := convertKey(nKey) + if err != nil { + return nil, err + } + return oKey, nil } return protoToKey(ref) From 8d736f0e4ff70890d5e12525810b964bdd93c370 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Wed, 6 Feb 2019 11:35:41 -0500 Subject: [PATCH 02/35] splitting logic into a new file --- datastore/key.go | 47 ------------------------ datastore/keycompatibility.go | 69 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 47 deletions(-) create mode 100644 datastore/keycompatibility.go diff --git a/datastore/key.go b/datastore/key.go index 49a0dee0..afea4c42 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -24,53 +24,6 @@ import ( nds "cloud.google.com/go/datastore" ) -var errKeyConversion string = "Key conversions must be enabled in the application.\n See for more details." -var convKey *KeyConverter - -// EnableKeyConversion enables forward key conversation -// abilities. Calling this function in a single handler -// will enable the feature for all handlers. This function can be called in -// the /_ah/start handler. -// Support for key converstion. Variable holds the appid -// so that key conversion can retrieve it without a context. -func EnableKeyConversion(ctx context) *KeyConverter { - convKey = *KeyConverter{ - appid: internal.FullyQualifiedAppID(ctx), - } - return -} - -// KeyConverter is the struct used to hold the appid -// for the process of key conversion -type KeyConverter struct { - appid string -} - -// convertKey takes at new datastore key type and -// returns a old key type -func convertKey(key *nds.Key) (*Key, error) { - // if key conerstion is not enabled return right away - if convKey == nil { - return nil, errors.New(errKeyConversion) - } - var pKey *Key - var err error - if key.Parent != nil { - pKey, err = convertKey(key.Parent) - if err != nil { - return nil, err - } - } - return &Key{ - intID: key.ID, - kind: key.Kind, - namespace: key.Namespace, - parent: pKey, - stringID: key.Name, - appID: convKey.appid, - }, nil -} - type KeyRangeCollisionError struct { start int64 end int64 diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go new file mode 100644 index 00000000..299052e9 --- /dev/null +++ b/datastore/keycompatibility.go @@ -0,0 +1,69 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +// This change adds additional compatibility to help customers +// transition from google.golang.org/appengine/datastore (oldds) to cloud.google.com/go/datastore (newds). +// Each lib (oldds and newds) contain functions Key.Encode() and Key.Decode(). These functions +// create base64 representations of a json marshalled type of datastore keys. Customers have been using +// these encoded values to communicate between services in appengine. The issue is that protobuf key types +// change between oldds and newds making the corresponding base64 key strings incompatible. +// Customer who attempt to upgrade to newds that use this pattern will fail. +// keycompatibility.go placed in oldds enables forward compatibility of newds encoded keys. +// An update to newds will also be necessary to enable backward compatibility. + +package datastore + +import ( + "errors" + + nds "cloud.google.com/go/datastore" + "google.golang.org/appengine/internal" +) + +var errKeyConversion string = "Key conversions must be enabled in the application.\n See for more details." +var convKey *keyConverter + +// EnableKeyConversion enables forward key conversation +// abilities. Calling this function in a single handler +// will enable the feature for all handlers. This function can be called in +// the /_ah/start handler. +// Support for key converstion. Variable holds the appid +// so that key conversion can retrieve it without a context. +func EnableKeyConversion(ctx context) { + convKey = *keyConverter{ + appid: internal.FullyQualifiedAppID(ctx), + } + return +} + +// KeyConverter is the struct used to hold the appid +// for the process of key conversion +type keyConverter struct { + appid string +} + +// convertKey takes at new datastore key type and +// returns a old key type +func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *nds.Key) (*Key, error) { + // if key conerstion is not enabled return right away + if convKey == nil { + return nil, errors.New(errKeyConversion) + } + var pKey *Key + var err error + if key.Parent != nil { + pKey, err = convertNewKeyFormatToOldKeyFormat(key.Parent) + if err != nil { + return nil, err + } + } + return &Key{ + intID: key.ID, + kind: key.Kind, + namespace: key.Namespace, + parent: pKey, + stringID: key.Name, + appID: c.appid, + }, nil +} From 71b3369d7860e563f02c6b59b38fc7a0ff3c8c46 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 7 Feb 2019 11:33:44 -0500 Subject: [PATCH 03/35] Added implemetnation check --- datastore/key.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/datastore/key.go b/datastore/key.go index afea4c42..e4c84788 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -260,16 +260,19 @@ func DecodeKey(encoded string) (*Key, error) { if err := proto.Unmarshal(b, ref); err != nil { // if there was an err on the key lets try // to decode the new key type by default - nKey, nerr := nds.DecodeKey(encoded) - if nerr != nil { - // returning the orginal error on purpose - return nil, err - } - oKey, err := convertKey(nKey) - if err != nil { - return nil, err + if convKey != nil { + nKey, nerr := nds.DecodeKey(encoded) + if nerr != nil { + // returning the orginal error on purpose + return nil, err + } + oKey, err := convKey.convertNewKeyFormatToOldKeyFormat(nKey) + if err != nil { + return nil, err + } + return oKey, nil } - return oKey, nil + } return protoToKey(ref) From 1a7bf35c294faa9b34d908d5f0dcfd00b70f98e8 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 7 Feb 2019 11:34:29 -0500 Subject: [PATCH 04/35] split code into seperate files and completed code review comments --- datastore/keycompatibility.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index 299052e9..ae42c989 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -18,6 +18,7 @@ import ( "errors" nds "cloud.google.com/go/datastore" + "golang.org/x/net/context" "google.golang.org/appengine/internal" ) @@ -30,8 +31,8 @@ var convKey *keyConverter // the /_ah/start handler. // Support for key converstion. Variable holds the appid // so that key conversion can retrieve it without a context. -func EnableKeyConversion(ctx context) { - convKey = *keyConverter{ +func EnableKeyConversion(ctx context.Context) { + convKey = &keyConverter{ appid: internal.FullyQualifiedAppID(ctx), } return @@ -53,7 +54,7 @@ func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *nds.Key) (*Key, er var pKey *Key var err error if key.Parent != nil { - pKey, err = convertNewKeyFormatToOldKeyFormat(key.Parent) + pKey, err = c.convertNewKeyFormatToOldKeyFormat(key.Parent) if err != nil { return nil, err } From 74ebe0f8c701f683deedc38b11e56f1d3d5a3ac6 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 7 Feb 2019 16:25:05 -0500 Subject: [PATCH 05/35] Add additional return to fix nil pointer exception --- datastore/key.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore/key.go b/datastore/key.go index e4c84788..3bd02265 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -260,6 +260,7 @@ func DecodeKey(encoded string) (*Key, error) { if err := proto.Unmarshal(b, ref); err != nil { // if there was an err on the key lets try // to decode the new key type by default + // Check to see if key converter has been implemented if convKey != nil { nKey, nerr := nds.DecodeKey(encoded) if nerr != nil { @@ -272,7 +273,7 @@ func DecodeKey(encoded string) (*Key, error) { } return oKey, nil } - + return nil, err } return protoToKey(ref) From 3b5c8ef3597b243f49b9e9d415a6a6988bbcc70d Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 7 Feb 2019 16:25:45 -0500 Subject: [PATCH 06/35] added key parent testing and as well as control tests --- datastore/keycompat_test.go | 87 +++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 datastore/keycompat_test.go diff --git a/datastore/keycompat_test.go b/datastore/keycompat_test.go new file mode 100644 index 00000000..50ff963d --- /dev/null +++ b/datastore/keycompat_test.go @@ -0,0 +1,87 @@ +package datastore + +import ( + "reflect" + "testing" +) + +var testCasesKeyCompat = []struct { + desc string + key *Key + encodedKey string +}{ + { + desc: "A control test for legacy to legacy key conversion int as the key", + key: &Key{ + kind: "Person", + intID: 1, + appID: "glibrary", + }, + encodedKey: "aghnbGlicmFyeXIMCxIGUGVyc29uGAEM", + }, + { + desc: "A control test for legacy to legacy key conversion string as the key", + key: &Key{ + kind: "Graph", + stringID: "graph:7-day-active", + appID: "glibrary", + }, + encodedKey: "aghnbGlicmFyeXIdCxIFR3JhcGgiEmdyYXBoOjctZGF5LWFjdGl2ZQw", + }, + + // These are keys encoded with cloud.google.com/go/datastore + // Standard int as the key + { + desc: "Convert new key format to old key with int id", + key: &Key{ + kind: "WordIndex", + intID: 1033, + appID: "glibrary", + }, + encodedKey: "Eg4KCVdvcmRJbmRleBCJCA", + }, + // These are keys encoded with cloud.google.com/go/datastore + // Standard string + { + desc: "Convert new key format to old key with string id", + key: &Key{ + kind: "WordIndex", + stringID: "IAmAnID", + appID: "glibrary", + }, + encodedKey: "EhQKCVdvcmRJbmRleBoHSUFtQW5JRA", + }, + + // These are keys encoded with cloud.google.com/go/datastore + // ID String with parent as string + { + desc: "Convert new key format to old key with string id with a parent", + key: &Key{ + kind: "WordIndex", + stringID: "IAmAnID", + appID: "glibrary", + parent: &Key{ + kind: "LetterIndex", + stringID: "IAmAnotherID", + appID: "glibrary", + }, + }, + encodedKey: "EhsKC0xldHRlckluZGV4GgxJQW1Bbm90aGVySUQSFAoJV29yZEluZGV4GgdJQW1BbklE", + }, +} + +func TestKeyCoversion(t *testing.T) { + // Simulate the key converter enablement + convKey = &keyConverter{appid: "glibrary"} + + for _, tc := range testCasesKeyCompat { + enc, err := DecodeKey(tc.encodedKey) + if err != nil { + t.Fatalf("%v", err.Error()) + } + if !reflect.DeepEqual(enc, tc.key) { + t.Errorf("%s: got %+v, want %+v", tc.desc, enc, tc.key) + } + + } +} From 572143289463b2796e93f11aae2e3cdf143a3c15 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 7 Feb 2019 16:36:24 -0500 Subject: [PATCH 07/35] Fix pointer preference per review comments --- datastore/keycompatibility.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index ae42c989..a14b3b32 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -48,7 +48,7 @@ type keyConverter struct { // returns a old key type func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *nds.Key) (*Key, error) { // if key conerstion is not enabled return right away - if convKey == nil { + if c == nil { return nil, errors.New(errKeyConversion) } var pKey *Key From ee4f33da7bf6bb7204217790f20d9543939658b3 Mon Sep 17 00:00:00 2001 From: Luke Date: Tue, 12 Feb 2019 20:19:28 +0000 Subject: [PATCH 08/35] Change variable name per comment --- datastore/keycompat_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datastore/keycompat_test.go b/datastore/keycompat_test.go index 50ff963d..169bdf38 100644 --- a/datastore/keycompat_test.go +++ b/datastore/keycompat_test.go @@ -75,12 +75,12 @@ func TestKeyCoversion(t *testing.T) { convKey = &keyConverter{appid: "glibrary"} for _, tc := range testCasesKeyCompat { - enc, err := DecodeKey(tc.encodedKey) + dk, err := DecodeKey(tc.encodedKey) if err != nil { t.Fatalf("%v", err.Error()) } - if !reflect.DeepEqual(enc, tc.key) { - t.Errorf("%s: got %+v, want %+v", tc.desc, enc, tc.key) + if !reflect.DeepEqual(dk, tc.key) { + t.Errorf("%s: got %+v, want %+v", tc.desc, dk, tc.key) } } From 8e1c38a440135ac01f59df0cf4f5d2a675e83ff4 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Fri, 15 Feb 2019 12:46:43 -0500 Subject: [PATCH 09/35] Added documentation for datastore key conversation --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index d86768a2..aeae2a3d 100644 --- a/README.md +++ b/README.md @@ -71,3 +71,17 @@ A few APIs were cleaned up, and there are some differences: [blobstore package](https://google.golang.org/appengine/blobstore). * `appengine/socket` is not required on App Engine flexible environment / Managed VMs. Use the standard `net` package instead. + +## Key Encode Decode compatibiltiy to help with datastore library migrations + +Key compatibility updates have been added to help customers transition from google.golang.org/appengine/datastore (oldds) to cloud.google.com/go/datastore (newds). +Each lib (oldds and newds) contain functions Key.Encode() and Key.Decode(). These functions +create base64 representations of a json marshalled type of datastore keys. Customers have been using +these encoded values to communicate between services in appengine. Protobuf key types +change between oldds and newds making the corresponding base64 key strings incompatible. +Customer who attempt to upgrade to newds that use this pattern will fail. +keycompatibility.go placed in oldds enables forward compatibility of newds encoded keys. +An update to newds will also be necessary to enable backward compatibility. + +### Enabling key conversation +<< Update with test code.>> \ No newline at end of file From 5d06d014ed8460e27324d58ff512aff3c3a00646 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Fri, 15 Feb 2019 12:47:26 -0500 Subject: [PATCH 10/35] fixed various comment nits --- datastore/key.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/datastore/key.go b/datastore/key.go index 3bd02265..ca3235b7 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -19,8 +19,7 @@ import ( "google.golang.org/appengine/internal" pb "google.golang.org/appengine/internal/datastore" - // Adding Support the new key encoding format - // in cloud.google.com/go/datastore + // Adding Support for the new key encoding format in cloud.google.com/go/datastore nds "cloud.google.com/go/datastore" ) @@ -258,9 +257,8 @@ func DecodeKey(encoded string) (*Key, error) { ref := new(pb.Reference) if err := proto.Unmarshal(b, ref); err != nil { - // if there was an err on the key lets try - // to decode the new key type by default - // Check to see if key converter has been implemented + // If there was an err on the key lets try to decode the new key type by default check to see if key converter + // has been implemented. if convKey != nil { nKey, nerr := nds.DecodeKey(encoded) if nerr != nil { From f4a94409e5f96724762d6c5def4832a4c13bc7d0 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Fri, 15 Feb 2019 12:48:09 -0500 Subject: [PATCH 11/35] fixed various comment nits and typos --- datastore/keycompatibility.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index a14b3b32..3d8e62e4 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -6,7 +6,7 @@ // transition from google.golang.org/appengine/datastore (oldds) to cloud.google.com/go/datastore (newds). // Each lib (oldds and newds) contain functions Key.Encode() and Key.Decode(). These functions // create base64 representations of a json marshalled type of datastore keys. Customers have been using -// these encoded values to communicate between services in appengine. The issue is that protobuf key types +// these encoded values to communicate between services in appengine. Protobuf key types // change between oldds and newds making the corresponding base64 key strings incompatible. // Customer who attempt to upgrade to newds that use this pattern will fail. // keycompatibility.go placed in oldds enables forward compatibility of newds encoded keys. @@ -22,15 +22,14 @@ import ( "google.golang.org/appengine/internal" ) -var errKeyConversion string = "Key conversions must be enabled in the application.\n See for more details." +var errKeyConversion string = `Key conversions must be enabled in the application.\n +See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to -help-with-datastore-library-migrations for more details.` + var convKey *keyConverter -// EnableKeyConversion enables forward key conversation -// abilities. Calling this function in a single handler -// will enable the feature for all handlers. This function can be called in -// the /_ah/start handler. -// Support for key converstion. Variable holds the appid -// so that key conversion can retrieve it without a context. +// EnableKeyConversion enables forward key conversion abilities. Calling this function in a single handler will enable +// the feature for all handlers. This function can be called in the /_ah/start handler. Support for key converstion. +// Variable holds the appid so that key conversion can retrieve it without a context. func EnableKeyConversion(ctx context.Context) { convKey = &keyConverter{ appid: internal.FullyQualifiedAppID(ctx), @@ -38,16 +37,14 @@ func EnableKeyConversion(ctx context.Context) { return } -// KeyConverter is the struct used to hold the appid -// for the process of key conversion +// keyConverter is the struct used to hold the appid for the process of key conversion type keyConverter struct { appid string } -// convertKey takes at new datastore key type and -// returns a old key type +// convertKey takes at new datastore key type and returns a old key type func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *nds.Key) (*Key, error) { - // if key conerstion is not enabled return right away + // if key conversion is not enabled return right away if c == nil { return nil, errors.New(errKeyConversion) } From 7789107addf9db35b4811c14935e337e846dd10e Mon Sep 17 00:00:00 2001 From: Luke Date: Mon, 25 Feb 2019 16:19:15 +0000 Subject: [PATCH 12/35] Updated variable name to make it more readable --- datastore/key.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/key.go b/datastore/key.go index ca3235b7..ccd4bc6b 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -260,8 +260,8 @@ func DecodeKey(encoded string) (*Key, error) { // If there was an err on the key lets try to decode the new key type by default check to see if key converter // has been implemented. if convKey != nil { - nKey, nerr := nds.DecodeKey(encoded) - if nerr != nil { + nKey, nLibKeyErr := nds.DecodeKey(encoded) + if nLibKeyErr != nil { // returning the orginal error on purpose return nil, err } From 7d7c32742ae5e4e76e07bd8b807bdc76e555b7dd Mon Sep 17 00:00:00 2001 From: Luke Date: Mon, 25 Feb 2019 16:21:27 +0000 Subject: [PATCH 13/35] Removed \n and spacing issue --- datastore/keycompatibility.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index 3d8e62e4..e4a9e2ea 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -22,8 +22,8 @@ import ( "google.golang.org/appengine/internal" ) -var errKeyConversion string = `Key conversions must be enabled in the application.\n -See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to -help-with-datastore-library-migrations for more details.` +var errKeyConversion string = `Key conversions must be enabled in the application. +See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.` var convKey *keyConverter From ea111b8b11fc5a3e1274909323dec70b3688d012 Mon Sep 17 00:00:00 2001 From: Luke Date: Mon, 25 Feb 2019 16:32:29 +0000 Subject: [PATCH 14/35] Update lib alias name to newnds so that the comments and the alias match. --- datastore/key.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/key.go b/datastore/key.go index ccd4bc6b..fd2b72f1 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -20,7 +20,7 @@ import ( pb "google.golang.org/appengine/internal/datastore" // Adding Support for the new key encoding format in cloud.google.com/go/datastore - nds "cloud.google.com/go/datastore" + newds "cloud.google.com/go/datastore" ) type KeyRangeCollisionError struct { @@ -260,7 +260,7 @@ func DecodeKey(encoded string) (*Key, error) { // If there was an err on the key lets try to decode the new key type by default check to see if key converter // has been implemented. if convKey != nil { - nKey, nLibKeyErr := nds.DecodeKey(encoded) + nKey, nLibKeyErr := newds.DecodeKey(encoded) if nLibKeyErr != nil { // returning the orginal error on purpose return nil, err From 300b945a8f6b2bacfdebd5babfe0cd48419534d0 Mon Sep 17 00:00:00 2001 From: Luke Date: Mon, 25 Feb 2019 16:32:45 +0000 Subject: [PATCH 15/35] Update lib alias name to newnds so that the comments and the alias match. --- datastore/keycompatibility.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index e4a9e2ea..9607f9f1 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -17,7 +17,7 @@ package datastore import ( "errors" - nds "cloud.google.com/go/datastore" + newds "cloud.google.com/go/datastore" "golang.org/x/net/context" "google.golang.org/appengine/internal" ) @@ -43,7 +43,7 @@ type keyConverter struct { } // convertKey takes at new datastore key type and returns a old key type -func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *nds.Key) (*Key, error) { +func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *newds.Key) (*Key, error) { // if key conversion is not enabled return right away if c == nil { return nil, errors.New(errKeyConversion) From dc956f742fc6a0a78e45ce0067e176a34ca50e87 Mon Sep 17 00:00:00 2001 From: Luke Date: Mon, 25 Feb 2019 16:43:57 +0000 Subject: [PATCH 16/35] Fixed various greammer issues and string spacing --- datastore/keycompatibility.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index 9607f9f1..365465e1 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -4,11 +4,11 @@ // This change adds additional compatibility to help customers // transition from google.golang.org/appengine/datastore (oldds) to cloud.google.com/go/datastore (newds). -// Each lib (oldds and newds) contain functions Key.Encode() and Key.Decode(). These functions +// Each lib (oldds and newds) contain the functions Key.Encode() and Key.Decode(). These functions // create base64 representations of a json marshalled type of datastore keys. Customers have been using -// these encoded values to communicate between services in appengine. Protobuf key types +// these encoded values to communicate between services in appengine. The protobuf key types // change between oldds and newds making the corresponding base64 key strings incompatible. -// Customer who attempt to upgrade to newds that use this pattern will fail. +// Customers who attempt to upgrade to newds that use this pattern will fail. // keycompatibility.go placed in oldds enables forward compatibility of newds encoded keys. // An update to newds will also be necessary to enable backward compatibility. @@ -22,7 +22,7 @@ import ( "google.golang.org/appengine/internal" ) -var errKeyConversion string = `Key conversions must be enabled in the application. +var errKeyConversion = `Key conversions must be enabled in the application. See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.` var convKey *keyConverter From 35a84644a4f81b3b965a1faf5506e45bb0befe2e Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 14 Mar 2019 12:51:56 -0400 Subject: [PATCH 17/35] passing unit tests --- datastore/key.go | 5 ++--- datastore/keycompatibility.go | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/datastore/key.go b/datastore/key.go index fd2b72f1..7befa8e4 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -18,9 +18,8 @@ import ( "google.golang.org/appengine/internal" pb "google.golang.org/appengine/internal/datastore" - // Adding Support for the new key encoding format in cloud.google.com/go/datastore - newds "cloud.google.com/go/datastore" + //newds "cloud.google.com/go/datastore" ) type KeyRangeCollisionError struct { @@ -260,7 +259,7 @@ func DecodeKey(encoded string) (*Key, error) { // If there was an err on the key lets try to decode the new key type by default check to see if key converter // has been implemented. if convKey != nil { - nKey, nLibKeyErr := newds.DecodeKey(encoded) + nKey, nLibKeyErr := decodeToNewKey(encoded) if nLibKeyErr != nil { // returning the orginal error on purpose return nil, err diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index 365465e1..3876971e 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -17,7 +17,6 @@ package datastore import ( "errors" - newds "cloud.google.com/go/datastore" "golang.org/x/net/context" "google.golang.org/appengine/internal" ) @@ -43,7 +42,7 @@ type keyConverter struct { } // convertKey takes at new datastore key type and returns a old key type -func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *newds.Key) (*Key, error) { +func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *NewFormatKey) (*Key, error) { // if key conversion is not enabled return right away if c == nil { return nil, errors.New(errKeyConversion) From 68002c3aaff43c037edda3320a58e3cd8aa95133 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 14 Mar 2019 12:57:38 -0400 Subject: [PATCH 18/35] removed dependancies on new libs --- datastore/go.mod | 7 +++++++ datastore/go.sum | 6 ++++++ 2 files changed, 13 insertions(+) create mode 100644 datastore/go.mod create mode 100644 datastore/go.sum diff --git a/datastore/go.mod b/datastore/go.mod new file mode 100644 index 00000000..2182bb0a --- /dev/null +++ b/datastore/go.mod @@ -0,0 +1,7 @@ +module google.golang.org/appengine + +require ( + github.com/golang/protobuf v1.2.0 + golang.org/x/net v0.0.0-20180724234803-3673e40ba225 + golang.org/x/text v0.3.0 +) \ No newline at end of file diff --git a/datastore/go.sum b/datastore/go.sum new file mode 100644 index 00000000..a2ebdfd5 --- /dev/null +++ b/datastore/go.sum @@ -0,0 +1,6 @@ +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= \ No newline at end of file From 00306098639f6a62d346c9e38f2f535d2b947908 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 14 Mar 2019 12:59:12 -0400 Subject: [PATCH 19/35] code duplication file - code copied from cloud.google.com/go/datastore --- datastore/keyDecodeCodeDup.go | 431 ++++++++++++++++++++++++++++++++++ 1 file changed, 431 insertions(+) create mode 100644 datastore/keyDecodeCodeDup.go diff --git a/datastore/keyDecodeCodeDup.go b/datastore/keyDecodeCodeDup.go new file mode 100644 index 00000000..e9ccc9ef --- /dev/null +++ b/datastore/keyDecodeCodeDup.go @@ -0,0 +1,431 @@ +package datastore + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/golang/protobuf/proto" +) + +// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. +// decodeToNewKey decodes a key from the opaque representation returned by Encode. +func decodeToNewKey(encoded string) (*NewFormatKey, error) { + // Re-add padding. + if m := len(encoded) % 4; m != 0 { + encoded += strings.Repeat("=", 4-m) + } + + b, err := base64.URLEncoding.DecodeString(encoded) + if err != nil { + return nil, err + } + + pKey := new(PBKey) + if err := proto.Unmarshal(b, pKey); err != nil { + return nil, err + } + return cgdProtoToKey(pKey) +} + +// protoToKey decodes a protocol buffer representation of a key into an +// equivalent *Key object. If the key is invalid, protoToKey will return the +// invalid key along with ErrInvalidKey. +func cgdProtoToKey(p *PBKey) (*NewFormatKey, error) { + var key *NewFormatKey + var namespace string + if partition := p.PartitionId; partition != nil { + namespace = partition.NamespaceId + } + for _, el := range p.Path { + key = &NewFormatKey{ + Namespace: namespace, + Kind: el.Kind, + ID: el.GetId(), + Name: el.GetName(), + Parent: key, + } + } + if !key.valid() { // Also detects key == nil. + return key, ErrInvalidKey + } + return key, nil +} + +// valid returns whether the key is valid. +func (k *NewFormatKey) valid() bool { + if k == nil { + return false + } + for ; k != nil; k = k.Parent { + if k.Kind == "" { + return false + } + if k.Name != "" && k.ID != 0 { + return false + } + if k.Parent != nil { + if k.Parent.Incomplete() { + return false + } + if k.Parent.Namespace != k.Namespace { + return false + } + } + } + return true +} + +// Incomplete reports whether the key does not refer to a stored entity. +func (k *NewFormatKey) Incomplete() bool { + return k.Name == "" && k.ID == 0 +} + +// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. +// Key represents the datastore key for a stored entity. +type NewFormatKey struct { + // Kind cannot be empty. + Kind string + // Either ID or Name must be zero for the Key to be valid. + // If both are zero, the Key is incomplete. + ID int64 + Name string + // Parent must either be a complete Key or nil. + Parent *NewFormatKey + + // Namespace provides the ability to partition your data for multiple + // tenants. In most cases, it is not necessary to specify a namespace. + // See docs on datastore multitenancy for details: + // https://cloud.google.com/datastore/docs/concepts/multitenancy + Namespace string +} + +// A partition ID identifies a grouping of entities. The grouping is always +// by project and namespace, however the namespace ID may be empty. +// +// A partition ID contains several dimensions: +// project ID and namespace ID. +// +// Partition dimensions: +// +// - May be `""`. +// - Must be valid UTF-8 bytes. +// - Must have values that match regex `[A-Za-z\d\.\-_]{1,100}` +// If the value of any dimension matches regex `__.*__`, the partition is +// reserved/read-only. +// A reserved/read-only partition ID is forbidden in certain documented +// contexts. +// +// Foreign partition IDs (in which the project ID does +// not match the context project ID ) are discouraged. +// Reads and writes of foreign partition IDs may fail if the project is not in +// an active state. +type PartitionId struct { + // The ID of the project to which the entities belong. + ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` + // If not empty, the ID of the namespace to which the entities belong. + NamespaceId string `protobuf:"bytes,4,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PartitionId) Reset() { *m = PartitionId{} } +func (m *PartitionId) String() string { return proto.CompactTextString(m) } +func (*PartitionId) ProtoMessage() {} +func (*PartitionId) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{0} +} +func (m *PartitionId) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PartitionId.Unmarshal(m, b) +} +func (m *PartitionId) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PartitionId.Marshal(b, m, deterministic) +} +func (dst *PartitionId) XXX_Merge(src proto.Message) { + xxx_messageInfo_PartitionId.Merge(dst, src) +} +func (m *PartitionId) XXX_Size() int { + return xxx_messageInfo_PartitionId.Size(m) +} +func (m *PartitionId) XXX_DiscardUnknown() { + xxx_messageInfo_PartitionId.DiscardUnknown(m) +} + +var xxx_messageInfo_PartitionId proto.InternalMessageInfo + +func (m *PartitionId) GetProjectId() string { + if m != nil { + return m.ProjectId + } + return "" +} + +func (m *PartitionId) GetNamespaceId() string { + if m != nil { + return m.NamespaceId + } + return "" +} + +// A unique identifier for an entity. +// If a key's partition ID or any of its path kinds or names are +// reserved/read-only, the key is reserved/read-only. +// A reserved/read-only key is forbidden in certain documented contexts. +type PBKey struct { + // Entities are partitioned into subsets, currently identified by a project + // ID and namespace ID. + // Queries are scoped to a single partition. + PartitionId *PartitionId `protobuf:"bytes,1,opt,name=partition_id,json=partitionId,proto3" json:"partition_id,omitempty"` + // The entity path. + // An entity path consists of one or more elements composed of a kind and a + // string or numerical identifier, which identify entities. The first + // element identifies a _root entity_, the second element identifies + // a _child_ of the root entity, the third element identifies a child of the + // second entity, and so forth. The entities identified by all prefixes of + // the path are called the element's _ancestors_. + // + // An entity path is always fully complete: *all* of the entity's ancestors + // are required to be in the path along with the entity identifier itself. + // The only exception is that in some documented cases, the identifier in the + // last path element (for the entity) itself may be omitted. For example, + // the last path element of the key of `Mutation.insert` may have no + // identifier. + // + // A path can never be empty, and a path can have at most 100 elements. + Path []*Key_PathElement `protobuf:"bytes,2,rep,name=path,proto3" json:"path,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PBKey) Reset() { *m = PBKey{} } +func (m *PBKey) String() string { return proto.CompactTextString(m) } +func (*PBKey) ProtoMessage() {} +func (*PBKey) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{1} +} +func (m *PBKey) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Key.Unmarshal(m, b) +} +func (m *PBKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Key.Marshal(b, m, deterministic) +} +func (dst *PBKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_Key.Merge(dst, src) +} +func (m *PBKey) XXX_Size() int { + return xxx_messageInfo_Key.Size(m) +} +func (m *PBKey) XXX_DiscardUnknown() { + xxx_messageInfo_Key.DiscardUnknown(m) +} + +// A (kind, ID/name) pair used to construct a key path. +// +// If either name or ID is set, the element is complete. +// If neither is set, the element is incomplete. +type Key_PathElement struct { + // The kind of the entity. + // A kind matching regex `__.*__` is reserved/read-only. + // A kind must not contain more than 1500 bytes when UTF-8 encoded. + // Cannot be `""`. + Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"` + // The type of ID. + // + // Types that are valid to be assigned to IdType: + // *Key_PathElement_Id + // *Key_PathElement_Name + IdType isKey_PathElement_IdType `protobuf_oneof:"id_type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Key_PathElement) Reset() { *m = Key_PathElement{} } +func (m *Key_PathElement) String() string { return proto.CompactTextString(m) } +func (*Key_PathElement) ProtoMessage() {} +func (*Key_PathElement) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{1, 0} +} +func (m *Key_PathElement) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Key_PathElement.Unmarshal(m, b) +} +func (m *Key_PathElement) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Key_PathElement.Marshal(b, m, deterministic) +} +func (dst *Key_PathElement) XXX_Merge(src proto.Message) { + xxx_messageInfo_Key_PathElement.Merge(dst, src) +} +func (m *Key_PathElement) XXX_Size() int { + return xxx_messageInfo_Key_PathElement.Size(m) +} +func (m *Key_PathElement) XXX_DiscardUnknown() { + xxx_messageInfo_Key_PathElement.DiscardUnknown(m) +} + +var xxx_messageInfo_Key_PathElement proto.InternalMessageInfo + +func (m *Key_PathElement) GetKind() string { + if m != nil { + return m.Kind + } + return "" +} + +type isKey_PathElement_IdType interface { + isKey_PathElement_IdType() +} + +type Key_PathElement_Id struct { + Id int64 `protobuf:"varint,2,opt,name=id,proto3,oneof"` +} + +type Key_PathElement_Name struct { + Name string `protobuf:"bytes,3,opt,name=name,proto3,oneof"` +} + +func (*Key_PathElement_Id) isKey_PathElement_IdType() {} + +func (*Key_PathElement_Name) isKey_PathElement_IdType() {} + +func (m *Key_PathElement) GetIdType() isKey_PathElement_IdType { + if m != nil { + return m.IdType + } + return nil +} + +func (m *Key_PathElement) GetId() int64 { + if x, ok := m.GetIdType().(*Key_PathElement_Id); ok { + return x.Id + } + return 0 +} + +func (m *Key_PathElement) GetName() string { + if x, ok := m.GetIdType().(*Key_PathElement_Name); ok { + return x.Name + } + return "" +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Key_PathElement) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Key_PathElement_OneofMarshaler, _Key_PathElement_OneofUnmarshaler, _Key_PathElement_OneofSizer, []interface{}{ + (*Key_PathElement_Id)(nil), + (*Key_PathElement_Name)(nil), + } +} + +func _Key_PathElement_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Key_PathElement) + // id_type + switch x := m.IdType.(type) { + case *Key_PathElement_Id: + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Id)) + case *Key_PathElement_Name: + b.EncodeVarint(3<<3 | proto.WireBytes) + b.EncodeStringBytes(x.Name) + case nil: + default: + return fmt.Errorf("Key_PathElement.IdType has unexpected type %T", x) + } + return nil +} + +func _Key_PathElement_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Key_PathElement) + switch tag { + case 2: // id_type.id + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.IdType = &Key_PathElement_Id{int64(x)} + return true, err + case 3: // id_type.name + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.IdType = &Key_PathElement_Name{x} + return true, err + default: + return false, nil + } +} + +func _Key_PathElement_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Key_PathElement) + // id_type + switch x := m.IdType.(type) { + case *Key_PathElement_Id: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Id)) + case *Key_PathElement_Name: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.Name))) + n += len(x.Name) + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +var fileDescriptor_entity_096a297364b049a5 = []byte{ + // 780 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xff, 0x6e, 0xdc, 0x44, + 0x10, 0xc7, 0xed, 0xbb, 0x5c, 0x1a, 0x8f, 0xdd, 0xa4, 0x6c, 0x2a, 0x61, 0x02, 0x28, 0x26, 0x80, + 0x74, 0x02, 0xc9, 0x6e, 0xc2, 0x1f, 0x54, 0x14, 0xa4, 0x72, 0x25, 0xe0, 0x28, 0x15, 0x9c, 0x56, + 0x55, 0x24, 0x50, 0xa4, 0xd3, 0xde, 0x79, 0xeb, 0x2e, 0x67, 0xef, 0x5a, 0xf6, 0x3a, 0xaa, 0xdf, + 0x05, 0xf1, 0x00, 0x3c, 0x0a, 0x8f, 0x80, 0x78, 0x18, 0xb4, 0x3f, 0xec, 0x0b, 0xed, 0x35, 0xff, + 0x79, 0x67, 0x3e, 0xdf, 0xd9, 0xef, 0xec, 0xce, 0x1a, 0xa2, 0x5c, 0x88, 0xbc, 0xa0, 0x49, 0x46, + 0x24, 0x69, 0xa4, 0xa8, 0x69, 0x72, 0x73, 0x9a, 0x50, 0x2e, 0x99, 0xec, 0xe2, 0xaa, 0x16, 0x52, + 0xa0, 0x43, 0x43, 0xc4, 0x03, 0x11, 0xdf, 0x9c, 0x1e, 0x7d, 0x64, 0x65, 0xa4, 0x62, 0x09, 0xe1, + 0x5c, 0x48, 0x22, 0x99, 0xe0, 0x8d, 0x91, 0x0c, 0x59, 0xbd, 0x5a, 0xb6, 0x2f, 0x93, 0x46, 0xd6, + 0xed, 0x4a, 0xda, 0xec, 0xf1, 0x9b, 0x59, 0xc9, 0x4a, 0xda, 0x48, 0x52, 0x56, 0x16, 0x08, 0x2d, + 0x20, 0xbb, 0x8a, 0x26, 0x05, 0x91, 0x05, 0xcf, 0x4d, 0xe6, 0xe4, 0x17, 0xf0, 0xe7, 0xa4, 0x96, + 0x4c, 0x6d, 0x76, 0x91, 0xa1, 0x8f, 0x01, 0xaa, 0x5a, 0xfc, 0x4e, 0x57, 0x72, 0xc1, 0xb2, 0x70, + 0x14, 0xb9, 0x53, 0x0f, 0x7b, 0x36, 0x72, 0x91, 0xa1, 0x4f, 0x20, 0xe0, 0xa4, 0xa4, 0x4d, 0x45, + 0x56, 0x54, 0x01, 0x3b, 0x1a, 0xf0, 0x87, 0xd8, 0x45, 0x76, 0xf2, 0x8f, 0x0b, 0xe3, 0x4b, 0xda, + 0xa1, 0x67, 0x10, 0x54, 0x7d, 0x61, 0x85, 0xba, 0x91, 0x3b, 0xf5, 0xcf, 0xa2, 0x78, 0x4b, 0xef, + 0xf1, 0x2d, 0x07, 0xd8, 0xaf, 0x6e, 0xd9, 0x79, 0x0c, 0x3b, 0x15, 0x91, 0xaf, 0xc2, 0x51, 0x34, + 0x9e, 0xfa, 0x67, 0x9f, 0x6d, 0x15, 0x5f, 0xd2, 0x2e, 0x9e, 0x13, 0xf9, 0xea, 0xbc, 0xa0, 0x25, + 0xe5, 0x12, 0x6b, 0xc5, 0xd1, 0x0b, 0xd5, 0xd7, 0x10, 0x44, 0x08, 0x76, 0xd6, 0x8c, 0x1b, 0x17, + 0x1e, 0xd6, 0xdf, 0xe8, 0x01, 0x8c, 0x6c, 0x8f, 0xe3, 0xd4, 0xc1, 0x23, 0x96, 0xa1, 0x87, 0xb0, + 0xa3, 0x5a, 0x09, 0xc7, 0x8a, 0x4a, 0x1d, 0xac, 0x57, 0x33, 0x0f, 0xee, 0xb1, 0x6c, 0xa1, 0x8e, + 0xee, 0xe4, 0x29, 0xc0, 0xf7, 0x75, 0x4d, 0xba, 0x2b, 0x52, 0xb4, 0x14, 0x9d, 0xc1, 0xee, 0x8d, + 0xfa, 0x68, 0x42, 0x57, 0xfb, 0x3b, 0xda, 0xea, 0x4f, 0xb3, 0xd8, 0x92, 0x27, 0x7f, 0x4c, 0x60, + 0x62, 0xd4, 0x4f, 0x00, 0x78, 0x5b, 0x14, 0x0b, 0x9d, 0x08, 0xfd, 0xc8, 0x9d, 0xee, 0x6f, 0x2a, + 0xf4, 0x37, 0x19, 0xff, 0xdc, 0x16, 0x85, 0xe6, 0x53, 0x07, 0x7b, 0xbc, 0x5f, 0xa0, 0xcf, 0xe1, + 0xfe, 0x52, 0x88, 0x82, 0x12, 0x6e, 0xf5, 0xaa, 0xb1, 0xbd, 0xd4, 0xc1, 0x81, 0x0d, 0x0f, 0x18, + 0xe3, 0x92, 0xe6, 0xb4, 0xb6, 0x58, 0xdf, 0x6d, 0x60, 0xc3, 0x06, 0xfb, 0x14, 0x82, 0x4c, 0xb4, + 0xcb, 0x82, 0x5a, 0x4a, 0xf5, 0xef, 0xa6, 0x0e, 0xf6, 0x4d, 0xd4, 0x40, 0xe7, 0x70, 0x30, 0x8c, + 0x95, 0xe5, 0x40, 0xdf, 0xe9, 0xdb, 0xa6, 0x5f, 0xf4, 0x5c, 0xea, 0xe0, 0xfd, 0x41, 0x64, 0xca, + 0x7c, 0x0d, 0xde, 0x9a, 0x76, 0xb6, 0xc0, 0x44, 0x17, 0x08, 0xdf, 0x75, 0xaf, 0xa9, 0x83, 0xf7, + 0xd6, 0xb4, 0x1b, 0x4c, 0x36, 0xb2, 0x66, 0x3c, 0xb7, 0xda, 0xf7, 0xec, 0x25, 0xf9, 0x26, 0x6a, + 0xa0, 0x63, 0x80, 0x65, 0x21, 0x96, 0x16, 0x41, 0x91, 0x3b, 0x0d, 0xd4, 0xc1, 0xa9, 0x98, 0x01, + 0xbe, 0x83, 0x83, 0x9c, 0x8a, 0x45, 0x25, 0x18, 0x97, 0x96, 0xda, 0xd3, 0x26, 0x0e, 0x7b, 0x13, + 0xea, 0xa2, 0xe3, 0xe7, 0x44, 0x3e, 0xe7, 0x79, 0xea, 0xe0, 0xfb, 0x39, 0x15, 0x73, 0x05, 0x1b, + 0xf9, 0x53, 0x08, 0xcc, 0x53, 0xb6, 0xda, 0x5d, 0xad, 0xfd, 0x70, 0x6b, 0x03, 0xe7, 0x1a, 0x54, + 0x0e, 0x8d, 0xc4, 0x54, 0x98, 0x81, 0x4f, 0xd4, 0x08, 0xd9, 0x02, 0x9e, 0x2e, 0x70, 0xbc, 0xb5, + 0xc0, 0x66, 0xd4, 0x52, 0x07, 0x03, 0xd9, 0x0c, 0x5e, 0x08, 0xf7, 0x4a, 0x4a, 0x38, 0xe3, 0x79, + 0xb8, 0x1f, 0xb9, 0xd3, 0x09, 0xee, 0x97, 0xe8, 0x11, 0x3c, 0xa4, 0xaf, 0x57, 0x45, 0x9b, 0xd1, + 0xc5, 0xcb, 0x5a, 0x94, 0x0b, 0xc6, 0x33, 0xfa, 0x9a, 0x36, 0xe1, 0xa1, 0x1a, 0x0f, 0x8c, 0x6c, + 0xee, 0xc7, 0x5a, 0x94, 0x17, 0x26, 0x33, 0x0b, 0x00, 0xb4, 0x13, 0x33, 0xe0, 0xff, 0xba, 0xb0, + 0x6b, 0x7c, 0xa3, 0x2f, 0x60, 0xbc, 0xa6, 0x9d, 0x7d, 0xb7, 0xef, 0xbc, 0x22, 0xac, 0x20, 0x74, + 0xa9, 0x7f, 0x1b, 0x15, 0xad, 0x25, 0xa3, 0x4d, 0x38, 0xd6, 0xaf, 0xe1, 0xcb, 0x3b, 0x0e, 0x25, + 0x9e, 0x0f, 0xf4, 0x39, 0x97, 0x75, 0x87, 0x6f, 0xc9, 0x8f, 0x7e, 0x85, 0x83, 0x37, 0xd2, 0xe8, + 0xc1, 0xc6, 0x8b, 0x67, 0x76, 0x7c, 0x04, 0x93, 0xcd, 0x44, 0xdf, 0xfd, 0xf4, 0x0c, 0xf8, 0xcd, + 0xe8, 0xb1, 0x3b, 0xfb, 0xd3, 0x85, 0xf7, 0x57, 0xa2, 0xdc, 0x06, 0xcf, 0x7c, 0x63, 0x6d, 0xae, + 0x86, 0x78, 0xee, 0xfe, 0xf6, 0xad, 0x65, 0x72, 0x51, 0x10, 0x9e, 0xc7, 0xa2, 0xce, 0x93, 0x9c, + 0x72, 0x3d, 0xe2, 0x89, 0x49, 0x91, 0x8a, 0x35, 0xff, 0xfb, 0xcb, 0x3f, 0x19, 0x16, 0x7f, 0x8d, + 0x3e, 0xf8, 0xc9, 0xc8, 0x9f, 0x15, 0xa2, 0xcd, 0xe2, 0x1f, 0x86, 0x8d, 0xae, 0x4e, 0xff, 0xee, + 0x73, 0xd7, 0x3a, 0x77, 0x3d, 0xe4, 0xae, 0xaf, 0x4e, 0x97, 0xbb, 0x7a, 0x83, 0xaf, 0xfe, 0x0b, + 0x00, 0x00, 0xff, 0xff, 0xf3, 0xdd, 0x11, 0x96, 0x45, 0x06, 0x00, 0x00, +} + +var xxx_messageInfo_Key proto.InternalMessageInfo From a66c27912e46a6c87d9c5231d9cb884ca79931b2 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 14 Mar 2019 14:47:23 -0400 Subject: [PATCH 20/35] Spacing and format changes --- datastore/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore/go.mod b/datastore/go.mod index 2182bb0a..f449359d 100644 --- a/datastore/go.mod +++ b/datastore/go.mod @@ -4,4 +4,4 @@ require ( github.com/golang/protobuf v1.2.0 golang.org/x/net v0.0.0-20180724234803-3673e40ba225 golang.org/x/text v0.3.0 -) \ No newline at end of file +) From cf0b49750d02e34ad7921e7d6e1c32db0f798aec Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Fri, 15 Mar 2019 17:23:45 -0400 Subject: [PATCH 21/35] adjusted formating --- datastore/keycompatibility.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index 3876971e..ddd8cec0 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -19,6 +19,7 @@ import ( "golang.org/x/net/context" "google.golang.org/appengine/internal" + "google.golang.org/appengine/log" ) var errKeyConversion = `Key conversions must be enabled in the application. @@ -30,9 +31,11 @@ var convKey *keyConverter // the feature for all handlers. This function can be called in the /_ah/start handler. Support for key converstion. // Variable holds the appid so that key conversion can retrieve it without a context. func EnableKeyConversion(ctx context.Context) { + convKey = &keyConverter{ appid: internal.FullyQualifiedAppID(ctx), } + log.Debugf(ctx, "AppID Is:%v", convKey.appid) return } From adf4e0fd57ea6059ccb84ca244866b229dfdc0f5 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 21 Mar 2019 11:45:34 -0400 Subject: [PATCH 22/35] added sync.Once --- datastore/keycompatibility.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index ddd8cec0..9bdc9d57 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -16,26 +16,30 @@ package datastore import ( "errors" + "sync" "golang.org/x/net/context" "google.golang.org/appengine/internal" - "google.golang.org/appengine/log" ) var errKeyConversion = `Key conversions must be enabled in the application. See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.` var convKey *keyConverter +var once sync.Once // EnableKeyConversion enables forward key conversion abilities. Calling this function in a single handler will enable // the feature for all handlers. This function can be called in the /_ah/start handler. Support for key converstion. // Variable holds the appid so that key conversion can retrieve it without a context. func EnableKeyConversion(ctx context.Context) { - convKey = &keyConverter{ - appid: internal.FullyQualifiedAppID(ctx), - } - log.Debugf(ctx, "AppID Is:%v", convKey.appid) + once.Do(func() { + convKey = &keyConverter{ + appid: internal.FullyQualifiedAppID(ctx), + } + + }) + return } From 0fbebe6351efcb39fc331a5f651ac4fb36252d51 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 21 Mar 2019 13:19:40 -0400 Subject: [PATCH 23/35] Updated Key conversion documentation in readme.md --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index aeae2a3d..78cd0c32 100644 --- a/README.md +++ b/README.md @@ -84,4 +84,15 @@ keycompatibility.go placed in oldds enables forward compatibility of newds encod An update to newds will also be necessary to enable backward compatibility. ### Enabling key conversation -<< Update with test code.>> \ No newline at end of file +Enable key conversion by calling `EnableKeyConversion(ctx)` in the `/_ah/startup` handler for basic and manual scaling or any handler in automatic scaling. + +### 1. Basic or manual scaling +This startup handler will enable key conversion for all handlers in the service. +``` + http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) { + datastore.EnableKeyConversion(appengine.NewContext(r)) + }) +``` +### 2. Automatic scaling +Since `/_ah/start` is not called in atomatic scaling and `/_ah/warmup` is not garenteed a call to `datastore.EnableKeyConversion(appengine.NewContext(r))` +is needed in each handler where key conversion is needed. `EnableKeyConversion` is safe for concurrent use. \ No newline at end of file From abdd3c6614f3777d20462410e5211b104190519c Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 21 Mar 2019 13:24:26 -0400 Subject: [PATCH 24/35] corrected spelling --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78cd0c32..524b7f68 100644 --- a/README.md +++ b/README.md @@ -94,5 +94,5 @@ This startup handler will enable key conversion for all handlers in the service. }) ``` ### 2. Automatic scaling -Since `/_ah/start` is not called in atomatic scaling and `/_ah/warmup` is not garenteed a call to `datastore.EnableKeyConversion(appengine.NewContext(r))` +Since `/_ah/start` is not called in automatic scaling and `/_ah/warmup` is not guaranteed, a call to `datastore.EnableKeyConversion(appengine.NewContext(r))` is needed in each handler where key conversion is needed. `EnableKeyConversion` is safe for concurrent use. \ No newline at end of file From ac84822427b329ee3316f19cf23a57b4982dbc97 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 21 Mar 2019 13:36:30 -0400 Subject: [PATCH 25/35] Updated description --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 524b7f68..1dbf0a1f 100644 --- a/README.md +++ b/README.md @@ -74,14 +74,8 @@ A few APIs were cleaned up, and there are some differences: ## Key Encode Decode compatibiltiy to help with datastore library migrations -Key compatibility updates have been added to help customers transition from google.golang.org/appengine/datastore (oldds) to cloud.google.com/go/datastore (newds). -Each lib (oldds and newds) contain functions Key.Encode() and Key.Decode(). These functions -create base64 representations of a json marshalled type of datastore keys. Customers have been using -these encoded values to communicate between services in appengine. Protobuf key types -change between oldds and newds making the corresponding base64 key strings incompatible. -Customer who attempt to upgrade to newds that use this pattern will fail. -keycompatibility.go placed in oldds enables forward compatibility of newds encoded keys. -An update to newds will also be necessary to enable backward compatibility. +Key compatibility updates have been added to help customers transition from google.golang.org/appengine/datastore to cloud.google.com/go/datastore. +the `EnableKeyConversion` enable automatic coversion from and key encoded with cloud.google.com/go/datastore to google.golang.org/appengine/datastore key type. ### Enabling key conversation Enable key conversion by calling `EnableKeyConversion(ctx)` in the `/_ah/startup` handler for basic and manual scaling or any handler in automatic scaling. From 5069ace93c218527a139b207236c44720f7e3ad8 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Fri, 22 Mar 2019 12:43:01 -0400 Subject: [PATCH 26/35] Fixed grammar issues --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1dbf0a1f..5cb7afb9 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ A few APIs were cleaned up, and there are some differences: ## Key Encode Decode compatibiltiy to help with datastore library migrations Key compatibility updates have been added to help customers transition from google.golang.org/appengine/datastore to cloud.google.com/go/datastore. -the `EnableKeyConversion` enable automatic coversion from and key encoded with cloud.google.com/go/datastore to google.golang.org/appengine/datastore key type. +The `EnableKeyConversion` enables automatic conversion from a key encoded with cloud.google.com/go/datastore to google.golang.org/appengine/datastore key type. ### Enabling key conversation Enable key conversion by calling `EnableKeyConversion(ctx)` in the `/_ah/startup` handler for basic and manual scaling or any handler in automatic scaling. From bb0e4f6a761ec2154a420cb15e6f3d1730cc81a9 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 2 May 2019 13:17:46 -0400 Subject: [PATCH 27/35] Renamed file --- datastore/keydecode_code_dup.go | 431 ++++++++++++++++++++++++++++++++ 1 file changed, 431 insertions(+) create mode 100644 datastore/keydecode_code_dup.go diff --git a/datastore/keydecode_code_dup.go b/datastore/keydecode_code_dup.go new file mode 100644 index 00000000..e9ccc9ef --- /dev/null +++ b/datastore/keydecode_code_dup.go @@ -0,0 +1,431 @@ +package datastore + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/golang/protobuf/proto" +) + +// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. +// decodeToNewKey decodes a key from the opaque representation returned by Encode. +func decodeToNewKey(encoded string) (*NewFormatKey, error) { + // Re-add padding. + if m := len(encoded) % 4; m != 0 { + encoded += strings.Repeat("=", 4-m) + } + + b, err := base64.URLEncoding.DecodeString(encoded) + if err != nil { + return nil, err + } + + pKey := new(PBKey) + if err := proto.Unmarshal(b, pKey); err != nil { + return nil, err + } + return cgdProtoToKey(pKey) +} + +// protoToKey decodes a protocol buffer representation of a key into an +// equivalent *Key object. If the key is invalid, protoToKey will return the +// invalid key along with ErrInvalidKey. +func cgdProtoToKey(p *PBKey) (*NewFormatKey, error) { + var key *NewFormatKey + var namespace string + if partition := p.PartitionId; partition != nil { + namespace = partition.NamespaceId + } + for _, el := range p.Path { + key = &NewFormatKey{ + Namespace: namespace, + Kind: el.Kind, + ID: el.GetId(), + Name: el.GetName(), + Parent: key, + } + } + if !key.valid() { // Also detects key == nil. + return key, ErrInvalidKey + } + return key, nil +} + +// valid returns whether the key is valid. +func (k *NewFormatKey) valid() bool { + if k == nil { + return false + } + for ; k != nil; k = k.Parent { + if k.Kind == "" { + return false + } + if k.Name != "" && k.ID != 0 { + return false + } + if k.Parent != nil { + if k.Parent.Incomplete() { + return false + } + if k.Parent.Namespace != k.Namespace { + return false + } + } + } + return true +} + +// Incomplete reports whether the key does not refer to a stored entity. +func (k *NewFormatKey) Incomplete() bool { + return k.Name == "" && k.ID == 0 +} + +// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. +// Key represents the datastore key for a stored entity. +type NewFormatKey struct { + // Kind cannot be empty. + Kind string + // Either ID or Name must be zero for the Key to be valid. + // If both are zero, the Key is incomplete. + ID int64 + Name string + // Parent must either be a complete Key or nil. + Parent *NewFormatKey + + // Namespace provides the ability to partition your data for multiple + // tenants. In most cases, it is not necessary to specify a namespace. + // See docs on datastore multitenancy for details: + // https://cloud.google.com/datastore/docs/concepts/multitenancy + Namespace string +} + +// A partition ID identifies a grouping of entities. The grouping is always +// by project and namespace, however the namespace ID may be empty. +// +// A partition ID contains several dimensions: +// project ID and namespace ID. +// +// Partition dimensions: +// +// - May be `""`. +// - Must be valid UTF-8 bytes. +// - Must have values that match regex `[A-Za-z\d\.\-_]{1,100}` +// If the value of any dimension matches regex `__.*__`, the partition is +// reserved/read-only. +// A reserved/read-only partition ID is forbidden in certain documented +// contexts. +// +// Foreign partition IDs (in which the project ID does +// not match the context project ID ) are discouraged. +// Reads and writes of foreign partition IDs may fail if the project is not in +// an active state. +type PartitionId struct { + // The ID of the project to which the entities belong. + ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` + // If not empty, the ID of the namespace to which the entities belong. + NamespaceId string `protobuf:"bytes,4,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PartitionId) Reset() { *m = PartitionId{} } +func (m *PartitionId) String() string { return proto.CompactTextString(m) } +func (*PartitionId) ProtoMessage() {} +func (*PartitionId) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{0} +} +func (m *PartitionId) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PartitionId.Unmarshal(m, b) +} +func (m *PartitionId) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PartitionId.Marshal(b, m, deterministic) +} +func (dst *PartitionId) XXX_Merge(src proto.Message) { + xxx_messageInfo_PartitionId.Merge(dst, src) +} +func (m *PartitionId) XXX_Size() int { + return xxx_messageInfo_PartitionId.Size(m) +} +func (m *PartitionId) XXX_DiscardUnknown() { + xxx_messageInfo_PartitionId.DiscardUnknown(m) +} + +var xxx_messageInfo_PartitionId proto.InternalMessageInfo + +func (m *PartitionId) GetProjectId() string { + if m != nil { + return m.ProjectId + } + return "" +} + +func (m *PartitionId) GetNamespaceId() string { + if m != nil { + return m.NamespaceId + } + return "" +} + +// A unique identifier for an entity. +// If a key's partition ID or any of its path kinds or names are +// reserved/read-only, the key is reserved/read-only. +// A reserved/read-only key is forbidden in certain documented contexts. +type PBKey struct { + // Entities are partitioned into subsets, currently identified by a project + // ID and namespace ID. + // Queries are scoped to a single partition. + PartitionId *PartitionId `protobuf:"bytes,1,opt,name=partition_id,json=partitionId,proto3" json:"partition_id,omitempty"` + // The entity path. + // An entity path consists of one or more elements composed of a kind and a + // string or numerical identifier, which identify entities. The first + // element identifies a _root entity_, the second element identifies + // a _child_ of the root entity, the third element identifies a child of the + // second entity, and so forth. The entities identified by all prefixes of + // the path are called the element's _ancestors_. + // + // An entity path is always fully complete: *all* of the entity's ancestors + // are required to be in the path along with the entity identifier itself. + // The only exception is that in some documented cases, the identifier in the + // last path element (for the entity) itself may be omitted. For example, + // the last path element of the key of `Mutation.insert` may have no + // identifier. + // + // A path can never be empty, and a path can have at most 100 elements. + Path []*Key_PathElement `protobuf:"bytes,2,rep,name=path,proto3" json:"path,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PBKey) Reset() { *m = PBKey{} } +func (m *PBKey) String() string { return proto.CompactTextString(m) } +func (*PBKey) ProtoMessage() {} +func (*PBKey) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{1} +} +func (m *PBKey) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Key.Unmarshal(m, b) +} +func (m *PBKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Key.Marshal(b, m, deterministic) +} +func (dst *PBKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_Key.Merge(dst, src) +} +func (m *PBKey) XXX_Size() int { + return xxx_messageInfo_Key.Size(m) +} +func (m *PBKey) XXX_DiscardUnknown() { + xxx_messageInfo_Key.DiscardUnknown(m) +} + +// A (kind, ID/name) pair used to construct a key path. +// +// If either name or ID is set, the element is complete. +// If neither is set, the element is incomplete. +type Key_PathElement struct { + // The kind of the entity. + // A kind matching regex `__.*__` is reserved/read-only. + // A kind must not contain more than 1500 bytes when UTF-8 encoded. + // Cannot be `""`. + Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"` + // The type of ID. + // + // Types that are valid to be assigned to IdType: + // *Key_PathElement_Id + // *Key_PathElement_Name + IdType isKey_PathElement_IdType `protobuf_oneof:"id_type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Key_PathElement) Reset() { *m = Key_PathElement{} } +func (m *Key_PathElement) String() string { return proto.CompactTextString(m) } +func (*Key_PathElement) ProtoMessage() {} +func (*Key_PathElement) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{1, 0} +} +func (m *Key_PathElement) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Key_PathElement.Unmarshal(m, b) +} +func (m *Key_PathElement) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Key_PathElement.Marshal(b, m, deterministic) +} +func (dst *Key_PathElement) XXX_Merge(src proto.Message) { + xxx_messageInfo_Key_PathElement.Merge(dst, src) +} +func (m *Key_PathElement) XXX_Size() int { + return xxx_messageInfo_Key_PathElement.Size(m) +} +func (m *Key_PathElement) XXX_DiscardUnknown() { + xxx_messageInfo_Key_PathElement.DiscardUnknown(m) +} + +var xxx_messageInfo_Key_PathElement proto.InternalMessageInfo + +func (m *Key_PathElement) GetKind() string { + if m != nil { + return m.Kind + } + return "" +} + +type isKey_PathElement_IdType interface { + isKey_PathElement_IdType() +} + +type Key_PathElement_Id struct { + Id int64 `protobuf:"varint,2,opt,name=id,proto3,oneof"` +} + +type Key_PathElement_Name struct { + Name string `protobuf:"bytes,3,opt,name=name,proto3,oneof"` +} + +func (*Key_PathElement_Id) isKey_PathElement_IdType() {} + +func (*Key_PathElement_Name) isKey_PathElement_IdType() {} + +func (m *Key_PathElement) GetIdType() isKey_PathElement_IdType { + if m != nil { + return m.IdType + } + return nil +} + +func (m *Key_PathElement) GetId() int64 { + if x, ok := m.GetIdType().(*Key_PathElement_Id); ok { + return x.Id + } + return 0 +} + +func (m *Key_PathElement) GetName() string { + if x, ok := m.GetIdType().(*Key_PathElement_Name); ok { + return x.Name + } + return "" +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Key_PathElement) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Key_PathElement_OneofMarshaler, _Key_PathElement_OneofUnmarshaler, _Key_PathElement_OneofSizer, []interface{}{ + (*Key_PathElement_Id)(nil), + (*Key_PathElement_Name)(nil), + } +} + +func _Key_PathElement_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Key_PathElement) + // id_type + switch x := m.IdType.(type) { + case *Key_PathElement_Id: + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Id)) + case *Key_PathElement_Name: + b.EncodeVarint(3<<3 | proto.WireBytes) + b.EncodeStringBytes(x.Name) + case nil: + default: + return fmt.Errorf("Key_PathElement.IdType has unexpected type %T", x) + } + return nil +} + +func _Key_PathElement_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Key_PathElement) + switch tag { + case 2: // id_type.id + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.IdType = &Key_PathElement_Id{int64(x)} + return true, err + case 3: // id_type.name + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.IdType = &Key_PathElement_Name{x} + return true, err + default: + return false, nil + } +} + +func _Key_PathElement_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Key_PathElement) + // id_type + switch x := m.IdType.(type) { + case *Key_PathElement_Id: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Id)) + case *Key_PathElement_Name: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.Name))) + n += len(x.Name) + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +var fileDescriptor_entity_096a297364b049a5 = []byte{ + // 780 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xff, 0x6e, 0xdc, 0x44, + 0x10, 0xc7, 0xed, 0xbb, 0x5c, 0x1a, 0x8f, 0xdd, 0xa4, 0x6c, 0x2a, 0x61, 0x02, 0x28, 0x26, 0x80, + 0x74, 0x02, 0xc9, 0x6e, 0xc2, 0x1f, 0x54, 0x14, 0xa4, 0x72, 0x25, 0xe0, 0x28, 0x15, 0x9c, 0x56, + 0x55, 0x24, 0x50, 0xa4, 0xd3, 0xde, 0x79, 0xeb, 0x2e, 0x67, 0xef, 0x5a, 0xf6, 0x3a, 0xaa, 0xdf, + 0x05, 0xf1, 0x00, 0x3c, 0x0a, 0x8f, 0x80, 0x78, 0x18, 0xb4, 0x3f, 0xec, 0x0b, 0xed, 0x35, 0xff, + 0x79, 0x67, 0x3e, 0xdf, 0xd9, 0xef, 0xec, 0xce, 0x1a, 0xa2, 0x5c, 0x88, 0xbc, 0xa0, 0x49, 0x46, + 0x24, 0x69, 0xa4, 0xa8, 0x69, 0x72, 0x73, 0x9a, 0x50, 0x2e, 0x99, 0xec, 0xe2, 0xaa, 0x16, 0x52, + 0xa0, 0x43, 0x43, 0xc4, 0x03, 0x11, 0xdf, 0x9c, 0x1e, 0x7d, 0x64, 0x65, 0xa4, 0x62, 0x09, 0xe1, + 0x5c, 0x48, 0x22, 0x99, 0xe0, 0x8d, 0x91, 0x0c, 0x59, 0xbd, 0x5a, 0xb6, 0x2f, 0x93, 0x46, 0xd6, + 0xed, 0x4a, 0xda, 0xec, 0xf1, 0x9b, 0x59, 0xc9, 0x4a, 0xda, 0x48, 0x52, 0x56, 0x16, 0x08, 0x2d, + 0x20, 0xbb, 0x8a, 0x26, 0x05, 0x91, 0x05, 0xcf, 0x4d, 0xe6, 0xe4, 0x17, 0xf0, 0xe7, 0xa4, 0x96, + 0x4c, 0x6d, 0x76, 0x91, 0xa1, 0x8f, 0x01, 0xaa, 0x5a, 0xfc, 0x4e, 0x57, 0x72, 0xc1, 0xb2, 0x70, + 0x14, 0xb9, 0x53, 0x0f, 0x7b, 0x36, 0x72, 0x91, 0xa1, 0x4f, 0x20, 0xe0, 0xa4, 0xa4, 0x4d, 0x45, + 0x56, 0x54, 0x01, 0x3b, 0x1a, 0xf0, 0x87, 0xd8, 0x45, 0x76, 0xf2, 0x8f, 0x0b, 0xe3, 0x4b, 0xda, + 0xa1, 0x67, 0x10, 0x54, 0x7d, 0x61, 0x85, 0xba, 0x91, 0x3b, 0xf5, 0xcf, 0xa2, 0x78, 0x4b, 0xef, + 0xf1, 0x2d, 0x07, 0xd8, 0xaf, 0x6e, 0xd9, 0x79, 0x0c, 0x3b, 0x15, 0x91, 0xaf, 0xc2, 0x51, 0x34, + 0x9e, 0xfa, 0x67, 0x9f, 0x6d, 0x15, 0x5f, 0xd2, 0x2e, 0x9e, 0x13, 0xf9, 0xea, 0xbc, 0xa0, 0x25, + 0xe5, 0x12, 0x6b, 0xc5, 0xd1, 0x0b, 0xd5, 0xd7, 0x10, 0x44, 0x08, 0x76, 0xd6, 0x8c, 0x1b, 0x17, + 0x1e, 0xd6, 0xdf, 0xe8, 0x01, 0x8c, 0x6c, 0x8f, 0xe3, 0xd4, 0xc1, 0x23, 0x96, 0xa1, 0x87, 0xb0, + 0xa3, 0x5a, 0x09, 0xc7, 0x8a, 0x4a, 0x1d, 0xac, 0x57, 0x33, 0x0f, 0xee, 0xb1, 0x6c, 0xa1, 0x8e, + 0xee, 0xe4, 0x29, 0xc0, 0xf7, 0x75, 0x4d, 0xba, 0x2b, 0x52, 0xb4, 0x14, 0x9d, 0xc1, 0xee, 0x8d, + 0xfa, 0x68, 0x42, 0x57, 0xfb, 0x3b, 0xda, 0xea, 0x4f, 0xb3, 0xd8, 0x92, 0x27, 0x7f, 0x4c, 0x60, + 0x62, 0xd4, 0x4f, 0x00, 0x78, 0x5b, 0x14, 0x0b, 0x9d, 0x08, 0xfd, 0xc8, 0x9d, 0xee, 0x6f, 0x2a, + 0xf4, 0x37, 0x19, 0xff, 0xdc, 0x16, 0x85, 0xe6, 0x53, 0x07, 0x7b, 0xbc, 0x5f, 0xa0, 0xcf, 0xe1, + 0xfe, 0x52, 0x88, 0x82, 0x12, 0x6e, 0xf5, 0xaa, 0xb1, 0xbd, 0xd4, 0xc1, 0x81, 0x0d, 0x0f, 0x18, + 0xe3, 0x92, 0xe6, 0xb4, 0xb6, 0x58, 0xdf, 0x6d, 0x60, 0xc3, 0x06, 0xfb, 0x14, 0x82, 0x4c, 0xb4, + 0xcb, 0x82, 0x5a, 0x4a, 0xf5, 0xef, 0xa6, 0x0e, 0xf6, 0x4d, 0xd4, 0x40, 0xe7, 0x70, 0x30, 0x8c, + 0x95, 0xe5, 0x40, 0xdf, 0xe9, 0xdb, 0xa6, 0x5f, 0xf4, 0x5c, 0xea, 0xe0, 0xfd, 0x41, 0x64, 0xca, + 0x7c, 0x0d, 0xde, 0x9a, 0x76, 0xb6, 0xc0, 0x44, 0x17, 0x08, 0xdf, 0x75, 0xaf, 0xa9, 0x83, 0xf7, + 0xd6, 0xb4, 0x1b, 0x4c, 0x36, 0xb2, 0x66, 0x3c, 0xb7, 0xda, 0xf7, 0xec, 0x25, 0xf9, 0x26, 0x6a, + 0xa0, 0x63, 0x80, 0x65, 0x21, 0x96, 0x16, 0x41, 0x91, 0x3b, 0x0d, 0xd4, 0xc1, 0xa9, 0x98, 0x01, + 0xbe, 0x83, 0x83, 0x9c, 0x8a, 0x45, 0x25, 0x18, 0x97, 0x96, 0xda, 0xd3, 0x26, 0x0e, 0x7b, 0x13, + 0xea, 0xa2, 0xe3, 0xe7, 0x44, 0x3e, 0xe7, 0x79, 0xea, 0xe0, 0xfb, 0x39, 0x15, 0x73, 0x05, 0x1b, + 0xf9, 0x53, 0x08, 0xcc, 0x53, 0xb6, 0xda, 0x5d, 0xad, 0xfd, 0x70, 0x6b, 0x03, 0xe7, 0x1a, 0x54, + 0x0e, 0x8d, 0xc4, 0x54, 0x98, 0x81, 0x4f, 0xd4, 0x08, 0xd9, 0x02, 0x9e, 0x2e, 0x70, 0xbc, 0xb5, + 0xc0, 0x66, 0xd4, 0x52, 0x07, 0x03, 0xd9, 0x0c, 0x5e, 0x08, 0xf7, 0x4a, 0x4a, 0x38, 0xe3, 0x79, + 0xb8, 0x1f, 0xb9, 0xd3, 0x09, 0xee, 0x97, 0xe8, 0x11, 0x3c, 0xa4, 0xaf, 0x57, 0x45, 0x9b, 0xd1, + 0xc5, 0xcb, 0x5a, 0x94, 0x0b, 0xc6, 0x33, 0xfa, 0x9a, 0x36, 0xe1, 0xa1, 0x1a, 0x0f, 0x8c, 0x6c, + 0xee, 0xc7, 0x5a, 0x94, 0x17, 0x26, 0x33, 0x0b, 0x00, 0xb4, 0x13, 0x33, 0xe0, 0xff, 0xba, 0xb0, + 0x6b, 0x7c, 0xa3, 0x2f, 0x60, 0xbc, 0xa6, 0x9d, 0x7d, 0xb7, 0xef, 0xbc, 0x22, 0xac, 0x20, 0x74, + 0xa9, 0x7f, 0x1b, 0x15, 0xad, 0x25, 0xa3, 0x4d, 0x38, 0xd6, 0xaf, 0xe1, 0xcb, 0x3b, 0x0e, 0x25, + 0x9e, 0x0f, 0xf4, 0x39, 0x97, 0x75, 0x87, 0x6f, 0xc9, 0x8f, 0x7e, 0x85, 0x83, 0x37, 0xd2, 0xe8, + 0xc1, 0xc6, 0x8b, 0x67, 0x76, 0x7c, 0x04, 0x93, 0xcd, 0x44, 0xdf, 0xfd, 0xf4, 0x0c, 0xf8, 0xcd, + 0xe8, 0xb1, 0x3b, 0xfb, 0xd3, 0x85, 0xf7, 0x57, 0xa2, 0xdc, 0x06, 0xcf, 0x7c, 0x63, 0x6d, 0xae, + 0x86, 0x78, 0xee, 0xfe, 0xf6, 0xad, 0x65, 0x72, 0x51, 0x10, 0x9e, 0xc7, 0xa2, 0xce, 0x93, 0x9c, + 0x72, 0x3d, 0xe2, 0x89, 0x49, 0x91, 0x8a, 0x35, 0xff, 0xfb, 0xcb, 0x3f, 0x19, 0x16, 0x7f, 0x8d, + 0x3e, 0xf8, 0xc9, 0xc8, 0x9f, 0x15, 0xa2, 0xcd, 0xe2, 0x1f, 0x86, 0x8d, 0xae, 0x4e, 0xff, 0xee, + 0x73, 0xd7, 0x3a, 0x77, 0x3d, 0xe4, 0xae, 0xaf, 0x4e, 0x97, 0xbb, 0x7a, 0x83, 0xaf, 0xfe, 0x0b, + 0x00, 0x00, 0xff, 0xff, 0xf3, 0xdd, 0x11, 0x96, 0x45, 0x06, 0x00, 0x00, +} + +var xxx_messageInfo_Key proto.InternalMessageInfo From a55d898aab1c870589fb18d898fca1cfbba713f0 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 2 May 2019 13:48:16 -0400 Subject: [PATCH 28/35] renaming file --- datastore/keyDecodeCodeDup.go | 431 ---------------------------------- 1 file changed, 431 deletions(-) delete mode 100644 datastore/keyDecodeCodeDup.go diff --git a/datastore/keyDecodeCodeDup.go b/datastore/keyDecodeCodeDup.go deleted file mode 100644 index e9ccc9ef..00000000 --- a/datastore/keyDecodeCodeDup.go +++ /dev/null @@ -1,431 +0,0 @@ -package datastore - -import ( - "encoding/base64" - "fmt" - "strings" - - "github.com/golang/protobuf/proto" -) - -// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. -// decodeToNewKey decodes a key from the opaque representation returned by Encode. -func decodeToNewKey(encoded string) (*NewFormatKey, error) { - // Re-add padding. - if m := len(encoded) % 4; m != 0 { - encoded += strings.Repeat("=", 4-m) - } - - b, err := base64.URLEncoding.DecodeString(encoded) - if err != nil { - return nil, err - } - - pKey := new(PBKey) - if err := proto.Unmarshal(b, pKey); err != nil { - return nil, err - } - return cgdProtoToKey(pKey) -} - -// protoToKey decodes a protocol buffer representation of a key into an -// equivalent *Key object. If the key is invalid, protoToKey will return the -// invalid key along with ErrInvalidKey. -func cgdProtoToKey(p *PBKey) (*NewFormatKey, error) { - var key *NewFormatKey - var namespace string - if partition := p.PartitionId; partition != nil { - namespace = partition.NamespaceId - } - for _, el := range p.Path { - key = &NewFormatKey{ - Namespace: namespace, - Kind: el.Kind, - ID: el.GetId(), - Name: el.GetName(), - Parent: key, - } - } - if !key.valid() { // Also detects key == nil. - return key, ErrInvalidKey - } - return key, nil -} - -// valid returns whether the key is valid. -func (k *NewFormatKey) valid() bool { - if k == nil { - return false - } - for ; k != nil; k = k.Parent { - if k.Kind == "" { - return false - } - if k.Name != "" && k.ID != 0 { - return false - } - if k.Parent != nil { - if k.Parent.Incomplete() { - return false - } - if k.Parent.Namespace != k.Namespace { - return false - } - } - } - return true -} - -// Incomplete reports whether the key does not refer to a stored entity. -func (k *NewFormatKey) Incomplete() bool { - return k.Name == "" && k.ID == 0 -} - -// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. -// Key represents the datastore key for a stored entity. -type NewFormatKey struct { - // Kind cannot be empty. - Kind string - // Either ID or Name must be zero for the Key to be valid. - // If both are zero, the Key is incomplete. - ID int64 - Name string - // Parent must either be a complete Key or nil. - Parent *NewFormatKey - - // Namespace provides the ability to partition your data for multiple - // tenants. In most cases, it is not necessary to specify a namespace. - // See docs on datastore multitenancy for details: - // https://cloud.google.com/datastore/docs/concepts/multitenancy - Namespace string -} - -// A partition ID identifies a grouping of entities. The grouping is always -// by project and namespace, however the namespace ID may be empty. -// -// A partition ID contains several dimensions: -// project ID and namespace ID. -// -// Partition dimensions: -// -// - May be `""`. -// - Must be valid UTF-8 bytes. -// - Must have values that match regex `[A-Za-z\d\.\-_]{1,100}` -// If the value of any dimension matches regex `__.*__`, the partition is -// reserved/read-only. -// A reserved/read-only partition ID is forbidden in certain documented -// contexts. -// -// Foreign partition IDs (in which the project ID does -// not match the context project ID ) are discouraged. -// Reads and writes of foreign partition IDs may fail if the project is not in -// an active state. -type PartitionId struct { - // The ID of the project to which the entities belong. - ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` - // If not empty, the ID of the namespace to which the entities belong. - NamespaceId string `protobuf:"bytes,4,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PartitionId) Reset() { *m = PartitionId{} } -func (m *PartitionId) String() string { return proto.CompactTextString(m) } -func (*PartitionId) ProtoMessage() {} -func (*PartitionId) Descriptor() ([]byte, []int) { - return fileDescriptor_entity_096a297364b049a5, []int{0} -} -func (m *PartitionId) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PartitionId.Unmarshal(m, b) -} -func (m *PartitionId) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PartitionId.Marshal(b, m, deterministic) -} -func (dst *PartitionId) XXX_Merge(src proto.Message) { - xxx_messageInfo_PartitionId.Merge(dst, src) -} -func (m *PartitionId) XXX_Size() int { - return xxx_messageInfo_PartitionId.Size(m) -} -func (m *PartitionId) XXX_DiscardUnknown() { - xxx_messageInfo_PartitionId.DiscardUnknown(m) -} - -var xxx_messageInfo_PartitionId proto.InternalMessageInfo - -func (m *PartitionId) GetProjectId() string { - if m != nil { - return m.ProjectId - } - return "" -} - -func (m *PartitionId) GetNamespaceId() string { - if m != nil { - return m.NamespaceId - } - return "" -} - -// A unique identifier for an entity. -// If a key's partition ID or any of its path kinds or names are -// reserved/read-only, the key is reserved/read-only. -// A reserved/read-only key is forbidden in certain documented contexts. -type PBKey struct { - // Entities are partitioned into subsets, currently identified by a project - // ID and namespace ID. - // Queries are scoped to a single partition. - PartitionId *PartitionId `protobuf:"bytes,1,opt,name=partition_id,json=partitionId,proto3" json:"partition_id,omitempty"` - // The entity path. - // An entity path consists of one or more elements composed of a kind and a - // string or numerical identifier, which identify entities. The first - // element identifies a _root entity_, the second element identifies - // a _child_ of the root entity, the third element identifies a child of the - // second entity, and so forth. The entities identified by all prefixes of - // the path are called the element's _ancestors_. - // - // An entity path is always fully complete: *all* of the entity's ancestors - // are required to be in the path along with the entity identifier itself. - // The only exception is that in some documented cases, the identifier in the - // last path element (for the entity) itself may be omitted. For example, - // the last path element of the key of `Mutation.insert` may have no - // identifier. - // - // A path can never be empty, and a path can have at most 100 elements. - Path []*Key_PathElement `protobuf:"bytes,2,rep,name=path,proto3" json:"path,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PBKey) Reset() { *m = PBKey{} } -func (m *PBKey) String() string { return proto.CompactTextString(m) } -func (*PBKey) ProtoMessage() {} -func (*PBKey) Descriptor() ([]byte, []int) { - return fileDescriptor_entity_096a297364b049a5, []int{1} -} -func (m *PBKey) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Key.Unmarshal(m, b) -} -func (m *PBKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Key.Marshal(b, m, deterministic) -} -func (dst *PBKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_Key.Merge(dst, src) -} -func (m *PBKey) XXX_Size() int { - return xxx_messageInfo_Key.Size(m) -} -func (m *PBKey) XXX_DiscardUnknown() { - xxx_messageInfo_Key.DiscardUnknown(m) -} - -// A (kind, ID/name) pair used to construct a key path. -// -// If either name or ID is set, the element is complete. -// If neither is set, the element is incomplete. -type Key_PathElement struct { - // The kind of the entity. - // A kind matching regex `__.*__` is reserved/read-only. - // A kind must not contain more than 1500 bytes when UTF-8 encoded. - // Cannot be `""`. - Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"` - // The type of ID. - // - // Types that are valid to be assigned to IdType: - // *Key_PathElement_Id - // *Key_PathElement_Name - IdType isKey_PathElement_IdType `protobuf_oneof:"id_type"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Key_PathElement) Reset() { *m = Key_PathElement{} } -func (m *Key_PathElement) String() string { return proto.CompactTextString(m) } -func (*Key_PathElement) ProtoMessage() {} -func (*Key_PathElement) Descriptor() ([]byte, []int) { - return fileDescriptor_entity_096a297364b049a5, []int{1, 0} -} -func (m *Key_PathElement) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Key_PathElement.Unmarshal(m, b) -} -func (m *Key_PathElement) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Key_PathElement.Marshal(b, m, deterministic) -} -func (dst *Key_PathElement) XXX_Merge(src proto.Message) { - xxx_messageInfo_Key_PathElement.Merge(dst, src) -} -func (m *Key_PathElement) XXX_Size() int { - return xxx_messageInfo_Key_PathElement.Size(m) -} -func (m *Key_PathElement) XXX_DiscardUnknown() { - xxx_messageInfo_Key_PathElement.DiscardUnknown(m) -} - -var xxx_messageInfo_Key_PathElement proto.InternalMessageInfo - -func (m *Key_PathElement) GetKind() string { - if m != nil { - return m.Kind - } - return "" -} - -type isKey_PathElement_IdType interface { - isKey_PathElement_IdType() -} - -type Key_PathElement_Id struct { - Id int64 `protobuf:"varint,2,opt,name=id,proto3,oneof"` -} - -type Key_PathElement_Name struct { - Name string `protobuf:"bytes,3,opt,name=name,proto3,oneof"` -} - -func (*Key_PathElement_Id) isKey_PathElement_IdType() {} - -func (*Key_PathElement_Name) isKey_PathElement_IdType() {} - -func (m *Key_PathElement) GetIdType() isKey_PathElement_IdType { - if m != nil { - return m.IdType - } - return nil -} - -func (m *Key_PathElement) GetId() int64 { - if x, ok := m.GetIdType().(*Key_PathElement_Id); ok { - return x.Id - } - return 0 -} - -func (m *Key_PathElement) GetName() string { - if x, ok := m.GetIdType().(*Key_PathElement_Name); ok { - return x.Name - } - return "" -} - -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Key_PathElement) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Key_PathElement_OneofMarshaler, _Key_PathElement_OneofUnmarshaler, _Key_PathElement_OneofSizer, []interface{}{ - (*Key_PathElement_Id)(nil), - (*Key_PathElement_Name)(nil), - } -} - -func _Key_PathElement_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Key_PathElement) - // id_type - switch x := m.IdType.(type) { - case *Key_PathElement_Id: - b.EncodeVarint(2<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.Id)) - case *Key_PathElement_Name: - b.EncodeVarint(3<<3 | proto.WireBytes) - b.EncodeStringBytes(x.Name) - case nil: - default: - return fmt.Errorf("Key_PathElement.IdType has unexpected type %T", x) - } - return nil -} - -func _Key_PathElement_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Key_PathElement) - switch tag { - case 2: // id_type.id - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.IdType = &Key_PathElement_Id{int64(x)} - return true, err - case 3: // id_type.name - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.IdType = &Key_PathElement_Name{x} - return true, err - default: - return false, nil - } -} - -func _Key_PathElement_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Key_PathElement) - // id_type - switch x := m.IdType.(type) { - case *Key_PathElement_Id: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(x.Id)) - case *Key_PathElement_Name: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.Name))) - n += len(x.Name) - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - -var fileDescriptor_entity_096a297364b049a5 = []byte{ - // 780 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xff, 0x6e, 0xdc, 0x44, - 0x10, 0xc7, 0xed, 0xbb, 0x5c, 0x1a, 0x8f, 0xdd, 0xa4, 0x6c, 0x2a, 0x61, 0x02, 0x28, 0x26, 0x80, - 0x74, 0x02, 0xc9, 0x6e, 0xc2, 0x1f, 0x54, 0x14, 0xa4, 0x72, 0x25, 0xe0, 0x28, 0x15, 0x9c, 0x56, - 0x55, 0x24, 0x50, 0xa4, 0xd3, 0xde, 0x79, 0xeb, 0x2e, 0x67, 0xef, 0x5a, 0xf6, 0x3a, 0xaa, 0xdf, - 0x05, 0xf1, 0x00, 0x3c, 0x0a, 0x8f, 0x80, 0x78, 0x18, 0xb4, 0x3f, 0xec, 0x0b, 0xed, 0x35, 0xff, - 0x79, 0x67, 0x3e, 0xdf, 0xd9, 0xef, 0xec, 0xce, 0x1a, 0xa2, 0x5c, 0x88, 0xbc, 0xa0, 0x49, 0x46, - 0x24, 0x69, 0xa4, 0xa8, 0x69, 0x72, 0x73, 0x9a, 0x50, 0x2e, 0x99, 0xec, 0xe2, 0xaa, 0x16, 0x52, - 0xa0, 0x43, 0x43, 0xc4, 0x03, 0x11, 0xdf, 0x9c, 0x1e, 0x7d, 0x64, 0x65, 0xa4, 0x62, 0x09, 0xe1, - 0x5c, 0x48, 0x22, 0x99, 0xe0, 0x8d, 0x91, 0x0c, 0x59, 0xbd, 0x5a, 0xb6, 0x2f, 0x93, 0x46, 0xd6, - 0xed, 0x4a, 0xda, 0xec, 0xf1, 0x9b, 0x59, 0xc9, 0x4a, 0xda, 0x48, 0x52, 0x56, 0x16, 0x08, 0x2d, - 0x20, 0xbb, 0x8a, 0x26, 0x05, 0x91, 0x05, 0xcf, 0x4d, 0xe6, 0xe4, 0x17, 0xf0, 0xe7, 0xa4, 0x96, - 0x4c, 0x6d, 0x76, 0x91, 0xa1, 0x8f, 0x01, 0xaa, 0x5a, 0xfc, 0x4e, 0x57, 0x72, 0xc1, 0xb2, 0x70, - 0x14, 0xb9, 0x53, 0x0f, 0x7b, 0x36, 0x72, 0x91, 0xa1, 0x4f, 0x20, 0xe0, 0xa4, 0xa4, 0x4d, 0x45, - 0x56, 0x54, 0x01, 0x3b, 0x1a, 0xf0, 0x87, 0xd8, 0x45, 0x76, 0xf2, 0x8f, 0x0b, 0xe3, 0x4b, 0xda, - 0xa1, 0x67, 0x10, 0x54, 0x7d, 0x61, 0x85, 0xba, 0x91, 0x3b, 0xf5, 0xcf, 0xa2, 0x78, 0x4b, 0xef, - 0xf1, 0x2d, 0x07, 0xd8, 0xaf, 0x6e, 0xd9, 0x79, 0x0c, 0x3b, 0x15, 0x91, 0xaf, 0xc2, 0x51, 0x34, - 0x9e, 0xfa, 0x67, 0x9f, 0x6d, 0x15, 0x5f, 0xd2, 0x2e, 0x9e, 0x13, 0xf9, 0xea, 0xbc, 0xa0, 0x25, - 0xe5, 0x12, 0x6b, 0xc5, 0xd1, 0x0b, 0xd5, 0xd7, 0x10, 0x44, 0x08, 0x76, 0xd6, 0x8c, 0x1b, 0x17, - 0x1e, 0xd6, 0xdf, 0xe8, 0x01, 0x8c, 0x6c, 0x8f, 0xe3, 0xd4, 0xc1, 0x23, 0x96, 0xa1, 0x87, 0xb0, - 0xa3, 0x5a, 0x09, 0xc7, 0x8a, 0x4a, 0x1d, 0xac, 0x57, 0x33, 0x0f, 0xee, 0xb1, 0x6c, 0xa1, 0x8e, - 0xee, 0xe4, 0x29, 0xc0, 0xf7, 0x75, 0x4d, 0xba, 0x2b, 0x52, 0xb4, 0x14, 0x9d, 0xc1, 0xee, 0x8d, - 0xfa, 0x68, 0x42, 0x57, 0xfb, 0x3b, 0xda, 0xea, 0x4f, 0xb3, 0xd8, 0x92, 0x27, 0x7f, 0x4c, 0x60, - 0x62, 0xd4, 0x4f, 0x00, 0x78, 0x5b, 0x14, 0x0b, 0x9d, 0x08, 0xfd, 0xc8, 0x9d, 0xee, 0x6f, 0x2a, - 0xf4, 0x37, 0x19, 0xff, 0xdc, 0x16, 0x85, 0xe6, 0x53, 0x07, 0x7b, 0xbc, 0x5f, 0xa0, 0xcf, 0xe1, - 0xfe, 0x52, 0x88, 0x82, 0x12, 0x6e, 0xf5, 0xaa, 0xb1, 0xbd, 0xd4, 0xc1, 0x81, 0x0d, 0x0f, 0x18, - 0xe3, 0x92, 0xe6, 0xb4, 0xb6, 0x58, 0xdf, 0x6d, 0x60, 0xc3, 0x06, 0xfb, 0x14, 0x82, 0x4c, 0xb4, - 0xcb, 0x82, 0x5a, 0x4a, 0xf5, 0xef, 0xa6, 0x0e, 0xf6, 0x4d, 0xd4, 0x40, 0xe7, 0x70, 0x30, 0x8c, - 0x95, 0xe5, 0x40, 0xdf, 0xe9, 0xdb, 0xa6, 0x5f, 0xf4, 0x5c, 0xea, 0xe0, 0xfd, 0x41, 0x64, 0xca, - 0x7c, 0x0d, 0xde, 0x9a, 0x76, 0xb6, 0xc0, 0x44, 0x17, 0x08, 0xdf, 0x75, 0xaf, 0xa9, 0x83, 0xf7, - 0xd6, 0xb4, 0x1b, 0x4c, 0x36, 0xb2, 0x66, 0x3c, 0xb7, 0xda, 0xf7, 0xec, 0x25, 0xf9, 0x26, 0x6a, - 0xa0, 0x63, 0x80, 0x65, 0x21, 0x96, 0x16, 0x41, 0x91, 0x3b, 0x0d, 0xd4, 0xc1, 0xa9, 0x98, 0x01, - 0xbe, 0x83, 0x83, 0x9c, 0x8a, 0x45, 0x25, 0x18, 0x97, 0x96, 0xda, 0xd3, 0x26, 0x0e, 0x7b, 0x13, - 0xea, 0xa2, 0xe3, 0xe7, 0x44, 0x3e, 0xe7, 0x79, 0xea, 0xe0, 0xfb, 0x39, 0x15, 0x73, 0x05, 0x1b, - 0xf9, 0x53, 0x08, 0xcc, 0x53, 0xb6, 0xda, 0x5d, 0xad, 0xfd, 0x70, 0x6b, 0x03, 0xe7, 0x1a, 0x54, - 0x0e, 0x8d, 0xc4, 0x54, 0x98, 0x81, 0x4f, 0xd4, 0x08, 0xd9, 0x02, 0x9e, 0x2e, 0x70, 0xbc, 0xb5, - 0xc0, 0x66, 0xd4, 0x52, 0x07, 0x03, 0xd9, 0x0c, 0x5e, 0x08, 0xf7, 0x4a, 0x4a, 0x38, 0xe3, 0x79, - 0xb8, 0x1f, 0xb9, 0xd3, 0x09, 0xee, 0x97, 0xe8, 0x11, 0x3c, 0xa4, 0xaf, 0x57, 0x45, 0x9b, 0xd1, - 0xc5, 0xcb, 0x5a, 0x94, 0x0b, 0xc6, 0x33, 0xfa, 0x9a, 0x36, 0xe1, 0xa1, 0x1a, 0x0f, 0x8c, 0x6c, - 0xee, 0xc7, 0x5a, 0x94, 0x17, 0x26, 0x33, 0x0b, 0x00, 0xb4, 0x13, 0x33, 0xe0, 0xff, 0xba, 0xb0, - 0x6b, 0x7c, 0xa3, 0x2f, 0x60, 0xbc, 0xa6, 0x9d, 0x7d, 0xb7, 0xef, 0xbc, 0x22, 0xac, 0x20, 0x74, - 0xa9, 0x7f, 0x1b, 0x15, 0xad, 0x25, 0xa3, 0x4d, 0x38, 0xd6, 0xaf, 0xe1, 0xcb, 0x3b, 0x0e, 0x25, - 0x9e, 0x0f, 0xf4, 0x39, 0x97, 0x75, 0x87, 0x6f, 0xc9, 0x8f, 0x7e, 0x85, 0x83, 0x37, 0xd2, 0xe8, - 0xc1, 0xc6, 0x8b, 0x67, 0x76, 0x7c, 0x04, 0x93, 0xcd, 0x44, 0xdf, 0xfd, 0xf4, 0x0c, 0xf8, 0xcd, - 0xe8, 0xb1, 0x3b, 0xfb, 0xd3, 0x85, 0xf7, 0x57, 0xa2, 0xdc, 0x06, 0xcf, 0x7c, 0x63, 0x6d, 0xae, - 0x86, 0x78, 0xee, 0xfe, 0xf6, 0xad, 0x65, 0x72, 0x51, 0x10, 0x9e, 0xc7, 0xa2, 0xce, 0x93, 0x9c, - 0x72, 0x3d, 0xe2, 0x89, 0x49, 0x91, 0x8a, 0x35, 0xff, 0xfb, 0xcb, 0x3f, 0x19, 0x16, 0x7f, 0x8d, - 0x3e, 0xf8, 0xc9, 0xc8, 0x9f, 0x15, 0xa2, 0xcd, 0xe2, 0x1f, 0x86, 0x8d, 0xae, 0x4e, 0xff, 0xee, - 0x73, 0xd7, 0x3a, 0x77, 0x3d, 0xe4, 0xae, 0xaf, 0x4e, 0x97, 0xbb, 0x7a, 0x83, 0xaf, 0xfe, 0x0b, - 0x00, 0x00, 0xff, 0xff, 0xf3, 0xdd, 0x11, 0x96, 0x45, 0x06, 0x00, 0x00, -} - -var xxx_messageInfo_Key proto.InternalMessageInfo From aca5ca69168d7cb5828e768f1be6438dc6ea5343 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 2 May 2019 15:37:40 -0400 Subject: [PATCH 29/35] restructured files to new internal package --- datastore/go.mod | 7 - datastore/go.sum | 6 - datastore/key.go | 9 +- datastore/keycompat_test.go | 2 +- datastore/keycompatibility.go | 46 ++-- datastore/keydecode_code_dup.go | 431 -------------------------------- 6 files changed, 24 insertions(+), 477 deletions(-) delete mode 100644 datastore/go.mod delete mode 100644 datastore/go.sum delete mode 100644 datastore/keydecode_code_dup.go diff --git a/datastore/go.mod b/datastore/go.mod deleted file mode 100644 index f449359d..00000000 --- a/datastore/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module google.golang.org/appengine - -require ( - github.com/golang/protobuf v1.2.0 - golang.org/x/net v0.0.0-20180724234803-3673e40ba225 - golang.org/x/text v0.3.0 -) diff --git a/datastore/go.sum b/datastore/go.sum deleted file mode 100644 index a2ebdfd5..00000000 --- a/datastore/go.sum +++ /dev/null @@ -1,6 +0,0 @@ -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= \ No newline at end of file diff --git a/datastore/key.go b/datastore/key.go index 7befa8e4..a4f640be 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -16,10 +16,9 @@ import ( "github.com/golang/protobuf/proto" "golang.org/x/net/context" + "google.golang.org/appengine/datastore/internal/keycompat" "google.golang.org/appengine/internal" pb "google.golang.org/appengine/internal/datastore" - // Adding Support for the new key encoding format in cloud.google.com/go/datastore - //newds "cloud.google.com/go/datastore" ) type KeyRangeCollisionError struct { @@ -258,13 +257,13 @@ func DecodeKey(encoded string) (*Key, error) { if err := proto.Unmarshal(b, ref); err != nil { // If there was an err on the key lets try to decode the new key type by default check to see if key converter // has been implemented. - if convKey != nil { - nKey, nLibKeyErr := decodeToNewKey(encoded) + if keyConversionProject == "" { + nKey, nLibKeyErr := keycompat.DecodeToNewKey(encoded) if nLibKeyErr != nil { // returning the orginal error on purpose return nil, err } - oKey, err := convKey.convertNewKeyFormatToOldKeyFormat(nKey) + oKey, err := convertNewKeyFormatToOldKeyFormat(nKey) if err != nil { return nil, err } diff --git a/datastore/keycompat_test.go b/datastore/keycompat_test.go index 169bdf38..40477e5f 100644 --- a/datastore/keycompat_test.go +++ b/datastore/keycompat_test.go @@ -72,7 +72,7 @@ var testCasesKeyCompat = []struct { func TestKeyCoversion(t *testing.T) { // Simulate the key converter enablement - convKey = &keyConverter{appid: "glibrary"} + keyConversionProject = "glibrary" for _, tc := range testCasesKeyCompat { dk, err := DecodeKey(tc.encodedKey) diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index 9bdc9d57..390b89a4 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -3,14 +3,14 @@ // license that can be found in the LICENSE file. // This change adds additional compatibility to help customers -// transition from google.golang.org/appengine/datastore (oldds) to cloud.google.com/go/datastore (newds). -// Each lib (oldds and newds) contain the functions Key.Encode() and Key.Decode(). These functions +// transition from google.golang.org/appengine/datastore to cloud.google.com/go/datastore. +// Each lib google.golang.org/appengine/datastore and cloud.google.com/go/datastore contain the functions Key.Encode() and Key.Decode(). These functions // create base64 representations of a json marshalled type of datastore keys. Customers have been using // these encoded values to communicate between services in appengine. The protobuf key types -// change between oldds and newds making the corresponding base64 key strings incompatible. -// Customers who attempt to upgrade to newds that use this pattern will fail. -// keycompatibility.go placed in oldds enables forward compatibility of newds encoded keys. -// An update to newds will also be necessary to enable backward compatibility. +// changed between google.golang.org/appengine/datastore and cloud.google.com/go/datastore making the corresponding base64 key strings incompatible. +// Customers who attempt to upgrade to cloud.google.com/go/datastore that use this pattern will fail. +// keycompatibility.go placed in google.golang.org/appengine/datastore enables forward compatibility of cloud.google.com/go/datastore encoded keys. +// An update to cloud.google.com/go/datastore will also be necessary to enable backward compatibility. package datastore @@ -19,45 +19,37 @@ import ( "sync" "golang.org/x/net/context" + + "google.golang.org/appengine/datastore/internal/keycompat" "google.golang.org/appengine/internal" ) -var errKeyConversion = `Key conversions must be enabled in the application. -See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.` - -var convKey *keyConverter -var once sync.Once +var ( + errKeyConversion = `Key conversions must be enabled in the application. + See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.` + keyConversionProject string + once sync.Once +) // EnableKeyConversion enables forward key conversion abilities. Calling this function in a single handler will enable // the feature for all handlers. This function can be called in the /_ah/start handler. Support for key converstion. // Variable holds the appid so that key conversion can retrieve it without a context. func EnableKeyConversion(ctx context.Context) { - once.Do(func() { - convKey = &keyConverter{ - appid: internal.FullyQualifiedAppID(ctx), - } - + keyConversionProject = internal.FullyQualifiedAppID(ctx) }) - - return -} - -// keyConverter is the struct used to hold the appid for the process of key conversion -type keyConverter struct { - appid string } // convertKey takes at new datastore key type and returns a old key type -func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *NewFormatKey) (*Key, error) { +func convertNewKeyFormatToOldKeyFormat(key *keycompat.NewFormatKey) (*Key, error) { // if key conversion is not enabled return right away - if c == nil { + if keyConversionProject == "" { return nil, errors.New(errKeyConversion) } var pKey *Key var err error if key.Parent != nil { - pKey, err = c.convertNewKeyFormatToOldKeyFormat(key.Parent) + pKey, err = convertNewKeyFormatToOldKeyFormat(key.Parent) if err != nil { return nil, err } @@ -68,6 +60,6 @@ func (c *keyConverter) convertNewKeyFormatToOldKeyFormat(key *NewFormatKey) (*Ke namespace: key.Namespace, parent: pKey, stringID: key.Name, - appID: c.appid, + appID: keyConversionProject, }, nil } diff --git a/datastore/keydecode_code_dup.go b/datastore/keydecode_code_dup.go deleted file mode 100644 index e9ccc9ef..00000000 --- a/datastore/keydecode_code_dup.go +++ /dev/null @@ -1,431 +0,0 @@ -package datastore - -import ( - "encoding/base64" - "fmt" - "strings" - - "github.com/golang/protobuf/proto" -) - -// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. -// decodeToNewKey decodes a key from the opaque representation returned by Encode. -func decodeToNewKey(encoded string) (*NewFormatKey, error) { - // Re-add padding. - if m := len(encoded) % 4; m != 0 { - encoded += strings.Repeat("=", 4-m) - } - - b, err := base64.URLEncoding.DecodeString(encoded) - if err != nil { - return nil, err - } - - pKey := new(PBKey) - if err := proto.Unmarshal(b, pKey); err != nil { - return nil, err - } - return cgdProtoToKey(pKey) -} - -// protoToKey decodes a protocol buffer representation of a key into an -// equivalent *Key object. If the key is invalid, protoToKey will return the -// invalid key along with ErrInvalidKey. -func cgdProtoToKey(p *PBKey) (*NewFormatKey, error) { - var key *NewFormatKey - var namespace string - if partition := p.PartitionId; partition != nil { - namespace = partition.NamespaceId - } - for _, el := range p.Path { - key = &NewFormatKey{ - Namespace: namespace, - Kind: el.Kind, - ID: el.GetId(), - Name: el.GetName(), - Parent: key, - } - } - if !key.valid() { // Also detects key == nil. - return key, ErrInvalidKey - } - return key, nil -} - -// valid returns whether the key is valid. -func (k *NewFormatKey) valid() bool { - if k == nil { - return false - } - for ; k != nil; k = k.Parent { - if k.Kind == "" { - return false - } - if k.Name != "" && k.ID != 0 { - return false - } - if k.Parent != nil { - if k.Parent.Incomplete() { - return false - } - if k.Parent.Namespace != k.Namespace { - return false - } - } - } - return true -} - -// Incomplete reports whether the key does not refer to a stored entity. -func (k *NewFormatKey) Incomplete() bool { - return k.Name == "" && k.ID == 0 -} - -// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. -// Key represents the datastore key for a stored entity. -type NewFormatKey struct { - // Kind cannot be empty. - Kind string - // Either ID or Name must be zero for the Key to be valid. - // If both are zero, the Key is incomplete. - ID int64 - Name string - // Parent must either be a complete Key or nil. - Parent *NewFormatKey - - // Namespace provides the ability to partition your data for multiple - // tenants. In most cases, it is not necessary to specify a namespace. - // See docs on datastore multitenancy for details: - // https://cloud.google.com/datastore/docs/concepts/multitenancy - Namespace string -} - -// A partition ID identifies a grouping of entities. The grouping is always -// by project and namespace, however the namespace ID may be empty. -// -// A partition ID contains several dimensions: -// project ID and namespace ID. -// -// Partition dimensions: -// -// - May be `""`. -// - Must be valid UTF-8 bytes. -// - Must have values that match regex `[A-Za-z\d\.\-_]{1,100}` -// If the value of any dimension matches regex `__.*__`, the partition is -// reserved/read-only. -// A reserved/read-only partition ID is forbidden in certain documented -// contexts. -// -// Foreign partition IDs (in which the project ID does -// not match the context project ID ) are discouraged. -// Reads and writes of foreign partition IDs may fail if the project is not in -// an active state. -type PartitionId struct { - // The ID of the project to which the entities belong. - ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` - // If not empty, the ID of the namespace to which the entities belong. - NamespaceId string `protobuf:"bytes,4,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PartitionId) Reset() { *m = PartitionId{} } -func (m *PartitionId) String() string { return proto.CompactTextString(m) } -func (*PartitionId) ProtoMessage() {} -func (*PartitionId) Descriptor() ([]byte, []int) { - return fileDescriptor_entity_096a297364b049a5, []int{0} -} -func (m *PartitionId) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PartitionId.Unmarshal(m, b) -} -func (m *PartitionId) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PartitionId.Marshal(b, m, deterministic) -} -func (dst *PartitionId) XXX_Merge(src proto.Message) { - xxx_messageInfo_PartitionId.Merge(dst, src) -} -func (m *PartitionId) XXX_Size() int { - return xxx_messageInfo_PartitionId.Size(m) -} -func (m *PartitionId) XXX_DiscardUnknown() { - xxx_messageInfo_PartitionId.DiscardUnknown(m) -} - -var xxx_messageInfo_PartitionId proto.InternalMessageInfo - -func (m *PartitionId) GetProjectId() string { - if m != nil { - return m.ProjectId - } - return "" -} - -func (m *PartitionId) GetNamespaceId() string { - if m != nil { - return m.NamespaceId - } - return "" -} - -// A unique identifier for an entity. -// If a key's partition ID or any of its path kinds or names are -// reserved/read-only, the key is reserved/read-only. -// A reserved/read-only key is forbidden in certain documented contexts. -type PBKey struct { - // Entities are partitioned into subsets, currently identified by a project - // ID and namespace ID. - // Queries are scoped to a single partition. - PartitionId *PartitionId `protobuf:"bytes,1,opt,name=partition_id,json=partitionId,proto3" json:"partition_id,omitempty"` - // The entity path. - // An entity path consists of one or more elements composed of a kind and a - // string or numerical identifier, which identify entities. The first - // element identifies a _root entity_, the second element identifies - // a _child_ of the root entity, the third element identifies a child of the - // second entity, and so forth. The entities identified by all prefixes of - // the path are called the element's _ancestors_. - // - // An entity path is always fully complete: *all* of the entity's ancestors - // are required to be in the path along with the entity identifier itself. - // The only exception is that in some documented cases, the identifier in the - // last path element (for the entity) itself may be omitted. For example, - // the last path element of the key of `Mutation.insert` may have no - // identifier. - // - // A path can never be empty, and a path can have at most 100 elements. - Path []*Key_PathElement `protobuf:"bytes,2,rep,name=path,proto3" json:"path,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PBKey) Reset() { *m = PBKey{} } -func (m *PBKey) String() string { return proto.CompactTextString(m) } -func (*PBKey) ProtoMessage() {} -func (*PBKey) Descriptor() ([]byte, []int) { - return fileDescriptor_entity_096a297364b049a5, []int{1} -} -func (m *PBKey) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Key.Unmarshal(m, b) -} -func (m *PBKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Key.Marshal(b, m, deterministic) -} -func (dst *PBKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_Key.Merge(dst, src) -} -func (m *PBKey) XXX_Size() int { - return xxx_messageInfo_Key.Size(m) -} -func (m *PBKey) XXX_DiscardUnknown() { - xxx_messageInfo_Key.DiscardUnknown(m) -} - -// A (kind, ID/name) pair used to construct a key path. -// -// If either name or ID is set, the element is complete. -// If neither is set, the element is incomplete. -type Key_PathElement struct { - // The kind of the entity. - // A kind matching regex `__.*__` is reserved/read-only. - // A kind must not contain more than 1500 bytes when UTF-8 encoded. - // Cannot be `""`. - Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"` - // The type of ID. - // - // Types that are valid to be assigned to IdType: - // *Key_PathElement_Id - // *Key_PathElement_Name - IdType isKey_PathElement_IdType `protobuf_oneof:"id_type"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Key_PathElement) Reset() { *m = Key_PathElement{} } -func (m *Key_PathElement) String() string { return proto.CompactTextString(m) } -func (*Key_PathElement) ProtoMessage() {} -func (*Key_PathElement) Descriptor() ([]byte, []int) { - return fileDescriptor_entity_096a297364b049a5, []int{1, 0} -} -func (m *Key_PathElement) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Key_PathElement.Unmarshal(m, b) -} -func (m *Key_PathElement) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Key_PathElement.Marshal(b, m, deterministic) -} -func (dst *Key_PathElement) XXX_Merge(src proto.Message) { - xxx_messageInfo_Key_PathElement.Merge(dst, src) -} -func (m *Key_PathElement) XXX_Size() int { - return xxx_messageInfo_Key_PathElement.Size(m) -} -func (m *Key_PathElement) XXX_DiscardUnknown() { - xxx_messageInfo_Key_PathElement.DiscardUnknown(m) -} - -var xxx_messageInfo_Key_PathElement proto.InternalMessageInfo - -func (m *Key_PathElement) GetKind() string { - if m != nil { - return m.Kind - } - return "" -} - -type isKey_PathElement_IdType interface { - isKey_PathElement_IdType() -} - -type Key_PathElement_Id struct { - Id int64 `protobuf:"varint,2,opt,name=id,proto3,oneof"` -} - -type Key_PathElement_Name struct { - Name string `protobuf:"bytes,3,opt,name=name,proto3,oneof"` -} - -func (*Key_PathElement_Id) isKey_PathElement_IdType() {} - -func (*Key_PathElement_Name) isKey_PathElement_IdType() {} - -func (m *Key_PathElement) GetIdType() isKey_PathElement_IdType { - if m != nil { - return m.IdType - } - return nil -} - -func (m *Key_PathElement) GetId() int64 { - if x, ok := m.GetIdType().(*Key_PathElement_Id); ok { - return x.Id - } - return 0 -} - -func (m *Key_PathElement) GetName() string { - if x, ok := m.GetIdType().(*Key_PathElement_Name); ok { - return x.Name - } - return "" -} - -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Key_PathElement) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Key_PathElement_OneofMarshaler, _Key_PathElement_OneofUnmarshaler, _Key_PathElement_OneofSizer, []interface{}{ - (*Key_PathElement_Id)(nil), - (*Key_PathElement_Name)(nil), - } -} - -func _Key_PathElement_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Key_PathElement) - // id_type - switch x := m.IdType.(type) { - case *Key_PathElement_Id: - b.EncodeVarint(2<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.Id)) - case *Key_PathElement_Name: - b.EncodeVarint(3<<3 | proto.WireBytes) - b.EncodeStringBytes(x.Name) - case nil: - default: - return fmt.Errorf("Key_PathElement.IdType has unexpected type %T", x) - } - return nil -} - -func _Key_PathElement_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Key_PathElement) - switch tag { - case 2: // id_type.id - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.IdType = &Key_PathElement_Id{int64(x)} - return true, err - case 3: // id_type.name - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.IdType = &Key_PathElement_Name{x} - return true, err - default: - return false, nil - } -} - -func _Key_PathElement_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Key_PathElement) - // id_type - switch x := m.IdType.(type) { - case *Key_PathElement_Id: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(x.Id)) - case *Key_PathElement_Name: - n += 1 // tag and wire - n += proto.SizeVarint(uint64(len(x.Name))) - n += len(x.Name) - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - -var fileDescriptor_entity_096a297364b049a5 = []byte{ - // 780 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xff, 0x6e, 0xdc, 0x44, - 0x10, 0xc7, 0xed, 0xbb, 0x5c, 0x1a, 0x8f, 0xdd, 0xa4, 0x6c, 0x2a, 0x61, 0x02, 0x28, 0x26, 0x80, - 0x74, 0x02, 0xc9, 0x6e, 0xc2, 0x1f, 0x54, 0x14, 0xa4, 0x72, 0x25, 0xe0, 0x28, 0x15, 0x9c, 0x56, - 0x55, 0x24, 0x50, 0xa4, 0xd3, 0xde, 0x79, 0xeb, 0x2e, 0x67, 0xef, 0x5a, 0xf6, 0x3a, 0xaa, 0xdf, - 0x05, 0xf1, 0x00, 0x3c, 0x0a, 0x8f, 0x80, 0x78, 0x18, 0xb4, 0x3f, 0xec, 0x0b, 0xed, 0x35, 0xff, - 0x79, 0x67, 0x3e, 0xdf, 0xd9, 0xef, 0xec, 0xce, 0x1a, 0xa2, 0x5c, 0x88, 0xbc, 0xa0, 0x49, 0x46, - 0x24, 0x69, 0xa4, 0xa8, 0x69, 0x72, 0x73, 0x9a, 0x50, 0x2e, 0x99, 0xec, 0xe2, 0xaa, 0x16, 0x52, - 0xa0, 0x43, 0x43, 0xc4, 0x03, 0x11, 0xdf, 0x9c, 0x1e, 0x7d, 0x64, 0x65, 0xa4, 0x62, 0x09, 0xe1, - 0x5c, 0x48, 0x22, 0x99, 0xe0, 0x8d, 0x91, 0x0c, 0x59, 0xbd, 0x5a, 0xb6, 0x2f, 0x93, 0x46, 0xd6, - 0xed, 0x4a, 0xda, 0xec, 0xf1, 0x9b, 0x59, 0xc9, 0x4a, 0xda, 0x48, 0x52, 0x56, 0x16, 0x08, 0x2d, - 0x20, 0xbb, 0x8a, 0x26, 0x05, 0x91, 0x05, 0xcf, 0x4d, 0xe6, 0xe4, 0x17, 0xf0, 0xe7, 0xa4, 0x96, - 0x4c, 0x6d, 0x76, 0x91, 0xa1, 0x8f, 0x01, 0xaa, 0x5a, 0xfc, 0x4e, 0x57, 0x72, 0xc1, 0xb2, 0x70, - 0x14, 0xb9, 0x53, 0x0f, 0x7b, 0x36, 0x72, 0x91, 0xa1, 0x4f, 0x20, 0xe0, 0xa4, 0xa4, 0x4d, 0x45, - 0x56, 0x54, 0x01, 0x3b, 0x1a, 0xf0, 0x87, 0xd8, 0x45, 0x76, 0xf2, 0x8f, 0x0b, 0xe3, 0x4b, 0xda, - 0xa1, 0x67, 0x10, 0x54, 0x7d, 0x61, 0x85, 0xba, 0x91, 0x3b, 0xf5, 0xcf, 0xa2, 0x78, 0x4b, 0xef, - 0xf1, 0x2d, 0x07, 0xd8, 0xaf, 0x6e, 0xd9, 0x79, 0x0c, 0x3b, 0x15, 0x91, 0xaf, 0xc2, 0x51, 0x34, - 0x9e, 0xfa, 0x67, 0x9f, 0x6d, 0x15, 0x5f, 0xd2, 0x2e, 0x9e, 0x13, 0xf9, 0xea, 0xbc, 0xa0, 0x25, - 0xe5, 0x12, 0x6b, 0xc5, 0xd1, 0x0b, 0xd5, 0xd7, 0x10, 0x44, 0x08, 0x76, 0xd6, 0x8c, 0x1b, 0x17, - 0x1e, 0xd6, 0xdf, 0xe8, 0x01, 0x8c, 0x6c, 0x8f, 0xe3, 0xd4, 0xc1, 0x23, 0x96, 0xa1, 0x87, 0xb0, - 0xa3, 0x5a, 0x09, 0xc7, 0x8a, 0x4a, 0x1d, 0xac, 0x57, 0x33, 0x0f, 0xee, 0xb1, 0x6c, 0xa1, 0x8e, - 0xee, 0xe4, 0x29, 0xc0, 0xf7, 0x75, 0x4d, 0xba, 0x2b, 0x52, 0xb4, 0x14, 0x9d, 0xc1, 0xee, 0x8d, - 0xfa, 0x68, 0x42, 0x57, 0xfb, 0x3b, 0xda, 0xea, 0x4f, 0xb3, 0xd8, 0x92, 0x27, 0x7f, 0x4c, 0x60, - 0x62, 0xd4, 0x4f, 0x00, 0x78, 0x5b, 0x14, 0x0b, 0x9d, 0x08, 0xfd, 0xc8, 0x9d, 0xee, 0x6f, 0x2a, - 0xf4, 0x37, 0x19, 0xff, 0xdc, 0x16, 0x85, 0xe6, 0x53, 0x07, 0x7b, 0xbc, 0x5f, 0xa0, 0xcf, 0xe1, - 0xfe, 0x52, 0x88, 0x82, 0x12, 0x6e, 0xf5, 0xaa, 0xb1, 0xbd, 0xd4, 0xc1, 0x81, 0x0d, 0x0f, 0x18, - 0xe3, 0x92, 0xe6, 0xb4, 0xb6, 0x58, 0xdf, 0x6d, 0x60, 0xc3, 0x06, 0xfb, 0x14, 0x82, 0x4c, 0xb4, - 0xcb, 0x82, 0x5a, 0x4a, 0xf5, 0xef, 0xa6, 0x0e, 0xf6, 0x4d, 0xd4, 0x40, 0xe7, 0x70, 0x30, 0x8c, - 0x95, 0xe5, 0x40, 0xdf, 0xe9, 0xdb, 0xa6, 0x5f, 0xf4, 0x5c, 0xea, 0xe0, 0xfd, 0x41, 0x64, 0xca, - 0x7c, 0x0d, 0xde, 0x9a, 0x76, 0xb6, 0xc0, 0x44, 0x17, 0x08, 0xdf, 0x75, 0xaf, 0xa9, 0x83, 0xf7, - 0xd6, 0xb4, 0x1b, 0x4c, 0x36, 0xb2, 0x66, 0x3c, 0xb7, 0xda, 0xf7, 0xec, 0x25, 0xf9, 0x26, 0x6a, - 0xa0, 0x63, 0x80, 0x65, 0x21, 0x96, 0x16, 0x41, 0x91, 0x3b, 0x0d, 0xd4, 0xc1, 0xa9, 0x98, 0x01, - 0xbe, 0x83, 0x83, 0x9c, 0x8a, 0x45, 0x25, 0x18, 0x97, 0x96, 0xda, 0xd3, 0x26, 0x0e, 0x7b, 0x13, - 0xea, 0xa2, 0xe3, 0xe7, 0x44, 0x3e, 0xe7, 0x79, 0xea, 0xe0, 0xfb, 0x39, 0x15, 0x73, 0x05, 0x1b, - 0xf9, 0x53, 0x08, 0xcc, 0x53, 0xb6, 0xda, 0x5d, 0xad, 0xfd, 0x70, 0x6b, 0x03, 0xe7, 0x1a, 0x54, - 0x0e, 0x8d, 0xc4, 0x54, 0x98, 0x81, 0x4f, 0xd4, 0x08, 0xd9, 0x02, 0x9e, 0x2e, 0x70, 0xbc, 0xb5, - 0xc0, 0x66, 0xd4, 0x52, 0x07, 0x03, 0xd9, 0x0c, 0x5e, 0x08, 0xf7, 0x4a, 0x4a, 0x38, 0xe3, 0x79, - 0xb8, 0x1f, 0xb9, 0xd3, 0x09, 0xee, 0x97, 0xe8, 0x11, 0x3c, 0xa4, 0xaf, 0x57, 0x45, 0x9b, 0xd1, - 0xc5, 0xcb, 0x5a, 0x94, 0x0b, 0xc6, 0x33, 0xfa, 0x9a, 0x36, 0xe1, 0xa1, 0x1a, 0x0f, 0x8c, 0x6c, - 0xee, 0xc7, 0x5a, 0x94, 0x17, 0x26, 0x33, 0x0b, 0x00, 0xb4, 0x13, 0x33, 0xe0, 0xff, 0xba, 0xb0, - 0x6b, 0x7c, 0xa3, 0x2f, 0x60, 0xbc, 0xa6, 0x9d, 0x7d, 0xb7, 0xef, 0xbc, 0x22, 0xac, 0x20, 0x74, - 0xa9, 0x7f, 0x1b, 0x15, 0xad, 0x25, 0xa3, 0x4d, 0x38, 0xd6, 0xaf, 0xe1, 0xcb, 0x3b, 0x0e, 0x25, - 0x9e, 0x0f, 0xf4, 0x39, 0x97, 0x75, 0x87, 0x6f, 0xc9, 0x8f, 0x7e, 0x85, 0x83, 0x37, 0xd2, 0xe8, - 0xc1, 0xc6, 0x8b, 0x67, 0x76, 0x7c, 0x04, 0x93, 0xcd, 0x44, 0xdf, 0xfd, 0xf4, 0x0c, 0xf8, 0xcd, - 0xe8, 0xb1, 0x3b, 0xfb, 0xd3, 0x85, 0xf7, 0x57, 0xa2, 0xdc, 0x06, 0xcf, 0x7c, 0x63, 0x6d, 0xae, - 0x86, 0x78, 0xee, 0xfe, 0xf6, 0xad, 0x65, 0x72, 0x51, 0x10, 0x9e, 0xc7, 0xa2, 0xce, 0x93, 0x9c, - 0x72, 0x3d, 0xe2, 0x89, 0x49, 0x91, 0x8a, 0x35, 0xff, 0xfb, 0xcb, 0x3f, 0x19, 0x16, 0x7f, 0x8d, - 0x3e, 0xf8, 0xc9, 0xc8, 0x9f, 0x15, 0xa2, 0xcd, 0xe2, 0x1f, 0x86, 0x8d, 0xae, 0x4e, 0xff, 0xee, - 0x73, 0xd7, 0x3a, 0x77, 0x3d, 0xe4, 0xae, 0xaf, 0x4e, 0x97, 0xbb, 0x7a, 0x83, 0xaf, 0xfe, 0x0b, - 0x00, 0x00, 0xff, 0xff, 0xf3, 0xdd, 0x11, 0x96, 0x45, 0x06, 0x00, 0x00, -} - -var xxx_messageInfo_Key proto.InternalMessageInfo From 097a587977f625405a71bd2eb8a7e6d03cc78446 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 2 May 2019 15:42:35 -0400 Subject: [PATCH 30/35] adding new package files --- .../internal/keycompat/clouddatastorepb.go | 356 ++++++++++++++++++ datastore/internal/keycompat/keycompat.go | 90 +++++ 2 files changed, 446 insertions(+) create mode 100644 datastore/internal/keycompat/clouddatastorepb.go create mode 100644 datastore/internal/keycompat/keycompat.go diff --git a/datastore/internal/keycompat/clouddatastorepb.go b/datastore/internal/keycompat/clouddatastorepb.go new file mode 100644 index 00000000..4f55a2ef --- /dev/null +++ b/datastore/internal/keycompat/clouddatastorepb.go @@ -0,0 +1,356 @@ +package keycompat + +import ( + "fmt" + + "github.com/golang/protobuf/proto" +) + +// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. +// Key represents the datastore key for a stored entity. +type NewFormatKey struct { + // Kind cannot be empty. + Kind string + // Either ID or Name must be zero for the Key to be valid. + // If both are zero, the Key is incomplete. + ID int64 + Name string + // Parent must either be a complete Key or nil. + Parent *NewFormatKey + + // Namespace provides the ability to partition your data for multiple + // tenants. In most cases, it is not necessary to specify a namespace. + // See docs on datastore multitenancy for details: + // https://cloud.google.com/datastore/docs/concepts/multitenancy + Namespace string +} + +// A partition ID identifies a grouping of entities. The grouping is always +// by project and namespace, however the namespace ID may be empty. +// +// A partition ID contains several dimensions: +// project ID and namespace ID. +// +// Partition dimensions: +// +// - May be `""`. +// - Must be valid UTF-8 bytes. +// - Must have values that match regex `[A-Za-z\d\.\-_]{1,100}` +// If the value of any dimension matches regex `__.*__`, the partition is +// reserved/read-only. +// A reserved/read-only partition ID is forbidden in certain documented +// contexts. +// +// Foreign partition IDs (in which the project ID does +// not match the context project ID ) are discouraged. +// Reads and writes of foreign partition IDs may fail if the project is not in +// an active state. +type PartitionId struct { + // The ID of the project to which the entities belong. + ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` + // If not empty, the ID of the namespace to which the entities belong. + NamespaceId string `protobuf:"bytes,4,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PartitionId) Reset() { *m = PartitionId{} } +func (m *PartitionId) String() string { return proto.CompactTextString(m) } +func (*PartitionId) ProtoMessage() {} +func (*PartitionId) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{0} +} +func (m *PartitionId) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PartitionId.Unmarshal(m, b) +} +func (m *PartitionId) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PartitionId.Marshal(b, m, deterministic) +} +func (dst *PartitionId) XXX_Merge(src proto.Message) { + xxx_messageInfo_PartitionId.Merge(dst, src) +} +func (m *PartitionId) XXX_Size() int { + return xxx_messageInfo_PartitionId.Size(m) +} +func (m *PartitionId) XXX_DiscardUnknown() { + xxx_messageInfo_PartitionId.DiscardUnknown(m) +} + +var xxx_messageInfo_PartitionId proto.InternalMessageInfo + +func (m *PartitionId) GetProjectId() string { + if m != nil { + return m.ProjectId + } + return "" +} + +func (m *PartitionId) GetNamespaceId() string { + if m != nil { + return m.NamespaceId + } + return "" +} + +// A unique identifier for an entity. +// If a key's partition ID or any of its path kinds or names are +// reserved/read-only, the key is reserved/read-only. +// A reserved/read-only key is forbidden in certain documented contexts. +type PBKey struct { + // Entities are partitioned into subsets, currently identified by a project + // ID and namespace ID. + // Queries are scoped to a single partition. + PartitionId *PartitionId `protobuf:"bytes,1,opt,name=partition_id,json=partitionId,proto3" json:"partition_id,omitempty"` + // The entity path. + // An entity path consists of one or more elements composed of a kind and a + // string or numerical identifier, which identify entities. The first + // element identifies a _root entity_, the second element identifies + // a _child_ of the root entity, the third element identifies a child of the + // second entity, and so forth. The entities identified by all prefixes of + // the path are called the element's _ancestors_. + // + // An entity path is always fully complete: *all* of the entity's ancestors + // are required to be in the path along with the entity identifier itself. + // The only exception is that in some documented cases, the identifier in the + // last path element (for the entity) itself may be omitted. For example, + // the last path element of the key of `Mutation.insert` may have no + // identifier. + // + // A path can never be empty, and a path can have at most 100 elements. + Path []*Key_PathElement `protobuf:"bytes,2,rep,name=path,proto3" json:"path,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PBKey) Reset() { *m = PBKey{} } +func (m *PBKey) String() string { return proto.CompactTextString(m) } +func (*PBKey) ProtoMessage() {} +func (*PBKey) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{1} +} +func (m *PBKey) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Key.Unmarshal(m, b) +} +func (m *PBKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Key.Marshal(b, m, deterministic) +} +func (dst *PBKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_Key.Merge(dst, src) +} +func (m *PBKey) XXX_Size() int { + return xxx_messageInfo_Key.Size(m) +} +func (m *PBKey) XXX_DiscardUnknown() { + xxx_messageInfo_Key.DiscardUnknown(m) +} + +// A (kind, ID/name) pair used to construct a key path. +// +// If either name or ID is set, the element is complete. +// If neither is set, the element is incomplete. +type Key_PathElement struct { + // The kind of the entity. + // A kind matching regex `__.*__` is reserved/read-only. + // A kind must not contain more than 1500 bytes when UTF-8 encoded. + // Cannot be `""`. + Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"` + // The type of ID. + // + // Types that are valid to be assigned to IdType: + // *Key_PathElement_Id + // *Key_PathElement_Name + IdType isKey_PathElement_IdType `protobuf_oneof:"id_type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Key_PathElement) Reset() { *m = Key_PathElement{} } +func (m *Key_PathElement) String() string { return proto.CompactTextString(m) } +func (*Key_PathElement) ProtoMessage() {} +func (*Key_PathElement) Descriptor() ([]byte, []int) { + return fileDescriptor_entity_096a297364b049a5, []int{1, 0} +} +func (m *Key_PathElement) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Key_PathElement.Unmarshal(m, b) +} +func (m *Key_PathElement) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Key_PathElement.Marshal(b, m, deterministic) +} +func (dst *Key_PathElement) XXX_Merge(src proto.Message) { + xxx_messageInfo_Key_PathElement.Merge(dst, src) +} +func (m *Key_PathElement) XXX_Size() int { + return xxx_messageInfo_Key_PathElement.Size(m) +} +func (m *Key_PathElement) XXX_DiscardUnknown() { + xxx_messageInfo_Key_PathElement.DiscardUnknown(m) +} + +var xxx_messageInfo_Key_PathElement proto.InternalMessageInfo + +func (m *Key_PathElement) GetKind() string { + if m != nil { + return m.Kind + } + return "" +} + +type isKey_PathElement_IdType interface { + isKey_PathElement_IdType() +} + +type Key_PathElement_Id struct { + Id int64 `protobuf:"varint,2,opt,name=id,proto3,oneof"` +} + +type Key_PathElement_Name struct { + Name string `protobuf:"bytes,3,opt,name=name,proto3,oneof"` +} + +func (*Key_PathElement_Id) isKey_PathElement_IdType() {} + +func (*Key_PathElement_Name) isKey_PathElement_IdType() {} + +func (m *Key_PathElement) GetIdType() isKey_PathElement_IdType { + if m != nil { + return m.IdType + } + return nil +} + +func (m *Key_PathElement) GetId() int64 { + if x, ok := m.GetIdType().(*Key_PathElement_Id); ok { + return x.Id + } + return 0 +} + +func (m *Key_PathElement) GetName() string { + if x, ok := m.GetIdType().(*Key_PathElement_Name); ok { + return x.Name + } + return "" +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Key_PathElement) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Key_PathElement_OneofMarshaler, _Key_PathElement_OneofUnmarshaler, _Key_PathElement_OneofSizer, []interface{}{ + (*Key_PathElement_Id)(nil), + (*Key_PathElement_Name)(nil), + } +} + +func _Key_PathElement_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Key_PathElement) + // id_type + switch x := m.IdType.(type) { + case *Key_PathElement_Id: + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Id)) + case *Key_PathElement_Name: + b.EncodeVarint(3<<3 | proto.WireBytes) + b.EncodeStringBytes(x.Name) + case nil: + default: + return fmt.Errorf("Key_PathElement.IdType has unexpected type %T", x) + } + return nil +} + +func _Key_PathElement_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Key_PathElement) + switch tag { + case 2: // id_type.id + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.IdType = &Key_PathElement_Id{int64(x)} + return true, err + case 3: // id_type.name + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.IdType = &Key_PathElement_Name{x} + return true, err + default: + return false, nil + } +} + +func _Key_PathElement_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Key_PathElement) + // id_type + switch x := m.IdType.(type) { + case *Key_PathElement_Id: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Id)) + case *Key_PathElement_Name: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.Name))) + n += len(x.Name) + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +var fileDescriptor_entity_096a297364b049a5 = []byte{ + // 780 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xff, 0x6e, 0xdc, 0x44, + 0x10, 0xc7, 0xed, 0xbb, 0x5c, 0x1a, 0x8f, 0xdd, 0xa4, 0x6c, 0x2a, 0x61, 0x02, 0x28, 0x26, 0x80, + 0x74, 0x02, 0xc9, 0x6e, 0xc2, 0x1f, 0x54, 0x14, 0xa4, 0x72, 0x25, 0xe0, 0x28, 0x15, 0x9c, 0x56, + 0x55, 0x24, 0x50, 0xa4, 0xd3, 0xde, 0x79, 0xeb, 0x2e, 0x67, 0xef, 0x5a, 0xf6, 0x3a, 0xaa, 0xdf, + 0x05, 0xf1, 0x00, 0x3c, 0x0a, 0x8f, 0x80, 0x78, 0x18, 0xb4, 0x3f, 0xec, 0x0b, 0xed, 0x35, 0xff, + 0x79, 0x67, 0x3e, 0xdf, 0xd9, 0xef, 0xec, 0xce, 0x1a, 0xa2, 0x5c, 0x88, 0xbc, 0xa0, 0x49, 0x46, + 0x24, 0x69, 0xa4, 0xa8, 0x69, 0x72, 0x73, 0x9a, 0x50, 0x2e, 0x99, 0xec, 0xe2, 0xaa, 0x16, 0x52, + 0xa0, 0x43, 0x43, 0xc4, 0x03, 0x11, 0xdf, 0x9c, 0x1e, 0x7d, 0x64, 0x65, 0xa4, 0x62, 0x09, 0xe1, + 0x5c, 0x48, 0x22, 0x99, 0xe0, 0x8d, 0x91, 0x0c, 0x59, 0xbd, 0x5a, 0xb6, 0x2f, 0x93, 0x46, 0xd6, + 0xed, 0x4a, 0xda, 0xec, 0xf1, 0x9b, 0x59, 0xc9, 0x4a, 0xda, 0x48, 0x52, 0x56, 0x16, 0x08, 0x2d, + 0x20, 0xbb, 0x8a, 0x26, 0x05, 0x91, 0x05, 0xcf, 0x4d, 0xe6, 0xe4, 0x17, 0xf0, 0xe7, 0xa4, 0x96, + 0x4c, 0x6d, 0x76, 0x91, 0xa1, 0x8f, 0x01, 0xaa, 0x5a, 0xfc, 0x4e, 0x57, 0x72, 0xc1, 0xb2, 0x70, + 0x14, 0xb9, 0x53, 0x0f, 0x7b, 0x36, 0x72, 0x91, 0xa1, 0x4f, 0x20, 0xe0, 0xa4, 0xa4, 0x4d, 0x45, + 0x56, 0x54, 0x01, 0x3b, 0x1a, 0xf0, 0x87, 0xd8, 0x45, 0x76, 0xf2, 0x8f, 0x0b, 0xe3, 0x4b, 0xda, + 0xa1, 0x67, 0x10, 0x54, 0x7d, 0x61, 0x85, 0xba, 0x91, 0x3b, 0xf5, 0xcf, 0xa2, 0x78, 0x4b, 0xef, + 0xf1, 0x2d, 0x07, 0xd8, 0xaf, 0x6e, 0xd9, 0x79, 0x0c, 0x3b, 0x15, 0x91, 0xaf, 0xc2, 0x51, 0x34, + 0x9e, 0xfa, 0x67, 0x9f, 0x6d, 0x15, 0x5f, 0xd2, 0x2e, 0x9e, 0x13, 0xf9, 0xea, 0xbc, 0xa0, 0x25, + 0xe5, 0x12, 0x6b, 0xc5, 0xd1, 0x0b, 0xd5, 0xd7, 0x10, 0x44, 0x08, 0x76, 0xd6, 0x8c, 0x1b, 0x17, + 0x1e, 0xd6, 0xdf, 0xe8, 0x01, 0x8c, 0x6c, 0x8f, 0xe3, 0xd4, 0xc1, 0x23, 0x96, 0xa1, 0x87, 0xb0, + 0xa3, 0x5a, 0x09, 0xc7, 0x8a, 0x4a, 0x1d, 0xac, 0x57, 0x33, 0x0f, 0xee, 0xb1, 0x6c, 0xa1, 0x8e, + 0xee, 0xe4, 0x29, 0xc0, 0xf7, 0x75, 0x4d, 0xba, 0x2b, 0x52, 0xb4, 0x14, 0x9d, 0xc1, 0xee, 0x8d, + 0xfa, 0x68, 0x42, 0x57, 0xfb, 0x3b, 0xda, 0xea, 0x4f, 0xb3, 0xd8, 0x92, 0x27, 0x7f, 0x4c, 0x60, + 0x62, 0xd4, 0x4f, 0x00, 0x78, 0x5b, 0x14, 0x0b, 0x9d, 0x08, 0xfd, 0xc8, 0x9d, 0xee, 0x6f, 0x2a, + 0xf4, 0x37, 0x19, 0xff, 0xdc, 0x16, 0x85, 0xe6, 0x53, 0x07, 0x7b, 0xbc, 0x5f, 0xa0, 0xcf, 0xe1, + 0xfe, 0x52, 0x88, 0x82, 0x12, 0x6e, 0xf5, 0xaa, 0xb1, 0xbd, 0xd4, 0xc1, 0x81, 0x0d, 0x0f, 0x18, + 0xe3, 0x92, 0xe6, 0xb4, 0xb6, 0x58, 0xdf, 0x6d, 0x60, 0xc3, 0x06, 0xfb, 0x14, 0x82, 0x4c, 0xb4, + 0xcb, 0x82, 0x5a, 0x4a, 0xf5, 0xef, 0xa6, 0x0e, 0xf6, 0x4d, 0xd4, 0x40, 0xe7, 0x70, 0x30, 0x8c, + 0x95, 0xe5, 0x40, 0xdf, 0xe9, 0xdb, 0xa6, 0x5f, 0xf4, 0x5c, 0xea, 0xe0, 0xfd, 0x41, 0x64, 0xca, + 0x7c, 0x0d, 0xde, 0x9a, 0x76, 0xb6, 0xc0, 0x44, 0x17, 0x08, 0xdf, 0x75, 0xaf, 0xa9, 0x83, 0xf7, + 0xd6, 0xb4, 0x1b, 0x4c, 0x36, 0xb2, 0x66, 0x3c, 0xb7, 0xda, 0xf7, 0xec, 0x25, 0xf9, 0x26, 0x6a, + 0xa0, 0x63, 0x80, 0x65, 0x21, 0x96, 0x16, 0x41, 0x91, 0x3b, 0x0d, 0xd4, 0xc1, 0xa9, 0x98, 0x01, + 0xbe, 0x83, 0x83, 0x9c, 0x8a, 0x45, 0x25, 0x18, 0x97, 0x96, 0xda, 0xd3, 0x26, 0x0e, 0x7b, 0x13, + 0xea, 0xa2, 0xe3, 0xe7, 0x44, 0x3e, 0xe7, 0x79, 0xea, 0xe0, 0xfb, 0x39, 0x15, 0x73, 0x05, 0x1b, + 0xf9, 0x53, 0x08, 0xcc, 0x53, 0xb6, 0xda, 0x5d, 0xad, 0xfd, 0x70, 0x6b, 0x03, 0xe7, 0x1a, 0x54, + 0x0e, 0x8d, 0xc4, 0x54, 0x98, 0x81, 0x4f, 0xd4, 0x08, 0xd9, 0x02, 0x9e, 0x2e, 0x70, 0xbc, 0xb5, + 0xc0, 0x66, 0xd4, 0x52, 0x07, 0x03, 0xd9, 0x0c, 0x5e, 0x08, 0xf7, 0x4a, 0x4a, 0x38, 0xe3, 0x79, + 0xb8, 0x1f, 0xb9, 0xd3, 0x09, 0xee, 0x97, 0xe8, 0x11, 0x3c, 0xa4, 0xaf, 0x57, 0x45, 0x9b, 0xd1, + 0xc5, 0xcb, 0x5a, 0x94, 0x0b, 0xc6, 0x33, 0xfa, 0x9a, 0x36, 0xe1, 0xa1, 0x1a, 0x0f, 0x8c, 0x6c, + 0xee, 0xc7, 0x5a, 0x94, 0x17, 0x26, 0x33, 0x0b, 0x00, 0xb4, 0x13, 0x33, 0xe0, 0xff, 0xba, 0xb0, + 0x6b, 0x7c, 0xa3, 0x2f, 0x60, 0xbc, 0xa6, 0x9d, 0x7d, 0xb7, 0xef, 0xbc, 0x22, 0xac, 0x20, 0x74, + 0xa9, 0x7f, 0x1b, 0x15, 0xad, 0x25, 0xa3, 0x4d, 0x38, 0xd6, 0xaf, 0xe1, 0xcb, 0x3b, 0x0e, 0x25, + 0x9e, 0x0f, 0xf4, 0x39, 0x97, 0x75, 0x87, 0x6f, 0xc9, 0x8f, 0x7e, 0x85, 0x83, 0x37, 0xd2, 0xe8, + 0xc1, 0xc6, 0x8b, 0x67, 0x76, 0x7c, 0x04, 0x93, 0xcd, 0x44, 0xdf, 0xfd, 0xf4, 0x0c, 0xf8, 0xcd, + 0xe8, 0xb1, 0x3b, 0xfb, 0xd3, 0x85, 0xf7, 0x57, 0xa2, 0xdc, 0x06, 0xcf, 0x7c, 0x63, 0x6d, 0xae, + 0x86, 0x78, 0xee, 0xfe, 0xf6, 0xad, 0x65, 0x72, 0x51, 0x10, 0x9e, 0xc7, 0xa2, 0xce, 0x93, 0x9c, + 0x72, 0x3d, 0xe2, 0x89, 0x49, 0x91, 0x8a, 0x35, 0xff, 0xfb, 0xcb, 0x3f, 0x19, 0x16, 0x7f, 0x8d, + 0x3e, 0xf8, 0xc9, 0xc8, 0x9f, 0x15, 0xa2, 0xcd, 0xe2, 0x1f, 0x86, 0x8d, 0xae, 0x4e, 0xff, 0xee, + 0x73, 0xd7, 0x3a, 0x77, 0x3d, 0xe4, 0xae, 0xaf, 0x4e, 0x97, 0xbb, 0x7a, 0x83, 0xaf, 0xfe, 0x0b, + 0x00, 0x00, 0xff, 0xff, 0xf3, 0xdd, 0x11, 0x96, 0x45, 0x06, 0x00, 0x00, +} + +var xxx_messageInfo_Key proto.InternalMessageInfo diff --git a/datastore/internal/keycompat/keycompat.go b/datastore/internal/keycompat/keycompat.go new file mode 100644 index 00000000..a8aaf9c5 --- /dev/null +++ b/datastore/internal/keycompat/keycompat.go @@ -0,0 +1,90 @@ +package keycompat + +import ( + "encoding/base64" + "errors" + "strings" + + "github.com/golang/protobuf/proto" +) + +var ( + // ErrInvalidKey is returned when an invalid key is presented. + ErrInvalidKey = errors.New("datastore: invalid key") + + ErrKeyConversion = `Key conversions must be enabled in the application. + See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.` +) + +// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. +// decodeToNewKey decodes a key from the opaque representation returned by Encode. +func DecodeToNewKey(encoded string) (*NewFormatKey, error) { + // Re-add padding. + if m := len(encoded) % 4; m != 0 { + encoded += strings.Repeat("=", 4-m) + } + + b, err := base64.URLEncoding.DecodeString(encoded) + if err != nil { + return nil, err + } + + pKey := new(PBKey) + if err := proto.Unmarshal(b, pKey); err != nil { + return nil, err + } + return ProtoToKey(pKey) +} + +// protoToKey decodes a protocol buffer representation of a key into an +// equivalent *Key object. If the key is invalid, protoToKey will return the +// invalid key along with ErrInvalidKey. +func ProtoToKey(p *PBKey) (*NewFormatKey, error) { + var key *NewFormatKey + var namespace string + if partition := p.PartitionId; partition != nil { + namespace = partition.NamespaceId + } + for _, el := range p.Path { + key = &NewFormatKey{ + Namespace: namespace, + Kind: el.Kind, + ID: el.GetId(), + Name: el.GetName(), + Parent: key, + } + } + if !key.valid() { // Also detects key == nil. + return key, ErrInvalidKey + } + return key, nil +} + +// valid returns whether the key is valid. +func (k *NewFormatKey) valid() bool { + if k == nil { + return false + } + for ; k != nil; k = k.Parent { + if k.Kind == "" { + return false + } + if k.Name != "" && k.ID != 0 { + return false + } + if k.Parent != nil { + if k.Parent.Incomplete() { + return false + } + if k.Parent.Namespace != k.Namespace { + return false + } + } + } + return true +} + +// Incomplete reports whether the key does not refer to a stored entity. +func (k *NewFormatKey) Incomplete() bool { + return k.Name == "" && k.ID == 0 +} From 65b66c16c5fa6cb35c542c6576214d298f29df74 Mon Sep 17 00:00:00 2001 From: Luke Mccoy Date: Thu, 2 May 2019 16:34:56 -0400 Subject: [PATCH 31/35] fixed various code review issues --- datastore/key.go | 12 ++++-------- datastore/keycompat_test.go | 2 -- datastore/keycompatibility.go | 17 ++++++++--------- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/datastore/key.go b/datastore/key.go index a4f640be..0622ac65 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -257,17 +257,13 @@ func DecodeKey(encoded string) (*Key, error) { if err := proto.Unmarshal(b, ref); err != nil { // If there was an err on the key lets try to decode the new key type by default check to see if key converter // has been implemented. - if keyConversionProject == "" { - nKey, nLibKeyErr := keycompat.DecodeToNewKey(encoded) - if nLibKeyErr != nil { + if keyConversionProject != "" { + newKey, conversionErr := keycompat.DecodeToNewKey(encoded) + if conversionErr != nil { // returning the orginal error on purpose return nil, err } - oKey, err := convertNewKeyFormatToOldKeyFormat(nKey) - if err != nil { - return nil, err - } - return oKey, nil + return convertNewKeyFormatToOldKeyFormat(newKey) } return nil, err } diff --git a/datastore/keycompat_test.go b/datastore/keycompat_test.go index 40477e5f..0b0da2ea 100644 --- a/datastore/keycompat_test.go +++ b/datastore/keycompat_test.go @@ -73,7 +73,6 @@ var testCasesKeyCompat = []struct { func TestKeyCoversion(t *testing.T) { // Simulate the key converter enablement keyConversionProject = "glibrary" - for _, tc := range testCasesKeyCompat { dk, err := DecodeKey(tc.encodedKey) if err != nil { @@ -82,6 +81,5 @@ func TestKeyCoversion(t *testing.T) { if !reflect.DeepEqual(dk, tc.key) { t.Errorf("%s: got %+v, want %+v", tc.desc, dk, tc.key) } - } } diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go index 390b89a4..e28d8259 100644 --- a/datastore/keycompatibility.go +++ b/datastore/keycompatibility.go @@ -25,26 +25,25 @@ import ( ) var ( - errKeyConversion = `Key conversions must be enabled in the application. - See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.` - keyConversionProject string - once sync.Once + errKeyConversion = errors.New(`Key conversions must be enabled in the application. See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.`) + keyConversionProject string + keyConversionEnableOnce sync.Once ) // EnableKeyConversion enables forward key conversion abilities. Calling this function in a single handler will enable -// the feature for all handlers. This function can be called in the /_ah/start handler. Support for key converstion. -// Variable holds the appid so that key conversion can retrieve it without a context. +// the feature for all handlers. This function can be called in the /_ah/start handler. +// The keyConversionProject variable holds the appid so that key conversion can retrieve it without a context. func EnableKeyConversion(ctx context.Context) { - once.Do(func() { + keyConversionEnableOnce.Do(func() { keyConversionProject = internal.FullyQualifiedAppID(ctx) }) } -// convertKey takes at new datastore key type and returns a old key type +// convertNewKeyFormatToOldKeyFormat takes at new datastore key type and returns a old key type func convertNewKeyFormatToOldKeyFormat(key *keycompat.NewFormatKey) (*Key, error) { // if key conversion is not enabled return right away if keyConversionProject == "" { - return nil, errors.New(errKeyConversion) + return nil, errKeyConversion } var pKey *Key var err error From fb6e7776b06d20797b61dee49c1bbd108bada054 Mon Sep 17 00:00:00 2001 From: Chris Broadfoot Date: Thu, 9 May 2019 13:48:04 -0700 Subject: [PATCH 32/35] datastore: cleanup keycompat --- README.md | 24 +++-- .../{keycompat => cloudkey}/keycompat.go | 91 ++++++++++++------- .../entity.pb.go} | 45 ++++----- datastore/key.go | 12 +-- datastore/keycompat.go | 80 ++++++++++++++++ datastore/keycompat_test.go | 6 +- datastore/keycompatibility.go | 64 ------------- 7 files changed, 176 insertions(+), 146 deletions(-) rename datastore/internal/{keycompat => cloudkey}/keycompat.go (53%) rename datastore/internal/{keycompat/clouddatastorepb.go => cloudpb/entity.pb.go} (91%) create mode 100644 datastore/keycompat.go delete mode 100644 datastore/keycompatibility.go diff --git a/README.md b/README.md index 5cb7afb9..1259d451 100644 --- a/README.md +++ b/README.md @@ -72,21 +72,29 @@ A few APIs were cleaned up, and there are some differences: * `appengine/socket` is not required on App Engine flexible environment / Managed VMs. Use the standard `net` package instead. -## Key Encode Decode compatibiltiy to help with datastore library migrations +### Key Encode/Decode compatibiltiy to help with datastore library migrations Key compatibility updates have been added to help customers transition from google.golang.org/appengine/datastore to cloud.google.com/go/datastore. The `EnableKeyConversion` enables automatic conversion from a key encoded with cloud.google.com/go/datastore to google.golang.org/appengine/datastore key type. -### Enabling key conversation +### Enabling key conversion + Enable key conversion by calling `EnableKeyConversion(ctx)` in the `/_ah/startup` handler for basic and manual scaling or any handler in automatic scaling. -### 1. Basic or manual scaling +#### 1. Basic or manual scaling + This startup handler will enable key conversion for all handlers in the service. + ``` - http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) { - datastore.EnableKeyConversion(appengine.NewContext(r)) + http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) { + datastore.EnableKeyConversion(appengine.NewContext(r)) }) ``` -### 2. Automatic scaling -Since `/_ah/start` is not called in automatic scaling and `/_ah/warmup` is not guaranteed, a call to `datastore.EnableKeyConversion(appengine.NewContext(r))` -is needed in each handler where key conversion is needed. `EnableKeyConversion` is safe for concurrent use. \ No newline at end of file + +#### 2. Automatic scaling + +`/_ah/start` is not supported for automatic scaling and `/_ah/warmup` is not guaranteed to run, so you must call `datastore.EnableKeyConversion(appengine.NewContext(r))` +before you use code that needs key conversion. + +You may want to add this to each of your handlers, or introduce middleware where it's called. +`EnableKeyConversion` is safe for concurrent use. Any call to it after the first is ignored. \ No newline at end of file diff --git a/datastore/internal/keycompat/keycompat.go b/datastore/internal/cloudkey/keycompat.go similarity index 53% rename from datastore/internal/keycompat/keycompat.go rename to datastore/internal/cloudkey/keycompat.go index a8aaf9c5..05c9f58a 100644 --- a/datastore/internal/keycompat/keycompat.go +++ b/datastore/internal/cloudkey/keycompat.go @@ -1,3 +1,7 @@ +// Copyright 2019 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + package keycompat import ( @@ -6,19 +10,38 @@ import ( "strings" "github.com/golang/protobuf/proto" + cloudpb "google.golang.org/appengine/datastore/internal/cloudpb" ) +// Code below is copied from https://github.com/googleapis/google-cloud-go/blob/master/datastore/datastore.go + var ( // ErrInvalidKey is returned when an invalid key is presented. ErrInvalidKey = errors.New("datastore: invalid key") - - ErrKeyConversion = `Key conversions must be enabled in the application. - See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.` ) -// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. -// decodeToNewKey decodes a key from the opaque representation returned by Encode. -func DecodeToNewKey(encoded string) (*NewFormatKey, error) { +// Code below is copied from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go + +// Key represents the datastore key for a stored entity. +type Key struct { + // Kind cannot be empty. + Kind string + // Either ID or Name must be zero for the Key to be valid. + // If both are zero, the Key is incomplete. + ID int64 + Name string + // Parent must either be a complete Key or nil. + Parent *Key + + // Namespace provides the ability to partition your data for multiple + // tenants. In most cases, it is not necessary to specify a namespace. + // See docs on datastore multitenancy for details: + // https://cloud.google.com/datastore/docs/concepts/multitenancy + Namespace string +} + +// DecodeKey decodes a key from the opaque representation returned by Encode. +func DecodeKey(encoded string) (*Key, error) { // Re-add padding. if m := len(encoded) % 4; m != 0 { encoded += strings.Repeat("=", 4-m) @@ -29,39 +52,15 @@ func DecodeToNewKey(encoded string) (*NewFormatKey, error) { return nil, err } - pKey := new(PBKey) + pKey := new(cloudpb.Key) if err := proto.Unmarshal(b, pKey); err != nil { return nil, err } - return ProtoToKey(pKey) -} - -// protoToKey decodes a protocol buffer representation of a key into an -// equivalent *Key object. If the key is invalid, protoToKey will return the -// invalid key along with ErrInvalidKey. -func ProtoToKey(p *PBKey) (*NewFormatKey, error) { - var key *NewFormatKey - var namespace string - if partition := p.PartitionId; partition != nil { - namespace = partition.NamespaceId - } - for _, el := range p.Path { - key = &NewFormatKey{ - Namespace: namespace, - Kind: el.Kind, - ID: el.GetId(), - Name: el.GetName(), - Parent: key, - } - } - if !key.valid() { // Also detects key == nil. - return key, ErrInvalidKey - } - return key, nil + return protoToKey(pKey) } // valid returns whether the key is valid. -func (k *NewFormatKey) valid() bool { +func (k *Key) valid() bool { if k == nil { return false } @@ -85,6 +84,30 @@ func (k *NewFormatKey) valid() bool { } // Incomplete reports whether the key does not refer to a stored entity. -func (k *NewFormatKey) Incomplete() bool { +func (k *Key) Incomplete() bool { return k.Name == "" && k.ID == 0 } + +// protoToKey decodes a protocol buffer representation of a key into an +// equivalent *Key object. If the key is invalid, protoToKey will return the +// invalid key along with ErrInvalidKey. +func protoToKey(p *cloudpb.Key) (*Key, error) { + var key *Key + var namespace string + if partition := p.PartitionId; partition != nil { + namespace = partition.NamespaceId + } + for _, el := range p.Path { + key = &Key{ + Namespace: namespace, + Kind: el.Kind, + ID: el.GetId(), + Name: el.GetName(), + Parent: key, + } + } + if !key.valid() { // Also detects key == nil. + return key, ErrInvalidKey + } + return key, nil +} diff --git a/datastore/internal/keycompat/clouddatastorepb.go b/datastore/internal/cloudpb/entity.pb.go similarity index 91% rename from datastore/internal/keycompat/clouddatastorepb.go rename to datastore/internal/cloudpb/entity.pb.go index 4f55a2ef..691c938f 100644 --- a/datastore/internal/keycompat/clouddatastorepb.go +++ b/datastore/internal/cloudpb/entity.pb.go @@ -1,4 +1,8 @@ -package keycompat +// Copyright 2019 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package cloudpb import ( "fmt" @@ -6,24 +10,7 @@ import ( "github.com/golang/protobuf/proto" ) -// This code is duplicated from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go with the method renamed. -// Key represents the datastore key for a stored entity. -type NewFormatKey struct { - // Kind cannot be empty. - Kind string - // Either ID or Name must be zero for the Key to be valid. - // If both are zero, the Key is incomplete. - ID int64 - Name string - // Parent must either be a complete Key or nil. - Parent *NewFormatKey - - // Namespace provides the ability to partition your data for multiple - // tenants. In most cases, it is not necessary to specify a namespace. - // See docs on datastore multitenancy for details: - // https://cloud.google.com/datastore/docs/concepts/multitenancy - Namespace string -} +// Code below is copied from google.golang.org/genproto/googleapis/datastore/v1 // A partition ID identifies a grouping of entities. The grouping is always // by project and namespace, however the namespace ID may be empty. @@ -97,7 +84,7 @@ func (m *PartitionId) GetNamespaceId() string { // If a key's partition ID or any of its path kinds or names are // reserved/read-only, the key is reserved/read-only. // A reserved/read-only key is forbidden in certain documented contexts. -type PBKey struct { +type Key struct { // Entities are partitioned into subsets, currently identified by a project // ID and namespace ID. // Queries are scoped to a single partition. @@ -124,25 +111,25 @@ type PBKey struct { XXX_sizecache int32 `json:"-"` } -func (m *PBKey) Reset() { *m = PBKey{} } -func (m *PBKey) String() string { return proto.CompactTextString(m) } -func (*PBKey) ProtoMessage() {} -func (*PBKey) Descriptor() ([]byte, []int) { +func (m *Key) Reset() { *m = Key{} } +func (m *Key) String() string { return proto.CompactTextString(m) } +func (*Key) ProtoMessage() {} +func (*Key) Descriptor() ([]byte, []int) { return fileDescriptor_entity_096a297364b049a5, []int{1} } -func (m *PBKey) XXX_Unmarshal(b []byte) error { +func (m *Key) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Key.Unmarshal(m, b) } -func (m *PBKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *Key) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Key.Marshal(b, m, deterministic) } -func (dst *PBKey) XXX_Merge(src proto.Message) { +func (dst *Key) XXX_Merge(src proto.Message) { xxx_messageInfo_Key.Merge(dst, src) } -func (m *PBKey) XXX_Size() int { +func (m *Key) XXX_Size() int { return xxx_messageInfo_Key.Size(m) } -func (m *PBKey) XXX_DiscardUnknown() { +func (m *Key) XXX_DiscardUnknown() { xxx_messageInfo_Key.DiscardUnknown(m) } diff --git a/datastore/key.go b/datastore/key.go index 0622ac65..c8caee25 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -16,7 +16,6 @@ import ( "github.com/golang/protobuf/proto" "golang.org/x/net/context" - "google.golang.org/appengine/datastore/internal/keycompat" "google.golang.org/appengine/internal" pb "google.golang.org/appengine/internal/datastore" ) @@ -255,15 +254,8 @@ func DecodeKey(encoded string) (*Key, error) { ref := new(pb.Reference) if err := proto.Unmarshal(b, ref); err != nil { - // If there was an err on the key lets try to decode the new key type by default check to see if key converter - // has been implemented. - if keyConversionProject != "" { - newKey, conversionErr := keycompat.DecodeToNewKey(encoded) - if conversionErr != nil { - // returning the orginal error on purpose - return nil, err - } - return convertNewKeyFormatToOldKeyFormat(newKey) + if k := decodeCloudKey(encoded); k != nil { + return k, nil } return nil, err } diff --git a/datastore/keycompat.go b/datastore/keycompat.go new file mode 100644 index 00000000..c48b751b --- /dev/null +++ b/datastore/keycompat.go @@ -0,0 +1,80 @@ +// Copyright 2019 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package datastore + +import ( + "sync" + + "golang.org/x/net/context" + + "google.golang.org/appengine/datastore/internal/cloudkey" + "google.golang.org/appengine/internal" +) + +var keyConversion struct { + mu sync.RWMutex + project string +} + +// EnableKeyConversion enables encoded key compatibility with the Cloud Datastore client library (cloud.google.com/go/datastore). +// Encoded keys generated by the Cloud Datastore client library will be decoded into App Engine datastore keys. +// +// The context provided must be an App Engine context if running in first generation App Engine. +// This can be called in the /_ah/start handler. It is safe to call multiple times, and is cheap to call, so can also be inserted as middleware. +// +// Enabling key compatibility not affect the encoding format used by Key.Encode, it only expands the type of keys that are able to be decoded with DecodeKey. +func EnableKeyConversion(ctx context.Context) { + if getKeyConversionAppID() == "" { + return + } + + keyConversion.mu.Lock() + if keyConversion.project == "" { + keyConversion.project = internal.FullyQualifiedAppID(ctx) + } + keyConversion.mu.Unlock() +} + +func getKeyConversionAppID() string { + keyConversion.mu.RLock() + project := keyConversion.project + keyConversion.mu.RUnlock() + return project +} + +// decodeCloudKey attempts to decode the given encoded key generated by the Cloud Datastore client library, returning nil if the key couldn't be decoded. +func decodeCloudKey(encoded string) *Key { + cloudKey, err := keycompat.DecodeKey(encoded) + if err != nil { + return nil + } + appID := getKeyConversionAppID() + converted, err := convertCloudKey(cloudKey, appID) + if err != nil { + return nil + } + return converted +} + +// convertCloudKey converts a Cloud Datastore key and converts it to an App Engine Datastore key. +// Cloud Datastore keys don't include the project/app ID, so we must add it back in. +func convertCloudKey(key *keycompat.Key, appID string) (*Key, error) { + var pKey *Key + var err error + if key.Parent != nil { + pKey, err = convertCloudKey(key.Parent, appID) + if err != nil { + return nil, err + } + } + return &Key{ + intID: key.ID, + kind: key.Kind, + namespace: key.Namespace, + parent: pKey, + stringID: key.Name, + appID: appID, + }, nil +} diff --git a/datastore/keycompat_test.go b/datastore/keycompat_test.go index 0b0da2ea..7d5d9af6 100644 --- a/datastore/keycompat_test.go +++ b/datastore/keycompat_test.go @@ -1,3 +1,7 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + package datastore import ( @@ -72,7 +76,7 @@ var testCasesKeyCompat = []struct { func TestKeyCoversion(t *testing.T) { // Simulate the key converter enablement - keyConversionProject = "glibrary" + keyConversion.project = "glibrary" for _, tc := range testCasesKeyCompat { dk, err := DecodeKey(tc.encodedKey) if err != nil { diff --git a/datastore/keycompatibility.go b/datastore/keycompatibility.go deleted file mode 100644 index e28d8259..00000000 --- a/datastore/keycompatibility.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2011 Google Inc. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -// This change adds additional compatibility to help customers -// transition from google.golang.org/appengine/datastore to cloud.google.com/go/datastore. -// Each lib google.golang.org/appengine/datastore and cloud.google.com/go/datastore contain the functions Key.Encode() and Key.Decode(). These functions -// create base64 representations of a json marshalled type of datastore keys. Customers have been using -// these encoded values to communicate between services in appengine. The protobuf key types -// changed between google.golang.org/appengine/datastore and cloud.google.com/go/datastore making the corresponding base64 key strings incompatible. -// Customers who attempt to upgrade to cloud.google.com/go/datastore that use this pattern will fail. -// keycompatibility.go placed in google.golang.org/appengine/datastore enables forward compatibility of cloud.google.com/go/datastore encoded keys. -// An update to cloud.google.com/go/datastore will also be necessary to enable backward compatibility. - -package datastore - -import ( - "errors" - "sync" - - "golang.org/x/net/context" - - "google.golang.org/appengine/datastore/internal/keycompat" - "google.golang.org/appengine/internal" -) - -var ( - errKeyConversion = errors.New(`Key conversions must be enabled in the application. See https://github.com/golang/appengine#key-encode-decode-compatibiltiy-to-help-with-datastore-library-migrations for more details.`) - keyConversionProject string - keyConversionEnableOnce sync.Once -) - -// EnableKeyConversion enables forward key conversion abilities. Calling this function in a single handler will enable -// the feature for all handlers. This function can be called in the /_ah/start handler. -// The keyConversionProject variable holds the appid so that key conversion can retrieve it without a context. -func EnableKeyConversion(ctx context.Context) { - keyConversionEnableOnce.Do(func() { - keyConversionProject = internal.FullyQualifiedAppID(ctx) - }) -} - -// convertNewKeyFormatToOldKeyFormat takes at new datastore key type and returns a old key type -func convertNewKeyFormatToOldKeyFormat(key *keycompat.NewFormatKey) (*Key, error) { - // if key conversion is not enabled return right away - if keyConversionProject == "" { - return nil, errKeyConversion - } - var pKey *Key - var err error - if key.Parent != nil { - pKey, err = convertNewKeyFormatToOldKeyFormat(key.Parent) - if err != nil { - return nil, err - } - } - return &Key{ - intID: key.ID, - kind: key.Kind, - namespace: key.Namespace, - parent: pKey, - stringID: key.Name, - appID: keyConversionProject, - }, nil -} From 2c45d7e2b5bd4e761bf18723e755bf5c5cf69060 Mon Sep 17 00:00:00 2001 From: Chris Broadfoot Date: Thu, 9 May 2019 14:06:52 -0700 Subject: [PATCH 33/35] README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1259d451..4376a376 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ A few APIs were cleaned up, and there are some differences: * `appengine/socket` is not required on App Engine flexible environment / Managed VMs. Use the standard `net` package instead. -### Key Encode/Decode compatibiltiy to help with datastore library migrations +## Key Encode/Decode compatibiltiy to help with datastore library migrations Key compatibility updates have been added to help customers transition from google.golang.org/appengine/datastore to cloud.google.com/go/datastore. The `EnableKeyConversion` enables automatic conversion from a key encoded with cloud.google.com/go/datastore to google.golang.org/appengine/datastore key type. @@ -86,9 +86,9 @@ Enable key conversion by calling `EnableKeyConversion(ctx)` in the `/_ah/startup This startup handler will enable key conversion for all handlers in the service. ``` - http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) { +http.HandleFunc("/_ah/start", func(w http.ResponseWriter, r *http.Request) { datastore.EnableKeyConversion(appengine.NewContext(r)) - }) +}) ``` #### 2. Automatic scaling From d3c996c3c9b6559f735e40bed344daf728a3810a Mon Sep 17 00:00:00 2001 From: Chris Broadfoot Date: Thu, 9 May 2019 15:03:10 -0700 Subject: [PATCH 34/35] keycompat -> cloudkey --- datastore/internal/cloudkey/{keycompat.go => cloudkey.go} | 2 +- datastore/keycompat.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename datastore/internal/cloudkey/{keycompat.go => cloudkey.go} (99%) diff --git a/datastore/internal/cloudkey/keycompat.go b/datastore/internal/cloudkey/cloudkey.go similarity index 99% rename from datastore/internal/cloudkey/keycompat.go rename to datastore/internal/cloudkey/cloudkey.go index 05c9f58a..438a0b58 100644 --- a/datastore/internal/cloudkey/keycompat.go +++ b/datastore/internal/cloudkey/cloudkey.go @@ -2,7 +2,7 @@ // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. -package keycompat +package cloudkey import ( "encoding/base64" diff --git a/datastore/keycompat.go b/datastore/keycompat.go index c48b751b..cbb6e817 100644 --- a/datastore/keycompat.go +++ b/datastore/keycompat.go @@ -46,7 +46,7 @@ func getKeyConversionAppID() string { // decodeCloudKey attempts to decode the given encoded key generated by the Cloud Datastore client library, returning nil if the key couldn't be decoded. func decodeCloudKey(encoded string) *Key { - cloudKey, err := keycompat.DecodeKey(encoded) + cloudKey, err := cloudkey.DecodeKey(encoded) if err != nil { return nil } @@ -60,7 +60,7 @@ func decodeCloudKey(encoded string) *Key { // convertCloudKey converts a Cloud Datastore key and converts it to an App Engine Datastore key. // Cloud Datastore keys don't include the project/app ID, so we must add it back in. -func convertCloudKey(key *keycompat.Key, appID string) (*Key, error) { +func convertCloudKey(key *cloudkey.Key, appID string) (*Key, error) { var pKey *Key var err error if key.Parent != nil { From 98afd7e61d4997d58d0eb0d746167499f3a5112d Mon Sep 17 00:00:00 2001 From: Chris Broadfoot Date: Mon, 13 May 2019 16:14:26 -0700 Subject: [PATCH 35/35] review comments --- datastore/internal/cloudkey/cloudkey.go | 7 ++ datastore/internal/cloudpb/entity.pb.go | 5 +- datastore/key.go | 1 + datastore/keycompat.go | 71 +++++++------- datastore/keycompat_test.go | 118 ++++++++++++------------ 5 files changed, 110 insertions(+), 92 deletions(-) diff --git a/datastore/internal/cloudkey/cloudkey.go b/datastore/internal/cloudkey/cloudkey.go index 438a0b58..643d4049 100644 --- a/datastore/internal/cloudkey/cloudkey.go +++ b/datastore/internal/cloudkey/cloudkey.go @@ -2,6 +2,9 @@ // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. +// Package cloudpb is a subset of types and functions, copied from cloud.google.com/go/datastore. +// +// They are copied here to provide compatibility to decode keys generated by the cloud.google.com/go/datastore package. package cloudkey import ( @@ -13,14 +16,18 @@ import ( cloudpb "google.golang.org/appengine/datastore/internal/cloudpb" ) +///////////////////////////////////////////////////////////////////// // Code below is copied from https://github.com/googleapis/google-cloud-go/blob/master/datastore/datastore.go +///////////////////////////////////////////////////////////////////// var ( // ErrInvalidKey is returned when an invalid key is presented. ErrInvalidKey = errors.New("datastore: invalid key") ) +///////////////////////////////////////////////////////////////////// // Code below is copied from https://github.com/googleapis/google-cloud-go/blob/master/datastore/key.go +///////////////////////////////////////////////////////////////////// // Key represents the datastore key for a stored entity. type Key struct { diff --git a/datastore/internal/cloudpb/entity.pb.go b/datastore/internal/cloudpb/entity.pb.go index 691c938f..af8195f3 100644 --- a/datastore/internal/cloudpb/entity.pb.go +++ b/datastore/internal/cloudpb/entity.pb.go @@ -2,6 +2,9 @@ // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. +// Package cloudpb is a subset of protobufs, copied from google.golang.org/genproto/googleapis/datastore/v1. +// +// They are copied here to provide compatibility to decode keys generated by the cloud.google.com/go/datastore package. package cloudpb import ( @@ -10,8 +13,6 @@ import ( "github.com/golang/protobuf/proto" ) -// Code below is copied from google.golang.org/genproto/googleapis/datastore/v1 - // A partition ID identifies a grouping of entities. The grouping is always // by project and namespace, however the namespace ID may be empty. // diff --git a/datastore/key.go b/datastore/key.go index c8caee25..fd598dc9 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -254,6 +254,7 @@ func DecodeKey(encoded string) (*Key, error) { ref := new(pb.Reference) if err := proto.Unmarshal(b, ref); err != nil { + // Couldn't decode it as an App Engine key, try decoding it as a key encoded by cloud.google.com/go/datastore. if k := decodeCloudKey(encoded); k != nil { return k, nil } diff --git a/datastore/keycompat.go b/datastore/keycompat.go index cbb6e817..371a64ee 100644 --- a/datastore/keycompat.go +++ b/datastore/keycompat.go @@ -14,67 +14,76 @@ import ( ) var keyConversion struct { - mu sync.RWMutex - project string + mu sync.RWMutex + appID string // read using getKeyConversionAppID } -// EnableKeyConversion enables encoded key compatibility with the Cloud Datastore client library (cloud.google.com/go/datastore). -// Encoded keys generated by the Cloud Datastore client library will be decoded into App Engine datastore keys. +// EnableKeyConversion enables encoded key compatibility with the Cloud +// Datastore client library (cloud.google.com/go/datastore). Encoded keys +// generated by the Cloud Datastore client library will be decoded into App +// Engine datastore keys. // -// The context provided must be an App Engine context if running in first generation App Engine. -// This can be called in the /_ah/start handler. It is safe to call multiple times, and is cheap to call, so can also be inserted as middleware. +// The context provided must be an App Engine context if running in App Engine +// first generation runtime. This can be called in the /_ah/start handler. It is +// safe to call multiple times, and is cheap to call, so can also be inserted as +// middleware. // -// Enabling key compatibility not affect the encoding format used by Key.Encode, it only expands the type of keys that are able to be decoded with DecodeKey. +// Enabling key compatibility does not affect the encoding format used by +// Key.Encode, it only expands the type of keys that are able to be decoded with +// DecodeKey. func EnableKeyConversion(ctx context.Context) { - if getKeyConversionAppID() == "" { + // Only attempt to set appID if it's unset. + // If already set, ignore. + if getKeyConversionAppID() != "" { return } keyConversion.mu.Lock() - if keyConversion.project == "" { - keyConversion.project = internal.FullyQualifiedAppID(ctx) + // Check again to avoid race where another goroutine set appID between the call + // to getKeyConversionAppID above and taking the write lock. + if keyConversion.appID == "" { + keyConversion.appID = internal.FullyQualifiedAppID(ctx) } keyConversion.mu.Unlock() } func getKeyConversionAppID() string { keyConversion.mu.RLock() - project := keyConversion.project + appID := keyConversion.appID keyConversion.mu.RUnlock() - return project + return appID } -// decodeCloudKey attempts to decode the given encoded key generated by the Cloud Datastore client library, returning nil if the key couldn't be decoded. +// decodeCloudKey attempts to decode the given encoded key generated by the +// Cloud Datastore client library (cloud.google.com/go/datastore), returning nil +// if the key couldn't be decoded. func decodeCloudKey(encoded string) *Key { - cloudKey, err := cloudkey.DecodeKey(encoded) - if err != nil { + appID := getKeyConversionAppID() + if appID == "" { return nil } - appID := getKeyConversionAppID() - converted, err := convertCloudKey(cloudKey, appID) + + k, err := cloudkey.DecodeKey(encoded) if err != nil { return nil } - return converted + return convertCloudKey(k, appID) } -// convertCloudKey converts a Cloud Datastore key and converts it to an App Engine Datastore key. -// Cloud Datastore keys don't include the project/app ID, so we must add it back in. -func convertCloudKey(key *cloudkey.Key, appID string) (*Key, error) { - var pKey *Key - var err error - if key.Parent != nil { - pKey, err = convertCloudKey(key.Parent, appID) - if err != nil { - return nil, err - } +// convertCloudKey converts a Cloud Datastore key and converts it to an App +// Engine Datastore key. Cloud Datastore keys don't include the project/app ID, +// so we must add it back in. +func convertCloudKey(key *cloudkey.Key, appID string) *Key { + if key == nil { + return nil } - return &Key{ + k := &Key{ intID: key.ID, kind: key.Kind, namespace: key.Namespace, - parent: pKey, + parent: convertCloudKey(key.Parent, appID), stringID: key.Name, appID: appID, - }, nil + } + return k } diff --git a/datastore/keycompat_test.go b/datastore/keycompat_test.go index 7d5d9af6..923fdac9 100644 --- a/datastore/keycompat_test.go +++ b/datastore/keycompat_test.go @@ -9,78 +9,78 @@ import ( "testing" ) -var testCasesKeyCompat = []struct { - desc string - key *Key - encodedKey string -}{ - { - desc: "A control test for legacy to legacy key conversion int as the key", - key: &Key{ - kind: "Person", - intID: 1, - appID: "glibrary", +func TestKeyConversion(t *testing.T) { + var tests = []struct { + desc string + key *Key + encodedKey string + }{ + { + desc: "A control test for legacy to legacy key conversion int as the key", + key: &Key{ + kind: "Person", + intID: 1, + appID: "glibrary", + }, + encodedKey: "aghnbGlicmFyeXIMCxIGUGVyc29uGAEM", }, - encodedKey: "aghnbGlicmFyeXIMCxIGUGVyc29uGAEM", - }, - { - desc: "A control test for legacy to legacy key conversion string as the key", - key: &Key{ - kind: "Graph", - stringID: "graph:7-day-active", - appID: "glibrary", + { + desc: "A control test for legacy to legacy key conversion string as the key", + key: &Key{ + kind: "Graph", + stringID: "graph:7-day-active", + appID: "glibrary", + }, + encodedKey: "aghnbGlicmFyeXIdCxIFR3JhcGgiEmdyYXBoOjctZGF5LWFjdGl2ZQw", }, - encodedKey: "aghnbGlicmFyeXIdCxIFR3JhcGgiEmdyYXBoOjctZGF5LWFjdGl2ZQw", - }, - // These are keys encoded with cloud.google.com/go/datastore - // Standard int as the key - { - desc: "Convert new key format to old key with int id", - key: &Key{ - kind: "WordIndex", - intID: 1033, - appID: "glibrary", + // These are keys encoded with cloud.google.com/go/datastore + // Standard int as the key + { + desc: "Convert new key format to old key with int id", + key: &Key{ + kind: "WordIndex", + intID: 1033, + appID: "glibrary", + }, + encodedKey: "Eg4KCVdvcmRJbmRleBCJCA", }, - encodedKey: "Eg4KCVdvcmRJbmRleBCJCA", - }, - // These are keys encoded with cloud.google.com/go/datastore - // Standard string - { - desc: "Convert new key format to old key with string id", - key: &Key{ - kind: "WordIndex", - stringID: "IAmAnID", - appID: "glibrary", + // These are keys encoded with cloud.google.com/go/datastore + // Standard string + { + desc: "Convert new key format to old key with string id", + key: &Key{ + kind: "WordIndex", + stringID: "IAmAnID", + appID: "glibrary", + }, + encodedKey: "EhQKCVdvcmRJbmRleBoHSUFtQW5JRA", }, - encodedKey: "EhQKCVdvcmRJbmRleBoHSUFtQW5JRA", - }, - // These are keys encoded with cloud.google.com/go/datastore - // ID String with parent as string - { - desc: "Convert new key format to old key with string id with a parent", - key: &Key{ - kind: "WordIndex", - stringID: "IAmAnID", - appID: "glibrary", - parent: &Key{ - kind: "LetterIndex", - stringID: "IAmAnotherID", + // These are keys encoded with cloud.google.com/go/datastore + // ID String with parent as string + { + desc: "Convert new key format to old key with string id with a parent", + key: &Key{ + kind: "WordIndex", + stringID: "IAmAnID", appID: "glibrary", + parent: &Key{ + kind: "LetterIndex", + stringID: "IAmAnotherID", + appID: "glibrary", + }, }, + encodedKey: "EhsKC0xldHRlckluZGV4GgxJQW1Bbm90aGVySUQSFAoJV29yZEluZGV4GgdJQW1BbklE", }, - encodedKey: "EhsKC0xldHRlckluZGV4GgxJQW1Bbm90aGVySUQSFAoJV29yZEluZGV4GgdJQW1BbklE", - }, -} + } -func TestKeyCoversion(t *testing.T) { // Simulate the key converter enablement - keyConversion.project = "glibrary" - for _, tc := range testCasesKeyCompat { + keyConversion.appID = "glibrary" + for _, tc := range tests { dk, err := DecodeKey(tc.encodedKey) if err != nil { - t.Fatalf("%v", err.Error()) + t.Fatalf("DecodeKey: %v", err) } if !reflect.DeepEqual(dk, tc.key) { t.Errorf("%s: got %+v, want %+v", tc.desc, dk, tc.key)