Skip to content

Commit 3d80f24

Browse files
justinsbjustinsb
authored
Extract out common parse-package logic (#3711)
We had this code duplicated in a few places also.
1 parent dc931ec commit 3d80f24

File tree

6 files changed

+192
-141
lines changed

6 files changed

+192
-141
lines changed

‎porch/controllers/klippy/pkg/controllers/klippy/klippy_controller.go

Lines changed: 4 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,11 @@ import (
1818
"context"
1919
"flag"
2020
"fmt"
21-
"path/filepath"
2221
"strings"
23-
"unicode"
2422

2523
api "github.com/GoogleContainerTools/kpt/porch/api/porch/v1alpha1"
2624
"github.com/GoogleContainerTools/kpt/porch/controllers/remoterootsyncsets/pkg/applyset"
25+
"github.com/GoogleContainerTools/kpt/porch/pkg/objects"
2726
"k8s.io/apimachinery/pkg/api/meta"
2827
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2928
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -34,7 +33,6 @@ import (
3433
ctrl "sigs.k8s.io/controller-runtime"
3534
"sigs.k8s.io/controller-runtime/pkg/client"
3635
"sigs.k8s.io/controller-runtime/pkg/log"
37-
"sigs.k8s.io/yaml"
3836
)
3937

4038
type Options struct {
@@ -98,15 +96,6 @@ func (r *KlippyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
9896
return ctrl.Result{}, nil
9997
}
10098

101-
func isWhitespace(s string) bool {
102-
for _, r := range s {
103-
if !unicode.IsSpace(r) {
104-
return false
105-
}
106-
}
107-
return true
108-
}
109-
11099
func (r *KlippyReconciler) reconcile(ctx context.Context, parent *api.PackageRevision, parentResources *api.PackageRevisionResources) error {
111100
log := log.FromContext(ctx)
112101

@@ -121,7 +110,7 @@ func (r *KlippyReconciler) reconcile(ctx context.Context, parent *api.PackageRev
121110
return err
122111
}
123112

124-
parentObjects, err := parseObjects(ctx, parentResources)
113+
parentObjects, err := objects.Parser{}.AsUnstructureds(parentResources.Spec.Resources)
125114
if err != nil {
126115
return err
127116
}
@@ -166,10 +155,11 @@ func (r *KlippyReconciler) parseBlueprints(ctx context.Context, packageRevisions
166155
return nil, fmt.Errorf("error fetching PackageRevisionResources %v: %w", id, err)
167156
}
168157

169-
objects, err := parseObjects(ctx, &packageRevisionResources)
158+
objects, err := objects.Parser{}.AsUnstructureds(packageRevisionResources.Spec.Resources)
170159
if err != nil {
171160
return nil, err
172161
}
162+
173163
blueprint := &blueprint{
174164
Objects: objects,
175165
ID: id,
@@ -180,41 +170,6 @@ func (r *KlippyReconciler) parseBlueprints(ctx context.Context, packageRevisions
180170
return blueprints, nil
181171
}
182172

183-
func parseObjects(ctx context.Context, subject *api.PackageRevisionResources) ([]*unstructured.Unstructured, error) {
184-
log := log.FromContext(ctx)
185-
186-
// TODO: Does this function exist somewhere?
187-
var objects []*unstructured.Unstructured
188-
189-
for path, contents := range subject.Spec.Resources {
190-
ext := filepath.Ext(path)
191-
ext = strings.ToLower(ext)
192-
193-
switch ext {
194-
case ".yaml", ".yml":
195-
// TODO: Use https://github.com/kubernetes-sigs/kustomize/blob/a5b61016bb40c30dd1b0a78290b28b2330a0383e/kyaml/kio/byteio_reader.go#L170 or similar?
196-
for _, s := range strings.Split(contents, "\n---\n") {
197-
if isWhitespace(s) {
198-
continue
199-
}
200-
201-
o := &unstructured.Unstructured{}
202-
if err := yaml.Unmarshal([]byte(s), o); err != nil {
203-
return nil, fmt.Errorf("error parsing package as kubernetes object: %w", err)
204-
}
205-
206-
// TODO: sync with kpt logic; skip objects marked with the local-only annotation
207-
objects = append(objects, o)
208-
}
209-
210-
default:
211-
log.V(2).Info("skipping file with unhandled extension", "path", path)
212-
}
213-
}
214-
215-
return objects, nil
216-
}
217-
218173
type bindingSlotRequirements struct {
219174
MatchLabels map[string]string
220175
}

‎porch/controllers/remoterootsyncsets/pkg/controllers/remoterootsyncset/remoterootsync_controller.go

Lines changed: 8 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,15 @@ import (
1919
"flag"
2020
"fmt"
2121
"os"
22-
"path"
23-
"strings"
24-
"unicode"
2522

2623
kptoci "github.com/GoogleContainerTools/kpt/pkg/oci"
2724
api "github.com/GoogleContainerTools/kpt/porch/controllers/remoterootsyncsets/api/v1alpha1"
2825
"github.com/GoogleContainerTools/kpt/porch/controllers/remoterootsyncsets/pkg/applyset"
2926
"github.com/GoogleContainerTools/kpt/porch/controllers/remoterootsyncsets/pkg/remoteclient"
27+
"github.com/GoogleContainerTools/kpt/porch/pkg/objects"
3028
"github.com/GoogleContainerTools/kpt/porch/pkg/oci"
3129
"k8s.io/apimachinery/pkg/api/meta"
3230
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33-
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
3431
"k8s.io/client-go/discovery"
3532
memory "k8s.io/client-go/discovery/cached"
3633
"k8s.io/client-go/dynamic"
@@ -40,7 +37,6 @@ import (
4037
ctrl "sigs.k8s.io/controller-runtime"
4138
"sigs.k8s.io/controller-runtime/pkg/client"
4239
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
43-
"sigs.k8s.io/yaml"
4440
)
4541

4642
var (
@@ -325,50 +321,16 @@ func (r *RemoteRootSyncSetReconciler) BuildObjectsToApply(ctx context.Context, s
325321
return nil, err
326322
}
327323

328-
var objects []applyset.ApplyableObject
329-
330-
for filePath, fileContents := range resources.Contents {
331-
ext := path.Ext(filePath)
332-
ext = strings.ToLower(ext)
333-
334-
parse := false
335-
switch ext {
336-
case ".yaml", ".yml":
337-
parse = true
338-
339-
default:
340-
klog.Warningf("ignoring non-yaml file %s", filePath)
341-
}
342-
343-
if !parse {
344-
continue
345-
}
346-
// TODO: Use https://github.com/kubernetes-sigs/kustomize/blob/a5b61016bb40c30dd1b0a78290b28b2330a0383e/kyaml/kio/byteio_reader.go#L170 or similar?
347-
for _, s := range strings.Split(fileContents, "\n---\n") {
348-
if isWhitespace(s) {
349-
continue
350-
}
351-
352-
o := &unstructured.Unstructured{}
353-
if err := yaml.Unmarshal([]byte(s), &o); err != nil {
354-
return nil, fmt.Errorf("error parsing yaml from %s: %w", filePath, err)
355-
}
356-
357-
// TODO: sync with kpt logic; skip objects marked with the local-only annotation
358-
objects = append(objects, o)
359-
}
324+
unstructureds, err := objects.Parser{}.AsUnstructureds(resources.Contents)
325+
if err != nil {
326+
return nil, err
360327
}
361328

362-
return objects, nil
363-
}
364-
365-
func isWhitespace(s string) bool {
366-
for _, r := range s {
367-
if !unicode.IsSpace(r) {
368-
return false
369-
}
329+
var applyables []applyset.ApplyableObject
330+
for _, u := range unstructureds {
331+
applyables = append(applyables, u)
370332
}
371-
return true
333+
return applyables, nil
372334
}
373335

374336
// SetupWithManager sets up the controller with the Manager.

‎porch/pkg/engine/engine.go

Lines changed: 11 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,9 @@ import (
2020
"fmt"
2121
"io/fs"
2222
"os"
23-
"path"
2423
"path/filepath"
2524
"reflect"
2625
"strings"
27-
"unicode"
2826

2927
"github.com/GoogleContainerTools/kpt/internal/builtins"
3028
"github.com/GoogleContainerTools/kpt/internal/fnruntime"
@@ -35,6 +33,7 @@ import (
3533
"github.com/GoogleContainerTools/kpt/porch/pkg/cache"
3634
"github.com/GoogleContainerTools/kpt/porch/pkg/kpt"
3735
"github.com/GoogleContainerTools/kpt/porch/pkg/meta"
36+
"github.com/GoogleContainerTools/kpt/porch/pkg/objects"
3837
"github.com/GoogleContainerTools/kpt/porch/pkg/repository"
3938
"go.opentelemetry.io/otel"
4039
"go.opentelemetry.io/otel/trace"
@@ -1385,42 +1384,17 @@ func (cad *cadEngine) recloneAndReplay(ctx context.Context, repo repository.Repo
13851384

13861385
// ExtractContextConfigMap returns the package-context configmap, if found
13871386
func ExtractContextConfigMap(resources map[string]string) (*unstructured.Unstructured, error) {
1388-
var matches []*unstructured.Unstructured
1389-
1390-
for itemPath, itemContents := range resources {
1391-
ext := path.Ext(itemPath)
1392-
ext = strings.ToLower(ext)
1393-
1394-
parse := false
1395-
switch ext {
1396-
case ".yaml", ".yml":
1397-
parse = true
1398-
1399-
default:
1400-
klog.Warningf("ignoring non-yaml file %s", itemPath)
1401-
}
1402-
1403-
if !parse {
1404-
continue
1405-
}
1406-
// TODO: Use https://github.com/kubernetes-sigs/kustomize/blob/a5b61016bb40c30dd1b0a78290b28b2330a0383e/kyaml/kio/byteio_reader.go#L170 or similar?
1407-
for _, s := range strings.Split(itemContents, "\n---\n") {
1408-
if isWhitespace(s) {
1409-
continue
1410-
}
1411-
1412-
o := &unstructured.Unstructured{}
1413-
if err := yaml.Unmarshal([]byte(s), &o); err != nil {
1414-
return nil, fmt.Errorf("error parsing yaml from %s: %w", itemPath, err)
1415-
}
1416-
1417-
// TODO: sync with kpt logic; skip objects marked with the local-only annotation
1387+
unstructureds, err := objects.Parser{}.AsUnstructureds(resources)
1388+
if err != nil {
1389+
return nil, err
1390+
}
14181391

1419-
configMapGK := schema.GroupKind{Kind: "ConfigMap"}
1420-
if o.GroupVersionKind().GroupKind() == configMapGK {
1421-
if o.GetName() == builtins.PkgContextName {
1422-
matches = append(matches, o)
1423-
}
1392+
var matches []*unstructured.Unstructured
1393+
for _, o := range unstructureds {
1394+
configMapGK := schema.GroupKind{Kind: "ConfigMap"}
1395+
if o.GroupVersionKind().GroupKind() == configMapGK {
1396+
if o.GetName() == builtins.PkgContextName {
1397+
matches = append(matches, o)
14241398
}
14251399
}
14261400
}
@@ -1434,12 +1408,3 @@ func ExtractContextConfigMap(resources map[string]string) (*unstructured.Unstruc
14341408

14351409
return matches[0], nil
14361410
}
1437-
1438-
func isWhitespace(s string) bool {
1439-
for _, r := range s {
1440-
if !unicode.IsSpace(r) {
1441-
return false
1442-
}
1443-
}
1444-
return true
1445-
}

‎porch/pkg/objects/extract.go

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package objects
16+
17+
import (
18+
"fmt"
19+
"path"
20+
"strings"
21+
"unicode"
22+
23+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
24+
"k8s.io/klog/v2"
25+
"sigs.k8s.io/yaml"
26+
)
27+
28+
type Parser struct {
29+
IncludeLocalConfig bool
30+
}
31+
32+
func (p Parser) AsUnstructureds(resources map[string]string) ([]*unstructured.Unstructured, error) {
33+
objectList, err := p.AsObjectList(resources)
34+
if err != nil {
35+
return nil, err
36+
}
37+
unstructureds, err := objectList.AsUnstructured()
38+
if err != nil {
39+
return nil, err
40+
}
41+
return unstructureds, nil
42+
}
43+
44+
func (p Parser) AsObjectList(resources map[string]string) (*ObjectList, error) {
45+
var items []*Object
46+
47+
for filePath, fileContents := range resources {
48+
ext := path.Ext(filePath)
49+
ext = strings.ToLower(ext)
50+
51+
parse := false
52+
switch ext {
53+
case ".yaml", ".yml":
54+
parse = true
55+
56+
default:
57+
klog.Warningf("ignoring non-yaml file %s", filePath)
58+
}
59+
60+
if !parse {
61+
continue
62+
}
63+
// TODO: Use https://github.com/kubernetes-sigs/kustomize/blob/a5b61016bb40c30dd1b0a78290b28b2330a0383e/kyaml/kio/byteio_reader.go#L170 or similar?
64+
for _, s := range strings.Split(fileContents, "\n---\n") {
65+
if isWhitespace(s) {
66+
continue
67+
}
68+
69+
raw := []byte(s)
70+
u := &unstructured.Unstructured{}
71+
if err := yaml.Unmarshal(raw, &u); err != nil {
72+
return nil, fmt.Errorf("error parsing yaml from %s: %w", filePath, err)
73+
}
74+
75+
annotations := u.GetAnnotations()
76+
77+
if !p.IncludeLocalConfig {
78+
s := annotations["config.kubernetes.io/local-config"]
79+
if s == "true" {
80+
continue
81+
}
82+
}
83+
84+
obj := &Object{
85+
filePath: filePath,
86+
raw: raw,
87+
u: u,
88+
}
89+
90+
// TODO: sync with kpt logic; skip objects marked with the local-only annotation
91+
items = append(items, obj)
92+
}
93+
}
94+
95+
return &ObjectList{Items: items}, nil
96+
}
97+
98+
func isWhitespace(s string) bool {
99+
for _, r := range s {
100+
if !unicode.IsSpace(r) {
101+
return false
102+
}
103+
}
104+
return true
105+
}

0 commit comments

Comments
 (0)