Skip to content

Commit 3b58b26

Browse files
voyvodovHristo Voyvodov
voyvodov
and
Hristo Voyvodov
authored
Add ingress-class-name controller flag (#1482)
* Add ingress-class-name controller flag Add a flag to set ingressClassName field for ingress objects created for Spark UI. This will make ingress compliant with Kubernetes >v1.19 and better utilizing multiple ingress controllers * Update Helm chart to v1.1.20/appVersion 1.3.4 Include ingressClassName changes Co-authored-by: Hristo Voyvodov <[email protected]>
1 parent 55732a6 commit 3b58b26

File tree

6 files changed

+95
-21
lines changed

6 files changed

+95
-21
lines changed

‎charts/spark-operator-chart/Chart.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
apiVersion: v2
22
name: spark-operator
33
description: A Helm chart for Spark on Kubernetes operator
4-
version: 1.1.19
5-
appVersion: v1beta2-1.3.3-3.1.1
4+
version: 1.1.20
5+
appVersion: v1beta2-1.3.4-3.1.1
66
keywords:
77
- spark
88
home: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator

‎main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ var (
7373
metricsPort = flag.String("metrics-port", "10254", "Port for the metrics endpoint.")
7474
metricsEndpoint = flag.String("metrics-endpoint", "/metrics", "Metrics endpoint.")
7575
metricsPrefix = flag.String("metrics-prefix", "", "Prefix for the metrics.")
76+
ingressClassName = flag.String("ingress-class-name", "", "Set ingressClassName for ingress resources created.")
7677
metricsLabels util.ArrayFlags
7778
metricsJobStartLatencyBuckets util.HistogramBuckets = util.DefaultJobStartLatencyBuckets
7879
)
@@ -184,7 +185,7 @@ func main() {
184185
}
185186

186187
applicationController := sparkapplication.NewController(
187-
crClient, kubeClient, crInformerFactory, podInformerFactory, metricConfig, *namespace, *ingressURLFormat, batchSchedulerMgr, *enableUIService)
188+
crClient, kubeClient, crInformerFactory, podInformerFactory, metricConfig, *namespace, *ingressURLFormat, *ingressClassName, batchSchedulerMgr, *enableUIService)
188189
scheduledApplicationController := scheduledsparkapplication.NewController(
189190
crClient, kubeClient, apiExtensionsClient, crInformerFactory, clock.RealClock{})
190191

‎pkg/controller/sparkapplication/controller.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ type Controller struct {
7676
applicationLister crdlisters.SparkApplicationLister
7777
podLister v1.PodLister
7878
ingressURLFormat string
79+
ingressClassName string
7980
batchSchedulerMgr *batchscheduler.SchedulerManager
8081
enableUIService bool
8182
}
@@ -89,6 +90,7 @@ func NewController(
8990
metricsConfig *util.MetricConfig,
9091
namespace string,
9192
ingressURLFormat string,
93+
ingressClassName string,
9294
batchSchedulerMgr *batchscheduler.SchedulerManager,
9395
enableUIService bool) *Controller {
9496
crdscheme.AddToScheme(scheme.Scheme)
@@ -100,7 +102,7 @@ func NewController(
100102
})
101103
recorder := eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "spark-operator"})
102104

103-
return newSparkApplicationController(crdClient, kubeClient, crdInformerFactory, podInformerFactory, recorder, metricsConfig, ingressURLFormat, batchSchedulerMgr, enableUIService)
105+
return newSparkApplicationController(crdClient, kubeClient, crdInformerFactory, podInformerFactory, recorder, metricsConfig, ingressURLFormat, ingressClassName, batchSchedulerMgr, enableUIService)
104106
}
105107

106108
func newSparkApplicationController(
@@ -111,6 +113,7 @@ func newSparkApplicationController(
111113
eventRecorder record.EventRecorder,
112114
metricsConfig *util.MetricConfig,
113115
ingressURLFormat string,
116+
ingressClassName string,
114117
batchSchedulerMgr *batchscheduler.SchedulerManager,
115118
enableUIService bool) *Controller {
116119
queue := workqueue.NewNamedRateLimitingQueue(&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(queueTokenRefillRate), queueTokenBucketSize)},
@@ -122,6 +125,7 @@ func newSparkApplicationController(
122125
recorder: eventRecorder,
123126
queue: queue,
124127
ingressURLFormat: ingressURLFormat,
128+
ingressClassName: ingressClassName,
125129
batchSchedulerMgr: batchSchedulerMgr,
126130
enableUIService: enableUIService,
127131
}
@@ -687,7 +691,7 @@ func (c *Controller) submitSparkApplication(app *v1beta2.SparkApplication) *v1be
687691
app.Spec.SparkConf["spark.ui.proxyBase"] = ingressURL.Path
688692
app.Spec.SparkConf["spark.ui.proxyRedirectUri"] = "/"
689693
}
690-
ingress, err := createSparkUIIngress(app, *service, ingressURL, c.kubeClient)
694+
ingress, err := createSparkUIIngress(app, *service, ingressURL, c.ingressClassName, c.kubeClient)
691695
if err != nil {
692696
glog.Errorf("failed to create UI Ingress for SparkApplication %s/%s: %v", app.Namespace, app.Name, err)
693697
} else {

‎pkg/controller/sparkapplication/controller_test.go

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func newFakeController(app *v1beta2.SparkApplication, pods ...*apiv1.Pod) (*Cont
6868

6969
podInformerFactory := informers.NewSharedInformerFactory(kubeClient, 0*time.Second)
7070
controller := newSparkApplicationController(crdClient, kubeClient, informerFactory, podInformerFactory, recorder,
71-
&util.MetricConfig{}, "", nil, true)
71+
&util.MetricConfig{}, "", "", nil, true)
7272

7373
informer := informerFactory.Sparkoperator().V1beta2().SparkApplications().Informer()
7474
if app != nil {
@@ -1619,6 +1619,52 @@ func TestIngressWithSubpathAffectsSparkConfiguration(t *testing.T) {
16191619
}
16201620
}
16211621

1622+
func TestIngressWithClassName(t *testing.T) {
1623+
os.Setenv(kubernetesServiceHostEnvVar, "localhost")
1624+
os.Setenv(kubernetesServicePortEnvVar, "443")
1625+
1626+
appName := "ingressaffectssparkconfig"
1627+
1628+
app := &v1beta2.SparkApplication{
1629+
ObjectMeta: metav1.ObjectMeta{
1630+
Name: appName,
1631+
Namespace: "test",
1632+
},
1633+
Spec: v1beta2.SparkApplicationSpec{
1634+
RestartPolicy: v1beta2.RestartPolicy{
1635+
Type: v1beta2.Never,
1636+
},
1637+
TimeToLiveSeconds: int64ptr(1),
1638+
},
1639+
Status: v1beta2.SparkApplicationStatus{},
1640+
}
1641+
1642+
ctrl, _ := newFakeController(app)
1643+
ctrl.ingressURLFormat = "{{$appNamespace}}.{{$appName}}.example.com"
1644+
ctrl.ingressClassName = "nginx"
1645+
ctrl.enableUIService = true
1646+
_, err := ctrl.crdClient.SparkoperatorV1beta2().SparkApplications(app.Namespace).Create(context.TODO(), app, metav1.CreateOptions{})
1647+
if err != nil {
1648+
t.Fatal(err)
1649+
}
1650+
err = ctrl.syncSparkApplication(fmt.Sprintf("%s/%s", app.Namespace, app.Name))
1651+
assert.Nil(t, err)
1652+
_, err = ctrl.crdClient.SparkoperatorV1beta2().SparkApplications(app.Namespace).Get(context.TODO(), app.Name, metav1.GetOptions{})
1653+
if err != nil {
1654+
t.Fatal(err)
1655+
}
1656+
ingresses, err := ctrl.kubeClient.NetworkingV1().Ingresses(app.Namespace).List(context.TODO(), metav1.ListOptions{})
1657+
if err != nil {
1658+
t.Fatal(err)
1659+
}
1660+
if ingresses == nil || ingresses.Items == nil || len(ingresses.Items) != 1 {
1661+
t.Fatal("The ingress does not exist, has no items, or wrong amount of items")
1662+
}
1663+
if ingresses.Items[0].Spec.IngressClassName == nil || *ingresses.Items[0].Spec.IngressClassName != "nginx" {
1664+
t.Fatal("The ingressClassName does not exists, or wrong value is set")
1665+
}
1666+
}
1667+
16221668
func stringptr(s string) *string {
16231669
return &s
16241670
}

‎pkg/controller/sparkapplication/sparkui.go

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -75,21 +75,22 @@ type SparkService struct {
7575

7676
// SparkIngress encapsulates information about the driver UI ingress.
7777
type SparkIngress struct {
78-
ingressName string
79-
ingressURL *url.URL
80-
annotations map[string]string
81-
ingressTLS []networkingv1.IngressTLS
78+
ingressName string
79+
ingressURL *url.URL
80+
ingressClassName string
81+
annotations map[string]string
82+
ingressTLS []networkingv1.IngressTLS
8283
}
8384

84-
func createSparkUIIngress(app *v1beta2.SparkApplication, service SparkService, ingressURL *url.URL, kubeClient clientset.Interface) (*SparkIngress, error) {
85+
func createSparkUIIngress(app *v1beta2.SparkApplication, service SparkService, ingressURL *url.URL, ingressClassName string, kubeClient clientset.Interface) (*SparkIngress, error) {
8586
if util.IngressCapabilities.Has("networking.k8s.io/v1") {
86-
return createSparkUIIngress_v1(app, service, ingressURL, kubeClient)
87+
return createSparkUIIngress_v1(app, service, ingressURL, ingressClassName, kubeClient)
8788
} else {
8889
return createSparkUIIngress_legacy(app, service, ingressURL, kubeClient)
8990
}
9091
}
9192

92-
func createSparkUIIngress_v1(app *v1beta2.SparkApplication, service SparkService, ingressURL *url.URL, kubeClient clientset.Interface) (*SparkIngress, error) {
93+
func createSparkUIIngress_v1(app *v1beta2.SparkApplication, service SparkService, ingressURL *url.URL, ingressClassName string, kubeClient clientset.Interface) (*SparkIngress, error) {
9394
ingressResourceAnnotations := getIngressResourceAnnotations(app)
9495
ingressTlsHosts := getIngressTlsHosts(app)
9596

@@ -145,16 +146,21 @@ func createSparkUIIngress_v1(app *v1beta2.SparkApplication, service SparkService
145146
if len(ingressTlsHosts) != 0 {
146147
ingress.Spec.TLS = ingressTlsHosts
147148
}
149+
if len(ingressClassName) != 0 {
150+
ingress.Spec.IngressClassName = &ingressClassName
151+
}
152+
148153
glog.Infof("Creating an Ingress %s for the Spark UI for application %s", ingress.Name, app.Name)
149154
_, err := kubeClient.NetworkingV1().Ingresses(ingress.Namespace).Create(context.TODO(), &ingress, metav1.CreateOptions{})
150155
if err != nil {
151156
return nil, err
152157
}
153158
return &SparkIngress{
154-
ingressName: ingress.Name,
155-
ingressURL: ingressURL,
156-
annotations: ingress.Annotations,
157-
ingressTLS: ingressTlsHosts,
159+
ingressName: ingress.Name,
160+
ingressURL: ingressURL,
161+
ingressClassName: ingressClassName,
162+
annotations: ingress.Annotations,
163+
ingressTLS: ingressTlsHosts,
158164
}, nil
159165
}
160166

‎pkg/controller/sparkapplication/sparkui_test.go

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ func TestCreateSparkUIIngress(t *testing.T) {
330330
expectError bool
331331
}
332332

333-
testFn := func(test testcase, t *testing.T, ingressURLFormat string) {
333+
testFn := func(test testcase, t *testing.T, ingressURLFormat string, ingressClassName string) {
334334
fakeClient := fake.NewSimpleClientset()
335335
sparkService, err := createSparkUIService(test.app, fakeClient)
336336
if err != nil {
@@ -340,7 +340,7 @@ func TestCreateSparkUIIngress(t *testing.T) {
340340
if err != nil {
341341
t.Fatal(err)
342342
}
343-
sparkIngress, err := createSparkUIIngress(test.app, *sparkService, ingressURL, fakeClient)
343+
sparkIngress, err := createSparkUIIngress(test.app, *sparkService, ingressURL, ingressClassName, fakeClient)
344344
if err != nil {
345345
if test.expectError {
346346
return
@@ -492,6 +492,7 @@ func TestCreateSparkUIIngress(t *testing.T) {
492492
},
493493
},
494494
}
495+
495496
testcases := []testcase{
496497
{
497498
name: "simple ingress object",
@@ -550,7 +551,7 @@ func TestCreateSparkUIIngress(t *testing.T) {
550551
}
551552

552553
for _, test := range testcases {
553-
testFn(test, t, "{{$appName}}.ingress.clusterName.com")
554+
testFn(test, t, "{{$appName}}.ingress.clusterName.com", "")
554555
}
555556

556557
testcases = []testcase{
@@ -569,7 +570,23 @@ func TestCreateSparkUIIngress(t *testing.T) {
569570
}
570571

571572
for _, test := range testcases {
572-
testFn(test, t, "ingress.clusterName.com/{{$appNamespace}}/{{$appName}}")
573+
testFn(test, t, "ingress.clusterName.com/{{$appNamespace}}/{{$appName}}", "")
574+
}
575+
576+
testcases = []testcase{
577+
{
578+
name: "simple ingress object with ingressClassName set",
579+
app: app1,
580+
expectedIngress: SparkIngress{
581+
ingressName: fmt.Sprintf("%s-ui-ingress", app1.GetName()),
582+
ingressURL: parseURLAndAssertError(app1.GetName()+".ingress.clusterName.com", t),
583+
ingressClassName: "nginx",
584+
},
585+
expectError: false,
586+
},
587+
}
588+
for _, test := range testcases {
589+
testFn(test, t, "{{$appName}}.ingress.clusterName.com", "nginx")
573590
}
574591
}
575592

0 commit comments

Comments
 (0)