Move TAS pod and TAS demo into their own namespaces

madalazar · uniemimu · commit 09906449702c · 2023-08-02T20:15:37.000+03:00
This PR will:
- move TAS pod and associated resources in the
'telemetry-aware-scheduling' namespace
- move health-metric demo and its associated resource in the
'health-metric-demo' namespace
- move power demo and its associated resources in the
'power-demo' namespace
- move stategy-labeling demo and its associated resources
in the 'labeling-demo' namespace
- add new step in BM WFs to clean-up TAS related resources
 before exiting
- update Kind E2E tests policy to match new TAS namespace
- update related docs
- update demo docs with supported descheduler versions

Signed-off-by: Madalina Lazar &lt;madalina.lazar@intel.com&gt;
diff --git a/.github/scripts/e2e_setup_cluster.sh b/.github/scripts/e2e_setup_cluster.sh
@@ -247,8 +247,8 @@ mkdir "${mount_dir}/certs"
 docker cp kind-control-plane:/etc/kubernetes/pki/ca.crt "${mount_dir}/certs/client.crt"
 docker cp kind-control-plane:/etc/kubernetes/pki/ca.key "${mount_dir}/certs/client.key"
 
-
-kubectl create secret tls extender-secret --cert "${mount_dir}/certs/client.crt" --key "${mount_dir}/certs/client.key"
+kubectl create namespace telemetry-aware-scheduling
+kubectl create secret tls extender-secret --cert "${mount_dir}/certs/client.crt" --key "${mount_dir}/certs/client.key" -n telemetry-aware-scheduling
 sed "s/intel\/telemetry-aware-scheduling.*$/tasextender/g" "${root}/telemetry-aware-scheduling/deploy/tas-deployment.yaml" -i
 set_node_affinity_and_tolerations
 kubectl apply -f "${root}/telemetry-aware-scheduling/deploy/"
diff --git a/.github/scripts/policies/policy.yaml b/.github/scripts/policies/policy.yaml
@@ -6,7 +6,7 @@
         },
         "extenders" : [
             {
-              "urlPrefix": "https://tas-service.default.svc.cluster.local:9001",             
+              "urlPrefix": "https://tas-service.telemetry-aware-scheduling.svc.cluster.local:9001",
               "prioritizeVerb": "scheduler/prioritize",
               "filterVerb": "scheduler/filter",
               "weight": 100,
diff --git a/.github/workflows/BM-end-to-end.yaml b/.github/workflows/BM-end-to-end.yaml
@@ -51,6 +51,8 @@ jobs:
       run: ansible-playbook -i ${{ env.CI_CNO_ANSIBLE_FOLDER }}/inventory_cni.ini ${{ env.CI_CNO_PLAYBOOK_TAS_FOLDER }}/build.yml -e ${{ env.ANSIBLE_TAS_FOLDER_DESTINATION_PARAM }}
     - name: BM Smoke Test
       run: ansible-playbook -i  ${{ env.CI_CNO_ANSIBLE_FOLDER }}/inventory_cni.ini ${{ env.CI_CNO_PLAYBOOK_TAS_FOLDER }}/smokeTest.yml -e ${{ env.ANSIBLE_TAS_FOLDER_DESTINATION_PARAM }}
+    - name: BM Clean-up TAS
+      run: ansible-playbook -i  ${{ env.CI_CNO_ANSIBLE_FOLDER }}/inventory_cni.ini ${{ env.CI_CNO_PLAYBOOK_TAS_FOLDER }}/clusterCleanup.yml -e ${{ env.ANSIBLE_TAS_FOLDER_DESTINATION_PARAM }}
     - name: BM Clean-up
       run: ansible-playbook -i  ${{ env.CI_CNO_ANSIBLE_FOLDER }}/inventory_cni.ini ${{ env.CI_CNO_PLAYBOOK_COMMON_FOLDER }}/commonCleanup.yml -e ${{ env.ANSIBLE_TELEMETRY_FOLDER_DESTINATION_PARAM }} 
 
diff --git a/.github/workflows/BM-static-analysis.yaml b/.github/workflows/BM-static-analysis.yaml
@@ -53,5 +53,7 @@ jobs:
         run: ansible-playbook -i  ${{ env.CI_CNO_ANSIBLE_FOLDER }}/inventory_cni.ini ${{ env.CI_CNO_PLAYBOOK_COMMON_FOLDER }}/commonTest.yml -e ${{ env.ANSIBLE_TELEMETRY_FOLDER_DESTINATION_PARAM }}
       - name: BM mtlsTest
         run: ansible-playbook -i  ${{ env.CI_CNO_ANSIBLE_FOLDER }}/inventory_cni.ini /${{ env.CI_CNO_PLAYBOOK_TAS_FOLDER }}/mtlsTest.yml -e ${{ env.ANSIBLE_TAS_FOLDER_DESTINATION_PARAM }}
+      - name: BM Clean-up TAS
+        run: ansible-playbook -i  ${{ env.CI_CNO_ANSIBLE_FOLDER }}/inventory_cni.ini ${{ env.CI_CNO_PLAYBOOK_TAS_FOLDER }}/clusterCleanup.yml -e ${{ env.ANSIBLE_TAS_FOLDER_DESTINATION_PARAM }}
       - name: BM Clean-up
         run: ansible-playbook -i  ${{ env.CI_CNO_ANSIBLE_FOLDER }}/inventory_cni.ini ${{ env.CI_CNO_PLAYBOOK_COMMON_FOLDER }}/commonCleanup.yml -e ${{ env.ANSIBLE_TELEMETRY_FOLDER_DESTINATION_PARAM }}
diff --git a/README.md b/README.md
@@ -28,7 +28,7 @@ kind: KubeSchedulerConfiguration
 clientConnection:
   kubeconfig: /etc/kubernetes/scheduler.conf
 extenders:
-  - urlPrefix: "https://tas-service.default.svc.cluster.local:9001"
+  - urlPrefix: "https://tas-service.telemetry-aware-scheduling.svc.cluster.local:9001"
     prioritizeVerb: "scheduler/prioritize"
     filterVerb: "scheduler/filter"
     weight: 1
diff --git a/telemetry-aware-scheduling/README.md b/telemetry-aware-scheduling/README.md
@@ -62,7 +62,7 @@ kind: KubeSchedulerConfiguration
 clientConnection:
   kubeconfig: /etc/kubernetes/scheduler.conf
 extenders:
-  - urlPrefix: "https://tas-service.default.svc.cluster.local:9001"
+  - urlPrefix: "https://tas-service.telemetry-aware-scheduling.svc.cluster.local:9001"
     prioritizeVerb: "scheduler/prioritize"
     filterVerb: "scheduler/filter"
     weight: 1
@@ -91,14 +91,19 @@ Telemetry Aware Scheduling uses go modules. It requires Go 1.16+ with modules en
 TAS current version has been tested with the recent Kubernetes version at the released date. It maintains support to the three most recent K8s versions. 
 TAS was tested on Intel® Server Boards S2600WF and S2600WT-Based Systems.
 
-A yaml file for TAS is contained in the deploy folder along with its service and RBAC roles and permissions.
+A yaml file for TAS is contained in the deploy folder along with its service and RBAC roles and permissions. 
+The TAS components all reside in a custom namespace: **telemetry-aware-scheduling**, which means this namespace needs to be created first:
+
+``
+kubectl create namespace telemetry-aware-scheduling
+``
 
 A secret called extender-secret will need to be created with the cert and key for the TLS endpoint. TAS will not deploy if there is no secret available with the given deployment file.
 
 The secret can be created with:
 
 ``
-kubectl create secret tls extender-secret --cert /etc/kubernetes/<PATH_TO_CERT> --key /etc/kubernetes/<PATH_TO_KEY> 
+kubectl create secret tls extender-secret --cert /etc/kubernetes/<PATH_TO_CERT> --key /etc/kubernetes/<PATH_TO_KEY> -n telemetry-aware-scheduling
 ``
 <details>
 <summary>Cert selection tip for <a href="https://github.com/intel/platform-aware-scheduling/blob/24f25a38613e326b4830f5e647211df16060fe70/telemetry-aware-scheduling/deploy/extender-configuration/configure-scheduler.sh#L136-L137">configure-scheduler.sh</a> users</summary>
@@ -147,7 +152,7 @@ apiVersion: telemetry.intel.com/v1alpha1
 kind: TASPolicy
 metadata:
   name: scheduling-policy
-  namespace: default
+  namespace: health-metric-demo
 spec:
   strategies:
     deschedule:
@@ -214,7 +219,7 @@ apiVersion: telemetry.intel.com/v1alpha1
 kind: TASPolicy
 metadata:
   name: multirules-policy
-  namespace: default
+  namespace: health-metric-demo
 spec:
   strategies:
     deschedule:
@@ -264,6 +269,7 @@ For example,  in a deployment file:
 apiVersion: apps/v1 
 kind: Deployment
 metadata:
+  namespace: health-metric-demo
   name: demo-app
   labels:
     app: demo
diff --git a/telemetry-aware-scheduling/deploy/extender-configuration/scheduler-config.yaml b/telemetry-aware-scheduling/deploy/extender-configuration/scheduler-config.yaml
@@ -3,7 +3,7 @@ kind: KubeSchedulerConfiguration
 clientConnection:
   kubeconfig: /etc/kubernetes/scheduler.conf
 extenders:
-  - urlPrefix: "https://tas-service.default.svc.cluster.local:9001"
+  - urlPrefix: "https://tas-service.telemetry-aware-scheduling.svc.cluster.local:9001"
     prioritizeVerb: "scheduler/prioritize"
     filterVerb: "scheduler/filter"
     weight: 1
diff --git a/telemetry-aware-scheduling/deploy/health-metric-demo/demo-pod.yaml b/telemetry-aware-scheduling/deploy/health-metric-demo/demo-pod.yaml
@@ -2,6 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: nginx-conf
+  namespace: health-metric-demo
 data:
   nginx.conf: |
     worker_processes  auto;
@@ -30,6 +31,7 @@ metadata:
   name: demo-app
   labels:
     app: demo
+  namespace: health-metric-demo
 spec:
   replicas: 1
   selector:
diff --git a/telemetry-aware-scheduling/deploy/health-metric-demo/health-policy.yaml b/telemetry-aware-scheduling/deploy/health-metric-demo/health-policy.yaml
@@ -2,7 +2,7 @@ apiVersion: telemetry.intel.com/v1alpha1
 kind: TASPolicy
 metadata:
   name: demo-policy
-  namespace: default
+  namespace: health-metric-demo
 spec:
   strategies:
     deschedule:
diff --git a/telemetry-aware-scheduling/deploy/tas-deployment.yaml b/telemetry-aware-scheduling/deploy/tas-deployment.yaml
@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: telemetry-aware-scheduling
-  namespace: default 
+  namespace: telemetry-aware-scheduling
   labels:
     app: tas
 spec:
diff --git a/telemetry-aware-scheduling/deploy/tas-rbac-accounts.yaml b/telemetry-aware-scheduling/deploy/tas-rbac-accounts.yaml
@@ -2,10 +2,11 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: handle-policies
+  namespace: telemetry-aware-scheduling
 subjects:
   - kind: ServiceAccount
     name: telemetry-aware-scheduling-service-account
-    namespace: default
+    namespace: telemetry-aware-scheduling
 roleRef:
   kind: ClusterRole
   name: policy-handler
@@ -15,6 +16,7 @@ roleRef:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  namespace: telemetry-aware-scheduling
   name: policy-handler
 rules:
 - apiGroups: ["telemetry.intel.com"]
@@ -34,4 +36,5 @@ rules:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: telemetry-aware-scheduling-service-account
+  name: telemetry-aware-scheduling-service-account
+  namespace: telemetry-aware-scheduling
diff --git a/telemetry-aware-scheduling/deploy/tas-service.yaml b/telemetry-aware-scheduling/deploy/tas-service.yaml
@@ -2,10 +2,10 @@ apiVersion: v1
 kind: Service
 metadata:
   name: tas-service
-  namespace: default
+  namespace: telemetry-aware-scheduling
 spec:
   selector:
     app: tas
   type: ClusterIP
   ports:
-    - port: 9001
+    - port: 9001
diff --git a/telemetry-aware-scheduling/docs/health-metric-example.md b/telemetry-aware-scheduling/docs/health-metric-example.md
@@ -45,6 +45,12 @@ sudo kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/nodes/*/health_metri
 ```
 It may take a couple of minutes for the metric to be initially scraped.
 
+## Setting the workspace
+
+``
+kubectl create namespace health-metric-demo
+``
+
 ## Declare a Telemetry Policy
 A Telemetry Policy should be declared using kubectl apply -f <NAME_OF_FILE>. Our [demo health metric policy](../deploy/health-metric-demo/health-policy.yaml) is:
 
@@ -53,7 +59,7 @@ apiVersion: telemetry.intel.com/v1alpha1
 kind: TASPolicy
 metadata:
   name: demo-policy
-  namespace: default
+  namespace: health-metric-demo
 spec:
   strategies:
     deschedule:
@@ -90,6 +96,7 @@ The pod spec here is:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  namespace: health-metric-demo
   name: demo-app
   labels:
     app: demo
@@ -145,7 +152,11 @@ This should produce extender logs like:
 We can see that only two of our three nodes was deemed suitable because it didn't break the dontschedule rule declared in our Telemetry Scheduling Policy.
 
 ### Deschedule
-Descheduler can be installed as a binary with the instructions from the [project repo](https://github.com/kubernetes-sigs/descheduler.). Please make sure to install version ***0.23.1*** of the Descheduler available [here](https://github.com/kubernetes-sigs/descheduler/tree/v0.23.1). You can use older versions, but make sure you do not use versions ***> 0.23.1  (0.24.0 and so on)*** as there seem to be compatibility issues between the two. https://github.com/intel/platform-awarescheduling/issues/90#issuecomment-1169012485 links to an issue cut to the Descheduler project team to try to find a solution to this problem.
+Descheduler can be installed as a binary with the instructions from the [project repo](https://github.com/kubernetes-sigs/descheduler). This demo has been tested with the following Descheduler versions:
+1. **v0.23.1** and older
+2. **v0.27.1**
+
+Telemetry Aware Scheduler and the following Descheduler versions **v0.24.x** to **v0.26.x** seem to have compatibility issues (https://github.com/intel/platform-aware-scheduling/issues/90#issuecomment-1169012485 links to an issue cut to the Descheduler project team). The problem seems to have been fixed in Descheduler **v0.27.1**.
 
 A policy file in the health-metric-demo should be passed to the descheduler as a flag.
 
@@ -222,3 +233,29 @@ sudo kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/" | jq . | grep node
 ```
 
 If this command doesn't return multiple metrics with paths like **/nodes/*/<METRIC_NAME> it's likely there is some issue with the scrape configuration for the custom metrics api. More info can be seen at [custom-metrics.md](custom-metrics.md)
+
+### Descheduler
+
+If the pods are not moving to the non-violating node when running the descheduler comment, please check the descheduler logs for any errors.
+
+#### Insufficient telemetry/scheduling
+
+Example error:
+````
+I0629 17:54:44.063789       1 node.go:168] "Pod does not fit on node" pod="labeling-demo/demo-app-label-6b9cc98bf4-rd9mx" node="NODE-A"
+I0629 17:54:44.063807       1 node.go:170] "insufficient telemetry/scheduling"
+````
+To address this error we need to patch all nodes in the cluster by adding the missing resource. To do so:
+
+1. On the control plane node, in a new window run:
+
+````
+kubectl proxy
+````
+2. On the control plane node, for each node run:
+
+````
+curl --header "Content-Type: application/json-patch+json" --request PATCH --data '[{"op": "add", "path": "/status/capacity/telemetry~1scheduling", "value": "10"}]' http://localhost:8001/api/v1/nodes/$NODE_NAME/status
+````
+
+**[NOTE]** The value chosen for this example is random. The only constraints are: the value has to be a positive integer (> 0) and high enough to be able to fulfill the resource specs. More details [here](https://github.com/intel/platform-aware-scheduling/tree/master/telemetry-aware-scheduling#linking-a-workload-to-a-policy)
diff --git a/telemetry-aware-scheduling/docs/power/README.md b/telemetry-aware-scheduling/docs/power/README.md
@@ -211,6 +211,10 @@ This command should return some json objects containing pods and associated powe
 ### 4: Create TAS Telemetry Policy for power
 On the Kubernetes BMRA set up Telemetry Aware Scheduling is already installed and integrated with the Kubernetes control plane. In order to allow our cluster to make scheduling decisions based on current power usage we simply need to create a telemetry policy and associate appropriate pods with that policy.
 
+``kubectl create namespace power-demo``
+
+then
+
 ``kubectl apply -f $POW_DIR/tas-policy.yaml``
 
 We can see our policy by running:
diff --git a/telemetry-aware-scheduling/docs/power/power-autoscaler.yaml b/telemetry-aware-scheduling/docs/power/power-autoscaler.yaml
@@ -2,7 +2,7 @@ apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
   name: power-autoscaler
-  namespace: default
+  namespace: power-demo
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
diff --git a/telemetry-aware-scheduling/docs/power/power-hungry-application.yaml b/telemetry-aware-scheduling/docs/power/power-hungry-application.yaml
@@ -1,6 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  namespace: power-demo
   name: power-hungry-application 
   labels:
     app: power-hungry-application 
diff --git a/telemetry-aware-scheduling/docs/power/tas-policy.yaml b/telemetry-aware-scheduling/docs/power/tas-policy.yaml
@@ -2,7 +2,7 @@ apiVersion: telemetry.intel.com/v1alpha1
 kind: TASPolicy
 metadata:
   name: power-sensitive-scheduling-policy
-  namespace: default
+  namespace: power-demo
 spec:
   strategies:
     dontschedule:
diff --git a/telemetry-aware-scheduling/docs/strategy-labeling-example.md b/telemetry-aware-scheduling/docs/strategy-labeling-example.md

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`	`},`
`7`	`7`	`"extenders" : [`
`8`	`8`	`{`
`9`		`- "urlPrefix": "https://tas-service.default.svc.cluster.local:9001",`
	`9`	`+ "urlPrefix": "https://tas-service.telemetry-aware-scheduling.svc.cluster.local:9001",`
`10`	`10`	`"prioritizeVerb": "scheduler/prioritize",`
`11`	`11`	`"filterVerb": "scheduler/filter",`
`12`	`12`	`"weight": 100,`