various updates (#1124)

Author: arjunsuresh

cm-mlops/automation/script/module.py

@@ -1940,7 +1940,7 @@ def _get_variation_tags_from_default_variations(self, variation_meta, variations
                     if tag_to_append not in variations:
                         variation_tag_static = self._get_name_for_dynamic_variation_tag(tag_to_append)
                         if not variation_tag_static or variation_tag_static not in variations:
-                            return {'return': 1, 'error': 'Invalid variation "{}" specified in default variations for the variation "{}" '.format(tag_to_append, variation_name)}
+                            return {'return': 1, 'error': 'Invalid variation "{}" specified in default variations for the variation "{}" '.format(tag_to_append, variation_meta)}
                     tmp_variation_tags.append(tag_to_append)
 
         return {'return': 0, 'variations_to_add': tmp_variation_tags}

cm-mlops/script/app-mlperf-inference-reference/_cm.yaml

@@ -563,7 +563,7 @@ prehook_deps:
     - remote-run-cmds
     tags: remote,run,cmds
     enable_if_env:
-      CM_SSH_RUN_COMMANDS:
+      CM_ASSH_RUN_COMMANDS:
       - "on"
 
 posthook_deps: 

cm-mlops/script/app-mlperf-inference/_cm.yaml

@@ -328,7 +328,7 @@ variations:
 
   kilt,qualcomm,bert-99.9:
     default_variations:
-      precision: fp16
+      precision: float16
 
   resnet50:
     group:

cm-mlops/script/benchmark-any-mlperf-inference-implementation/_cm.yaml

@@ -291,3 +291,27 @@ variations:
           glow:
             offline_target_qps: 350
             server_target_qps: 300
+  rb6:
+    group: sut
+    default_env:
+      CATEGORY: edge
+      DIVISION: closed
+    default_variations:
+      implementation: qualcomm
+
+  rb6,qualcomm:
+    default_env:
+      EXTRA_ARGS: " --adr.mlperf-inference-implementation.tags=_rb6 --env.CM_MLPERF_SHORT_RANGING_RUN=no"
+    state:
+      resnet50:
+        qaic:
+          glow:
+            offline_target_qps: 6800
+      retinanet:
+        qaic:
+          glow:
+            offline_target_qps: 125
+      bert-99:
+        qaic:
+          glow:
+            offline_target_qps: 255

cm-mlops/script/calibrate-model-for.qaic/_cm.json

@@ -74,7 +74,6 @@
   "variations": {
     "bs.1": {
       "group": "batch-size",
-      "default": true,
       "env": {
         "CM_QAIC_MODEL_BATCH_SIZE": "1",
         "CM_CREATE_INPUT_BATCH": "yes"

cm-mlops/script/compile-model-for.qaic/_cm.json

@@ -5,7 +5,7 @@
   "cache": true,
   "category": "AI/ML optimization",
   "new_env_keys": [
-    "CM_QAIC_*",
+    "CM_QAIC_MODEL*",
     "CM_ML_MODEL_FILE_WITH_PATH"
   ],
   "input_mapping": {
@@ -204,7 +204,8 @@
     },
     "resnet50,multistream": {
       "env": {
-        "CM_QAIC_MODEL_COMPILER_ARGS": ""
+        "CM_QAIC_MODEL_COMPILER_ARGS": "",
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=4 -mos=1 -ols=1"
       }
     },
     "resnet50,server": {
@@ -214,13 +215,13 @@
     "bert-99": {
       "adr": {
         "model-src": {
-          "tags": "bert-large,_onnx"
+          "tags": "bert-large,_onnx,_packed"
         }
       },
       "env": {
         "CM_COMPILE_BERT": "on",
         "CM_QAIC_MODEL_TO_CONVERT": "calibrate_bert_mlperf",
-        "CM_QAIC_MODEL_COMPILER_PARAMS_BASE": "-aic-hw -aic-hw-version=2.0 -execute-nodes-in-fp16=Mul,Sqrt,Div,Add,ReduceMean,Softmax,Sub,Gather,Erf,Pow,Concat,Tile,LayerNormalization -quantization-schema=symmetric_with_uint8 -quantization-precision=Int8 -quantization-precision-bias=Int32 -vvv -compile-only -onnx-define-symbol=batch_size,1 -onnx-define-symbol=seg_length,384 -multicast-weights -combine-inputs=false -combine-outputs=false",
+        "CM_QAIC_MODEL_COMPILER_PARAMS_BASE": "-aic-hw -aic-hw-version=2.0 -execute-nodes-in-fp16=Add,Div,Erf,Softmax -quantization-schema=symmetric_with_uint8 -quantization-precision=Int8 -quantization-precision-bias=Int32 -vvv -compile-only -onnx-define-symbol=batch_size,1 -onnx-define-symbol=seg_length,384 -multicast-weights -combine-inputs=false -combine-outputs=false",
         "CM_QAIC_MODEL_COMPILER_ARGS": ""
       },
       "deps": [
@@ -236,7 +237,12 @@
     "bert-99,offline": {
       "env": {
         "CM_QAIC_MODEL_COMPILER_ARGS": "-allocator-dealloc-delay=2 -size-split-granularity=1536 -vtcm-working-set-limit-ratio=1",
-        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=1 -mos=1 -ols=3"
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=1 -mos=1 -ols=2"
+      }
+    },
+    "bert-99,offline,nsp.16": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=1 -mos=1 -ols=2"
       }
     },
     "bert-99,offline,nsp.14": {
@@ -266,6 +272,47 @@
         "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=8 -mos=8 -ols=1"
       }
     },
+    "bert-99.9": {
+      "base": [
+        "no-quantized"
+      ],
+      "adr": {
+        "model-src": {
+          "tags": "bert-large,_onnx,_packed"
+        }
+      },
+      "env": {
+        "CM_COMPILE_BERT": "on",
+        "CM_QAIC_MODEL_TO_CONVERT": "bert_mlperf",
+        "CM_QAIC_MODEL_COMPILER_PARAMS_BASE": "-aic-hw -aic-hw-version=2.0 -convert-to-fp16 -vvv -compile-only -onnx-define-symbol=batch_size,1 -onnx-define-symbol=seg_length,384 -combine-inputs=false -combine-outputs=false",
+        "CM_QAIC_MODEL_COMPILER_ARGS": ""
+      }
+    },
+    "bert-99.9,offline": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=2 -mos=1 -ols=2"
+      }
+    },
+    "bert-99.9,offline,nsp.16": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=2 -mos=1 -ols=2"
+      }
+    },
+    "bert-99.9,offline,nsp.14": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=2 -mos=1 -ols=2"
+      }
+    },
+    "bert-99.9,server": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=4"
+      }
+    },
+    "bert-99.9,server,nsp.14": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=4"
+      }
+    },
     "tf": {
       "group": "model-framework"
     },

cm-mlops/script/compile-model-for.qaic/customize.py

@@ -22,6 +22,8 @@ def preprocess(i):
 
         print("Compiling from "+ os.getcwd())
 
+        env['CM_QAIC_MODEL_FINAL_COMPILATION_CMD'] = cmd
+
         env['CM_RUN_CMD'] = cmd
     else:
         import shutil

cm-mlops/script/compile-model-for.qaic/run.sh

@@ -24,5 +24,5 @@ function run() {
 }
 
 #Add your run commands here...
-#run "rm -rf elfs" 
+run "rm -rf elfs" 
 run "$CM_RUN_CMD"

cm-mlops/script/compile-program/customize.py

@@ -16,7 +16,8 @@ def preprocess(i):
     env['CM_F_INCLUDE_PATH'] = " -I".join([" "] + env.get('+F_INCLUDE_PATH', []) + CPATH)
 
     # If windows, need to extend it more ...
-    if os_info['platform'] == 'windows':
+    if os_info['platform'] == 'windows' and env.get('CM_COMPILER_FAMILY','')!='LLVM':
+        print ("WARNING: compile-program script should be extended to support flags for non-LLVM compilers on Windows")
         return {'return':0}
 
     LDFLAGS = env.get('+ LDFLAGS', [])

cm-mlops/script/compile-program/run.bat

@@ -22,14 +22,14 @@ cd %CM_SOURCE_FOLDER_PATH%
 IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
 
 if not "%CM_C_SOURCE_FILES%" == "" (
-  echo %CM_C_COMPILER_WITH_PATH% %CM_C_COMPILER_FLAGS% %CM_C_INCLUDE_PATH% %CM_C_SOURCE_FILES% %CM_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
-  "%CM_C_COMPILER_WITH_PATH%" %CM_C_COMPILER_FLAGS% %CM_C_INCLUDE_PATH% %CM_C_SOURCE_FILES% %CM_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
+  echo %CM_C_COMPILER_WITH_PATH% %CM_C_COMPILER_FLAGS% %CM_C_INCLUDE_PATH% %CM_C_SOURCE_FILES% %CM_LD_LIBRARY_PATH% %LDCFLAGS% %CM_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
+  "%CM_C_COMPILER_WITH_PATH%" %CM_C_COMPILER_FLAGS% %CM_C_INCLUDE_PATH% %CM_C_SOURCE_FILES% %CM_LD_LIBRARY_PATH% %LDCFLAGS% %CM_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
   IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
 )
 
 if not "%CM_CXX_SOURCE_FILES%" == "" (
-  echo %CM_CXX_COMPILER_WITH_PATH% %CM_CXX_COMPILER_FLAGS% %CM_CXX_INCLUDE_PATH% %CM_CXX_SOURCE_FILES% %CM_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
-  "%CM_CXX_COMPILER_WITH_PATH%" %CM_CXX_COMPILER_FLAGS% %CM_CXX_INCLUDE_PATH% %CM_CXX_SOURCE_FILES% %CM_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
+  echo %CM_CXX_COMPILER_WITH_PATH%  %CM_CXX_SOURCE_FILES% %CM_CXX_COMPILER_FLAGS% %CM_CPLUS_INCLUDE_PATH% %CM_LD_LIBRARY_PATH% %LDCXXFLAGS% %CM_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
+  "%CM_CXX_COMPILER_WITH_PATH%"  %CM_CXX_SOURCE_FILES% %CM_CXX_COMPILER_FLAGS% %CM_CPLUS_INCLUDE_PATH% %CM_LD_LIBRARY_PATH% %LDCXXFLAGS% %CM_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
   IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
 )