[Commit Slider] AC / e2e compatibility (openvinotoolkit#24410)

yury-intel · AsyaPronina · commit a998c2864ea1 · 2024-07-01T14:40:00.000Z
### Details:
 - *tool - specific substitutions moved to config*
diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py
@@ -142,7 +142,7 @@ def testBmSteppedBreak2(self):
         )
 
     @skip_commit_slider_devtest
-    def testForsubstitutionRule(self):
+    def testForMapSubstitutionRule(self):
         from utils.helpers import applySubstitutionRules
         cfg = {
                 "serviceConfig": {
@@ -228,6 +228,74 @@ def applyByRef(cfg: map, rules: list, substitution: str):
             "{commitHash1} is unchanged"
         )
 
+    @skip_commit_slider_devtest
+    def testForStaticSubstitutionRule(self):
+        from utils.helpers import applySubstitutionRules
+        cfg = {
+                "serviceConfig": {
+                    "previousKey": "previousValue"
+                },
+                "wrongDst": "{pathOne} is unchanged",
+                "dst": {
+                    "complex": {
+                        "path": [
+                            "{pathOne} is natural number",
+                            "{pathTwo} is natural number",
+                            "{pathOne} is not {pathTwo}"
+                        ]
+                    }
+                },
+                "src": {
+                    "complex": {
+                        "path": {
+                            "one": "1",
+                            "two": "2"
+                        }
+                    }
+                }
+        }
+        rules = [
+            {
+                "name": "testRule1",
+                "enabled": True,
+                "type": "static",
+                "placeholder": "pathOne",
+                "from": "$.src.complex.path.one",
+                "to": "$.dst.complex.path"
+            },
+            {
+                "name": "testRule2",
+                "enabled": True,
+                "type": "static",
+                "placeholder": "pathTwo",
+                "from": "$.src.complex.path.two",
+                "to": "$.dst.complex.path"
+            }
+        ]
+        def applyByRef(cfg: map, rules: list, substitution: str):
+            applySubstitutionRules(cfg, rules, substitution)
+
+        applyByRef(cfg, rules, "mustBeIgnored")
+
+        # assert substitutions
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][0],
+            "1 is natural number"
+        )
+
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][1],
+            "2 is natural number"
+        )
+        self.assertEqual(
+            cfg["dst"]["complex"]["path"][2],
+            "1 is not 2"
+        )
+        self.assertEqual(
+            cfg["wrongDst"],
+            "{pathOne} is unchanged"
+        )
+
     @skip_commit_slider_devtest
     def testForDeepUpdate(self):
         from utils.helpers import deepMapUpdate
diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py
@@ -9,14 +9,26 @@ class CfgManager():
     def __init__(self, cfg) -> None:
         self.cfg = cfg
 
+    @staticmethod
+    def multistepStrFormat(input: str, placeholder: str, substitution: str):
+        return input.replace(
+            '{}{}{}'.format('{', placeholder, '}'),
+            substitution
+        )
+
     def applyTemplate(self):
         if not "template" in self.cfg:
             return self.cfg
         logPath = self.cfg["logPath"]
         tmplName = self.cfg["template"]["name"]
         fullCfg = {}
+        # todo: generalize tmplcfg generator
         if tmplName == "bm_simple":
             fullCfg = self.generatebmSimpleTemplate()
+        elif tmplName == "e2e":
+            fullCfg = self.generateE2ETemplate()
+        elif tmplName == "bm_functional":
+            fullCfg = self.generatebmFunctionalTemplate()
         else:
             raise Exception(
                 "Unknown template '{}'".format(tmplName)
@@ -32,6 +44,45 @@ def readJsonTmpl(self, tmplFileName: str):
             tmplJSON = json.load(cfgFile)
             return tmplJSON
 
+    def generateE2ETemplate(self):
+        tmpl = self.cfg["template"]
+        tmpJSON = self.readJsonTmpl("e2e_for_CI.json")
+
+        if "errorPattern" in tmpl and\
+            "precommitPath" in tmpl and\
+            "testCmd" in tmpl:
+            tmpJSON["runConfig"]["stopPattern"] = tmpl["errorPattern"]
+            tmpJSON["dlbConfig"]["commonPath"] = tmpl["precommitPath"]
+            tmpJSON["cachedPathConfig"]["commonPath"] = tmpl["precommitPath"]
+            tmpJSON["appCmd"] = CfgManager.multistepStrFormat(
+                tmpJSON["appCmd"],
+                tmpl["appCmd"],
+                "testCmd"
+            )
+        else:
+            raise("Template is incomplete.")
+        subPath = "private_linux_manylinux2014_release/"
+        if "subPath" in tmpl:
+            subPath = tmpl["subPath"]
+        tmpJSON["dlbConfig"]["subPath"] = subPath
+        tmpJSON["cachedPathConfig"]["subPath"] = subPath
+
+        return tmpJSON
+
+    def generatebmFunctionalTemplate(self):
+        tmpl = self.cfg["template"]
+        tmpJSON = self.readJsonTmpl("bm_output.json")
+        stopPattern = "stopPattern"
+        if "appCmd" in tmpl:
+            tmpJSON["appCmd"] = tmpl["appCmd"]
+        else:
+            raise("No 'appcmd' in template")
+        if stopPattern in tmpl:
+            tmpJSON["runConfig"][stopPattern] = tmpl[stopPattern]
+        else:
+            raise("No 'stopPattern' in template")
+        return tmpJSON
+
     def generatebmSimpleTemplate(self):
         tmpl = self.cfg["template"]
         tmpJSON = self.readJsonTmpl("bm_perf_for_CI.json")
diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/AC_for_CI.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/AC_for_CI.json
@@ -0,0 +1,83 @@
+{
+    "appCmd":"source {venvName}/bin/activate && cd {appPath} && {py} -m pip install --upgrade pip && {py} -m pip install openvino-dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=={wheelVersion} --find-links={precommitPath}wheels/ && {py} -m pip install -r requirements_dev.txt && {testCmd}",
+    "venvCfg":{
+       "venvEnabled":true,
+       "venvDir":"{workPath}/venv/",
+       "venvName":"tempVenv"
+    },
+    "commandList":[
+ 
+    ],
+    "runConfig":{
+       "mode":"checkOutput",
+       "traversal":"firstFailedVersion",
+       "stopPattern":"{ErrorPattern}"
+    },
+    "dlbConfig":{
+       "launchedAsJob":true,
+       "toolName":"ac",
+       "wheelVersionsMap":{
+ 
+       },
+       "commonPath":"{precommitPath}",
+       "subPath":"{subPath}",
+       "appPath":"",
+       "appCmd":""
+    },
+    "cachedPathConfig":{
+       "enabled":true,
+       "scheme":"mandatory",
+       "passCmdList":false,
+       "changeAppPath":false,
+       "commonPath":"{precommitPath}",
+       "subPath":"{subPath}",
+       "cashMap":{
+ 
+       }
+    },
+    "substitutionRules":[
+       {
+          "name":"precommitPath",
+          "enabled":true,
+          "type":"map",
+          "placeholder":"precommitPath",
+          "from":"$.cachedPathConfig.cashMap",
+          "to":"$.appCmd"
+       },
+       {
+          "name":"wheelVersion",
+          "enabled":true,
+          "type":"map",
+          "placeholder":"wheelVersion",
+          "from":"$.dlbConfig.wheelVersionsMap",
+          "to":"$.appCmd"
+       },
+       {
+        "name":"command line for client application",
+        "enabled":true,
+        "type":"static",
+        "placeholder":"appCmd",
+        "from":"$.dlbConfig.appCmd",
+        "to":"$.appCmd"
+        },
+        {
+        "name":"path to AC requirements",
+        "enabled":true,
+        "type":"static",
+        "placeholder":"toolPath",
+        "from":"$.dlbConfig.toolPath",
+        "to":"$.appCmd"
+        }
+    ],
+    "subscriptions":[
+       {
+          "name":"wheelPathsMap",
+          "enabled":true
+       },
+       {
+          "name":"wheelVersionsMap",
+          "enabled":true
+       }
+    ],
+    "verboseOutput":false
+ }
diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json
@@ -1,12 +1,13 @@
 {
-    "appCmd" : "./benchmark_app -m <model_path> -d CPU -t 10",
+    "appCmd" : "{appCmd}",
     "makeCmd" : "cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=OFF -DTHREADING=TBB -DENABLE_INTEL_GPU=OFF -DENABLE_SAMPLES=ON -DENABLE_TESTS=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_CPU_DEBUG_CAPS=OFF -DENABLE_DEBUG_CAPS=OFF -DENABLE_OPENVINO_DEBUG=OFF -DCMAKE_CXX_FLAGS=-Wno-deprecated -DCMAKE_C_FLAGS=-Wno-deprecated -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_FLAGS=-Wno-deprecated-declarations ..",
     "runConfig" : {
         "commitList" : {
             "getCommitListCmd" : "git log <start_commit>..<end_commit> --boundary --pretty=\"%h\""
         },
         "mode" : "checkOutput",
         "traversal" : "firstFailedVersion",
-        "stopPattern" : "(.)*<bm_error_message>(.)*"
-    }
+        "stopPattern" : "(.)*{bm_error_message}(.)*"
+    },
+    "extendBuildCommand":true
 }
diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json
@@ -0,0 +1,67 @@
+{
+    "appCmd":"source {venvName}/bin/activate && cd {appPath} && {py} -m pip install --upgrade pip && {py} -m pip install openvino-dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=={wheelVersion} --find-links={precommitPath}wheels/ && {py} -m pip install -r requirements_dev.txt && {testCmd}",
+    "venvCfg":{
+       "venvEnabled":true,
+       "venvDir":"{workPath}/venv/",
+       "venvName":"tempVenv"
+    },
+    "commandList":[
+ 
+    ],
+    "runConfig":{
+       "mode":"checkOutput",
+       "traversal":"firstFailedVersion",
+       "stopPattern":"{ErrorPattern}"
+    },
+    "dlbConfig":{
+       "launchedAsJob":true,
+       "toolName":"e2e",
+       "wheelVersionsMap":{
+ 
+       },
+       "commonPath":"{precommitPath}",
+       "subPath":"{subPath}",
+       "appPath":"",
+       "appCmd":""
+    },
+    "cachedPathConfig":{
+       "enabled":true,
+       "scheme":"mandatory",
+       "passCmdList":false,
+       "changeAppPath":false,
+       "commonPath":"{precommitPath}",
+       "subPath":"{subPath}",
+       "cashMap":{
+ 
+       }
+    },
+    "substitutionRules":[
+       {
+          "name":"precommitPath",
+          "enabled":true,
+          "type":"map",
+          "placeholder":"precommitPath",
+          "from":"$.cachedPathConfig.cashMap",
+          "to":"$.appCmd"
+       },
+       {
+          "name":"wheelVersion",
+          "enabled":true,
+          "type":"map",
+          "placeholder":"wheelVersion",
+          "from":"$.dlbConfig.wheelVersionsMap",
+          "to":"$.appCmd"
+       }
+    ],
+    "subscriptions":[
+       {
+          "name":"wheelPathsMap",
+          "enabled":true
+       },
+       {
+          "name":"wheelVersionsMap",
+          "enabled":true
+       }
+    ],
+    "verboseOutput":false
+ }
diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py b/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py
@@ -290,12 +290,9 @@ def fetchAppOutput(cfg, commit):
         for item in [
                 {"src": cfg["venvCfg"]["venvName"], "dst": "venvName"},
                 {"src": cfg["appPath"], "dst": "appPath"},
-                {"src": sys.executable, "dst": "py"},
-                # for AC case
-                {"src": cfg["dlbConfig"]["appCmd"], "dst": "appCmd"},
-                {"src": cfg["dlbConfig"]["toolPath"], "dst": "toolPath"}
+                {"src": sys.executable, "dst": "py"}
                 ]:
-            appCmd = multistepStrFormat(
+            appCmd = CfgManager.multistepStrFormat(
                 appCmd,
                 item["dst"],
                 item["src"]
@@ -646,7 +643,8 @@ def formatJSON(content, formatLambda):
     return content
 
 def applySubstitutionRules(cfg: map, rules: list, commit: str=None):
-    # if commit is None, the rule is considered as static,
+    # if commit is None or rule['type'] == 'static',
+    # the rule is considered as static,
     # substitution proceeds as simple string replacing
 
     serviceCfg = cfg["serviceConfig"]
@@ -682,14 +680,15 @@ def applySubstitutionRules(cfg: map, rules: list, commit: str=None):
             srcPos = srcPos[item]
         for item in pathToDst:
             dstPos = dstPos[item]
+        ruleIsStatic = True if rule["type"] == "static" else False
         dstPos = formatJSON(
             dstPos,
             lambda content:
-            multistepStrFormat(
+            CfgManager.multistepStrFormat(
                 content,
                 rule["placeholder"],
                 getMapValueByShortHash(srcPos, commit)\
-                    if commit is not None\
+                    if commit is not None and not ruleIsStatic\
                     else srcPos
             )
         )
@@ -704,12 +703,6 @@ def getMapValueByShortHash(map: dict, commit: str):
         commit, map.keys()
     ))
 
-def multistepStrFormat(input: str, placeholder: str, substitution: str):
-    return input.replace(
-        '{}{}{}'.format('{', placeholder, '}'),
-        substitution
-    )
-
 def deepMapUpdate(content: map, path: list, substitution):
     if not path:
         return substitution
