Bug 1959703 - deactivate should alert in perf test r=padenot,perftest-reviewers,sparky

Differential Revision: https://phabricator.services.mozilla.com/D245083

testing/perfdocs/generated/mozperftest.rst

@@ -392,7 +392,7 @@ browser_ml_smart_tab_clustering_perf.js
 ::
 
  --perfherder
- --perfherder-metrics name:latency,unit:ms,shouldAlert:True, name:memory,unit:MiB,shouldAlert:True
+ --perfherder-metrics name:latency,unit:ms,shouldAlert:False, name:memory,unit:MiB,shouldAlert:False
  --verbose
  --manifest perftest.toml
  --manifest-flavor browser-chrome
@@ -410,7 +410,7 @@ browser_ml_autofill_perf.js
 ::
 
  --perfherder
- --perfherder-metrics name:AUTOFILL-pipeline-ready-latency,unit:ms,shouldAlert:True, name:AUTOFILL-initialization-latency,unit:ms,shouldAlert:True, name:AUTOFILL-model-run-latency,unit:ms,shouldAlert:True, name:AUTOFILL-total-memory-usage,unit:MiB,shouldAlert:True, name:tokenSpeed,unit:tokens/s,shouldAlert:True,lowerIsBetter:False, name:charactersSpeed,unit:chars/s,shouldAlert:True,lowerIsBetter:False
+ --perfherder-metrics name:AUTOFILL-pipeline-ready-latency,unit:ms,shouldAlert:False, name:AUTOFILL-initialization-latency,unit:ms,shouldAlert:False, name:AUTOFILL-model-run-latency,unit:ms,shouldAlert:False, name:AUTOFILL-total-memory-usage,unit:MiB,shouldAlert:False, name:tokenSpeed,unit:tokens/s,shouldAlert:False,lowerIsBetter:False, name:charactersSpeed,unit:chars/s,shouldAlert:False,lowerIsBetter:False
  --verbose
  --manifest perftest.toml
  --manifest-flavor browser-chrome
@@ -428,7 +428,7 @@ browser_ml_engine_multi_perf.js
 ::
 
  --perfherder
- --perfherder-metrics name:latency,unit:ms,shouldAlert:True, name:memory,unit:MiB,shouldAlert:True
+ --perfherder-metrics name:latency,unit:ms,shouldAlert:False, name:memory,unit:MiB,shouldAlert:False
  --verbose
  --manifest perftest.toml
  --manifest-flavor browser-chrome
@@ -446,7 +446,7 @@ browser_ml_engine_perf.js
 ::
 
  --perfherder
- --perfherder-metrics name:latency,unit:ms,shouldAlert:True, name:memory,unit:MiB,shouldAlert:True, name:tokenSpeed,unit:tokens/s,shouldAlert:True,lowerIsBetter:False, name:charactersSpeed,unit:chars/s,shouldAlert:True,lowerIsBetter:False
+ --perfherder-metrics name:latency,unit:ms,shouldAlert:False, name:memory,unit:MiB,shouldAlert:False, name:tokenSpeed,unit:tokens/s,shouldAlert:False,lowerIsBetter:False, name:charactersSpeed,unit:chars/s,shouldAlert:False,lowerIsBetter:False
  --verbose
  --manifest perftest.toml
  --manifest-flavor browser-chrome
@@ -464,7 +464,7 @@ browser_ml_llama_summarizer_perf.js
 ::
 
  --perfherder
- --perfherder-metrics name:latency,unit:ms,shouldAlert:True, name:memory,unit:MB,shouldAlert:True, name:tokenSpeed,unit:tokens/s,shouldAlert:True,lowerIsBetter:False, name:charactersSpeed,unit:chars/s,shouldAlert:True,lowerIsBetter:False
+ --perfherder-metrics name:latency,unit:ms,shouldAlert:False, name:memory,unit:MB,shouldAlert:False, name:tokenSpeed,unit:tokens/s,shouldAlert:False,lowerIsBetter:False, name:charactersSpeed,unit:chars/s,shouldAlert:False,lowerIsBetter:False
  --verbose
  --manifest perftest.toml
  --manifest-flavor browser-chrome
@@ -482,7 +482,7 @@ browser_ml_smart_tab_perf.js
 ::
 
  --perfherder
- --perfherder-metrics name:latency,unit:ms,shouldAlert:True, name:memory,unit:MiB,shouldAlert:True, name:tokenSpeed,unit:tokens/s,shouldAlert:True,lowerIsBetter:False, name:charactersSpeed,unit:chars/s,shouldAlert:True,lowerIsBetter:False
+ --perfherder-metrics name:latency,unit:ms,shouldAlert:False, name:memory,unit:MiB,shouldAlert:False, name:tokenSpeed,unit:tokens/s,shouldAlert:False,lowerIsBetter:False, name:charactersSpeed,unit:chars/s,shouldAlert:False,lowerIsBetter:False
  --verbose
  --manifest perftest.toml
  --manifest-flavor browser-chrome
@@ -518,7 +518,7 @@ browser_ml_suggest_inference.js
 ::
 
  --perfherder
- --perfherder-metrics name:inference-pipeline-ready-latency,unit:ms,shouldAlert:True, name:inference-initialization-latency,unit:ms,shouldAlert:True, name:inference-model-run-latency,unit:ms,shouldAlert:True, name:inference-total-memory-usage,unit:ms,shouldAlert:True
+ --perfherder-metrics name:inference-pipeline-ready-latency,unit:ms,shouldAlert:False, name:inference-initialization-latency,unit:ms,shouldAlert:False, name:inference-model-run-latency,unit:ms,shouldAlert:False, name:inference-total-memory-usage,unit:ms,shouldAlert:False
  --verbose
  --manifest perftest.toml
  --manifest-flavor browser-chrome

toolkit/components/ml/tests/browser/browser_ml_autofill_perf.js

@@ -13,25 +13,33 @@ const perfMetadata = {
         {
           name: "AUTOFILL-pipeline-ready-latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "AUTOFILL-initialization-latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
+        },
+        {
+          name: "AUTOFILL-model-run-latency",
+          unit: "ms",
+          shouldAlert: false,
+        },
+        {
+          name: "AUTOFILL-total-memory-usage",
+          unit: "MiB",
+          shouldAlert: false,
         },
-        { name: "AUTOFILL-model-run-latency", unit: "ms", shouldAlert: true },
-        { name: "AUTOFILL-total-memory-usage", unit: "MiB", shouldAlert: true },
         {
           name: "tokenSpeed",
           unit: "tokens/s",
-          shouldAlert: true,
+          shouldAlert: false,
           lowerIsBetter: false,
         },
         {
           name: "charactersSpeed",
           unit: "chars/s",
-          shouldAlert: true,
+          shouldAlert: false,
           lowerIsBetter: false,
         },
       ],

toolkit/components/ml/tests/browser/browser_ml_engine_multi_perf.js

@@ -90,12 +90,12 @@ const perfMetadata = {
         {
           name: "latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "memory",
           unit: "MiB",
-          shouldAlert: true,
+          shouldAlert: false,
         },
       ],
       verbose: true,
@@ -110,7 +110,7 @@ for (let metric of METRICS) {
   perfMetadata.options.default.perfherder_metrics.push({
     name: metric,
     unit: metric.includes("latency") ? "ms" : "MiB",
-    shouldAlert: true,
+    shouldAlert: false,
   });
 }
 

toolkit/components/ml/tests/browser/browser_ml_engine_perf.js

@@ -13,23 +13,23 @@ const perfMetadata = {
         {
           name: "latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "memory",
           unit: "MiB",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "tokenSpeed",
           unit: "tokens/s",
-          shouldAlert: true,
+          shouldAlert: false,
           lowerIsBetter: false,
         },
         {
           name: "charactersSpeed",
           unit: "chars/s",
-          shouldAlert: true,
+          shouldAlert: false,
           lowerIsBetter: false,
         },
       ],

toolkit/components/ml/tests/browser/browser_ml_llama_summarizer_perf.js

@@ -71,23 +71,23 @@ const perfMetadata = {
         {
           name: "latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "memory",
           unit: "MB",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "tokenSpeed",
           unit: "tokens/s",
-          shouldAlert: true,
+          shouldAlert: false,
           lowerIsBetter: false,
         },
         {
           name: "charactersSpeed",
           unit: "chars/s",
-          shouldAlert: true,
+          shouldAlert: false,
           lowerIsBetter: false,
         },
       ],

toolkit/components/ml/tests/browser/browser_ml_smart_tab_clustering_perf.js

@@ -13,12 +13,12 @@ const perfMetadata = {
         {
           name: "latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "memory",
           unit: "MiB",
-          shouldAlert: true,
+          shouldAlert: false,
         },
       ],
       verbose: true,

toolkit/components/ml/tests/browser/browser_ml_smart_tab_perf.js

@@ -13,23 +13,23 @@ const perfMetadata = {
         {
           name: "latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "memory",
           unit: "MiB",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "tokenSpeed",
           unit: "tokens/s",
-          shouldAlert: true,
+          shouldAlert: false,
           lowerIsBetter: false,
         },
         {
           name: "charactersSpeed",
           unit: "chars/s",
-          shouldAlert: true,
+          shouldAlert: false,
           lowerIsBetter: false,
         },
       ],

toolkit/components/ml/tests/browser/browser_ml_suggest_inference.js

@@ -25,15 +25,23 @@ const perfMetadata = {
         {
           name: "inference-pipeline-ready-latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
         },
         {
           name: "inference-initialization-latency",
           unit: "ms",
-          shouldAlert: true,
+          shouldAlert: false,
+        },
+        {
+          name: "inference-model-run-latency",
+          unit: "ms",
+          shouldAlert: false,
+        },
+        {
+          name: "inference-total-memory-usage",
+          unit: "ms",
+          shouldAlert: false,
         },
-        { name: "inference-model-run-latency", unit: "ms", shouldAlert: true },
-        { name: "inference-total-memory-usage", unit: "ms", shouldAlert: true },
       ],
       verbose: true,
       manifest: "perftest.toml",