corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1928224 - Add intent and NER performance test in local r=sparky,tarek,perftest-reviewers

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D227416
This commit is contained in:
Chidam Gopal
2024-11-07 14:45:12 +00:00
parent e54344c713
commit 1c3139c502
5 changed files with 188 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/mozperftest/perfdocs/config.yml
View File

@@ -58,6 +58,8 @@ suites:
description: "Performance tests running through Mochitest for ML Models"
tests:
"ML Test Model": ""
"ML Suggest Intent Model": ""
"ML Suggest NER Model": ""
intl/benchmarks/test/xpcshell:
description: "Performance tests running through XPCShell for Intl code"

36
testing/perfdocs/generated/mozperftest.rst
View File

@@ -391,6 +391,42 @@ toolkit/components/ml/tests/browser
-----------------------------------
Performance tests running through Mochitest for ML Models
browser_ml_suggest_intent_perf.js
=================================
:owner: GenAI Team
:name: ML Suggest Intent Model
:Default options:
::
--perfherder
--perfherder-metrics name:intent-pipeline-ready-latency,unit:ms,shouldAlert:True, name:intent-initialization-latency,unit:ms,shouldAlert:True, name:intent-model-run-latency,unit:ms,shouldAlert:True, name:intent-pipeline-ready-memory,unit:MB,shouldAlert:True, name:intent-initialization-memory,unit:MB,shouldAlert:True, name:intent-model-run-memory,unit:MB,shouldAlert:True
--verbose
--manifest perftest.toml
--manifest-flavor browser-chrome
--try-platform linux, mac, win
**Template test for latency for ML suggest Intent Model**
browser_ml_suggest_ner_perf.js
==============================
:owner: GenAI Team
:name: ML Suggest NER Model
:Default options:
::
--perfherder
--perfherder-metrics name:NER-pipeline-ready-latency,unit:ms,shouldAlert:True, name:NER-initialization-latency,unit:ms,shouldAlert:True, name:NER-model-run-latency,unit:ms,shouldAlert:True, name:NER-pipeline-ready-memory,unit:MB,shouldAlert:True, name:NER-initialization-memory,unit:MB,shouldAlert:True, name:NER-model-run-memory,unit:MB,shouldAlert:True
--verbose
--manifest perftest.toml
--manifest-flavor browser-chrome
--try-platform linux, mac, win
**Template test for latency for ML suggest NER model**
browser_ml_engine_perf.js
=========================

76
toolkit/components/ml/tests/browser/browser_ml_suggest_intent_perf.js Normal file
View File

@@ -0,0 +1,76 @@
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
const ITERATIONS = 10;
const PREFIX = "intent";
const METRICS = [
`${PREFIX}_${PIPELINE_READY_LATENCY}`,
`${PREFIX}_${INITIALIZATION_LATENCY}`,
`${PREFIX}_${MODEL_RUN_LATENCY}`,
`${PREFIX}_${PIPELINE_READY_MEMORY}`,
`${PREFIX}_${INITIALIZATION_MEMORY}`,
`${PREFIX}_${MODEL_RUN_MEMORY}`,
];
const journal = {};
for (let metric of METRICS) {
journal[metric] = [];
}
const perfMetadata = {
owner: "GenAI Team",
name: "ML Suggest Intent Model",
description: "Template test for latency for ML suggest Intent Model",
options: {
default: {
perfherder: true,
perfherder_metrics: [
{
name: "intent-pipeline-ready-latency",
unit: "ms",
shouldAlert: true,
},
{
name: "intent-initialization-latency",
unit: "ms",
shouldAlert: true,
},
{ name: "intent-model-run-latency", unit: "ms", shouldAlert: true },
{ name: "intent-pipeline-ready-memory", unit: "MB", shouldAlert: true },
{ name: "intent-initialization-memory", unit: "MB", shouldAlert: true },
{ name: "intent-model-run-memory", unit: "MB", shouldAlert: true },
],
verbose: true,
manifest: "perftest.toml",
manifest_flavor: "browser-chrome",
try_platform: ["linux", "mac", "win"],
},
},
};
requestLongerTimeout(120);
/**
* Tests local suggest intent model
*/
add_task(async function test_ml_generic_pipeline() {
const options = new PipelineOptions({
taskName: "text-classification",
modelId: "Mozilla/mobilebert-uncased-finetuned-LoRA-intent-classifier",
modelHubUrlTemplate: "{model}/{revision}",
modelRevision: "main",
dtype: "q8",
});
const args = ["restaurants in seattle, wa"];
for (let i = 0; i < ITERATIONS; i++) {
let metrics = await runInference(options, args);
for (let [metricName, metricVal] of Object.entries(metrics)) {
Assert.ok(metricVal >= 0, "Metric should be non-negative.");
journal[`${PREFIX}_${metricName}`].push(metricVal);
}
}
reportMetrics(journal);
});

68
toolkit/components/ml/tests/browser/browser_ml_suggest_ner_perf.js Normal file
View File

@@ -0,0 +1,68 @@
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
const ITERATIONS = 10;
const PREFIX = "NER";
const METRICS = [
`${PREFIX}_${PIPELINE_READY_LATENCY}`,
`${PREFIX}_${INITIALIZATION_LATENCY}`,
`${PREFIX}_${MODEL_RUN_LATENCY}`,
`${PREFIX}_${PIPELINE_READY_MEMORY}`,
`${PREFIX}_${INITIALIZATION_MEMORY}`,
`${PREFIX}_${MODEL_RUN_MEMORY}`,
];
const journal = {};
for (let metric of METRICS) {
journal[metric] = [];
}
const perfMetadata = {
owner: "GenAI Team",
name: "ML Suggest NER Model",
description: "Template test for latency for ML suggest NER model",
options: {
default: {
perfherder: true,
perfherder_metrics: [
{ name: "NER-pipeline-ready-latency", unit: "ms", shouldAlert: true },
{ name: "NER-initialization-latency", unit: "ms", shouldAlert: true },
{ name: "NER-model-run-latency", unit: "ms", shouldAlert: true },
{ name: "NER-pipeline-ready-memory", unit: "MB", shouldAlert: true },
{ name: "NER-initialization-memory", unit: "MB", shouldAlert: true },
{ name: "NER-model-run-memory", unit: "MB", shouldAlert: true },
],
verbose: true,
manifest: "perftest.toml",
manifest_flavor: "browser-chrome",
try_platform: ["linux", "mac", "win"],
},
},
};
requestLongerTimeout(120);
/**
* Tests local suggest NER model
*/
add_task(async function test_ml_generic_pipeline() {
const options = new PipelineOptions({
taskName: "token-classification",
modelId: "Mozilla/distilbert-uncased-NER-LoRA",
modelHubUrlTemplate: "{model}/{revision}",
modelRevision: "main",
dtype: "q8",
});
const args = ["restaurants in seattle, wa"];
for (let i = 0; i < ITERATIONS; i++) {
let metrics = await runInference(options, args);
for (let [metricName, metricVal] of Object.entries(metrics)) {
Assert.ok(metricVal >= 0, "Metric should be non-negative.");
journal[`${PREFIX}_${metricName}`].push(metricVal);
}
}
reportMetrics(journal);
});

6
toolkit/components/ml/tests/browser/perftest.toml
View File

@@ -5,3 +5,9 @@ support-files = [
["browser_ml_engine_perf.js"]
skip-if = true # Disabled as we want to run this only as perftest, not regular CI
["browser_ml_suggest_intent_perf.js"]
skip-if = true # Disabled as we want to run this only as perftest, not regular CI
["browser_ml_suggest_ner_perf.js"]
skip-if = true # Disabled as we want to run this only as perftest, not regular CI