corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1599424 - Convert vismet task output to a perfherder artifact r=sparky,rwood

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D55393
This commit is contained in:
Tarek Ziadé
2019-12-06 07:57:50 +00:00
parent 4db781bed6
commit fd743f2653
9 changed files with 148 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
build/sparse-profiles/docker-image
View File

@@ -10,6 +10,7 @@ path:browser/config/version.txt
# Result from `grep -hr %include taskcluster/docker | grep -v " taskcluster/" | sort -u`
path:python/mozbuild/mozbuild/action/tooltool.py
path:testing/config/tooltool-manifests/linux64/releng.manifest
path:testing/mozharness/external_tools/performance-artifact-schema.json
path:testing/mozharness/external_tools/robustcheckout.py
path:tools/lint/spell/codespell_requirements.txt
path:tools/lint/eslint/eslint-plugin-mozilla/manifest.tt

7
taskcluster/ci/visual-metrics-dep/kind.yml
View File

@@ -32,8 +32,11 @@ job-template:
max-run-time: 900
artifacts:
- type: file
name: public/visual-metrics.tar.xz
path: /builds/worker/artifacts/visual-metrics.tar.xz
name: public/perfherder-data.json
path: /builds/worker/artifacts/perfherder-data.json
- type: file
name: public/summary.json
path: /builds/worker/artifacts/summary.json
fetches:
fetch:
- visual-metrics

8
taskcluster/ci/visual-metrics/kind.yml
View File

@@ -30,9 +30,11 @@ jobs:
max-run-time: 9000
artifacts:
- type: file
name: public/visual-metrics.tar.xz
path: /builds/worker/artifacts/visual-metrics.tar.xz
name: public/perfherder-data.json
path: /builds/worker/artifacts/perfherder-data.json
- type: file
name: public/summary.json
path: /builds/worker/artifacts/summary.json
fetches:
fetch:
- visual-metrics

6
taskcluster/docker/visual-metrics/Dockerfile
View File

@@ -12,7 +12,13 @@ RUN apt-get update && \
python3 \
python3-pip
WORKDIR /builds/worker
# %include testing/mozharness/external_tools/performance-artifact-schema.json
ADD topsrcdir/testing/mozharness/external_tools/performance-artifact-schema.json /builds/worker/performance-artifact-schema.json
COPY requirements.txt /builds/worker/requirements.txt
RUN pip3 install setuptools==42.0.2
RUN pip3 install --require-hashes -r /builds/worker/requirements.txt && \
rm /builds/worker/requirements.txt

5
taskcluster/docker/visual-metrics/requirements.txt
View File

@@ -3,10 +3,15 @@ attrs==19.1.0 --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc081
requests==2.22.0 --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31
structlog==19.1.0 --hash=sha256:db441b81c65b0f104a7ce5d86c5432be099956b98b8a2c8be0b3fb3a7a0b1536
voluptuous==0.11.5 --hash=sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1
jsonschema==3.2.0 --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163
# Transitive dependencies
certifi==2019.6.16 --hash=sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939
chardet==3.0.4 --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
idna==2.8 --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
importlib_metadata==1.1.0 --hash=sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742
more_itertools==8.0.0 --hash=sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45
pyrsistent==0.15.6 --hash=sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b
six==1.12.0 --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c
urllib3==1.25.3 --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1
zipp==0.6.0 --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335

147
taskcluster/docker/visual-metrics/run-visual-metrics.py
View File

@@ -22,6 +22,8 @@ import attr
import requests
import structlog
import subprocess
from jsonschema import validate
from voluptuous import Required, Schema
#: The workspace directory where files will be downloaded, etc.
@@ -36,6 +38,9 @@ OUTPUT_DIR = Path("/", "builds", "worker", "artifacts")
#: A job to process through visualmetrics.py
@attr.s
class Job:
#: The name of the test.
test_name = attr.ib(type=str)
#: The directory for all the files pertaining to the job.
job_dir = attr.ib(type=Path)
@@ -58,11 +63,19 @@ class Job:
JOB_SCHEMA = Schema(
{
Required("jobs"): [
{Required("json_location"): str, Required("video_location"): str}
{
Required("test_name"): str,
Required("json_location"): str,
Required("video_location"): str,
}
]
}
)
PERFHERDER_SCHEMA = Path("/", "builds", "worker", "performance-artifact-schema.json")
with PERFHERDER_SCHEMA.open() as f:
PERFHERDER_SCHEMA = json.loads(f.read())
def run_command(log, cmd):
"""Run a command using subprocess.check_output
@@ -84,6 +97,71 @@ def run_command(log, cmd):
return e.returncode, e.output
def append_result(log, suites, test_name, name, result):
"""Appends a ``name`` metrics result in the ``test_name`` suite.
Args:
log: The structlog logger instance.
suites: A mapping containing the suites.
test_name: The name of the test.
name: The name of the metrics.
result: The value to append.
"""
if name.endswith("Progress"):
return
try:
result = int(result)
except ValueError:
log.error("Could not convert value", name=name)
log.error("%s" % result)
result = 0
if test_name not in suites:
suites[test_name] = {"name": test_name, "subtests": {}}
subtests = suites[test_name]["subtests"]
if name not in subtests:
subtests[name] = {
"name": name,
"replicates": [result],
"lowerIsBetter": True,
"unit": "ms",
}
else:
subtests[name]["replicates"].append(result)
def compute_median(subtest):
"""Adds in the subtest the ``value`` field, which is the average of all
replicates.
Args:
subtest: The subtest containing all replicates.
Returns:
The subtest.
"""
if "replicates" not in subtest:
return subtest
series = subtest["replicates"][1:]
subtest["value"] = float(sum(series)) / float(len(series))
return subtest
def get_suite(suite):
"""Returns the suite with computed medians in its subtests.
Args:
suite: The suite to convert.
Returns:
The suite.
"""
suite["subtests"] = [
compute_median(subtest) for subtest in suite["subtests"].values()
]
return suite
def main(log, args):
"""Run visualmetrics.py in parallel.
@@ -136,12 +214,13 @@ def main(log, args):
return 1
try:
downloaded_jobs, failed_jobs = download_inputs(log, jobs_json["jobs"])
downloaded_jobs, failed_downloads = download_inputs(log, jobs_json["jobs"])
except Exception as e:
log.error("Failed to download jobs: %s" % e, exc_info=True)
return 1
runs_failed = 0
failed_runs = 0
suites = {}
with ProcessPoolExecutor(max_workers=cpu_count()) as executor:
for job, result in zip(
@@ -162,48 +241,52 @@ def main(log, args):
video_location=job.video_location,
error=res,
)
runs_failed += 1
failed_runs += 1
else:
path = job.job_dir / "visual-metrics.json"
with path.open("wb") as f:
log.info("Writing job result", path=path)
f.write(res)
# Python 3.5 requires a str object (not 3.6+)
res = json.loads(res.decode("utf8"))
for name, value in res.items():
append_result(log, suites, job.test_name, name, value)
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
suites = [get_suite(suite) for suite in suites.values()]
perf_data = {
"framework": {"name": "browsertime"},
"type": "vismet",
"suites": suites,
}
with Path(WORKSPACE_DIR, "jobs.json").open("w") as f:
# Validates the perf data complies with perfherder schema.
# The perfherder schema uses jsonschema so we can't use voluptuous here.
validate(perf_data, PERFHERDER_SCHEMA)
raw_perf_data = json.dumps(perf_data)
with Path(OUTPUT_DIR, "perfherder-data.json").open("w") as f:
f.write(raw_perf_data)
# Prints the data in logs for Perfherder to pick it up.
log.info("PERFHERDER_DATA: %s" % raw_perf_data)
# Lists the number of processed jobs, failures, and successes.
with Path(OUTPUT_DIR, "summary.json").open("w") as f:
json.dump(
{
"successful_jobs": [
"total_jobs": len(downloaded_jobs) + len(failed_downloads),
"successful_runs": len(downloaded_jobs) - failed_runs,
"failed_runs": failed_runs,
"failed_downloads": [
{
"video_location": job.video_location,
"json_location": job.json_location,
"path": (str(job.job_dir.relative_to(WORKSPACE_DIR)) + "/"),
"test_name": job.test_name,
}
for job in downloaded_jobs
],
"failed_jobs": [
{
"video_location": job.video_location,
"json_location": job.json_location,
}
for job in failed_jobs
for job in failed_downloads
],
},
f,
)
archive = OUTPUT_DIR / "visual-metrics.tar.xz"
log.info("Creating the tarfile", tarfile=archive)
returncode, res = run_command(
log, ["tar", "cJf", str(archive), "-C", str(WORKSPACE_DIR), "."]
)
if returncode != 0:
raise Exception("Could not tar the results")
# If there's one failure along the way, we want to return > 0
# to trigger a red job in TC.
return len(failed_jobs) + runs_failed
return len(failed_downloads) + failed_runs
def download_inputs(log, raw_jobs):
@@ -224,6 +307,7 @@ def download_inputs(log, raw_jobs):
job_dir.mkdir(parents=True, exist_ok=True)
pending_jobs.append(
Job(
job["test_name"],
job_dir,
job_dir / "browsertime.json",
job["json_location"],
@@ -315,6 +399,11 @@ def download_or_copy(url_or_location, path):
if os.path.exists(url_or_location):
shutil.copyfile(url_or_location, str(path))
return
elif not url_or_location.startswith("http"):
raise IOError(
"%s does not seem to be an URL or an existing file" % url_or_location
)
download(url_or_location, path)

1
taskcluster/taskgraph/decision.py
View File

@@ -120,6 +120,7 @@ PER_PROJECT_PARAMETERS = {
visual_metrics_jobs_schema = Schema({
Required('jobs'): [
{
Required('test_name'): str,
Required('json_location'): str,
Required('video_location'): str,
}

3
taskcluster/taskgraph/util/schema.py
View File

@@ -129,7 +129,8 @@ def resolve_keyed_by(item, field, item_name, **extra_values):
WHITELISTED_SCHEMA_IDENTIFIERS = [
# upstream-artifacts are handed directly to scriptWorker, which expects interCaps
lambda path: "[u'upstream-artifacts']" in path,
lambda path: "[u'json_location']" in path or "[u'video_location']" in path,
lambda path: ("[u'test_name']" in path or "[u'json_location']" in path
or "[u'video_location']" in path),
]

8
testing/raptor/raptor/results.py
View File

@@ -442,7 +442,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
return results
def _extract_vmetrics(self, browsertime_json, browsertime_results):
def _extract_vmetrics(self, test_name, browsertime_json, browsertime_results):
# The visual metrics task expects posix paths.
def _normalized_join(*args):
path = os.path.join(*args)
@@ -456,7 +456,8 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
# mapping expected by the visual metrics task
vfiles = res.get("files", {}).get("video", [])
return [{"json_location": _normalized_join(reldir, "browsertime.json"),
"video_location": _normalized_join(reldir, vfile)}
"video_location": _normalized_join(reldir, vfile),
"test_name": test_name}
for vfile in vfiles]
vmetrics = []
@@ -497,6 +498,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
run_local = test_config.get('run_local', False)
for test in tests:
test_name = test['name']
bt_res_json = os.path.join(self.result_dir_for_test(test), 'browsertime.json')
if os.path.exists(bt_res_json):
LOG.info("found browsertime results at %s" % bt_res_json)
@@ -514,7 +516,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
raise
if not run_local:
video_files = self._extract_vmetrics(bt_res_json, raw_btresults)
video_files = self._extract_vmetrics(test_name, bt_res_json, raw_btresults)
if video_files:
video_jobs.extend(video_files)