3ee37077ea
This patch adds the ability to document metrics dynamically. The metrics are documented in the config YAML file for each framework, and then defined in a metrics.rst file. Two things are expected to be found in the framework docs now if they want to use dynamic metric documentation:
* A metrics.rst file that contains a `{metrics_documentation}` entry somewhere to insert the built documentation.
* A `{metrics_rst_name}` entry somewhere in the top-level/index.rst file so that a link can be produced to the metrics documentation.
The metrics documentation is specific to each framework. See the schema for the metrics for what is expected in this field. At the same time, the dynamic documentation for Raptor is generated here. Each metric in Raptor contains a list of all the tests that use (alert on) the specific metric. The tests link back to the test definition, and the metrics defined in the tests also link to the metric definitions.
Differential Revision: https://phabricator.services.mozilla.com/D206950
318 lines
12 KiB
Python
318 lines
12 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
|
|
from perfdocs.logger import PerfDocLogger
|
|
from perfdocs.utils import (
|
|
ON_TRY,
|
|
are_dirs_equal,
|
|
get_changed_files,
|
|
read_file,
|
|
read_yaml,
|
|
save_file,
|
|
)
|
|
|
|
logger = PerfDocLogger()
|
|
|
|
|
|
class Generator(object):
|
|
"""
|
|
After each perfdocs directory was validated, the generator uses the templates
|
|
for each framework, fills them with the test descriptions in config and saves
|
|
the perfdocs in the form index.rst as index file and suite_name.rst for
|
|
each suite of tests in the framework.
|
|
"""
|
|
|
|
def __init__(self, verifier, workspace, generate=False):
|
|
"""
|
|
Initialize the Generator.
|
|
|
|
:param verifier: Verifier object. It should not be a fresh Verifier object,
|
|
but an initialized one with validate_tree() method already called
|
|
:param workspace: Path to the top-level checkout directory.
|
|
:param generate: Flag for generating the documentation
|
|
"""
|
|
self._workspace = workspace
|
|
if not self._workspace:
|
|
raise Exception("PerfDocs Generator requires a workspace directory.")
|
|
# Template documents without added information reside here
|
|
self.templates_path = pathlib.Path(
|
|
self._workspace, "tools", "lint", "perfdocs", "templates"
|
|
)
|
|
self.perfdocs_path = pathlib.Path(
|
|
self._workspace, "testing", "perfdocs", "generated"
|
|
)
|
|
|
|
self._generate = generate
|
|
self._verifier = verifier
|
|
self._perfdocs_tree = self._verifier._gatherer.perfdocs_tree
|
|
|
|
def build_perfdocs_from_tree(self):
|
|
"""
|
|
Builds up a document for each framework that was found.
|
|
|
|
:return dict: A dictionary containing a mapping from each framework
|
|
to the document that was built for it, i.e:
|
|
{
|
|
framework_name: framework_document,
|
|
...
|
|
}
|
|
"""
|
|
|
|
# Using the verified `perfdocs_tree`, build up the documentation.
|
|
frameworks_info = {}
|
|
for framework in self._perfdocs_tree:
|
|
yaml_content = read_yaml(pathlib.Path(framework["path"], framework["yml"]))
|
|
rst_content = read_file(
|
|
pathlib.Path(framework["path"], framework["rst"]), stringify=True
|
|
)
|
|
gatherer = self._verifier._gatherer.framework_gatherers[
|
|
yaml_content["name"]
|
|
]
|
|
|
|
metrics_rst_content = None
|
|
metrics_info = self._verifier.metrics_info[yaml_content["name"]]
|
|
if framework.get("metrics"):
|
|
metrics_rst_content = read_file(
|
|
pathlib.Path(framework["path"], framework["metrics"]),
|
|
stringify=True,
|
|
)
|
|
|
|
# Gather all tests and descriptions and format them into
|
|
# documentation content
|
|
documentation = []
|
|
suites = yaml_content["suites"]
|
|
for suite_name in sorted(suites.keys()):
|
|
suite_info = suites[suite_name]
|
|
|
|
# Add the suite section
|
|
documentation.extend(
|
|
self._verifier._gatherer.framework_gatherers[
|
|
yaml_content["name"]
|
|
].build_suite_section(suite_name, suite_info["description"])
|
|
)
|
|
|
|
tests = suite_info.get("tests", {})
|
|
for test_name in sorted(tests.keys()):
|
|
test_description = gatherer.build_test_description(
|
|
test_name,
|
|
test_description=tests[test_name],
|
|
suite_name=suite_name,
|
|
metrics_info=metrics_info,
|
|
)
|
|
documentation.extend(test_description)
|
|
documentation.append("")
|
|
|
|
# For each framework, we may want to organize the metrics differently
|
|
# so delegate the complete setup of the metric documentation to the
|
|
# framework-specific gatherers
|
|
metrics_rst = ""
|
|
if metrics_rst_content:
|
|
metrics_documentation = gatherer.build_metrics_documentation(
|
|
metrics_info
|
|
)
|
|
metrics_rst = re.sub(
|
|
r"{metrics_documentation}",
|
|
"\n".join(metrics_documentation),
|
|
metrics_rst_content,
|
|
)
|
|
rst_content = re.sub(
|
|
r"{metrics_rst_name}",
|
|
f"{yaml_content['name']}-metrics",
|
|
rst_content,
|
|
)
|
|
|
|
# Insert documentation into `.rst` file
|
|
framework_rst = re.sub(
|
|
r"{documentation}", "\n".join(documentation), rst_content
|
|
)
|
|
frameworks_info[yaml_content["name"]] = {
|
|
"dynamic": framework_rst,
|
|
"metrics": metrics_rst,
|
|
"static": [],
|
|
}
|
|
|
|
# For static `.rst` file
|
|
for static_file in framework["static"]:
|
|
if static_file.endswith("rst"):
|
|
frameworks_info[yaml_content["name"]]["static"].append(
|
|
{
|
|
"file": static_file,
|
|
"content": read_file(
|
|
pathlib.Path(framework["path"], static_file),
|
|
stringify=True,
|
|
),
|
|
}
|
|
)
|
|
else:
|
|
frameworks_info[yaml_content["name"]]["static"].append(
|
|
{
|
|
"file": static_file,
|
|
"content": pathlib.Path(framework["path"], static_file),
|
|
}
|
|
)
|
|
|
|
return frameworks_info
|
|
|
|
def _create_temp_dir(self):
|
|
"""
|
|
Create a temp directory as preparation of saving the documentation tree.
|
|
:return: str the location of perfdocs_tmpdir
|
|
"""
|
|
# Build the directory that will contain the final result (a tmp dir
|
|
# that will be moved to the final location afterwards)
|
|
try:
|
|
tmpdir = pathlib.Path(tempfile.mkdtemp())
|
|
perfdocs_tmpdir = pathlib.Path(tmpdir, "generated")
|
|
perfdocs_tmpdir.mkdir(parents=True, exist_ok=True)
|
|
perfdocs_tmpdir.chmod(0o766)
|
|
except OSError as e:
|
|
logger.critical("Error creating temp file: {}".format(e))
|
|
|
|
if perfdocs_tmpdir.is_dir():
|
|
return perfdocs_tmpdir
|
|
return False
|
|
|
|
def _create_perfdocs(self):
|
|
"""
|
|
Creates the perfdocs documentation.
|
|
:return: str path of the temp dir it is saved in
|
|
"""
|
|
# All directories that are kept in the perfdocs tree are valid,
|
|
# so use it to build up the documentation.
|
|
framework_docs = self.build_perfdocs_from_tree()
|
|
perfdocs_tmpdir = self._create_temp_dir()
|
|
|
|
# Save the documentation files
|
|
frameworks = []
|
|
for framework_name in sorted(framework_docs.keys()):
|
|
frameworks.append(framework_name)
|
|
save_file(
|
|
framework_docs[framework_name]["dynamic"],
|
|
pathlib.Path(perfdocs_tmpdir, framework_name),
|
|
)
|
|
|
|
if framework_docs[framework_name]["metrics"]:
|
|
save_file(
|
|
framework_docs[framework_name]["metrics"],
|
|
pathlib.Path(perfdocs_tmpdir, f"{framework_name}-metrics"),
|
|
)
|
|
|
|
for static_name in framework_docs[framework_name]["static"]:
|
|
if static_name["file"].endswith(".rst"):
|
|
# XXX Replace this with a shutil.copy call (like below)
|
|
save_file(
|
|
static_name["content"],
|
|
pathlib.Path(
|
|
perfdocs_tmpdir, static_name["file"].split(".")[0]
|
|
),
|
|
)
|
|
else:
|
|
shutil.copy(
|
|
static_name["content"],
|
|
pathlib.Path(perfdocs_tmpdir, static_name["file"]),
|
|
)
|
|
|
|
# Get the main page and add the framework links to it
|
|
mainpage = read_file(
|
|
pathlib.Path(self.templates_path, "index.rst"), stringify=True
|
|
)
|
|
|
|
fmt_frameworks = "\n".join([" * :doc:`%s`" % name for name in frameworks])
|
|
fmt_toctree = "\n".join([" %s" % name for name in frameworks])
|
|
|
|
fmt_mainpage = re.sub(r"{toctree_documentation}", fmt_toctree, mainpage)
|
|
fmt_mainpage = re.sub(r"{test_documentation}", fmt_frameworks, fmt_mainpage)
|
|
|
|
save_file(fmt_mainpage, pathlib.Path(perfdocs_tmpdir, "index"))
|
|
|
|
return perfdocs_tmpdir
|
|
|
|
def _save_perfdocs(self, perfdocs_tmpdir):
|
|
"""
|
|
Copies the perfdocs tree after it was saved into the perfdocs_tmpdir
|
|
:param perfdocs_tmpdir: str location of the temp dir where the
|
|
perfdocs was saved
|
|
"""
|
|
# Remove the old docs and copy the new version there without
|
|
# checking if they need to be regenerated.
|
|
logger.log("Regenerating perfdocs...")
|
|
|
|
if self.perfdocs_path.exists():
|
|
shutil.rmtree(str(self.perfdocs_path))
|
|
|
|
try:
|
|
saved = shutil.copytree(str(perfdocs_tmpdir), str(self.perfdocs_path))
|
|
if saved:
|
|
logger.log(
|
|
"Documentation saved to {}/".format(
|
|
re.sub(".*testing", "testing", str(self.perfdocs_path))
|
|
)
|
|
)
|
|
except Exception as e:
|
|
logger.critical(
|
|
"There was an error while saving the documentation: {}".format(e)
|
|
)
|
|
|
|
def generate_perfdocs(self):
|
|
"""
|
|
Generate the performance documentation.
|
|
|
|
If `self._generate` is True, then the documentation will be regenerated
|
|
without any checks. Otherwise, if it is False, the new documentation will be
|
|
prepare and compare with the existing documentation to determine if
|
|
it should be regenerated.
|
|
|
|
:return bool: True/False - For True, if `self._generate` is True, then the
|
|
docs were regenerated. If `self._generate` is False, then True will mean
|
|
that the docs should be regenerated, and False means that they do not
|
|
need to be regenerated.
|
|
"""
|
|
|
|
def get_possibly_changed_files():
|
|
"""
|
|
Returns files that might have been modified
|
|
(used to output a linter warning for regeneration)
|
|
:return: list - files that might have been modified
|
|
"""
|
|
# Returns files that might have been modified
|
|
# (used to output a linter warning for regeneration)
|
|
files = []
|
|
for entry in self._perfdocs_tree:
|
|
files.extend(
|
|
[
|
|
pathlib.Path(entry["path"], entry["yml"]),
|
|
pathlib.Path(entry["path"], entry["rst"]),
|
|
]
|
|
)
|
|
return files
|
|
|
|
# Throw a warning if there's no need for generating
|
|
if not self.perfdocs_path.exists() and not self._generate:
|
|
# If they don't exist and we are not generating, then throw
|
|
# a linting error and exit.
|
|
logger.warning(
|
|
"PerfDocs need to be regenerated.", files=get_possibly_changed_files()
|
|
)
|
|
return True
|
|
|
|
perfdocs_tmpdir = self._create_perfdocs()
|
|
if self._generate:
|
|
self._save_perfdocs(perfdocs_tmpdir)
|
|
elif not are_dirs_equal(perfdocs_tmpdir, self.perfdocs_path):
|
|
# If we are not generating, then at least check if they
|
|
# should be regenerated by comparing the directories.
|
|
logger.warning(
|
|
"PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` "
|
|
+ "to update them. You can also apply the "
|
|
+ f"{'perfdocs.diff' if ON_TRY else 'diff.txt'} patch file "
|
|
+ f"{'produced from this reviewbot test ' if ON_TRY else ''}"
|
|
+ "to fix the issue.",
|
|
files=get_changed_files(self._workspace),
|
|
restricted=False,
|
|
)
|