3ee37077ea
This patch adds the ability to document metrics dynamically. The metrics are documented in the config YAML file for each framework, and then defined in a metrics.rst file. Two things are expected to be found in the framework docs now if they want to use dynamic metric documentation:
* A metrics.rst file that contains a `{metrics_documentation}` entry somewhere to insert the built documentation.
* A `{metrics_rst_name}` entry somewhere in the top-level/index.rst file so that a link can be produced to the metrics documentation.
The metrics documentation is specific to each framework. See the schema for the metrics for what is expected in this field. At the same time, the dynamic documentation for Raptor is generated here. Each metric in Raptor contains a list of all the tests that use (alert on) the specific metric. The tests link back to the test definition, and the metrics defined in the tests also link to the metric definitions.
Differential Revision: https://phabricator.services.mozilla.com/D206950
162 lines
5.3 KiB
Python
162 lines
5.3 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
import os
|
|
import pathlib
|
|
|
|
from perfdocs.framework_gatherers import (
|
|
AwsyGatherer,
|
|
MozperftestGatherer,
|
|
RaptorGatherer,
|
|
StaticGatherer,
|
|
TalosGatherer,
|
|
)
|
|
from perfdocs.logger import PerfDocLogger
|
|
from perfdocs.utils import read_yaml
|
|
|
|
logger = PerfDocLogger()
|
|
|
|
# TODO: Implement decorator/searcher to find the classes.
|
|
frameworks = {
|
|
"raptor": RaptorGatherer,
|
|
"mozperftest": MozperftestGatherer,
|
|
"talos": TalosGatherer,
|
|
"awsy": AwsyGatherer,
|
|
}
|
|
|
|
# List of file types allowed to be used as static files
|
|
ALLOWED_STATIC_FILETYPES = ("rst", "png")
|
|
|
|
|
|
class Gatherer(object):
|
|
"""
|
|
Gatherer produces the tree of the perfdoc's entries found
|
|
and can obtain manifest-based test lists. Used by the Verifier.
|
|
"""
|
|
|
|
def __init__(self, workspace_dir, taskgraph=None):
|
|
"""
|
|
Initialzie the Gatherer.
|
|
|
|
:param str workspace_dir: Path to the gecko checkout.
|
|
"""
|
|
self.workspace_dir = workspace_dir
|
|
self.taskgraph = taskgraph
|
|
self._perfdocs_tree = []
|
|
self._test_list = []
|
|
self.framework_gatherers = {}
|
|
|
|
@property
|
|
def perfdocs_tree(self):
|
|
"""
|
|
Returns the perfdocs_tree, and computes it
|
|
if it doesn't exist.
|
|
|
|
:return dict: The perfdocs tree containing all
|
|
framework perfdoc entries. See `fetch_perfdocs_tree`
|
|
for information on the data structure.
|
|
"""
|
|
if self._perfdocs_tree:
|
|
return self._perfdocs_tree
|
|
else:
|
|
self.fetch_perfdocs_tree()
|
|
return self._perfdocs_tree
|
|
|
|
def fetch_perfdocs_tree(self):
|
|
"""
|
|
Creates the perfdocs tree with the following structure:
|
|
[
|
|
{
|
|
"path": Path to the perfdocs directory.
|
|
"yml": Name of the configuration YAML file.
|
|
"rst": Name of the RST file.
|
|
"static": Name of the static file.
|
|
}, ...
|
|
]
|
|
|
|
This method doesn't return anything. The result can be found in
|
|
the perfdocs_tree attribute.
|
|
"""
|
|
exclude_dir = [
|
|
str(pathlib.Path(self.workspace_dir, ".hg")),
|
|
str(pathlib.Path("tools", "lint")),
|
|
str(pathlib.Path("testing", "perfdocs")),
|
|
]
|
|
|
|
for path in pathlib.Path(self.workspace_dir).rglob("perfdocs"):
|
|
if any(d in str(path.resolve()) for d in exclude_dir):
|
|
continue
|
|
files = [f for f in os.listdir(path)]
|
|
|
|
# Metrics are optional so it's only added to the matched if we
|
|
# find the `metrics.rst` file in the perfdocs folder
|
|
matched = {"path": str(path), "yml": "", "rst": "", "static": []}
|
|
|
|
for file in files:
|
|
# Add the yml/rst/static file to its key if re finds the searched file
|
|
if file == "config.yml" or file == "config.yaml":
|
|
matched["yml"] = file
|
|
elif file == "index.rst":
|
|
matched["rst"] = file
|
|
elif file == "metrics.rst":
|
|
matched["metrics"] = file
|
|
elif file.split(".")[-1] in ALLOWED_STATIC_FILETYPES:
|
|
matched["static"].append(file)
|
|
|
|
# Append to structdocs if all the searched files were found
|
|
if all(val for val in matched.values() if not type(val) == list):
|
|
self._perfdocs_tree.append(matched)
|
|
|
|
logger.log(
|
|
"Found {} perfdocs directories in {}".format(
|
|
len(self._perfdocs_tree),
|
|
[d["path"] for d in self._perfdocs_tree],
|
|
)
|
|
)
|
|
|
|
def get_test_list(self, sdt_entry):
|
|
"""
|
|
Use a perfdocs_tree entry to find the test list for
|
|
the framework that was found.
|
|
|
|
:return: A framework info dictionary with fields: {
|
|
'yml_path': Path to YAML,
|
|
'yml_content': Content of YAML,
|
|
'name': Name of framework,
|
|
'test_list': Test list found for the framework
|
|
}
|
|
"""
|
|
|
|
# If it was computed before, return it
|
|
yaml_path = pathlib.Path(sdt_entry["path"], sdt_entry["yml"])
|
|
for entry in self._test_list:
|
|
if entry["yml_path"] == yaml_path:
|
|
return entry
|
|
|
|
# Set up framework entry with meta data
|
|
yaml_content = read_yaml(yaml_path)
|
|
framework = {
|
|
"yml_content": yaml_content,
|
|
"yml_path": yaml_path,
|
|
"name": yaml_content["name"],
|
|
"test_list": {},
|
|
}
|
|
|
|
if yaml_content["static-only"]:
|
|
framework_gatherer_cls = StaticGatherer
|
|
else:
|
|
framework_gatherer_cls = frameworks[framework["name"]]
|
|
|
|
# Get and then store the frameworks tests
|
|
framework_gatherer = self.framework_gatherers[
|
|
framework["name"]
|
|
] = framework_gatherer_cls(
|
|
framework["yml_path"], self.workspace_dir, self.taskgraph
|
|
)
|
|
|
|
if not yaml_content["static-only"]:
|
|
framework["test_list"] = framework_gatherer.get_test_list()
|
|
|
|
self._test_list.append(framework)
|
|
return framework
|