Bug 1654339 - Add Glean metrics autodocs to moztreedocs r=janerik,ahal

Also define a scheme for storing the index of Glean definitions files in a file
separate from the build system for consumption by
* mach build
* mach doc
* (future) mozilla/probe-scraper

Differential Revision: https://phabricator.services.mozilla.com/D87600

build/sparse-profiles/sphinx-docs

@@ -28,3 +28,9 @@ glob:**/moz.build
 
 # Read to set the version of the docs.
 path:config/milestone.txt
+
+# metrics.yaml and pings.yaml files (and their index) are needed to generate
+# Glean autodocs
+glob:**/metrics.yaml
+glob:**/pings.yaml
+path:toolkit/components/glean/metrics_index.py

docs/conf.py

@@ -27,6 +27,7 @@ EXTRA_PATHS = (
     "third_party/python/futures",
     "third_party/python/jsmin",
     "third_party/python/which",
+    "toolkit/components/glean/sphinx",
 )
 
 sys.path[:0] = [os.path.join(topsrcdir, p) for p in EXTRA_PATHS]
@@ -46,6 +47,7 @@ extensions = [
     "recommonmark",
     "sphinx_copybutton",
     "sphinx_markdown_tables",
+    "glean",
 ]
 
 # JSDoc must run successfully for dirs specified, so running

docs/config.yml

@@ -48,6 +48,8 @@ categories:
         - python
     fennec_doc:
         - mobile/android
+    metrics_doc:
+        - metrics
 
 redirects:
     browser/browser: browser

docs/index.rst

@@ -44,6 +44,12 @@ Firefox Source Tree Documentation
 
    {python_doc}
 
+.. toctree::
+   :caption: Metrics Collected in Firefox
+   :maxdepth: 1
+
+   {metrics_doc}
+
 
 Indices and tables
 ==================

toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py

@@ -11,11 +11,23 @@ from glean_parser import lint, parser, util
 from pathlib import Path
 
 
-def main(output_fd, *metrics_yamls):
+def main(output_fd, metrics_index_path, which_array):
+
+    # Source the list of input files from `metrics_index.py`
+    sys.path.append(str(Path(metrics_index_path).parent))
+    from metrics_index import METRICS, PINGS
+    if which_array == 'METRICS':
+        input_files = METRICS
+    elif which_array == 'PINGS':
+        input_files = PINGS
+    else:
+        print("Build system's asking for unknown array {}".format(which_array))
+        sys.exit(1)
+
     # Derived heavily from glean_parser.translate.translate.
     # Adapted to how mozbuild sends us a fd.
     options = {"allow_reserved": False}
-    input_files = [Path(x) for x in metrics_yamls]
+    input_files = [Path(x) for x in input_files]
 
     all_objs = parser.parse_objects(input_files, options)
     if util.report_validation_errors(all_objs):

toolkit/components/glean/docs/new_definitions_file.md

@@ -27,13 +27,16 @@ But where can you find `metrics.yaml` or `pings.yaml`?
 If you're not the first person in your component to ask that question,
 the answer is likely "in the root of your component".
 Look for the definitions files near to where you are instrumenting your code.
+Or you can look in
+`toolkit/components/glean/metrics_index.py`
+to see the list of all currently-known definitions files.
 
 If you _are_ the first person in your component to ask that question,
 you get to choose where to start them!
 We recommend adding them in the root of your component, next to a `moz.build`.
 
-When you do so, be sure to edit `toolkit/components/glean/moz.build`,
+When you do so, be sure to edit `toolkit/components/glean/metrics_index.py`,
-adding your definitions files to the `GeneratedFile` directives therein.
+adding your definitions files to the Python lists therein.
 If you don't, no API will be generated for your metrics and your build will fail.
 
 In addition, do not forget to file a bug in `Data Platform and Tools :: General`

toolkit/components/glean/metrics_index.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# The list of all Glean metrics.yaml files, relative to the top src dir.
+# New additions should be added to the bottom of the list.
+METRICS = [
+    'toolkit/components/glean/metrics.yaml',
+]
+
+# The list of all Glean pings.yaml files, relative to the top src dir.
+# New additions should be added to the bottom of the list.
+PINGS = [
+    'toolkit/components/glean/pings.yaml',
+]

toolkit/components/glean/moz.build

@@ -30,11 +30,13 @@ if CONFIG['MOZ_GLEAN']:
 
     GeneratedFile('api/src/metrics.rs',
                   script='build_scripts/glean_parser_ext/run_glean_parser.py',
-                  inputs=['metrics.yaml'])
+                  flags=['METRICS'],
+                  inputs=['metrics_index.py'])
 
     GeneratedFile('api/src/pings.rs',
                   script='build_scripts/glean_parser_ext/run_glean_parser.py',
-                  inputs=['pings.yaml'])
+                  flags=['PINGS'],
+                  inputs=['metrics_index.py'])
 
     DIRS += [
         'aboutGlean',

toolkit/components/glean/sphinx/glean.py

@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from pathlib import Path
+import sys
+
+
+def setup(app):
+    from moztreedocs import manager
+
+    # Import the list of metrics and ping files
+    glean_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
+    sys.path.append(glean_dir)
+    from metrics_index import METRICS, PINGS
+    input_files = [Path(os.path.join(manager.topsrcdir, x)) for x in METRICS]
+    input_files += [Path(os.path.join(manager.topsrcdir, x)) for x in PINGS]
+
+    # Generate the autodocs.
+    from glean_parser import translate
+    out_path = Path(os.path.join(manager.staging_dir, 'metrics'))
+    translate.translate(input_files, 'markdown', out_path, {'project_title': 'Firefox'})
+
+    # Rename the generated docfile to index so Sphinx finds it
+    os.rename(os.path.join(out_path, 'metrics.md'), os.path.join(out_path, 'index.md'))