Bug 1943355 - Sort type IDs and categories to keep a stable order in generated files r=chutten

Differential Revision: https://phabricator.services.mozilla.com/D235443

toolkit/components/glean/build_scripts/glean_parser_ext/jog.py

@@ -137,7 +137,6 @@ def output_factory(objs, output_fd, options={}):
 
     output_fd.write(
         template.render(
-            all_objs=objs,
             common_metric_data_args=common_metric_data_args,
             extra_args=util.extra_args,
             metric_types=metric_types,

toolkit/components/glean/build_scripts/glean_parser_ext/util.py

@@ -109,4 +109,6 @@ def type_ids_and_categories(objs) -> Tuple[Dict[str, Tuple[int, List[str]]], Lis
                         args.append(arg_name)
                 metric_type_ids[metric.type] = {"id": type_id, "args": args}
 
+    metric_type_ids = dict(sorted(metric_type_ids.items()))
+    categories = sorted(categories)
     return (metric_type_ids, categories)

toolkit/components/glean/tests/pytest/jogfactory_output

@@ -0,0 +1,539 @@
+// -*- mode: Rust -*-
+
+// AUTOGENERATED BY glean_parser.  DO NOT EDIT.
+
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/// This file contains factory implementation information for the
+/// JOG Runtime Registration module.
+/// It is responsible for being able to build metrics and pings described at runtime.
+/// It is generated to keep it in sync with how the runtime definitions are defined.
+
+use std::borrow::Cow;
+use std::sync::atomic::{AtomicU32, Ordering};
+use crate::private::{
+    CommonMetricData,
+    LabeledMetricData,
+    Lifetime,
+    MemoryUnit,
+    TimeUnit,
+    Ping,
+    LabeledMetric,
+    MetricId,
+    BooleanMetric,
+    CounterMetric,
+    CustomDistributionMetric,
+    DatetimeMetric,
+    DenominatorMetric,
+    EventMetric,
+    MemoryDistributionMetric,
+    NumeratorMetric,
+    QuantityMetric,
+    RateMetric,
+    StringMetric,
+    StringListMetric,
+    TextMetric,
+    TimespanMetric,
+    TimingDistributionMetric,
+    UuidMetric,
+};
+use crate::private::traits::HistogramType;
+
+pub(crate) static DYNAMIC_METRIC_BIT: u32 = 26;
+// 2**DYNAMIC_METRIC_BIT + 1 (+1 because we reserve the 0 metric id)
+static NEXT_METRIC_ID: AtomicU32 = AtomicU32::new(67108865);
+#[cfg(feature = "with_gecko")] // only used in submit_ping_by_id, which is gecko-only.
+pub(crate) static DYNAMIC_PING_BIT: u32 = 15;
+// 2**DYNAMIC_PING_BIT + 1 (+1 because we reserve the 0 ping id)
+static NEXT_PING_ID: AtomicU32 = AtomicU32::new(32769);
+
+pub(crate) mod __jog_metric_maps {
+    use crate::metrics::DynamicLabel;
+    use crate::private::MetricId;
+    use crate::private::{
+        Ping,
+        LabeledMetric,
+        NoExtraKeys,
+        BooleanMetric,
+        CounterMetric,
+        CustomDistributionMetric,
+        DatetimeMetric,
+        DenominatorMetric,
+        EventMetric,
+        LabeledBooleanMetric,
+        LabeledCounterMetric,
+        LabeledStringMetric,
+        MemoryDistributionMetric,
+        NumeratorMetric,
+        ObjectMetric,
+        QuantityMetric,
+        RateMetric,
+        StringMetric,
+        StringListMetric,
+        TextMetric,
+        TimespanMetric,
+        TimingDistributionMetric,
+        UuidMetric,
+    };
+    use once_cell::sync::Lazy;
+    use std::collections::HashMap;
+    use std::sync::{Arc, RwLock};
+
+    pub static BOOLEAN_MAP: Lazy<Arc<RwLock<HashMap<MetricId, BooleanMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static COUNTER_MAP: Lazy<Arc<RwLock<HashMap<MetricId, CounterMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static CUSTOM_DISTRIBUTION_MAP: Lazy<Arc<RwLock<HashMap<MetricId, CustomDistributionMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static DATETIME_MAP: Lazy<Arc<RwLock<HashMap<MetricId, DatetimeMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static DENOMINATOR_MAP: Lazy<Arc<RwLock<HashMap<MetricId, DenominatorMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static MEMORY_DISTRIBUTION_MAP: Lazy<Arc<RwLock<HashMap<MetricId, MemoryDistributionMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static NUMERATOR_MAP: Lazy<Arc<RwLock<HashMap<MetricId, NumeratorMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static QUANTITY_MAP: Lazy<Arc<RwLock<HashMap<MetricId, QuantityMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static RATE_MAP: Lazy<Arc<RwLock<HashMap<MetricId, RateMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static STRING_MAP: Lazy<Arc<RwLock<HashMap<MetricId, StringMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static STRING_LIST_MAP: Lazy<Arc<RwLock<HashMap<MetricId, StringListMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static TEXT_MAP: Lazy<Arc<RwLock<HashMap<MetricId, TextMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static TIMESPAN_MAP: Lazy<Arc<RwLock<HashMap<MetricId, TimespanMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static TIMING_DISTRIBUTION_MAP: Lazy<Arc<RwLock<HashMap<MetricId, TimingDistributionMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static UUID_MAP: Lazy<Arc<RwLock<HashMap<MetricId, UuidMetric>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static LABELED_BOOLEAN_MAP: Lazy<Arc<RwLock<HashMap<MetricId, LabeledMetric<LabeledBooleanMetric, DynamicLabel>>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static LABELED_COUNTER_MAP: Lazy<Arc<RwLock<HashMap<MetricId, LabeledMetric<LabeledCounterMetric, DynamicLabel>>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static LABELED_STRING_MAP: Lazy<Arc<RwLock<HashMap<MetricId, LabeledMetric<LabeledStringMetric, DynamicLabel>>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static PING_MAP: Lazy<Arc<RwLock<HashMap<u32, Ping>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+
+    pub static EVENT_MAP: Lazy<Arc<RwLock<HashMap<MetricId, EventMetric<NoExtraKeys>>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+    #[allow(dead_code)]
+    pub static OBJECT_MAP: Lazy<Arc<RwLock<HashMap<MetricId, ObjectMetric<()>>>>> =
+        Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
+}
+
+#[derive(Debug)]
+struct MetricTypeNotFoundError(String);
+impl std::error::Error for MetricTypeNotFoundError {}
+impl std::fmt::Display for MetricTypeNotFoundError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "Metric type {} not found", self.0)
+    }
+}
+
+/// Creates and registers a metric, returning its type+id.
+pub fn create_and_register_metric(
+    metric_type: &str,
+    category: String,
+    name: String,
+    send_in_pings: Vec<String>,
+    lifetime: Lifetime,
+    disabled: bool,
+    time_unit: Option<TimeUnit>,
+    memory_unit: Option<MemoryUnit>,
+    allowed_extra_keys: Option<Vec<String>>,
+    range_min: Option<i64>,
+    range_max: Option<i64>,
+    bucket_count: Option<i64>,
+    histogram_type: Option<HistogramType>,
+    numerators: Option<Vec<CommonMetricData>>,
+    labels: Option<Vec<Cow<'static, str>>>,
+    permit_non_commutative_operations_over_ipc: Option<bool>,
+) -> Result<(u32, u32), Box<dyn std::error::Error>> {
+    let metric_id = NEXT_METRIC_ID.fetch_add(1, Ordering::SeqCst);
+    let metric32 = match metric_type {
+        "boolean" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = if permit_non_commutative_operations_over_ipc.unwrap_or(false) {
+                BooleanMetric::with_unordered_ipc(MetricId(metric_id), meta)
+            } else {
+                BooleanMetric::new(MetricId(metric_id), meta)
+            };
+            let metric32: u32 = (1 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::BOOLEAN_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "counter" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = CounterMetric::new(MetricId(metric_id), meta);
+            let metric32: u32 = (2 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::COUNTER_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "custom_distribution" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = CustomDistributionMetric::new(MetricId(metric_id), meta, range_min.unwrap(), range_max.unwrap(), bucket_count.unwrap(), histogram_type.unwrap());
+            let metric32: u32 = (3 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::CUSTOM_DISTRIBUTION_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "datetime" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = DatetimeMetric::new(MetricId(metric_id), meta, time_unit.unwrap());
+            let metric32: u32 = (14 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::DATETIME_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "denominator" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = DenominatorMetric::new(MetricId(metric_id), meta, numerators.unwrap());
+            let metric32: u32 = (16 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::DENOMINATOR_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "event" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = EventMetric::with_runtime_extra_keys(MetricId(metric_id), meta, allowed_extra_keys.unwrap());
+            let metric32: u32 = (15 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::EVENT_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "labeled_boolean" => {
+            let meta = LabeledMetricData::Common {
+                    cmd: CommonMetricData {
+                    name,
+                    category,
+                    send_in_pings,
+                    lifetime,
+                    disabled,
+                    ..Default::default()
+                }
+                };            let metric = if permit_non_commutative_operations_over_ipc.unwrap_or(false) {
+                LabeledMetric::with_unordered_ipc(MetricId(metric_id), meta, labels)
+            } else {
+                LabeledMetric::new(MetricId(metric_id), meta, labels)
+            };
+            let metric32: u32 = (4 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::LABELED_BOOLEAN_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "labeled_counter" => {
+            let meta = LabeledMetricData::Common {
+                    cmd: CommonMetricData {
+                    name,
+                    category,
+                    send_in_pings,
+                    lifetime,
+                    disabled,
+                    ..Default::default()
+                }
+                };            let metric = LabeledMetric::new(MetricId(metric_id), meta, labels);
+            let metric32: u32 = (5 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::LABELED_COUNTER_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "labeled_string" => {
+            let meta = LabeledMetricData::Common {
+                    cmd: CommonMetricData {
+                    name,
+                    category,
+                    send_in_pings,
+                    lifetime,
+                    disabled,
+                    ..Default::default()
+                }
+                };            let metric = LabeledMetric::new(MetricId(metric_id), meta, labels);
+            let metric32: u32 = (6 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::LABELED_STRING_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "memory_distribution" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = MemoryDistributionMetric::new(MetricId(metric_id), meta, memory_unit.unwrap());
+            let metric32: u32 = (7 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::MEMORY_DISTRIBUTION_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "numerator" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = NumeratorMetric::new(MetricId(metric_id), meta);
+            let metric32: u32 = (19 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::NUMERATOR_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "object" => {
+            return Err(Box::new(MetricTypeNotFoundError(metric_type.to_string())));
+        }
+        "quantity" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = QuantityMetric::new(MetricId(metric_id), meta);
+            let metric32: u32 = (17 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::QUANTITY_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "rate" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = RateMetric::new(MetricId(metric_id), meta);
+            let metric32: u32 = (18 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::RATE_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "string" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = StringMetric::new(MetricId(metric_id), meta);
+            let metric32: u32 = (9 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::STRING_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "string_list" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = StringListMetric::new(MetricId(metric_id), meta);
+            let metric32: u32 = (8 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::STRING_LIST_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "text" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = TextMetric::new(MetricId(metric_id), meta);
+            let metric32: u32 = (10 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::TEXT_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "timespan" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = TimespanMetric::new(MetricId(metric_id), meta, time_unit.unwrap());
+            let metric32: u32 = (11 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::TIMESPAN_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "timing_distribution" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = TimingDistributionMetric::new(MetricId(metric_id), meta, time_unit.unwrap());
+            let metric32: u32 = (12 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::TIMING_DISTRIBUTION_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        "uuid" => {
+            let meta = CommonMetricData {
+                name,
+                category,
+                send_in_pings,
+                lifetime,
+                disabled,
+                ..Default::default()
+            };
+            let metric = UuidMetric::new(MetricId(metric_id), meta);
+            let metric32: u32 = (20 << 27) | metric_id;
+            assert!(
+                __jog_metric_maps::UUID_MAP.write()?.insert(MetricId(metric_id), metric).is_none(),
+                "We should never insert a runtime metric with an already-used id."
+            );
+            metric32
+        }
+        _ => return Err(Box::new(MetricTypeNotFoundError(metric_type.to_string())))
+    };
+    Ok((metric32, metric_id))
+}
+
+/// Creates and registers a ping, returning its id.
+pub fn create_and_register_ping(
+    ping_name: String,
+    include_client_id: bool,
+    send_if_empty: bool,
+    precise_timestamps: bool,
+    include_info_sections: bool,
+    enabled: bool,
+    schedules_pings: Vec<String>,
+    reason_codes: Vec<String>,
+    follows_collection_enabled: bool,
+) -> Result<u32, Box<dyn std::error::Error>> {
+    let ping_id = NEXT_PING_ID.fetch_add(1, Ordering::SeqCst);
+    let ping = Ping::new(ping_name, include_client_id, send_if_empty, precise_timestamps, include_info_sections, enabled, schedules_pings, reason_codes, follows_collection_enabled);
+    assert!(
+        __jog_metric_maps::PING_MAP.write()?.insert(ping_id.into(), ping).is_none(),
+        "We should never insert a runtime ping with an already-used id."
+    );
+    Ok(ping_id)
+}

toolkit/components/glean/tests/pytest/test_jogfile_output.py

@@ -46,5 +46,28 @@ def test_jogfile_output():
     expect(here_path / "jogfile_output", output_fd.getvalue())
 
 
+def test_jogfactory_output():
+    """
+    A regression test. Very fragile.
+    It generates a jog factory for metrics_test.yaml and compares it
+    byte-for-byte with an expected output file.
+
+    To generate new expected output files, set `UPDATE_EXPECT=1` when running the test suite:
+
+    UPDATE_EXPECT=1 mach test toolkit/components/glean/tests/pytest
+    """
+
+    options = {"allow_reserved": False}
+    here_path = Path(path.dirname(__file__))
+    input_files = [here_path / "metrics_test.yaml", here_path / "pings_test.yaml"]
+
+    all_objs, options = run_glean_parser.parse_with_options(input_files, options)
+
+    output_fd = io.StringIO()
+    jog.output_factory(all_objs, output_fd, options)
+
+    expect(here_path / "jogfactory_output", output_fd.getvalue())
+
+
 if __name__ == "__main__":
     mozunit.main()