Bug 1784176 - Add a feature for manual logins for recording sites. r=perftest-reviewers,AlexandruIonescu

This patch adds the ability to run manual logins for our websites since it can be simpler, and quicker in some cases. At the same time, a bug with the options handling is fixed.

Differential Revision: https://phabricator.services.mozilla.com/D164590

python/mozperftest/mozperftest/test/browsertime/__init__.py

@@ -14,6 +14,6 @@ def add_option(env, name, value, overwrite=False):
     env.set_arg("browsertime-extra-options", options)
 
 
-def add_options(env, options):
+def add_options(env, options, overwrite=False):
-    for name, value in options:
+    for i, (name, value) in enumerate(options):
-        add_option(env, name, value)
+        add_option(env, name, value, overwrite=overwrite and i == 0)

python/mozperftest/mozperftest/test/browsertime/runner.py

@@ -435,6 +435,7 @@ class BrowsertimeRunner(NodeRunner):
                 if name == "browsertime.login" and value:
                     is_login_site = True
 
+                self.info(f"Adding extra browsertime argument: --{name} {value}")
                 args += ["--" + name, value]
 
         if self.get_arg("android"):

testing/performance/hooks_recording.py

@@ -8,8 +8,12 @@ from pathlib import Path
 
 from mozperftest.test.browsertime import add_option, add_options
 
+# Uncomment the manual_login line if you need to do a manual login.
+# The extra browsertime arguments get overwritten below so they
+# need to be set here. The value is the time you need to do a login.
 options = [
     ("pageCompleteWaitTime", "10000"),
+    # ("browsertime.manual_login", 100000),
 ]
 
 next_site = None
@@ -128,11 +132,14 @@ def before_runs(env):
             recording_file,
         )
 
-        add_option(env, "browsertime.url", test_site.get("test_url"), overwrite=True)
+        add_options(env, options, overwrite=True)
+        add_option(env, "browsertime.url", test_site.get("test_url"))
         add_option(env, "browsertime.screenshot", "true")
         add_option(env, "browsertime.testName", test_site.get("name"))
         add_option(env, "browsertime.testType", test_site.get("type", "pageload"))
-        add_option(env, "browsertime.login", test_site.get("login", "false"))
+        add_option(
+            env, "browsertime.login", "true" if test_site.get("login") else "false"
+        )
 
         prefs = test_site.get("preferences", {})
         for pref, val in prefs.items():

testing/performance/perftest_record.js

@@ -361,11 +361,22 @@ async function pageload_test(context, commands) {
   let dismissPrompt = context.options.browsertime.dismiss_cookie_prompt || "";
   context.log.info(context.options.browsertime);
 
+  // Wait for browser to settle
+  await commands.wait.byTime(1000);
+
   // If the user has RAPTOR_LOGINS configured correctly, a local login pageload
   // test can be attempted. Otherwise if attempting it in CI, only sites with the
   // associated MOZ_SCM_LEVEL will be attempted (e.g. Try = 1, autoland = 3)
   if (context.options.browsertime.login) {
-    if (
+    if (context.options.browsertime.manual_login) {
+      // Perform a manual login using the value given in manual_login
+      // as the amount of time to wait
+      await commands.navigate(testUrl);
+      context.log.info(
+        `Waiting ${context.options.browsertime.manual_login}ms for login...`
+      );
+      await commands.wait.byTime(context.options.browsertime.manual_login);
+    } else if (
       process.env.RAPTOR_LOGINS ||
       process.env.MOZ_SCM_LEVEL == 3 ||
       SCM_1_LOGIN_SITES.includes(testName)
@@ -378,12 +389,14 @@ async function pageload_test(context, commands) {
         );
         context.log.info("Error:" + err);
       }
+    } else {
+      context.log.info(`
+        NOTE: This is a login test but a manual login was not requested, and
+        we cannot find any logins defined in RAPTOR_LOGINS.
+      `);
     }
   }
 
-  // Wait for browser to settle
-  await commands.wait.byTime(1000);
-
   await commands.measure.start(testUrl);
   await commands.wait.byTime(40000);
   if (dismissPrompt) {