03bc4efb44
This officially makes 'moztest.resolve' the source of truth when it comes to suite names. It aligns that file with the names used in both the desktop_unittest and android_emulator_unittest scripts. Differential Revision: https://phabricator.services.mozilla.com/D27555
463 lines
18 KiB
Python
463 lines
18 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
# flake8: noqa: E501
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import cPickle as pickle
|
|
import os
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
|
|
import mozpack.path as mozpath
|
|
import mozunit
|
|
from mozbuild.base import MozbuildObject
|
|
from mozfile import NamedTemporaryFile
|
|
|
|
from moztest.resolve import (
|
|
TestMetadata,
|
|
TestResolver,
|
|
TEST_SUITES,
|
|
)
|
|
|
|
|
|
ALL_TESTS = {
|
|
"accessible/tests/mochitest/actions/test_anchors.html": [
|
|
{
|
|
"dir_relpath": "accessible/tests/mochitest/actions",
|
|
"expected": "pass",
|
|
"file_relpath": "accessible/tests/mochitest/actions/test_anchors.html",
|
|
"flavor": "a11y",
|
|
"here": "/firefox/accessible/tests/mochitest/actions",
|
|
"manifest": "/firefox/accessible/tests/mochitest/actions/a11y.ini",
|
|
"name": "test_anchors.html",
|
|
"path": "/firefox/accessible/tests/mochitest/actions/test_anchors.html",
|
|
"relpath": "test_anchors.html"
|
|
}
|
|
],
|
|
"services/common/tests/unit/test_async_chain.js": [
|
|
{
|
|
"dir_relpath": "services/common/tests/unit",
|
|
"file_relpath": "services/common/tests/unit/test_async_chain.js",
|
|
"firefox-appdir": "browser",
|
|
"flavor": "xpcshell",
|
|
"head": "head_global.js head_helpers.js head_http.js",
|
|
"here": "/firefox/services/common/tests/unit",
|
|
"manifest": "/firefox/services/common/tests/unit/xpcshell.ini",
|
|
"name": "test_async_chain.js",
|
|
"path": "/firefox/services/common/tests/unit/test_async_chain.js",
|
|
"relpath": "test_async_chain.js",
|
|
}
|
|
],
|
|
"services/common/tests/unit/test_async_querySpinningly.js": [
|
|
{
|
|
"dir_relpath": "services/common/tests/unit",
|
|
"file_relpath": "services/common/tests/unit/test_async_querySpinningly.js",
|
|
"firefox-appdir": "browser",
|
|
"flavor": "xpcshell",
|
|
"head": "head_global.js head_helpers.js head_http.js",
|
|
"here": "/firefox/services/common/tests/unit",
|
|
"manifest": "/firefox/services/common/tests/unit/xpcshell.ini",
|
|
"name": "test_async_querySpinningly.js",
|
|
"path": "/firefox/services/common/tests/unit/test_async_querySpinningly.js",
|
|
"relpath": "test_async_querySpinningly.js",
|
|
}
|
|
],
|
|
"toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js": [
|
|
{
|
|
"dir_relpath": "toolkit/mozapps/update/test/unit",
|
|
"file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
|
|
"flavor": "xpcshell",
|
|
"generated-files": "head_update.js",
|
|
"head": "head_update.js",
|
|
"here": "/firefox/toolkit/mozapps/update/test/unit",
|
|
"manifest": "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
|
|
"name": "test_0201_app_launch_apply_update.js",
|
|
"path": "/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
|
|
"reason": "bug 820380",
|
|
"relpath": "test_0201_app_launch_apply_update.js",
|
|
"run-sequentially": "Launches application.",
|
|
"skip-if": "os == 'android'",
|
|
},
|
|
{
|
|
"dir_relpath": "toolkit/mozapps/update/test/unit",
|
|
"file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
|
|
"flavor": "xpcshell",
|
|
"generated-files": "head_update.js",
|
|
"head": "head_update.js head2.js",
|
|
"here": "/firefox/toolkit/mozapps/update/test/unit",
|
|
"manifest": "/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
|
|
"name": "test_0201_app_launch_apply_update.js",
|
|
"path": "/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
|
|
"reason": "bug 820380",
|
|
"relpath": "test_0201_app_launch_apply_update.js",
|
|
"run-sequentially": "Launches application.",
|
|
"skip-if": "os == 'android'",
|
|
}
|
|
],
|
|
"mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java": [
|
|
{
|
|
"dir_relpath": "mobile/android/tests/background/junit3/src/common",
|
|
"file_relpath": "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java",
|
|
"flavor": "instrumentation",
|
|
"here": "/firefox/mobile/android/tests/background/junit3",
|
|
"manifest": "/firefox/mobile/android/tests/background/junit3/instrumentation.ini",
|
|
"name": "src/common/TestAndroidLogWriters.java",
|
|
"path": "/firefox/mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java",
|
|
"relpath": "src/common/TestAndroidLogWriters.java",
|
|
"subsuite": "background"
|
|
}
|
|
],
|
|
"mobile/android/tests/browser/junit3/src/TestDistribution.java": [
|
|
{
|
|
"dir_relpath": "mobile/android/tests/browser/junit3/src",
|
|
"file_relpath": "mobile/android/tests/browser/junit3/src/TestDistribution.java",
|
|
"flavor": "instrumentation",
|
|
"here": "/firefox/mobile/android/tests/browser/junit3",
|
|
"manifest": "/firefox/mobile/android/tests/browser/junit3/instrumentation.ini",
|
|
"name": "src/TestDistribution.java",
|
|
"path": "/firefox/mobile/android/tests/browser/junit3/src/TestDistribution.java",
|
|
"relpath": "src/TestDistribution.java",
|
|
"subsuite": "browser"
|
|
}
|
|
],
|
|
"image/test/browser/browser_bug666317.js": [
|
|
{
|
|
"dir_relpath": "image/test/browser",
|
|
"file_relpath": "image/test/browser/browser_bug666317.js",
|
|
"flavor": "browser-chrome",
|
|
"here": "/firefox/testing/mochitest/browser/image/test/browser",
|
|
"manifest": "/firefox/image/test/browser/browser.ini",
|
|
"name": "browser_bug666317.js",
|
|
"path": "/firefox/testing/mochitest/browser/image/test/browser/browser_bug666317.js",
|
|
"relpath": "image/test/browser/browser_bug666317.js",
|
|
"skip-if": "e10s # Bug 948194 - Decoded Images seem to not be discarded on memory-pressure notification with e10s enabled",
|
|
"subsuite": ""
|
|
}
|
|
],
|
|
"devtools/client/markupview/test/browser_markupview_copy_image_data.js": [
|
|
{
|
|
"dir_relpath": "devtools/client/markupview/test",
|
|
"file_relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
|
|
"flavor": "browser-chrome",
|
|
"here": "/firefox/testing/mochitest/browser/devtools/client/markupview/test",
|
|
"manifest": "/firefox/devtools/client/markupview/test/browser.ini",
|
|
"name": "browser_markupview_copy_image_data.js",
|
|
"path": "/firefox/testing/mochitest/browser/devtools/client/markupview/test/browser_markupview_copy_image_data.js",
|
|
"relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
|
|
"subsuite": "devtools",
|
|
"tags": "devtools"
|
|
}
|
|
]
|
|
}
|
|
|
|
TEST_DEFAULTS = {
|
|
"/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\ndata/**\nxpcshell_updater.ini"}
|
|
}
|
|
|
|
TASK_LABELS = [
|
|
'test-linux64/opt-browser-screenshots-1',
|
|
'test-linux64/opt-browser-screenshots-e10s-1',
|
|
'test-linux64/opt-marionette',
|
|
'test-linux64/opt-mochitest',
|
|
'test-linux64/debug-mochitest-e10s',
|
|
'test-linux64/opt-mochitest-a11y',
|
|
'test-linux64/opt-mochitest-browser',
|
|
'test-linux64/opt-mochitest-browser-chrome',
|
|
'test-linux64/opt-mochitest-browser-chrome-e10s',
|
|
'test-linux64/opt-mochitest-browser-chrome-e10s-11',
|
|
'test-linux64/opt-mochitest-chrome',
|
|
'test-linux64/opt-mochitest-clipboard',
|
|
'test-linux64/opt-mochitest-devtools',
|
|
'test-linux64/opt-mochitest-devtools-chrome',
|
|
'test-linux64/opt-mochitest-gpu',
|
|
'test-linux64/opt-mochitest-gpu-e10s',
|
|
'test-linux64/opt-mochitest-media-e10s-1',
|
|
'test-linux64/opt-mochitest-media-e10s-11',
|
|
'test-linux64/opt-mochitest-plain',
|
|
'test-linux64/opt-mochitest-screenshots-1',
|
|
'test-linux64/opt-reftest',
|
|
'test-linux64/debug-reftest-e10s-1',
|
|
'test-linux64/debug-reftest-e10s-11',
|
|
'test-linux64/opt-robocop',
|
|
'test-linux64/opt-robocop-1',
|
|
'test-linux64/opt-robocop-e10s',
|
|
'test-linux64/opt-robocop-e10s-1',
|
|
'test-linux64/opt-robocop-e10s-11',
|
|
'test-linux64/opt-web-platform-tests-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-reftests-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-reftest-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-1',
|
|
'test-linux64/opt-web-platform-test-e10s-1',
|
|
'test-linux64/opt-xpcshell',
|
|
'test-linux64/opt-xpcshell-1',
|
|
'test-linux64/opt-xpcshell-2',
|
|
]
|
|
|
|
|
|
class Base(unittest.TestCase):
|
|
def setUp(self):
|
|
self._temp_files = []
|
|
|
|
def tearDown(self):
|
|
for f in self._temp_files:
|
|
del f
|
|
|
|
self._temp_files = []
|
|
|
|
def _get_test_metadata(self):
|
|
all_tests = NamedTemporaryFile(mode='wb')
|
|
pickle.dump(ALL_TESTS, all_tests)
|
|
all_tests.flush()
|
|
self._temp_files.append(all_tests)
|
|
|
|
test_defaults = NamedTemporaryFile(mode='wb')
|
|
pickle.dump(TEST_DEFAULTS, test_defaults)
|
|
test_defaults.flush()
|
|
self._temp_files.append(test_defaults)
|
|
|
|
rv = TestMetadata(all_tests.name, "/firefox/", test_defaults=test_defaults.name)
|
|
rv._wpt_loaded = True # Don't try to load the wpt manifest
|
|
return rv
|
|
|
|
|
|
class TestTestMetadata(Base):
|
|
def test_load(self):
|
|
t = self._get_test_metadata()
|
|
self.assertEqual(len(t._tests_by_path), 8)
|
|
|
|
self.assertEqual(len(list(t.tests_with_flavor('xpcshell'))), 3)
|
|
self.assertEqual(len(list(t.tests_with_flavor('mochitest-plain'))), 0)
|
|
|
|
def test_resolve_all(self):
|
|
t = self._get_test_metadata()
|
|
self.assertEqual(len(list(t.resolve_tests())), 9)
|
|
|
|
def test_resolve_filter_flavor(self):
|
|
t = self._get_test_metadata()
|
|
self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell'))), 4)
|
|
|
|
def test_resolve_by_dir(self):
|
|
t = self._get_test_metadata()
|
|
self.assertEqual(len(list(t.resolve_tests(paths=['services/common']))), 2)
|
|
|
|
def test_resolve_under_path(self):
|
|
t = self._get_test_metadata()
|
|
self.assertEqual(len(list(t.resolve_tests(under_path='services'))), 2)
|
|
|
|
self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell',
|
|
under_path='services'))), 2)
|
|
|
|
def test_resolve_multiple_paths(self):
|
|
t = self._get_test_metadata()
|
|
result = list(t.resolve_tests(paths=['services', 'toolkit']))
|
|
self.assertEqual(len(result), 4)
|
|
|
|
def test_resolve_support_files(self):
|
|
expected_support_files = "\ndata/**\nxpcshell_updater.ini"
|
|
t = self._get_test_metadata()
|
|
result = list(t.resolve_tests(paths=['toolkit']))
|
|
self.assertEqual(len(result), 2)
|
|
|
|
for test in result:
|
|
self.assertEqual(test['support-files'],
|
|
expected_support_files)
|
|
|
|
def test_resolve_path_prefix(self):
|
|
t = self._get_test_metadata()
|
|
result = list(t.resolve_tests(paths=['image']))
|
|
self.assertEqual(len(result), 1)
|
|
|
|
|
|
class TestTestResolver(Base):
|
|
FAKE_TOPSRCDIR = '/firefox'
|
|
|
|
def setUp(self):
|
|
Base.setUp(self)
|
|
|
|
self._temp_dirs = []
|
|
|
|
def tearDown(self):
|
|
Base.tearDown(self)
|
|
|
|
for d in self._temp_dirs:
|
|
shutil.rmtree(d)
|
|
|
|
def _get_resolver(self):
|
|
topobjdir = tempfile.mkdtemp()
|
|
self._temp_dirs.append(topobjdir)
|
|
|
|
with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
|
|
pickle.dump(ALL_TESTS, fh)
|
|
with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
|
|
pickle.dump(TEST_DEFAULTS, fh)
|
|
|
|
o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)
|
|
|
|
# Monkey patch the test resolver to avoid tests failing to find make
|
|
# due to our fake topscrdir.
|
|
TestResolver._run_make = lambda *a, **b: None
|
|
|
|
return o._spawn(TestResolver)
|
|
|
|
def test_cwd_children_only(self):
|
|
"""If cwd is defined, only resolve tests under the specified cwd."""
|
|
r = self._get_resolver()
|
|
|
|
# Pretend we're under '/services' and ask for 'common'. This should
|
|
# pick up all tests from '/services/common'
|
|
tests = list(r.resolve_tests(paths=['common'], cwd=os.path.join(r.topsrcdir,
|
|
'services')))
|
|
|
|
self.assertEqual(len(tests), 2)
|
|
|
|
# Tests should be rewritten to objdir.
|
|
for t in tests:
|
|
self.assertEqual(t['here'], mozpath.join(r.topobjdir,
|
|
'_tests/xpcshell/services/common/tests/unit'))
|
|
|
|
def test_various_cwd(self):
|
|
"""Test various cwd conditions are all equal."""
|
|
|
|
r = self._get_resolver()
|
|
|
|
expected = list(r.resolve_tests(paths=['services']))
|
|
actual = list(r.resolve_tests(paths=['services'], cwd='/'))
|
|
self.assertEqual(actual, expected)
|
|
|
|
actual = list(r.resolve_tests(paths=['services'], cwd=r.topsrcdir))
|
|
self.assertEqual(actual, expected)
|
|
|
|
actual = list(r.resolve_tests(paths=['services'], cwd=r.topobjdir))
|
|
self.assertEqual(actual, expected)
|
|
|
|
def test_subsuites(self):
|
|
"""Test filtering by subsuite."""
|
|
|
|
r = self._get_resolver()
|
|
|
|
tests = list(r.resolve_tests(paths=['mobile']))
|
|
self.assertEqual(len(tests), 2)
|
|
|
|
tests = list(r.resolve_tests(paths=['mobile'], subsuite='browser'))
|
|
self.assertEqual(len(tests), 1)
|
|
self.assertEqual(tests[0]['name'], 'src/TestDistribution.java')
|
|
|
|
tests = list(r.resolve_tests(paths=['mobile'], subsuite='background'))
|
|
self.assertEqual(len(tests), 1)
|
|
self.assertEqual(tests[0]['name'], 'src/common/TestAndroidLogWriters.java')
|
|
|
|
def test_wildcard_patterns(self):
|
|
"""Test matching paths by wildcard."""
|
|
|
|
r = self._get_resolver()
|
|
|
|
tests = list(r.resolve_tests(paths=['mobile/**']))
|
|
self.assertEqual(len(tests), 2)
|
|
for t in tests:
|
|
self.assertTrue(t['file_relpath'].startswith('mobile'))
|
|
|
|
tests = list(r.resolve_tests(paths=['**/**.js', 'accessible/**']))
|
|
self.assertEqual(len(tests), 7)
|
|
for t in tests:
|
|
path = t['file_relpath']
|
|
self.assertTrue(path.startswith('accessible') or path.endswith('.js'))
|
|
|
|
def test_resolve_metadata(self):
|
|
"""Test finding metadata from outgoing files."""
|
|
r = self._get_resolver()
|
|
|
|
suites, tests = r.resolve_metadata(['bc'])
|
|
assert suites == {'mochitest-browser-chrome'}
|
|
assert tests == []
|
|
|
|
suites, tests = r.resolve_metadata(['mochitest-a11y', '/browser', 'xpcshell'])
|
|
assert suites == {'mochitest-a11y', 'xpcshell'}
|
|
assert sorted(t['file_relpath'] for t in tests) == [
|
|
'devtools/client/markupview/test/browser_markupview_copy_image_data.js',
|
|
'image/test/browser/browser_bug666317.js',
|
|
'mobile/android/tests/browser/junit3/src/TestDistribution.java',
|
|
]
|
|
|
|
def test_task_regexes(self):
|
|
"""Test the task_regexes defined in TEST_SUITES."""
|
|
|
|
test_cases = {
|
|
'mochitest-browser-chrome': [
|
|
'test-linux64/opt-mochitest-browser-chrome',
|
|
'test-linux64/opt-mochitest-browser-chrome-e10s',
|
|
],
|
|
'mochitest-chrome': [
|
|
'test-linux64/opt-mochitest-chrome',
|
|
],
|
|
'mochitest-devtools-chrome': [
|
|
'test-linux64/opt-mochitest-devtools-chrome',
|
|
],
|
|
'mochitest-media': [
|
|
'test-linux64/opt-mochitest-media-e10s-1',
|
|
],
|
|
'mochitest-plain': [
|
|
'test-linux64/opt-mochitest',
|
|
'test-linux64/debug-mochitest-e10s',
|
|
# this isn't a real task but the regex would match it if it were
|
|
'test-linux64/opt-mochitest-plain',
|
|
],
|
|
'mochitest-plain-gpu': [
|
|
'test-linux64/opt-mochitest-gpu',
|
|
'test-linux64/opt-mochitest-gpu-e10s',
|
|
],
|
|
'mochitest-browser-chrome-screenshots': [
|
|
'test-linux64/opt-browser-screenshots-1',
|
|
'test-linux64/opt-browser-screenshots-e10s-1',
|
|
],
|
|
'reftest': [
|
|
'test-linux64/opt-reftest',
|
|
'test-linux64/debug-reftest-e10s-1',
|
|
],
|
|
'robocop': [
|
|
'test-linux64/opt-robocop',
|
|
'test-linux64/opt-robocop-1',
|
|
'test-linux64/opt-robocop-e10s',
|
|
'test-linux64/opt-robocop-e10s-1',
|
|
],
|
|
'web-platform-tests': [
|
|
'test-linux64/opt-web-platform-tests-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-reftests-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-reftest-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-1',
|
|
],
|
|
'web-platform-tests-testharness': [
|
|
'test-linux64/opt-web-platform-tests-e10s-1',
|
|
'test-linux64/opt-web-platform-tests-1',
|
|
],
|
|
'web-platform-tests-reftest': [
|
|
'test-linux64/opt-web-platform-tests-reftests-e10s-1',
|
|
],
|
|
'web-platform-tests-wdspec': [
|
|
'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
|
|
],
|
|
'xpcshell': [
|
|
'test-linux64/opt-xpcshell',
|
|
'test-linux64/opt-xpcshell-1',
|
|
],
|
|
}
|
|
|
|
regexes = []
|
|
|
|
def match_task(task):
|
|
return any(re.search(pattern, task) for pattern in regexes)
|
|
|
|
for suite, expected in sorted(test_cases.items()):
|
|
print(suite)
|
|
regexes = TEST_SUITES[suite]['task_regex']
|
|
assert set(filter(match_task, TASK_LABELS)) == set(expected)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
mozunit.main()
|