Bug 1966468 - Make mozinstall handle archives when cross-compiling. r=firefox-build-system-reviewers,ahal,glandium

Differential Revision: https://phabricator.services.mozilla.com/D249445

python/mozbuild/mozbuild/artifacts.py

@@ -693,7 +693,6 @@ class MacArtifactJob(ArtifactJob):
 
     def process_package_artifact(self, filename, processed_filename):
         tempdir = tempfile.mkdtemp()
-        oldcwd = os.getcwd()
         try:
             self.log(
                 logging.DEBUG,
@@ -701,24 +700,6 @@ class MacArtifactJob(ArtifactJob):
                 {"tempdir": tempdir},
                 "Unpacking DMG into {tempdir}",
             )
-            if self._substs["HOST_OS_ARCH"] == "Linux":
-                # This is a cross build, use hfsplus and dmg tools to extract the dmg.
-                os.chdir(tempdir)
-                with open(os.devnull, "wb") as devnull:
-                    subprocess.check_call(
-                        [
-                            self._substs["DMG_TOOL"],
-                            "extract",
-                            filename,
-                            "extracted_img",
-                        ],
-                        stdout=devnull,
-                    )
-                    subprocess.check_call(
-                        [self._substs["HFS_TOOL"], "extracted_img", "extractall"],
-                        stdout=devnull,
-                    )
-            else:
             mozinstall.install(filename, tempdir)
 
             bundle_dirs = glob.glob(mozpath.join(tempdir, "*.app"))
@@ -769,7 +750,6 @@ class MacArtifactJob(ArtifactJob):
                             writer.add(destpath.encode("utf-8"), f.open(), mode=f.mode)
 
         finally:
-            os.chdir(oldcwd)
             try:
                 shutil.rmtree(tempdir)
             except OSError:

testing/mozbase/mozinstall/mozinstall/mozinstall.py

@@ -38,14 +38,7 @@ class InvalidBinary(Exception):
 
 
 class InvalidSource(Exception):
-    """Thrown when the specified source is not a recognized file type.
+    """Thrown when the specified source is not a recognized file type."""
-
-    Supported types:
-    Linux:   tar.gz, tar.bz2, tar.xz
-    Mac:     dmg
-    Windows: zip, exe
-
-    """
 
 
 class UninstallError(Exception):
@@ -128,14 +121,17 @@ def install(src, dest):
     trbk = None
     try:
         install_dir = None
-        if src.lower().endswith(".dmg"):
+        if src.lower().endswith(".msix"):
-            install_dir = _install_dmg(src, dest)
+            # MSIX packages _are_ ZIP files, so we need to look for them first.
-        elif src.lower().endswith(".exe"):
-            install_dir = _install_exe(src, dest)
-        elif src.lower().endswith(".msix"):
             install_dir = _install_msix(src)
         elif zipfile.is_zipfile(src) or tarfile.is_tarfile(src):
             install_dir = mozfile.extract(src, dest)[0]
+        elif src.lower().endswith(".dmg"):
+            install_dir = _install_dmg(src, dest)
+        elif src.lower().endswith(".exe"):
+            install_dir = _install_exe(src, dest)
+        else:
+            raise InvalidSource(f"{src} is not a valid installer file")
 
         return install_dir
 
@@ -168,9 +164,9 @@ def is_installer(src):
     """Tests if the given file is a valid installer package.
 
     Supported types:
-    Linux:   tar.gz, tar.bz2, tar.xz
+    All: zip, tar.gz, tar.bz2, tar.xz
-    Mac:     dmg
+    Linux, Mac: dmg
-    Windows: zip, exe
+    Windows: exe, msix
 
     On Windows pefile will be used to determine if the executable is the
     right type, if it is installed on the system.
@@ -182,14 +178,16 @@ def is_installer(src):
     if not os.path.isfile(src):
         return False
 
-    if mozinfo.isLinux:
+    if mozinfo.isWin and src.lower().endswith(".msix"):
-        return tarfile.is_tarfile(src)
+        # MSIX packages _are_ ZIP files, so look for them first.
-    elif mozinfo.isMac:
+        return True
-        return src.lower().endswith(".dmg")
-    elif mozinfo.isWin:
     if zipfile.is_zipfile(src):
         return True
-
+    if tarfile.is_tarfile(src):
+        return True
+    if mozinfo.isMac or mozinfo.isLinux:
+        return src.lower().endswith(".dmg")
+    if mozinfo.isWin:
         if os.access(src, os.X_OK) and src.lower().endswith(".exe"):
             if has_pefile:
                 # try to determine if binary is actually a gecko installer
@@ -298,6 +296,9 @@ def _install_dmg(src, dest):
     dest -- the path to extract to
 
     """
+    if mozinfo.isLinux:
+        return _install_dmg_cross(src, dest)
+
     appDir = None
     try:
         # According to the Apple doc, the hdiutil output is stable and is based on the tab
@@ -336,6 +337,49 @@ def _install_dmg(src, dest):
     return dest
 
 
+def _install_dmg_cross(src, dest):
+    # This is a cross build, use hfsplus and dmg tools to extract the dmg.
+    try:
+        import buildconfig
+
+        dmg_tool = buildconfig.substs.get("DMG_TOOL")
+        hfs_tool = buildconfig.substs.get("HFS_TOOL")
+    except ImportError:
+        pass
+
+    if not dmg_tool:
+        dmg_tool = os.environ.get("DMG_TOOL")
+    if not dmg_tool:
+        raise InstallError("No DMG_TOOL in environment")
+
+    if not hfs_tool:
+        hfs_tool = os.environ.get("HFS_TOOL")
+    if not hfs_tool:
+        raise InstallError("No HFS_TOOL in environment")
+
+    oldcwd = os.getcwd()
+    try:
+        os.chdir(dest)
+        with open(os.devnull, "wb") as devnull:
+            subprocess.check_call(
+                [
+                    dmg_tool,
+                    "extract",
+                    src,
+                    "extracted_img",
+                ],
+                stdout=devnull,
+            )
+            subprocess.check_call(
+                [hfs_tool, "extracted_img", "extractall"],
+                stdout=devnull,
+            )
+    finally:
+        os.chdir(oldcwd)
+
+    return dest
+
+
 def _install_exe(src, dest):
     """Run the MSI installer to silently install the application into the
     destination folder. Return the folder path.

testing/mozbase/mozinstall/tests/test_install.py

@@ -12,12 +12,11 @@ import pytest
 )
 def test_is_installer(request, get_installer):
     """Test that we can identify a correct installer."""
-    if mozinfo.isLinux:
     assert mozinstall.is_installer(get_installer("tar.xz"))
+    assert mozinstall.is_installer(get_installer("zip"))
 
     if mozinfo.isWin:
-        # test zip installer
+        assert mozinstall.is_installer(get_installer("msix"))
-        assert mozinstall.is_installer(get_installer("zip"))
 
         # test exe installer
         assert mozinstall.is_installer(get_installer("exe"))
@@ -34,23 +33,23 @@ def test_is_installer(request, get_installer):
         except ImportError:
             pass
 
-    if mozinfo.isMac:
+    if mozinfo.isMac or mozinfo.isLinux:
         assert mozinstall.is_installer(get_installer("dmg"))
 
 
 def test_invalid_source_error(get_installer):
     """Test that InvalidSource error is raised with an incorrect installer."""
-    if mozinfo.isLinux:
+    if mozinfo.isWin:
         with pytest.raises(mozinstall.InvalidSource):
             mozinstall.install(get_installer("dmg"), "firefox")
 
-    elif mozinfo.isWin:
+    elif mozinfo.isLinux:
         with pytest.raises(mozinstall.InvalidSource):
-            mozinstall.install(get_installer("tar.xz"), "firefox")
+            mozinstall.install(get_installer("msix"), "firefox")
 
     elif mozinfo.isMac:
         with pytest.raises(mozinstall.InvalidSource):
-            mozinstall.install(get_installer("tar.xz"), "firefox")
+            mozinstall.install(get_installer("exe"), "firefox")
 
     # Test an invalid url handler
     with pytest.raises(mozinstall.InvalidSource):
@@ -63,6 +62,11 @@ def test_invalid_source_error(get_installer):
 )
 def test_install(tmpdir, get_installer):
     """Test to install an installer."""
+    installdir_zip = mozinstall.install(
+        get_installer("zip"), tmpdir.join("zip").strpath
+    )
+    assert installdir_zip == tmpdir.join("zip", "firefox").strpath
+
     if mozinfo.isLinux:
         installdir = mozinstall.install(get_installer("tar.xz"), tmpdir.strpath)
         assert installdir == tmpdir.join("firefox").strpath
@@ -73,11 +77,6 @@ def test_install(tmpdir, get_installer):
         )
         assert installdir_exe == tmpdir.join("exe", "firefox").strpath
 
-        installdir_zip = mozinstall.install(
-            get_installer("zip"), tmpdir.join("zip").strpath
-        )
-        assert installdir_zip == tmpdir.join("zip", "firefox").strpath
-
     elif mozinfo.isMac:
         installdir = mozinstall.install(get_installer("dmg"), tmpdir.strpath)
         assert installdir == tmpdir.realpath().join("Firefox Stub.app").strpath

testing/mozbase/mozinstall/tests/test_is_installer.py

@@ -10,12 +10,12 @@ import pytest
 )
 def test_is_installer(request, get_installer):
     """Test that we can identify a correct installer."""
-    if mozinfo.isLinux:
+
     assert mozinstall.is_installer(get_installer("tar.xz"))
+    assert mozinstall.is_installer(get_installer("zip"))
 
     if mozinfo.isWin:
-        # test zip installer
+        assert mozinstall.is_installer(get_installer("msix"))
-        assert mozinstall.is_installer(get_installer("zip"))
 
         # test exe installer
         assert mozinstall.is_installer(get_installer("exe"))
@@ -32,7 +32,7 @@ def test_is_installer(request, get_installer):
         except ImportError:
             pass
 
-    if mozinfo.isMac:
+    if mozinfo.isMac or mozinfo.isLinux:
         assert mozinstall.is_installer(get_installer("dmg"))