Bug 1726251 - add mach repackage flatpak command. r=releng-reviewers,gerard-majax,bhearsum

Copy commands from taskcluster/docker/firefox-flatpak/runme.sh into a
standalone repackage command that can be pointed at a firefox build and
spit out a flatpak repository.

The browser/installer/linux/app/flatpak directory contains templates and
extra files that get injected into the flatpak build process.

Differential Revision: https://phabricator.services.mozilla.com/D238800

browser/installer/linux/app/flatpak/files/bin/firefox

@@ -0,0 +1,3 @@
+#!/bin/bash
+export TMPDIR=$XDG_CACHE_HOME/tmp
+exec /app/lib/firefox/firefox --name org.mozilla.firefox "$@"

browser/installer/linux/app/flatpak/files/share/appdata/org.mozilla.firefox.appdata.xml.in

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<component type="desktop-application">
+  <id>org.mozilla.firefox</id>
+  <launchable type="desktop-id">org.mozilla.firefox.desktop</launchable>
+  <name>Firefox</name>
+  <developer_name>Mozilla</developer_name>
+  <summary>Fast, Private &amp; Safe Web Browser</summary>
+  <metadata_license>CC0-1.0</metadata_license>
+  <project_license>MPL-2.0</project_license>
+  <description>
+    <p>When it comes to your life online, you have a choice: accept the factory settings or put your privacy first. When you choose Firefox as your default browser, you’re choosing to protect your data while supporting an independent tech company. Firefox is also the only major browser backed by a non-profit fighting to give you more openness, transparency and control of your life online. Join hundreds of millions of people who choose to protect what's important by choosing Firefox - a web browser designed to be fast, easy to use, customizable and private.</p>
+  </description>
+  <releases>
+	<release version="$VERSION" date="$DATE"/>
+  </releases>
+  <keywords>
+    <keyword>mozilla</keyword>
+    <keyword>internet</keyword>
+    <keyword>web</keyword>
+  </keywords>
+  <content_rating type="oars-1.1" />
+  <url type="homepage">https://www.mozilla.org/firefox/</url>
+  <url type="donation">https://donate.mozilla.org/</url>
+  <url type="bugtracker">https://bugzilla.mozilla.org/</url>
+  <url type="help">https://support.mozilla.org/</url>
+  <url type="translate">https://wiki.mozilla.org/L10n:Starting_a_localization</url>
+
+  <screenshots>
+    <screenshot type="default">
+      <image type="source">https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image1.png</image>
+      <caption>The “New Tab” page</caption>
+    </screenshot>
+    <screenshot>
+      <image type="source">https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image2.png</image>
+      <caption>A Wikipedia article displayed in Firefox</caption>
+    </screenshot>
+    <screenshot>
+      <image type="source">https://raw.githubusercontent.com/mozilla-releng/scriptworker-scripts/master/pushflatpakscript/media/screenshots/image3.png</image>
+      <caption>The “Welcome to Firefox” page: “Open up an amazing internet”</caption>
+    </screenshot>
+  </screenshots>
+  <custom>
+    <value key="flathub::manifest">https://hg.mozilla.org/mozilla-central/file/tip/browser/installer/linux/app/flatpak</value>
+  </custom>
+</component>

browser/installer/linux/app/flatpak/files/share/icons/hicolor/symbolic/apps/org.mozilla.firefox-symbolic.svg

@@ -0,0 +1,6 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<svg id="Assets" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512">
+  <path d="M190.368 150.591c0.157 0.009 0.079 0.003 0 0zm-57.874-28.933c0.158 0.008 0.079 0.003 0 0zm346.228 44.674c-10.445-25.123-31.6-52.248-48.211-60.82 13.52 26.5 21.345 53.093 24.335 72.935 0 0.04 0.015 0.136 0.047 0.4-27.175-67.732-73.254-95.047-110.886-154.512-1.9-3.008-3.805-6.022-5.661-9.2a73.237 73.237 0 0 1-2.646-4.972 43.757 43.757 0 0 1-3.585-9.5 0.625 0.625 0 0 0-0.546-0.644 0.8 0.8 0 0 0-0.451 0c-0.033 0.011-0.084 0.051-0.119 0.065-0.053 0.02-0.12 0.069-0.176 0.095 0.026-0.036 0.083-0.117 0.1-0.135-53.437 31.3-75.587 86.093-81.282 120.97a128.057 128.057 0 0 0-47.624 12.153 6.144 6.144 0 0 0-3.041 7.63 6.034 6.034 0 0 0 8.192 3.525 116.175 116.175 0 0 1 41.481-10.826c0.468-0.033 0.937-0.062 1.405-0.1a117.624 117.624 0 0 1 5.932-0.211 120.831 120.831 0 0 1 34.491 4.777c0.654 0.192 1.295 0.414 1.946 0.616a120.15 120.15 0 0 1 5.539 1.842 121.852 121.852 0 0 1 3.992 1.564c1.074 0.434 2.148 0.868 3.206 1.331a118.453 118.453 0 0 1 4.9 2.307c0.743 0.368 1.485 0.735 2.217 1.117a120.535 120.535 0 0 1 4.675 2.587 107.785 107.785 0 0 1 2.952 1.776 123.018 123.018 0 0 1 42.028 43.477c-12.833-9.015-35.81-17.918-57.947-14.068 86.441 43.214 63.234 192.027-56.545 186.408a106.7 106.7 0 0 1-31.271-6.031 132.461 132.461 0 0 1-7.059-2.886c-1.356-0.618-2.711-1.243-4.051-1.935-29.349-15.168-53.583-43.833-56.611-78.643 0 0 11.093-41.335 79.433-41.335 7.388 0 28.508-20.614 28.9-26.593-0.09-1.953-41.917-18.59-58.223-34.656-8.714-8.585-12.851-12.723-16.514-15.829a71.7 71.7 0 0 0-6.225-4.7 111.335 111.335 0 0 1-0.675-58.733c-24.687 11.242-43.89 29.011-57.849 44.7h-0.111c-9.528-12.067-8.855-51.873-8.312-60.184-0.114-0.516-7.107 3.63-8.024 4.254a175.21 175.21 0 0 0-23.486 20.12 210.5 210.5 0 0 0-22.443 26.913c0 0.012-0.007 0.025-0.011 0.037 0-0.012 0.007-0.025 0.011-0.038a202.837 202.837 0 0 0-32.244 72.81c-0.058 0.265-2.29 10.054-3.92 22.147a265.794 265.794 0 0 0-0.769 5.651c-0.558 3.636-0.992 7.6-1.42 13.767-0.019 0.239-0.031 0.474-0.048 0.712a591.152 591.152 0 0 0-0.481 7.995c0 0.411-0.025 0.816-0.025 1.227 0 132.709 107.6 240.29 240.324 240.29 118.865 0 217.559-86.288 236.882-199.63 0.407-3.075 0.732-6.168 1.092-9.27 4.777-41.21-0.53-84.525-15.588-120.747zm-164.068 72.1z" fill="#fff"/>
+</svg>

browser/installer/linux/app/flatpak/metadata.in

@@ -0,0 +1,19 @@
+[Application]
+name=org.mozilla.firefox
+runtime=org.freedesktop.Platform/${ARCH}/${FREEDESKTOP_VERSION}
+sdk=org.freedesktop.Sdk/${ARCH}/${FREEDESKTOP_VERSION}
+base=app/org.mozilla.firefox.BaseApp/${ARCH}/${FIREFOX_BASEAPP_CHANNEL}
+[Extension org.mozilla.firefox.Locale]
+directory=share/runtime/langpack
+autodelete=true
+locale-subset=true
+
+[Extension org.freedesktop.Platform.ffmpeg-full]
+directory=lib/ffmpeg
+add-ld-path=.
+no-autodownload=true
+version=${FREEDESKTOP_VERSION}
+
+[Extension org.mozilla.firefox.systemconfig]
+directory=etc/firefox
+no-autodownload=true

browser/installer/linux/app/flatpak/metadata.locale.in

@@ -0,0 +1,5 @@
+[Runtime]
+name=org.mozilla.firefox.Locale
+
+[ExtensionOf]
+ref=app/org.mozilla.firefox/${ARCH}/${FLATPAK_BRANCH}

python/mozbuild/mozbuild/mach_commands.py

@@ -3482,3 +3482,69 @@ def _prepend_debugger_args(args, debugger, debugger_args):
     # Prepend the debugger args.
     args = [debuggerInfo.path] + debuggerInfo.args + args
     return args
+
+
+@SubCommand(
+    "repackage",
+    "flatpak",
+    description="Repackage a tar file into a flatpak",
+    virtualenv_name="repackage-desktop-file",
+)
+@CommandArgument("--input", "-i", type=str, required=True, help="Input filename")
+@CommandArgument("--output", "-o", type=str, required=True, help="Output filename")
+@CommandArgument("--name", type=str, required=True, help="flatpak package name")
+@CommandArgument("--arch", type=str, required=True, help="flatpak architecture")
+@CommandArgument("--version", type=str, required=True, help="package version")
+@CommandArgument("--product", type=str, required=True, help="release product")
+@CommandArgument(
+    "--release-type",
+    type=str,
+    required=True,
+    help="The release being shipped. Used to disambiguate nightly/try etc.",
+)
+@CommandArgument(
+    "--flatpak-branch",
+    type=str,
+    required=True,
+    help="flatpak branch",
+)
+@CommandArgument(
+    "--template-dir", type=str, required=True, help="path to template directory"
+)
+@CommandArgument(
+    "--langpack-pattern",
+    type=str,
+    help="shell pattern matching language packs to include in the flatpak",
+)
+def repackage_flatpak(
+    command_context,
+    input,
+    output,
+    name,
+    arch,
+    version,
+    product,
+    release_type,
+    flatpak_branch,
+    template_dir,
+    langpack_pattern,
+):
+    if not os.path.exists(input):
+        print("Input file does not exist: %s" % input)
+        return 1
+
+    from mozbuild.repackaging.flatpak import repackage_flatpak
+
+    repackage_flatpak(
+        command_context.log,
+        input,
+        output,
+        arch,
+        version,
+        product,
+        release_type,
+        name,
+        flatpak_branch,
+        template_dir,
+        langpack_pattern,
+    )

python/mozbuild/mozbuild/repackaging/flatpak.py

@@ -0,0 +1,306 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+import glob
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import zipfile
+from pathlib import Path
+from string import Template
+
+# When updating this, please make sure to keep in sync the script for symbol
+# scraping at
+# https://github.com/mozilla/symbol-scrapers/blob/master/firefox-flatpak/script.sh
+FREEDESKTOP_VERSION = "23.08"
+# The base app is shared by firefox and thunderbird
+FIREFOX_BASEAPP = "org.mozilla.firefox.BaseApp"
+FIREFOX_BASEAPP_CHANNEL = "23.08"
+
+
+def _inject_flatpak_distribution_ini(target):
+    with tempfile.TemporaryDirectory() as git_clone_dir:
+        subprocess.check_call(
+            [
+                "git",
+                "clone",
+                "https://github.com/mozilla-partners/flatpak.git",
+                git_clone_dir,
+            ],
+        )
+        shutil.copyfile(
+            os.path.join(
+                git_clone_dir, "desktop/flatpak/distribution/distribution.ini"
+            ),
+            target,
+        )
+
+
+def _langpack_manifest(xpi):
+    with zipfile.ZipFile(xpi) as f:
+        return json.load(f.open("manifest.json"))
+
+
+def _render_template(source, dest, variables):
+    if source.endswith(".in"):
+        with open(source) as f:
+            template = Template(f.read())
+        with open(dest[:-3], "w") as f:
+            f.write(template.substitute(variables))
+    else:
+        shutil.copy(source, dest)
+
+
+def _render_flatpak_templates(template_dir, build_dir, variables):
+    for root, dirs, files in os.walk(template_dir):
+        relative = os.path.relpath(root, template_dir)
+        for d in dirs:
+            os.makedirs(build_dir / relative / d, exist_ok=True)
+        for f in files:
+            _render_template(
+                os.path.join(root, f), os.path.join(build_dir, relative, f), variables
+            )
+
+
+def repackage_flatpak(
+    log,
+    infile,
+    output,
+    arch,
+    version,
+    release_product,
+    release_type,
+    flatpak_name,
+    flatpak_branch,
+    template_dir,
+    langpack_pattern,
+):
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir = Path(tmpdir)
+        build_dir = tmpdir / "build"
+        app_dir = build_dir / "files"
+        lib_dir = app_dir / "lib"
+
+        # Fetch and install the base app
+        subprocess.run(
+            [
+                "flatpak",
+                "remote-add",
+                "--user",
+                "--if-not-exists",
+                "--from",
+                "flathub",
+                "https://dl.flathub.org/repo/flathub.flatpakrepo",
+            ],
+            check=True,
+        )
+        subprocess.run(
+            [
+                "flatpak",
+                "install",
+                "--user",
+                "-y",
+                "flathub",
+                f"{FIREFOX_BASEAPP}/{arch}/{FIREFOX_BASEAPP_CHANNEL}",
+                "--no-deps",
+            ],
+            check=True,
+        )
+        # Copy files from the base app to our build dir
+        base = (
+            Path.home()
+            / f".local/share/flatpak/app/{FIREFOX_BASEAPP}/{arch}/{FIREFOX_BASEAPP_CHANNEL}/active/files"
+        )
+        shutil.copytree(base, app_dir, symlinks=True)
+
+        # Extract our build to the app dir
+        lib_dir.mkdir(exist_ok=True)
+        subprocess.check_call(["tar", "xf", os.path.abspath(infile)], cwd=lib_dir)
+
+        if release_product == "firefox":
+            distribution_ini = lib_dir / "firefox" / "distribution" / "distribution.ini"
+            distribution_ini.parent.mkdir(parents=True)
+            _inject_flatpak_distribution_ini(distribution_ini)
+
+        date = datetime.date.today().strftime("%Y-%m-%d")
+        variables = {
+            "ARCH": arch,
+            "FREEDESKTOP_VERSION": FREEDESKTOP_VERSION,
+            "FIREFOX_BASEAPP_CHANNEL": FIREFOX_BASEAPP_CHANNEL,
+            "FLATPAK_BRANCH": flatpak_branch,
+            "VERSION": version,
+            "DATE": date,
+            "DEB_PKG_NAME": release_product,
+            "DBusActivatable": "false",
+            "Icon": flatpak_name,
+            "StartupWMClass": release_product,
+        }
+        _render_flatpak_templates(template_dir, build_dir, variables)
+
+        from fluent.runtime.fallback import FluentLocalization, FluentResourceLoader
+
+        from mozbuild.repackaging.desktop_file import generate_browser_desktop_entry
+
+        desktop = generate_browser_desktop_entry(
+            log,
+            variables,
+            release_product,
+            release_type,
+            FluentLocalization,
+            FluentResourceLoader,
+        )
+        desktop_dir = app_dir / "share" / "applications"
+        desktop_dir.mkdir(parents=True, exist_ok=True)
+        desktop_file_name = desktop_dir / f"{flatpak_name}.desktop"
+        with desktop_file_name.open("w") as f:
+            for line in desktop:
+                print(line, file=f)
+
+        if release_product == "firefox":
+            icon_path = "lib/firefox/browser/chrome/icons/default"
+        elif release_product == "thunderbird":
+            icon_path = "lib/thunderbird/chrome/icons/default"
+        else:
+            raise NotImplementedError()
+
+        for size in (16, 32, 48, 64, 128):
+            os.makedirs(
+                app_dir / f"share/icons/hicolor/{size}x{size}/apps", exist_ok=True
+            )
+            shutil.copy(
+                app_dir / icon_path / f"default{size}.png",
+                app_dir / f"share/icons/hicolor/{size}x{size}/apps/{flatpak_name}.png",
+            )
+
+        subprocess.run(
+            [
+                "appstream-compose",
+                f"--prefix={app_dir}",
+                "--origin=flatpak",
+                f"--basename={flatpak_name}",
+                flatpak_name,
+            ],
+            check=True,
+        )
+        subprocess.run(
+            [
+                "appstream-util",
+                "mirror-screenshots",
+                f"{app_dir}/share/app-info/xmls/{flatpak_name}.xml.gz",
+                f"https://dl.flathub.org/repo/screenshots/{flatpak_name}-{flatpak_branch}",
+                "build/screenshots",
+                f"build/screenshots/{flatpak_name}-{flatpak_branch}",
+            ],
+            check=True,
+            cwd=tmpdir,
+        )
+
+        os.makedirs(
+            app_dir / f"lib/{release_product}/distribution/extensions", exist_ok=True
+        )
+        for langpack in glob.iglob(langpack_pattern):
+            manifest = _langpack_manifest(langpack)
+            locale = manifest["langpack_id"]
+            name = manifest["browser_specific_settings"]["gecko"]["id"]
+
+            lang = locale.split("-", 1)[0]
+            os.makedirs(app_dir / "share/runtime/langpack" / lang, exist_ok=True)
+            shutil.copy(
+                langpack, app_dir / "share/runtime/langpack" / lang / f"{name}.xpi"
+            )
+            os.symlink(
+                f"/app/share/runtime/langpack/{lang}/{name}.xpi",
+                app_dir / f"lib/{release_product}/distribution/extensions/{name}.xpi",
+            )
+
+        subprocess.run(
+            [
+                "flatpak",
+                "build-finish",
+                "build",
+                "--allow=devel",
+                "--share=ipc",
+                "--share=network",
+                "--socket=pulseaudio",
+                "--socket=wayland",
+                "--socket=fallback-x11",
+                "--socket=pcsc",
+                "--socket=cups",
+                "--require-version=0.11.1",
+                "--persist=.mozilla",
+                "--env=DICPATH=/usr/share/hunspell",
+                "--filesystem=xdg-download:rw",
+                "--filesystem=/run/.heim_org.h5l.kcm-socket",
+                "--filesystem=xdg-run/speech-dispatcher:ro",
+                "--device=all",
+                "--talk-name=org.freedesktop.FileManager1",
+                "--system-talk-name=org.freedesktop.NetworkManager",
+                "--talk-name=org.a11y.Bus",
+                "--talk-name=org.gtk.vfs.*",
+                "--own-name=org.mpris.MediaPlayer2.firefox.*",
+                "--own-name=org.mozilla.firefox.*",
+                "--own-name=org.mozilla.firefox_beta.*",
+                "--command=firefox",
+            ],
+            check=True,
+            cwd=tmpdir,
+        )
+
+        subprocess.run(
+            [
+                "flatpak",
+                "build-export",
+                "--disable-sandbox",
+                "--no-update-summary",
+                "--exclude=/share/runtime/langpack/*/*",
+                "repo",
+                "build",
+                flatpak_branch,
+            ],
+            check=True,
+            cwd=tmpdir,
+        )
+        subprocess.run(
+            [
+                "flatpak",
+                "build-export",
+                "--disable-sandbox",
+                "--no-update-summary",
+                "--metadata=metadata.locale",
+                "--files=files/share/runtime/langpack",
+                "repo",
+                "build",
+                flatpak_branch,
+            ],
+            check=True,
+            cwd=tmpdir,
+        )
+        subprocess.run(
+            [
+                "ostree",
+                "commit",
+                "--repo=repo",
+                "--canonical-permissions",
+                "--branch=screenshots/x86_64",
+                "build/screenshots",
+            ],
+            check=True,
+            cwd=tmpdir,
+        )
+        subprocess.run(
+            ["flatpak", "build-update-repo", "--generate-static-deltas", "repo"],
+            check=True,
+            cwd=tmpdir,
+        )
+        env = os.environ.copy()
+        env["XZ_OPT"] = "-e9"
+        subprocess.run(
+            ["tar", "cvfJ", os.path.abspath(output), "repo"],
+            check=True,
+            env=env,
+            cwd=tmpdir,
+        )