Bug 1108035 - Add a preference that allows us to disable Flash protected mode from within Firefox. We do this by hooking CreateFileW and replacing the Flash config file with our own. r=aklotz

dom/plugins/base/nsNPAPIPlugin.cpp

@@ -409,7 +409,7 @@ GetNewPluginLibrary(nsPluginTag *aPluginTag)
   }
 
   if (nsNPAPIPlugin::RunPluginOOP(aPluginTag)) {
-    return PluginModuleChromeParent::LoadModule(aPluginTag->mFullPath.get(), aPluginTag->mId);
+    return PluginModuleChromeParent::LoadModule(aPluginTag->mFullPath.get(), aPluginTag->mId, aPluginTag);
   }
   return new PluginPRLibrary(aPluginTag->mFullPath.get(), aPluginTag->mLibrary);
 }

dom/plugins/ipc/PPluginModule.ipdl

@@ -45,6 +45,8 @@ both:
   async ProcessNativeEventsInInterruptCall();
 
 child:
+  async DisableFlashProtectedMode();
+
   // Forces the child process to update its plugin function table.
   intr NP_GetEntryPoints()
     returns (NPError rv);

dom/plugins/ipc/PluginModuleChild.cpp

@@ -76,6 +76,16 @@ static PRLibrary *sGtkLib = nullptr;
 #endif
 
 #ifdef XP_WIN
+// Hooking CreateFileW for protected-mode magic
+static WindowsDllInterceptor sKernel32Intercept;
+typedef HANDLE (WINAPI *CreateFileWPtr)(LPCWSTR fname, DWORD access,
+                                        DWORD share,
+                                        LPSECURITY_ATTRIBUTES security,
+                                        DWORD creation, DWORD flags,
+                                        HANDLE ftemplate);
+static CreateFileWPtr sCreateFileWStub = nullptr;
+static WCHAR* sReplacementConfigFile;
+
 // Used with fix for flash fullscreen window loosing focus.
 static bool gDelayFlashFocusReplyUntilEval = false;
 // Used to fix GetWindowInfo problems with internal flash settings dialogs
@@ -219,6 +229,18 @@ PluginModuleChild::InitForContent(base::ProcessHandle aParentProcessHandle,
     return true;
 }
 
+bool
+PluginModuleChild::RecvDisableFlashProtectedMode()
+{
+    MOZ_ASSERT(mIsChrome);
+#ifdef XP_WIN
+    HookProtectedMode();
+#else
+    MOZ_ASSERT(false, "Should not be called");
+#endif
+    return true;
+}
+
 bool
 PluginModuleChild::InitForChrome(const std::string& aPluginFilename,
                                  base::ProcessHandle aParentProcessHandle,
@@ -1887,9 +1909,99 @@ PluginModuleChild::AnswerNP_Initialize(const PluginSettings& aSettings, NPError*
 #else
 #  error Please implement me for your platform
 #endif
+
+#ifdef XP_WIN
+    CleanupProtectedModeHook();
+#endif
 }
 
 #if defined(XP_WIN)
+
+HANDLE WINAPI
+CreateFileHookFn(LPCWSTR fname, DWORD access, DWORD share,
+                 LPSECURITY_ATTRIBUTES security, DWORD creation, DWORD flags,
+                 HANDLE ftemplate)
+{
+    static const WCHAR kConfigFile[] = L"mms.cfg";
+    static const size_t kConfigLength = ArrayLength(kConfigFile) - 1;
+
+    while (true) { // goto out, in sheep's clothing
+        size_t len = wcslen(fname);
+        if (len < kConfigLength) {
+            break;
+        }
+        if (wcscmp(fname + len - kConfigLength, kConfigFile) != 0) {
+            break;
+        }
+
+        // This is the config file we want to rewrite
+        WCHAR tempPath[MAX_PATH+1];
+        if (GetTempPathW(MAX_PATH, tempPath) == 0) {
+            break;
+        }
+        WCHAR tempFile[MAX_PATH+1];
+        if (GetTempFileNameW(tempPath, L"fx", 0, tempFile) == 0) {
+            break;
+        }
+        HANDLE replacement =
+            sCreateFileWStub(tempFile, GENERIC_READ | GENERIC_WRITE, share,
+                             security, TRUNCATE_EXISTING,
+                             FILE_ATTRIBUTE_TEMPORARY,
+                             NULL);
+        if (replacement == INVALID_HANDLE_VALUE) {
+            break;
+        }
+
+        HANDLE original = sCreateFileWStub(fname, access, share, security,
+                                           creation, flags, ftemplate);
+        if (original != INVALID_HANDLE_VALUE) {
+            // copy original to replacement
+            static const size_t kBufferSize = 1024;
+            char buffer[kBufferSize];
+            DWORD bytes;
+            while (ReadFile(original, buffer, kBufferSize, &bytes, NULL)) {
+                if (bytes == 0) {
+                    break;
+                }
+                DWORD wbytes;
+                WriteFile(replacement, buffer, bytes, &wbytes, NULL);
+                if (bytes < kBufferSize) {
+                    break;
+                }
+            }
+            CloseHandle(original);
+        }
+        static const char kSettingString[] = "\nProtectedMode=0\n";
+        DWORD wbytes;
+        WriteFile(replacement, static_cast<const void*>(kSettingString),
+                  sizeof(kSettingString) - 1, &wbytes, NULL);
+        SetFilePointer(replacement, 0, NULL, FILE_BEGIN);
+        sReplacementConfigFile = _wcsdup(tempFile);
+        return replacement;
+    }
+    return sCreateFileWStub(fname, access, share, security, creation, flags,
+                            ftemplate);
+}
+
+void
+PluginModuleChild::HookProtectedMode()
+{
+    sKernel32Intercept.Init("kernel32.dll");
+    sKernel32Intercept.AddHook("CreateFileW",
+                               reinterpret_cast<intptr_t>(CreateFileHookFn),
+                               (void**) &sCreateFileWStub);
+}
+
+void
+PluginModuleChild::CleanupProtectedModeHook()
+{
+    if (sReplacementConfigFile) {
+        DeleteFile(sReplacementConfigFile);
+        free(sReplacementConfigFile);
+        sReplacementConfigFile = nullptr;
+    }
+}
+
 BOOL WINAPI
 PMCGetWindowInfoHook(HWND hWnd, PWINDOWINFO pwi)
 {

dom/plugins/ipc/PluginModuleChild.h

@@ -75,6 +75,7 @@ protected:
     virtual bool RecvSettingChanged(const PluginSettings& aSettings) MOZ_OVERRIDE;
 
     // Implement the PPluginModuleChild interface
+    virtual bool RecvDisableFlashProtectedMode() MOZ_OVERRIDE;
     virtual bool AnswerNP_GetEntryPoints(NPError* rv) MOZ_OVERRIDE;
     virtual bool AnswerNP_Initialize(const PluginSettings& aSettings, NPError* rv) MOZ_OVERRIDE;
 
@@ -293,6 +294,12 @@ private:
     void InitQuirksModes(const nsCString& aMimeType);
     bool InitGraphics();
     void DeinitGraphics();
+
+#if defined(OS_WIN)
+    void HookProtectedMode();
+    void CleanupProtectedModeHook();
+#endif
+
 #if defined(MOZ_WIDGET_GTK)
     static gboolean DetectNestedEventLoop(gpointer data);
     static gboolean ProcessBrowserEvents(gpointer data);

dom/plugins/ipc/PluginModuleParent.cpp

@@ -31,6 +31,7 @@
 #include "nsPrintfCString.h"
 #include "prsystem.h"
 #include "GeckoProfiler.h"
+#include "nsPluginTags.h"
 
 #ifdef XP_WIN
 #include "PluginHangUIParent.h"
@@ -153,7 +154,8 @@ PluginModuleContentParent::Create(mozilla::ipc::Transport* aTransport,
 
 // static
 PluginLibrary*
-PluginModuleChromeParent::LoadModule(const char* aFilePath, uint32_t aPluginId)
+PluginModuleChromeParent::LoadModule(const char* aFilePath, uint32_t aPluginId,
+                                     nsPluginTag* aPluginTag)
 {
     PLUGIN_LOG_DEBUG_FUNCTION;
 
@@ -192,6 +194,13 @@ PluginModuleChromeParent::LoadModule(const char* aFilePath, uint32_t aPluginId)
 #endif
 #endif
 
+#ifdef XP_WIN
+    if (aPluginTag->mIsFlashPlugin &&
+        Preferences::GetBool("dom.ipc.plugins.flash.disable-protected-mode", false)) {
+        parent->SendDisableFlashProtectedMode();
+    }
+#endif
+
     return parent.forget();
 }
 

dom/plugins/ipc/PluginModuleParent.h

@@ -28,6 +28,8 @@
 #include "nsExceptionHandler.h"
 #endif
 
+class nsPluginTag;
+
 namespace mozilla {
 namespace dom {
 class PCrashReporterParent;
@@ -295,7 +297,8 @@ class PluginModuleChromeParent
      * This may or may not launch a plugin child process,
      * and may or may not be very expensive.
      */
-    static PluginLibrary* LoadModule(const char* aFilePath, uint32_t aPluginId);
+    static PluginLibrary* LoadModule(const char* aFilePath, uint32_t aPluginId,
+                                     nsPluginTag* aPluginTag);
 
     virtual ~PluginModuleChromeParent();
 

modules/libpref/init/all.js

@@ -2354,6 +2354,8 @@ pref("dom.ipc.tabs.shutdownTimeoutSecs", 0);
 pref("dom.ipc.plugins.java.enabled", false);
 #endif
 
+pref("dom.ipc.plugins.flash.disable-protected-mode", false);
+
 pref("dom.ipc.plugins.flash.subprocess.crashreporter.enabled", true);
 pref("dom.ipc.plugins.reportCrashURL", true);