Bug 1551276 - Autodetect legacy encodings on unlabeled pages. r=emk

Differential Revision: https://phabricator.services.mozilla.com/D56362

--HG--
extra : moz-landing-system : lando

Author: hsivonen

Cargo.lock

@@ -3,6 +3,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+/* import-globals-from ../../base/content/utilityOverlay.js */
 /* import-globals-from ../../../toolkit/mozapps/preferences/fontbuilder.js */
 
 // browser.display.languageList LOCK ALL when LOCKED
@@ -29,6 +30,10 @@ Preferences.addAll([
   { id: "intl.charset.fallback.override", type: "string" },
 ]);
 
+document.getElementById("FallbackGroupbox").hidden = Services.prefs.getBoolPref(
+  "intl.charset.detector.ng.enabled"
+);
+
 var gFontsDialog = {
   _selectLanguageGroupPromise: Promise.resolve(),
 

browser/components/preferences/fonts.js

@@ -234,7 +234,7 @@
   </groupbox>
 
   <!-- Text Encoding -->
-  <groupbox>
+  <groupbox id="FallbackGroupbox">
     <label><html:h2 data-l10n-id="fonts-languages-fallback-header"/></label>
     <description data-l10n-id="fonts-languages-fallback-desc"/>
     <hbox align="center">

browser/components/preferences/fonts.xhtml

@@ -1290,7 +1290,8 @@ nsDocShell::GatherCharsetMenuTelemetry() {
             Telemetry::LABELS_ENCODING_OVERRIDE_SITUATION::RemoteNonTld);
       }
       break;
-    case kCharsetFromAutoDetection:
+    case kCharsetFromInitialAutoDetection:
+    case kCharsetFromFinalAutoDetection:
       // Changing charset on unlabeled doc where chardet fired
       if (isFileURL) {
         Telemetry::AccumulateCategorical(
@@ -1921,7 +1922,8 @@ nsDocShell::GetCharsetAutodetected(bool* aCharsetAutodetected) {
   }
   int32_t source = doc->GetDocumentCharacterSetSource();
 
-  if (source == kCharsetFromAutoDetection ||
+  if (source == kCharsetFromInitialAutoDetection ||
+      source == kCharsetFromFinalAutoDetection ||
       source == kCharsetFromUserForcedAutoDetection) {
     *aCharsetAutodetected = true;
   }

docshell/base/nsDocShell.cpp

@@ -10,15 +10,15 @@ function test() {
 
 function afterOpen() {
   is(
-    content.document.documentElement.textContent.indexOf("\u00A4"),
+    content.document.documentElement.textContent.indexOf("\u0434"),
     131,
-    "Parent doc should be windows-1252 initially"
+    "Parent doc should be IBM866 initially"
   );
 
   is(
-    content.frames[0].document.documentElement.textContent.indexOf("\u00A4"),
+    content.frames[0].document.documentElement.textContent.indexOf("\u0434"),
     87,
-    "Child doc should be windows-1252 initially"
+    "Child doc should be IBM866 initially"
   );
 }
 

docshell/test/browser/browser_bug1543077-1.js

@@ -10,15 +10,15 @@ function test() {
 
 function afterOpen() {
   is(
-    content.document.documentElement.textContent.indexOf("\u201A"),
+    content.document.documentElement.textContent.indexOf("\u0412"),
     134,
-    "Parent doc should be windows-1252 initially"
+    "Parent doc should be IBM866 initially"
   );
 
   is(
-    content.frames[0].document.documentElement.textContent.indexOf("\u201A"),
+    content.frames[0].document.documentElement.textContent.indexOf("\u0412"),
     90,
-    "Child doc should be windows-1252 initially"
+    "Child doc should be IBM866 initially"
   );
 }
 

docshell/test/browser/browser_bug1543077-2.js

@@ -10,15 +10,15 @@ function test() {
 
 function afterOpen() {
   is(
-    content.document.documentElement.textContent.indexOf("\u001B"),
+    content.document.documentElement.textContent.indexOf("\u3042"),
     136,
-    "Parent doc should be windows-1252 initially"
+    "Parent doc should be ISO-2022-JP initially"
   );
 
   is(
-    content.frames[0].document.documentElement.textContent.indexOf("\u001B"),
+    content.frames[0].document.documentElement.textContent.indexOf("\u3042"),
     92,
-    "Child doc should be windows-1252 initially"
+    "Child doc should be ISO-2022-JP initially"
   );
 }
 

docshell/test/browser/browser_bug1543077-3.js

@@ -10,15 +10,15 @@ function test() {
 
 function afterOpen() {
   is(
-    content.document.documentElement.textContent.indexOf("\u00A4"),
+    content.document.documentElement.textContent.indexOf("\u0434"),
     131,
-    "Parent doc should be windows-1252 initially"
+    "Parent doc should be IBM866 initially"
   );
 
   is(
-    content.frames[0].document.documentElement.textContent.indexOf("\u201A"),
+    content.frames[0].document.documentElement.textContent.indexOf("\u0412"),
     90,
-    "Child doc should be windows-1252 initially"
+    "Child doc should be IBM866 initially"
   );
 }
 

docshell/test/browser/browser_bug1543077-4.js

@@ -150,7 +150,6 @@
 #include "nsIAsyncVerifyRedirectCallback.h"
 #include "nsICategoryManager.h"
 #include "nsIChannelEventSink.h"
-#include "nsICharsetDetectionObserver.h"
 #include "nsIConsoleService.h"
 #include "nsIContent.h"
 #include "nsIContentInlines.h"

dom/base/nsContentUtils.cpp

@@ -288,7 +288,8 @@ void nsHTMLDocument::TryParentCharset(nsIDocShell* aDocShell,
     return;
   }
   if (kCharsetFromParentForced == parentSource ||
-      kCharsetFromUserForced == parentSource) {
+      kCharsetFromUserForced == parentSource ||
+      kCharsetFromUserForcedAutoDetection == parentSource) {
     if (WillIgnoreCharsetOverride() ||
         !IsAsciiCompatible(aEncoding) ||  // if channel said UTF-16
         !IsAsciiCompatible(parentCharset)) {