Bug 1461690 Part 1: Write uninstall ping. r=chutten,Gijs

This covers the in-Firefox portions of the install ping,
except for the otherInstalls count, which is added in
part 2. Later parts will cover the uninstaller.

Differential Revision: https://phabricator.services.mozilla.com/D92522

Author: agashlin

toolkit/components/telemetry/app/TelemetryControllerParent.jsm

@@ -42,6 +42,7 @@ const NEWPROFILE_PING_DEFAULT_DELAY = 30 * 60 * 1000;
 // Ping types.
 const PING_TYPE_MAIN = "main";
 const PING_TYPE_DELETION_REQUEST = "deletion-request";
+const PING_TYPE_UNINSTALL = "uninstall";
 
 // Session ping reasons.
 const REASON_GATHER_PAYLOAD = "gather-payload";
@@ -253,6 +254,19 @@ var TelemetryController = Object.freeze({
     return Impl.removeAbortedSessionPing();
   },
 
+  /**
+   * Create an uninstall ping and write it to disk, replacing any already present.
+   * This is stored independently from other pings, and only read by
+   * the Windows uninstaller.
+   *
+   * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+   *
+   * @return {Promise} Resolved when the ping has been saved.
+   */
+  saveUninstallPing() {
+    return Impl.saveUninstallPing();
+  },
+
   /**
    * Allows waiting for TelemetryControllers delayed initialization to complete.
    * The returned promise is guaranteed to resolve before TelemetryController is shutting down.
@@ -689,6 +703,30 @@ var Impl = {
     return TelemetryStorage.removeAbortedSessionPing();
   },
 
+  _countOtherInstalls() {
+    // TODO
+    throw new Error("_countOtherInstalls - not implemented");
+  },
+
+  async saveUninstallPing() {
+    if (AppConstants.platform != "win") {
+      return undefined;
+    }
+
+    this._log.trace("saveUninstallPing");
+
+    let payload = {};
+    try {
+      payload.otherInstalls = this._countOtherInstalls();
+    } catch (e) {
+      this._log.warn("saveUninstallPing - _countOtherInstalls failed", e);
+    }
+    const options = { addClientId: true, addEnvironment: true };
+    const pingData = this.assemblePing(PING_TYPE_UNINSTALL, payload, options);
+
+    return TelemetryStorage.saveUninstallPing(pingData);
+  },
+
   /**
    * This triggers basic telemetry initialization and schedules a full initialized for later
    * for performance reasons.
@@ -836,6 +874,16 @@ var Impl = {
           EcosystemTelemetry.startup();
           TelemetryPrioPing.startup();
 
+          if (uploadEnabled) {
+            await this.saveUninstallPing().catch(e =>
+              this._log.warn("_delayedInitTask - saveUninstallPing failed", e)
+            );
+          } else {
+            await TelemetryStorage.removeUninstallPings().catch(e =>
+              this._log.warn("_delayedInitTask - saveUninstallPing", e)
+            );
+          }
+
           this._delayedInitTaskDeferred.resolve();
         } catch (e) {
           this._delayedInitTaskDeferred.reject(e);
@@ -1004,6 +1052,10 @@ var Impl = {
         let id = await ClientID.getClientID();
         this._clientID = id;
         Telemetry.scalarSet("telemetry.data_upload_optin", true);
+
+        await this.saveUninstallPing().catch(e =>
+          this._log.warn("_onUploadPrefChange - saveUninstallPing failed", e)
+        );
       })();
 
       this._shutdownBarrier.client.addBlocker(
@@ -1023,6 +1075,7 @@ var Impl = {
         // 3. Remove all pending pings
         await TelemetryStorage.removeAppDataPings();
         await TelemetryStorage.runRemovePendingPingsTask();
+        await TelemetryStorage.removeUninstallPings();
       } catch (e) {
         this._log.error(
           "_onUploadPrefChange - error clearing pending pings",

toolkit/components/telemetry/app/TelemetryStorage.jsm

@@ -110,6 +110,26 @@ var Policy = {
     AppConstants.platform == "android"
       ? PENDING_PINGS_QUOTA_BYTES_MOBILE
       : PENDING_PINGS_QUOTA_BYTES_DESKTOP,
+  /**
+   * @param {string} id The ID of the ping that will be written into the file. Can be "*" to
+   *                    make a pattern to find all pings for this installation.
+   * @return
+   *         {
+   *           directory: <nsIFile>, // Directory to save pings
+   *           file: <string>, // File name for this ping (or pattern for all pings)
+   *         }
+   */
+  getUninstallPingPath: id => {
+    // UpdRootD is e.g. C:\ProgramData\Mozilla\updates\<PATH HASH>
+    const updateDirectory = Services.dirsvc.get("UpdRootD", Ci.nsIFile);
+    const installPathHash = updateDirectory.leafName;
+
+    return {
+      // e.g. C:\ProgramData\Mozilla
+      directory: updateDirectory.parent.parent.clone(),
+      file: `uninstall_ping_${installPathHash}_${id}.json`,
+    };
+  },
 };
 
 /**
@@ -346,6 +366,31 @@ var TelemetryStorage = {
     return TelemetryStorageImpl.removeAbortedSessionPing();
   },
 
+  /**
+   * Save an uninstall ping to disk, removing any old ones from this
+   * installation first.
+   * This is stored independently from other pings, and only read by
+   * the Windows uninstaller.
+   *
+   * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+   *
+   * @return {promise} Promise that is resolved when the ping has been saved.
+   */
+  saveUninstallPing(ping) {
+    return TelemetryStorageImpl.saveUninstallPing(ping);
+  },
+
+  /**
+   * Remove all uninstall pings from this installation.
+   *
+   * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+   *
+   * @return {promise} Promise that is resolved when the pings have been removed.
+   */
+  removeUninstallPings() {
+    return TelemetryStorageImpl.removeUninstallPings();
+  },
+
   /**
    * Save a single ping to a file.
    *
@@ -1975,6 +2020,49 @@ var TelemetryStorageImpl = {
     });
   },
 
+  async saveUninstallPing(ping) {
+    if (AppConstants.platform != "win") {
+      return;
+    }
+
+    // Remove any old pings from this install first.
+    await this.removeUninstallPings();
+
+    let { directory: pingFile, file } = Policy.getUninstallPingPath(ping.id);
+    pingFile.append(file);
+
+    await this.savePingToFile(ping, pingFile.path, /* overwrite */ true);
+  },
+
+  async removeUninstallPings() {
+    if (AppConstants.platform != "win") {
+      return;
+    }
+
+    const { directory, file } = Policy.getUninstallPingPath("*");
+
+    const iteratorOptions = { winPattern: file };
+    const iterator = new OS.File.DirectoryIterator(
+      directory.path,
+      iteratorOptions
+    );
+
+    await iterator.forEach(async entry => {
+      this._log.trace("removeUninstallPings - removing", entry.path);
+      try {
+        await OS.File.remove(entry.path);
+        this._log.trace("removeUninstallPings - success");
+      } catch (ex) {
+        if (ex.becauseNoSuchFile) {
+          this._log.trace("removeUninstallPings - no such file");
+        } else {
+          this._log.error("removeUninstallPings - error removing ping", ex);
+        }
+      }
+    });
+    iterator.close();
+  },
+
   /**
    * Remove FHR database files. This is temporary and will be dropped in
    * the future.

toolkit/components/telemetry/docs/data/uninstall-ping.rst

@@ -0,0 +1,36 @@
+
+"uninstall" ping
+================
+
+This opt-out ping is sent from the Windows uninstaller when the uninstall finishes. Notably it includes ``clientId`` and the :doc:`Telemetry Environment <environment>`. It follows the :doc:`common ping format <common-ping>`.
+
+Structure:
+
+.. code-block:: js
+
+    {
+      type: "uninstall",
+      ... common ping data
+      clientId: <UUID>,
+      environment: { ... },
+      payload: {
+        otherInstalls: <integer>, // Optional, number of other installs on the system, max 11.
+      }
+    }
+
+See also the `JSON schema <https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/master/templates/telemetry/uninstall/uninstall.4.schema.json>`_. These pings are recorded in the ``telemetry.uninstall`` table in Redash, using the default "Telemetry (BigQuery)" data source.
+
+payload.otherInstalls
+---------------------
+This is a count of how many other installs of Firefox were present on the system at the time the ping was written. It is the number of values in the ``Software\Mozilla\Firefox\TaskBarIDs`` registry key, for both 32-bit and 64-bit architectures, for both HKCU and HKLM, excluding duplicates, and excluding a value for this install (if present). For example, if this is the only install on the system, the value will be 0. It may be missing in case of an error.
+
+This count is capped at 11. This avoids introducing a high-resolution identifier in case of a system with a large, unique number of installs.
+
+Uninstall Ping storage and lifetime
+-----------------------------------
+
+On delayed Telemetry init (about 1 minute into each run of Firefox), if opt-out telemetry is enabled, this ping is written to disk. There is a single ping for each install, any uninstall pings from the same install are removed before the new ping is written.
+
+The ping is removed if Firefox notices that opt-out telemetry has been disabled, either when the ``datareporting.healthreport.uploadEnabled`` pref goes false or when it is false on delayed init. Conversely, when opt-out telemetry is re-enabled, the ping is written as Telemetry is setting itself up again.
+
+The ping is sent by the uninstaller some arbitrary time after it is written to disk by Firefox, so it could be significantly out of date when it is submitted. There should be little impact from stale data, since analysis is likely to focus on clients that uninstalled soon after running Firefox, and this ping mostly changes when Firefox itself is updated.

toolkit/components/telemetry/tests/unit/head.js

@@ -439,6 +439,20 @@ function fakeIntlReady() {
   Services.obs.notifyObservers(null, "browser-delayed-startup-finished");
 }
 
+// Override the uninstall ping file names
+function fakeUninstallPingPath(aPathFcn) {
+  const m = ChromeUtils.import(
+    "resource://gre/modules/TelemetryStorage.jsm",
+    null
+  );
+  m.Policy.getUninstallPingPath =
+    aPathFcn ||
+    (id => ({
+      directory: new FileUtils.File(OS.Constants.Path.profileDir),
+      file: `uninstall_ping_0123456789ABCDEF_${id}.json`,
+    }));
+}
+
 // Return a date that is |offset| ms in the future from |date|.
 function futureDate(date, offset) {
   return new Date(date.getTime() + offset);
@@ -570,3 +584,6 @@ fakeSchedulerTimer(
 );
 // Make pind sending predictable.
 fakeMidnightPingFuzzingDelay(0);
+
+// Avoid using the directory service, which is not registered in some tests.
+fakeUninstallPingPath();

toolkit/components/telemetry/tests/unit/test_UninstallPing.js

@@ -0,0 +1,127 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+"use strict";
+
+const { TelemetryStorage } = ChromeUtils.import(
+  "resource://gre/modules/TelemetryStorage.jsm"
+);
+const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm");
+const { FileUtils } = ChromeUtils.import(
+  "resource://gre/modules/FileUtils.jsm"
+);
+
+const gFakeInstallPathHash = "0123456789ABCDEF";
+let gFakeVendorDirectory;
+let gFakeGetUninstallPingPath;
+
+add_task(async function setup() {
+  do_get_profile();
+
+  let fakeVendorDirectoryNSFile = new FileUtils.File(
+    OS.Path.join(OS.Constants.Path.profileDir, "uninstall-ping-test")
+  );
+  fakeVendorDirectoryNSFile.createUnique(
+    Ci.nsIFile.DIRECTORY_TYPE,
+    FileUtils.PERMS_DIRECTORY
+  );
+  gFakeVendorDirectory = fakeVendorDirectoryNSFile.path;
+
+  gFakeGetUninstallPingPath = id => ({
+    directory: fakeVendorDirectoryNSFile.clone(),
+    file: `uninstall_ping_${gFakeInstallPathHash}_${id}.json`,
+  });
+
+  fakeUninstallPingPath(gFakeGetUninstallPingPath);
+
+  registerCleanupFunction(() => {
+    OS.File.removeDir(gFakeVendorDirectory);
+  });
+});
+
+function ping_path(ping) {
+  let { directory: pingFile, file } = gFakeGetUninstallPingPath(ping.id);
+  pingFile.append(file);
+  return pingFile.path;
+}
+
+add_task(async function test_store_ping() {
+  // Remove shouldn't throw on an empty dir.
+  await TelemetryStorage.removeUninstallPings();
+
+  // Write ping
+  const ping1 = {
+    id: "58b63aac-999e-4efb-9d5a-20f368670721",
+    payload: { some: "thing" },
+  };
+  const ping1Path = ping_path(ping1);
+  await TelemetryStorage.saveUninstallPing(ping1);
+
+  // Check the ping
+  Assert.ok(await OS.File.exists(ping1Path));
+  const readPing1 = JSON.parse(
+    await OS.File.read(ping1Path, { encoding: "utf-8" })
+  );
+  Assert.deepEqual(ping1, readPing1);
+
+  // Write another file that shouldn't match the pattern
+  const otherFilePath = OS.Path.join(gFakeVendorDirectory, "other_file.json");
+  await OS.File.writeAtomic(otherFilePath, "");
+  Assert.ok(await OS.File.exists(otherFilePath));
+
+  // Write another ping, should remove the earlier one
+  const ping2 = {
+    id: "7202c564-8f23-41b4-8a50-1744e9549260",
+    payload: { another: "thing" },
+  };
+  const ping2Path = ping_path(ping2);
+  await TelemetryStorage.saveUninstallPing(ping2);
+
+  Assert.ok(!(await OS.File.exists(ping1Path)));
+  Assert.ok(await OS.File.exists(ping2Path));
+  Assert.ok(await OS.File.exists(otherFilePath));
+
+  // Write an additional file manually so there are multiple matching pings to remove
+  const ping3 = { id: "yada-yada" };
+  const ping3Path = ping_path(ping3);
+
+  await OS.File.writeAtomic(ping3Path, "");
+  Assert.ok(await OS.File.exists(ping3Path));
+
+  // Remove pings
+  await TelemetryStorage.removeUninstallPings();
+
+  // Check our pings are removed but other file isn't
+  Assert.ok(!(await OS.File.exists(ping1Path)));
+  Assert.ok(!(await OS.File.exists(ping2Path)));
+  Assert.ok(!(await OS.File.exists(ping3Path)));
+  Assert.ok(await OS.File.exists(otherFilePath));
+
+  // Remove again, confirming that the remove doesn't cause an error if nothing to remove
+  await TelemetryStorage.removeUninstallPings();
+
+  const ping4 = {
+    id: "1f113673-753c-4fbe-9143-fe197f936036",
+    payload: { any: "thing" },
+  };
+  const ping4Path = ping_path(ping4);
+  await TelemetryStorage.saveUninstallPing(ping4);
+
+  // Open the ping without FILE_SHARE_DELETE, so a delete should fail.
+  const ping4File = await OS.File.open(
+    ping4Path,
+    { read: true, existing: true },
+    { winShare: OS.Constants.Win.FILE_SHARE_READ }
+  );
+
+  // Check that there is no error if the file can't be removed.
+  await TelemetryStorage.removeUninstallPings();
+
+  // And file should still exist.
+  Assert.ok(await OS.File.exists(ping4Path));
+
+  // Close the file, it should be possible to remove now.
+  ping4File.close();
+  await TelemetryStorage.removeUninstallPings();
+  Assert.ok(!(await OS.File.exists(ping4Path)));
+});