Bug 1550108 - Change StartupCache format from zip to custom r=froydnj

I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:

- Obviously the contents of the cache are compressed. I used lz4
  for this as I hit the same file size as deflate at a compression
  level of 1, which is what the StartupCache was using previously,
  while decompressing an order of magnitude faster. Seemed like
  the most conservative change to make. I think it's worth
  investigating what the impact of slower algs with higher ratios
  would be, but for right now I settled on this. We'd probably
  want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
  strictly necessary, but has the benefit of not requiring as
  much memory for large buffers, as well as giving us a built-in
  checksum, rather than relying on the much slower CRC that we
  were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
  jump back to add the offset and compressed size, which would
  make the nice Code(...) method for the ScriptPreloader stuff
  rather more complex. Open to cleaner solutions, but moving it
  out just felt like extra hoops for the reader to jump through
  to understand without the benefit of being more concise.

Differential Revision: https://phabricator.services.mozilla.com/D34652

--HG--
extra : moz-landing-system : lando

Author: squarewave

mfbt/Compression.cpp

@@ -13,7 +13,9 @@
 #include <string>
 
 #include "lz4/lz4.h"
+#include "lz4/lz4frame.h"
 
+using namespace mozilla;
 using namespace mozilla::Compression;
 
 /* Our wrappers */
@@ -78,3 +80,113 @@ bool LZ4::decompressPartial(const char* aSource, size_t aInputSize, char* aDest,
   *aOutputSize = 0;
   return false;
 }
+
+LZ4FrameCompressionContext::LZ4FrameCompressionContext(int aCompressionLevel,
+                                                       size_t aMaxSrcSize,
+                                                       bool aChecksum,
+                                                       bool aStableSrc)
+    : mContext(nullptr),
+      mCompressionLevel(aCompressionLevel),
+      mGenerateChecksum(aChecksum),
+      mStableSrc(aStableSrc),
+      mMaxSrcSize(aMaxSrcSize),
+      mWriteBufLen(0) {
+  LZ4F_contentChecksum_t checksum =
+      mGenerateChecksum ? LZ4F_contentChecksumEnabled : LZ4F_noContentChecksum;
+  LZ4F_preferences_t prefs = {
+      {
+          LZ4F_max256KB,
+          LZ4F_blockLinked,
+          checksum,
+      },
+      mCompressionLevel,
+  };
+  mWriteBufLen = LZ4F_compressBound(mMaxSrcSize, &prefs);
+  LZ4F_errorCode_t err = LZ4F_createCompressionContext(&mContext, LZ4F_VERSION);
+  MOZ_RELEASE_ASSERT(!LZ4F_isError(err));
+}
+
+LZ4FrameCompressionContext::~LZ4FrameCompressionContext() {
+  LZ4F_freeCompressionContext(mContext);
+}
+
+Result<Span<const char>, size_t> LZ4FrameCompressionContext::BeginCompressing(
+    Span<char> aWriteBuffer) {
+  mWriteBuffer = aWriteBuffer;
+  LZ4F_contentChecksum_t checksum =
+      mGenerateChecksum ? LZ4F_contentChecksumEnabled : LZ4F_noContentChecksum;
+  LZ4F_preferences_t prefs = {
+      {
+          LZ4F_max256KB,
+          LZ4F_blockLinked,
+          checksum,
+      },
+      mCompressionLevel,
+  };
+  size_t headerSize = LZ4F_compressBegin(mContext, mWriteBuffer.Elements(),
+                                         mWriteBufLen, &prefs);
+  if (LZ4F_isError(headerSize)) {
+    return Err(headerSize);
+  }
+
+  return MakeSpan(static_cast<const char*>(mWriteBuffer.Elements()),
+                  headerSize);
+}
+
+Result<Span<const char>, size_t>
+LZ4FrameCompressionContext::ContinueCompressing(Span<const char> aInput) {
+  LZ4F_compressOptions_t opts = {};
+  opts.stableSrc = (uint32_t)mStableSrc;
+  size_t outputSize =
+      LZ4F_compressUpdate(mContext, mWriteBuffer.Elements(), mWriteBufLen,
+                          aInput.Elements(), aInput.Length(), &opts);
+  if (LZ4F_isError(outputSize)) {
+    return Err(outputSize);
+  }
+
+  return MakeSpan(static_cast<const char*>(mWriteBuffer.Elements()),
+                  outputSize);
+}
+
+Result<Span<const char>, size_t> LZ4FrameCompressionContext::EndCompressing() {
+  size_t outputSize =
+      LZ4F_compressEnd(mContext, mWriteBuffer.Elements(), mWriteBufLen,
+                       /* options */ nullptr);
+  if (LZ4F_isError(outputSize)) {
+    return Err(outputSize);
+  }
+
+  return MakeSpan(static_cast<const char*>(mWriteBuffer.Elements()),
+                  outputSize);
+}
+
+LZ4FrameDecompressionContext::LZ4FrameDecompressionContext(bool aStableDest)
+    : mContext(nullptr), mStableDest(aStableDest) {
+  LZ4F_errorCode_t err =
+      LZ4F_createDecompressionContext(&mContext, LZ4F_VERSION);
+  MOZ_RELEASE_ASSERT(!LZ4F_isError(err));
+}
+
+LZ4FrameDecompressionContext::~LZ4FrameDecompressionContext() {
+  LZ4F_freeDecompressionContext(mContext);
+}
+
+Result<LZ4FrameDecompressionResult, size_t>
+LZ4FrameDecompressionContext::Decompress(Span<char> aOutput,
+                                         Span<const char> aInput) {
+  LZ4F_decompressOptions_t opts = {};
+  opts.stableDst = (uint32_t)mStableDest;
+  size_t outBytes = aOutput.Length();
+  size_t inBytes = aInput.Length();
+  size_t result = LZ4F_decompress(mContext, aOutput.Elements(), &outBytes,
+                                  aInput.Elements(), &inBytes, &opts);
+  if (LZ4F_isError(result)) {
+    return Err(result);
+  }
+
+  LZ4FrameDecompressionResult decompressionResult = {};
+  decompressionResult.mFinished = !result;
+  decompressionResult.mSizeRead = inBytes;
+  decompressionResult.mSizeWritten = outBytes;
+  return decompressionResult;
+}

mfbt/Compression.h

@@ -11,6 +11,12 @@
 
 #include "mozilla/Assertions.h"
 #include "mozilla/Types.h"
+#include "mozilla/Result.h"
+#include "mozilla/Span.h"
+#include "mozilla/UniquePtr.h"
+
+struct LZ4F_cctx_s;  // compression context
+struct LZ4F_dctx_s;  // decompression context
 
 namespace mozilla {
 namespace Compression {
@@ -137,6 +143,91 @@ class LZ4 {
   }
 };
 
+/**
+ * Context for LZ4 Frame-based streaming compression. Use this if you
+ * want to incrementally compress something or if you want to compress
+ * something such that another application can read it.
+ */
+class LZ4FrameCompressionContext final {
+ public:
+  MFBT_API LZ4FrameCompressionContext(int aCompressionLevel, size_t aMaxSrcSize,
+                                      bool aChecksum, bool aStableSrc = false);
+
+  MFBT_API ~LZ4FrameCompressionContext();
+
+  size_t GetRequiredWriteBufferLength() { return mWriteBufLen; }
+
+  /**
+   * Begin streaming frame-based compression.
+   *
+   * @return a Result with a Span containing the frame header, or an lz4 error
+   * code (size_t).
+   */
+  MFBT_API Result<Span<const char>, size_t> BeginCompressing(
+      Span<char> aWriteBuffer);
+
+  /**
+   * Continue streaming frame-based compression with the provided input.
+   *
+   * @param aInput input buffer to be compressed.
+   * @return a Result with a Span containing compressed output, or an lz4 error
+   * code (size_t).
+   */
+  MFBT_API Result<Span<const char>, size_t> ContinueCompressing(
+      Span<const char> aInput);
+
+  /**
+   * Finalize streaming frame-based compression with the provided input.
+   *
+   * @return a Result with a Span containing compressed output and the frame
+   * footer, or an lz4 error code (size_t).
+   */
+  MFBT_API Result<Span<const char>, size_t> EndCompressing();
+
+ private:
+  LZ4F_cctx_s* mContext;
+  int mCompressionLevel;
+  bool mGenerateChecksum;
+  bool mStableSrc;
+  size_t mMaxSrcSize;
+  size_t mWriteBufLen;
+  Span<char> mWriteBuffer;
+};
+
+struct LZ4FrameDecompressionResult {
+  size_t mSizeRead;
+  size_t mSizeWritten;
+  bool mFinished;
+};
+
+/**
+ * Context for LZ4 Frame-based streaming decompression. Use this if you
+ * want to decompress something compressed by LZ4FrameCompressionContext
+ * or by another application.
+ */
+class LZ4FrameDecompressionContext final {
+ public:
+  explicit MFBT_API LZ4FrameDecompressionContext(bool aStableDest = false);
+  MFBT_API ~LZ4FrameDecompressionContext();
+
+  /**
+   * Decompress a buffer/part of a buffer compressed with
+   * LZ4FrameCompressionContext or another application.
+   *
+   * @param aOutput output buffer to be write results into.
+   * @param aInput input buffer to be decompressed.
+   * @return a Result with information on bytes read/written and whether we
+   * completely decompressed the input into the output, or an lz4 error code
+   * (size_t).
+   */
+  MFBT_API Result<LZ4FrameDecompressionResult, size_t> Decompress(
+      Span<char> aOutput, Span<const char> aInput);
+
+ private:
+  LZ4F_dctx_s* mContext;
+  bool mStableDest;
+};
+
 } /* namespace Compression */
 } /* namespace mozilla */