compile cache: one file per module makes the cache cost more than it saves on Windows; a packed layout fixes it #66584

diff --git a/src/compile_cache.cc b/src/compile_cache.cc
index dd097ac..d275049 100644
--- a/src/compile_cache.cc
+++ b/src/compile_cache.cc
@@ -1,5 +1,6 @@
 #include "compile_cache.h"
 #include <string>
+#include <unordered_set>
 #include "debug_utils-inl.h"
 #include "env-inl.h"
 #include "node_file.h"
@@ -297,11 +298,298 @@ CompileCacheEntry* CompileCacheHandler::GetOrInsert(Local<String> code,
 
   // TODO(joyeecheung): if we fail enough times, stop trying for any future
   // files.
-  ReadCacheFile(result);
+  if (packed_) {
+    // A pack miss or mismatch falls back to a loose per-file entry, which is
+    // where PersistPack() puts a few changed entries without rewriting the
+    // pack (see there).
+    if (!ReadFromPack(result)) {
+      ReadCacheFile(result);
+      if (result->cache != nullptr) {
+        loose_hits_.insert(key);
+      }
+    }
+  } else {
+    ReadCacheFile(result);
+  }
 
   return result;
 }
 
+// Used for identifying and verifying a packed compile cache file.
+constexpr uint32_t kPackMagicNumber = 0x8adfdbb3;
+constexpr size_t kPackEntryHeaderCount = 5;  // key, code size/hash, cache size/hash
+// Changed plus loose entries from which a persist rewrites the pack.
+constexpr size_t kPackFoldThreshold = 16;
+
+std::string CompileCacheHandler::pack_filename() const {
+  return compile_cache_dir_ + kPathSeparator + "pack";
+}
+
+// Layout of the pack file:
+//   [uint32_t] magic number
+//   [uint32_t] entry count
+//   per entry:
+//     [uint32_t] cache key   [uint32_t] code size   [uint32_t] code hash
+//     [uint32_t] cache size  [uint32_t] cache hash
+//     .... compile cache content ....
+void CompileCacheHandler::LoadPack() {
+  pack_loaded_ = true;
+  std::string path = pack_filename();
+  if (ReadPackFile(path, &pack_buf_, &pack_index_)) {
+    Debug("[compile cache] loaded pack %s: %d entries\n",
+          path,
+          pack_index_.size());
+  }
+}
+
+// Reads and indexes a whole pack file. On failure, *buf and *index are left
+// empty and the cache behaves as if there were no pack.
+bool CompileCacheHandler::ReadPackFile(
+    const std::string& path,
+    std::unique_ptr<uint8_t[]>* out_buf,
+    std::unordered_map<uint32_t, PackedEntry>* out_index) const {
+  out_buf->reset();
+  out_index->clear();
+  uv_fs_t req;
+  auto defer_req_cleanup = OnScopeLeave([&req]() { uv_fs_req_cleanup(&req); });
+  uv_file file = uv_fs_open(nullptr, &req, path.c_str(), O_RDONLY, 0, nullptr);
+  if (req.result < 0) {
+    Debug("[compile cache] no pack at %s: %s\n", path, uv_strerror(req.result));
+    return false;
+  }
+  uv_fs_req_cleanup(&req);
+  auto defer_close = OnScopeLeave([file]() {
+    uv_fs_t close_req;
+    uv_fs_close(nullptr, &close_req, file, nullptr);
+    uv_fs_req_cleanup(&close_req);
+  });
+
+  if (uv_fs_fstat(nullptr, &req, file, nullptr) < 0) {
+    return false;
+  }
+  size_t size = static_cast<size_t>(req.statbuf.st_size);
+  uv_fs_req_cleanup(&req);
+  if (size < 2 * sizeof(uint32_t)) {
+    return false;
+  }
+
+  std::unique_ptr<uint8_t[]> buf(new uint8_t[size]);
+  size_t got = 0;
+  while (got < size) {
+    uv_buf_t iov =
+        uv_buf_init(reinterpret_cast<char*>(buf.get() + got), size - got);
+    int n = uv_fs_read(nullptr, &req, file, &iov, 1, got, nullptr);
+    uv_fs_req_cleanup(&req);
+    if (n <= 0) {
+      Debug("[compile cache] reading pack %s failed\n", path);
+      return false;
+    }
+    got += n;
+  }
+
+  uint32_t header[2];
+  memcpy(header, buf.get(), sizeof(header));
+  if (header[0] != kPackMagicNumber) {
+    Debug("[compile cache] pack %s: magic number mismatch\n", path);
+    return false;
+  }
+  size_t offset = sizeof(header);
+  for (uint32_t i = 0; i < header[1]; i++) {
+    uint32_t fields[kPackEntryHeaderCount];
+    if (offset + sizeof(fields) > size) {
+      out_index->clear();
+      return false;
+    }
+    memcpy(fields, buf.get() + offset, sizeof(fields));
+    offset += sizeof(fields);
+    if (fields[3] > size - offset) {
+      Debug("[compile cache] pack %s is truncated\n", path);
+      out_index->clear();
+      return false;
+    }
+    (*out_index)[fields[0]] =
+        PackedEntry{fields[1], fields[2], fields[3], fields[4], offset};
+    offset += fields[3];
+  }
+  *out_buf = std::move(buf);
+  return true;
+}
+
+bool CompileCacheHandler::ReadFromPack(CompileCacheEntry* entry) {
+  if (!pack_loaded_) {
+    LoadPack();
+  }
+  Debug("[compile cache] looking up %s %s in pack...",
+        entry->type_name(),
+        entry->source_filename);
+  auto it = pack_index_.find(entry->cache_key);
+  if (it == pack_index_.end()) {
+    Debug(" not found\n");
+    return false;
+  }
+  const PackedEntry& p = it->second;
+  if (p.code_size != entry->code_size || p.code_hash != entry->code_hash) {
+    Debug(" code size or hash mismatch\n");
+    return false;
+  }
+  const uint8_t* src = pack_buf_.get() + p.offset;
+  if (GetHash(reinterpret_cast<const char*>(src), p.cache_size) !=
+      p.cache_hash) {
+    Debug(" cache hash mismatch\n");
+    return false;
+  }
+  uint8_t* data = new uint8_t[p.cache_size];
+  memcpy(data, src, p.cache_size);
+  entry->cache.reset(new ScriptCompiler::CachedData(
+      data, p.cache_size, ScriptCompiler::CachedData::BufferOwned));
+  Debug(" success, size=%d\n", p.cache_size);
+  return true;
+}
+
+// Returns false when the dirty entries should be written as loose per-file
+// entries by the caller, true when the pack was written or nothing changed.
+//
+// Rewriting the pack costs as much as the whole pack, so a few changed entries
+// (a `node -e` script, an edited module) go to loose files like in the
+// per-file layout, and lookups find them after a pack miss. When enough
+// entries are changed or loose, they are folded into a new pack and the loose
+// files this process read are removed.
+bool CompileCacheHandler::PersistPack() {
+  size_t dirty = 0;
+  for (auto& pair : compiler_cache_store_) {
+    auto* entry = pair.second.get();
+    if (entry->cache != nullptr && entry->refreshed && !entry->persisted) {
+      dirty++;
+    }
+  }
+  if (dirty == 0) {
+    Debug("[compile cache] pack unchanged, not writing\n");
+    return true;
+  }
+  if (!pack_index_.empty() &&
+      dirty + loose_hits_.size() < kPackFoldThreshold) {
+    Debug("[compile cache] %d changed and %d loose entries, writing loose\n",
+          dirty,
+          loose_hits_.size());
+    return false;
+  }
+
+  // Merge this process's entries with the pack as it is on disk now, not as
+  // it was at the first lookup: a child process (npx -> tsc) may have
+  // written it since. Entries of this process win. Writers that overlap
+  // exactly: last rename wins, and the other's new entries are recompiled
+  // and written by a later run.
+  std::unique_ptr<uint8_t[]> disk_buf;
+  std::unordered_map<uint32_t, PackedEntry> disk_index;
+  ReadPackFile(pack_filename(), &disk_buf, &disk_index);
+
+  struct Out {
+    uint32_t key;
+    uint32_t code_size;
+    uint32_t code_hash;
+    uint32_t cache_size;
+    const uint8_t* data;
+  };
+  std::vector<Out> outs;
+  outs.reserve(compiler_cache_store_.size() + disk_index.size());
+  std::unordered_set<uint32_t> written;
+  for (auto& pair : compiler_cache_store_) {
+    auto* entry = pair.second.get();
+    if (entry->cache == nullptr) continue;
+    outs.push_back({pair.first,
+                    entry->code_size,
+                    entry->code_hash,
+                    static_cast<uint32_t>(entry->cache->length),
+                    entry->cache->data});
+    written.insert(pair.first);
+  }
+  for (auto& pair : disk_index) {
+    if (written.count(pair.first) != 0) continue;
+    const PackedEntry& p = pair.second;
+    outs.push_back({pair.first,
+                    p.code_size,
+                    p.code_hash,
+                    p.cache_size,
+                    disk_buf.get() + p.offset});
+  }
+
+  size_t total = 2 * sizeof(uint32_t);
+  for (const Out& o : outs) {
+    total += kPackEntryHeaderCount * sizeof(uint32_t) + o.cache_size;
+  }
+  std::vector<char> blob(total);
+  char* w = blob.data();
+  uint32_t header[2] = {kPackMagicNumber, static_cast<uint32_t>(outs.size())};
+  memcpy(w, header, sizeof(header));
+  w += sizeof(header);
+  for (const Out& o : outs) {
+    uint32_t fields[kPackEntryHeaderCount] = {
+        o.key,
+        o.code_size,
+        o.code_hash,
+        o.cache_size,
+        GetHash(reinterpret_cast<const char*>(o.data), o.cache_size)};
+    memcpy(w, fields, sizeof(fields));
+    w += sizeof(fields);
+    memcpy(w, o.data, o.cache_size);
+    w += o.cache_size;
+  }
+
+  uv_fs_t mkstemp_req;
+  auto cleanup_mkstemp =
+      OnScopeLeave([&mkstemp_req]() { uv_fs_req_cleanup(&mkstemp_req); });
+  std::string tmp = pack_filename() + ".XXXXXX";
+  int err = uv_fs_mkstemp(nullptr, &mkstemp_req, tmp.c_str(), nullptr);
+  if (err < 0) {
+    Debug("[compile cache] creating temporary pack failed: %s\n",
+          uv_strerror(err));
+    return true;
+  }
+  uv_buf_t buf = uv_buf_init(blob.data(), blob.size());
+  uv_fs_t write_req;
+  err = uv_fs_write(
+      nullptr, &write_req, mkstemp_req.result, &buf, 1, 0, nullptr);
+  uv_fs_req_cleanup(&write_req);
+  uv_fs_t close_req;
+  uv_fs_close(nullptr, &close_req, mkstemp_req.result, nullptr);
+  uv_fs_req_cleanup(&close_req);
+  if (err < 0 || static_cast<size_t>(err) != blob.size()) {
+    Debug("[compile cache] writing pack failed\n");
+    uv_fs_t unlink_req;
+    uv_fs_unlink(nullptr, &unlink_req, mkstemp_req.path, nullptr);
+    uv_fs_req_cleanup(&unlink_req);
+    return true;
+  }
+  uv_fs_t rename_req;
+  err = uv_fs_rename(
+      nullptr, &rename_req, mkstemp_req.path, pack_filename().c_str(), nullptr);
+  uv_fs_req_cleanup(&rename_req);
+  if (err < 0) {
+    Debug("[compile cache] renaming pack failed: %s\n", uv_strerror(err));
+    uv_fs_t unlink_req;
+    uv_fs_unlink(nullptr, &unlink_req, mkstemp_req.path, nullptr);
+    uv_fs_req_cleanup(&unlink_req);
+    return true;
+  }
+  for (auto& pair : compiler_cache_store_) {
+    pair.second->persisted = true;
+    if (loose_hits_.count(pair.first) != 0) {
+      uv_fs_t unlink_req;
+      uv_fs_unlink(nullptr,
+                   &unlink_req,
+                   pair.second->cache_filename.c_str(),
+                   nullptr);
+      uv_fs_req_cleanup(&unlink_req);
+    }
+  }
+  loose_hits_.clear();
+  Debug("[compile cache] wrote pack %s: %d entries, %d bytes\n",
+        pack_filename(),
+        outs.size(),
+        blob.size());
+  return true;
+}
+
 ScriptCompiler::CachedData* SerializeCodeCache(Local<Function> func) {
   return ScriptCompiler::CreateCodeCacheForFunction(func);
 }
@@ -382,6 +670,10 @@ void CompileCacheHandler::MaybeSave(CompileCacheEntry* entry,
 void CompileCacheHandler::Persist() {
   DCHECK(!compile_cache_dir_.empty());
 
+  if (packed_ && PersistPack()) {
+    return;
+  }
+
   // TODO(joyeecheung): do this using a separate event loop to utilize the
   // libuv thread pool and do the file system operations concurrently.
   // TODO(joyeecheung): Currently flushing is triggered by either process
@@ -584,6 +876,12 @@ CompileCacheEnableResult CompileCacheHandler::Enable(Environment* env,
   result.cache_directory = absolute_cache_dir_base;
   compile_cache_dir_ = cache_dir_with_tag;
   portable_ = option;
+  std::string packed;
+  if (credentials::SafeGetenv("NODE_COMPILE_CACHE_PACKED", &packed, env) &&
+      packed == "1") {
+    packed_ = true;
+    Debug("[compile cache] using the packed layout\n");
+  }
   if (option == EnableOption::PORTABLE) {
     normalized_compile_cache_dir_ =
         NormalizeFileURLOrPath(env, compile_cache_dir_);
diff --git a/src/compile_cache.h b/src/compile_cache.h
index 6293433..736cdc6 100644
--- a/src/compile_cache.h
+++ b/src/compile_cache.h
@@ -8,6 +8,7 @@
 #include <string>
 #include <string_view>
 #include <unordered_map>
+#include <unordered_set>
 #include "v8.h"
 
 namespace node {
@@ -86,6 +87,31 @@ class CompileCacheHandler {
  private:
   void ReadCacheFile(CompileCacheEntry* entry);
 
+  // Packed layout (NODE_COMPILE_CACHE_PACKED=1): the entries of the cache
+  // directory live in one file, read once at the first lookup and rewritten
+  // on persist when many entries changed, instead of one file opened per
+  // module. A few changed entries are written as loose per-file entries,
+  // which lookups fall back to (see PersistPack()).
+  struct PackedEntry {
+    uint32_t code_size;
+    uint32_t code_hash;
+    uint32_t cache_size;
+    uint32_t cache_hash;
+    size_t offset;  // into pack_buf_
+  };
+  void LoadPack();
+  bool ReadPackFile(const std::string& path,
+                    std::unique_ptr<uint8_t[]>* out_buf,
+                    std::unordered_map<uint32_t, PackedEntry>* out_index) const;
+  bool ReadFromPack(CompileCacheEntry* entry);
+  bool PersistPack();
+  std::string pack_filename() const;
+  bool packed_ = false;
+  bool pack_loaded_ = false;
+  std::unique_ptr<uint8_t[]> pack_buf_;
+  std::unordered_map<uint32_t, PackedEntry> pack_index_;
+  std::unordered_set<uint32_t> loose_hits_;  // keys read from loose files
+
   template <typename T>
   void MaybeSaveImpl(CompileCacheEntry* entry,
                      v8::Local<T> func_or_mod,
diff --git a/test/parallel/test-compile-cache-packed.js b/test/parallel/test-compile-cache-packed.js
new file mode 100644
index 0000000..2a711e2
--- /dev/null
+++ b/test/parallel/test-compile-cache-packed.js
@@ -0,0 +1,199 @@
+'use strict';
+
+// This tests NODE_COMPILE_CACHE_PACKED=1: one pack file per cache directory.
+
+require('../common');
+const { spawnSyncAndAssert } = require('../common/child_process');
+const assert = require('assert');
+const fixtures = require('../common/fixtures');
+const tmpdir = require('../common/tmpdir');
+const fs = require('fs');
+const path = require('path');
+
+function env(dir) {
+  return {
+    ...process.env,
+    NODE_DEBUG_NATIVE: 'COMPILE_CACHE',
+    NODE_COMPILE_CACHE: dir,
+    NODE_COMPILE_CACHE_PACKED: '1',
+  };
+}
+
+function packOf(dir, allowLoose = false) {
+  const sub = fs.readdirSync(dir);
+  assert.strictEqual(sub.length, 1);
+  const entries = fs.readdirSync(path.join(dir, sub[0]));
+  if (allowLoose) {
+    assert.ok(entries.includes('pack'), entries.join());
+  } else {
+    assert.deepStrictEqual(entries, ['pack']);
+  }
+  return path.join(dir, sub[0], 'pack');
+}
+
+{
+  // First run writes one pack file; the second run reads from it and does not rewrite it.
+  tmpdir.refresh();
+  const dir = tmpdir.resolve('.compile_cache_dir');
+  const script = fixtures.path('snapshot', 'typescript.js');
+
+  spawnSyncAndAssert(process.execPath, [script], { env: env(dir), cwd: tmpdir.path }, {
+    stderr(output) {
+      assert.match(output, /looking up .*typescript\.js in pack\.\.\. not found/);
+      assert.match(output, /wrote pack .*: \d+ entries/);
+      return true;
+    },
+  });
+  const pack = packOf(dir);
+  const first = fs.readFileSync(pack);
+  assert.strictEqual(first.readUInt32LE(0), 0x8adfdbb3);
+
+  spawnSyncAndAssert(process.execPath, [script], { env: env(dir), cwd: tmpdir.path }, {
+    stderr(output) {
+      assert.match(output, /looking up .*typescript\.js in pack\.\.\. success/);
+      assert.match(output, /cache for .*typescript\.js was accepted/);
+      assert.match(output, /pack unchanged, not writing/);
+      return true;
+    },
+  });
+  assert.ok(fs.readFileSync(pack).equals(first), 'the second run must not rewrite the pack');
+
+  // A truncated pack is ignored and rewritten.
+  fs.writeFileSync(pack, first.subarray(0, first.length - 1));
+  spawnSyncAndAssert(process.execPath, [script], { env: env(dir), cwd: tmpdir.path }, {
+    stderr(output) {
+      assert.match(output, /pack .* is truncated/);
+      assert.match(output, /wrote pack/);
+      return true;
+    },
+  });
+  const rewritten = fs.readFileSync(pack);
+  assert.strictEqual(rewritten.readUInt32LE(0), 0x8adfdbb3);
+  assert.strictEqual(rewritten.readUInt32LE(4), first.readUInt32LE(4));
+}
+
+{
+  // A child process that writes the pack while the parent runs: the parent
+  // merges with the pack on disk, so both keep their entries after one run.
+  tmpdir.refresh();
+  const dir = tmpdir.resolve('.compile_cache_dir');
+  fs.writeFileSync(tmpdir.resolve('a.js'), 'module.exports = function a() { return 1; };\n');
+  fs.writeFileSync(tmpdir.resolve('b.js'), 'module.exports = function b() { return 2; };\n');
+  fs.writeFileSync(tmpdir.resolve('child.js'), "require('./b');\n");
+  fs.writeFileSync(tmpdir.resolve('parent.js'),
+                   "require('./a');\n" +
+                   "const { spawnSync } = require('child_process');\n" +
+                   "const r = spawnSync(process.execPath, [require('path').join(__dirname, 'child.js')]," +
+                   " { stdio: 'inherit' });\n" +
+                   'process.exitCode = r.status;\n');
+
+  spawnSyncAndAssert(process.execPath, ['parent.js'], { env: env(dir), cwd: tmpdir.path }, {
+    stderr: /wrote pack/,
+  });
+  packOf(dir);
+  spawnSyncAndAssert(process.execPath, ['parent.js'], { env: env(dir), cwd: tmpdir.path }, {
+    stderr(output) {
+      assert.match(output, /looking up .*a\.js in pack\.\.\. success/);
+      assert.match(output, /looking up .*b\.js in pack\.\.\. success/);
+      assert.match(output, /looking up .*parent\.js in pack\.\.\. success/);
+      assert.match(output, /looking up .*child\.js in pack\.\.\. success/);
+      assert.doesNotMatch(output, /wrote pack/);
+      return true;
+    },
+  });
+
+  // An edited module: its new cache goes to a loose file, the pack is kept,
+  // and the next run reads the module from the loose file.
+  fs.writeFileSync(tmpdir.resolve('b.js'), 'module.exports = function b() { return 3; };\n');
+  spawnSyncAndAssert(process.execPath, ['parent.js'], { env: env(dir), cwd: tmpdir.path }, {
+    stderr(output) {
+      assert.match(output, /looking up .*b\.js in pack\.\.\. code size or hash mismatch/);
+      assert.match(output, /1 changed and 0 loose entries, writing loose/);
+      assert.match(output, /writing cache for .*b\.js.*success/);
+      assert.doesNotMatch(output, /wrote pack/);
+      return true;
+    },
+  });
+  spawnSyncAndAssert(process.execPath, ['parent.js'], { env: env(dir), cwd: tmpdir.path }, {
+    stderr(output) {
+      assert.match(output, /reading cache from .*for CommonJS .*b\.js\.\.\..*success/);
+      assert.match(output, /looking up .*a\.js in pack\.\.\. success/);
+      assert.doesNotMatch(output, /wrote pack|writing cache for/);
+      return true;
+    },
+  });
+}
+
+{
+  // Many changed entries are folded into a new pack, and the loose files
+  // read on the way are removed.
+  tmpdir.refresh();
+  const dir = tmpdir.resolve('.compile_cache_dir');
+  const N = 20;
+  const write = (v) => {
+    for (let i = 0; i < N; i++) {
+      fs.writeFileSync(tmpdir.resolve(`m${i}.js`), `module.exports = ${v} + ${i};\n`);
+    }
+  };
+  fs.writeFileSync(tmpdir.resolve('main.js'),
+                   `for (let i = 0; i < ${N}; i++) require('./m' + i);\n`);
+  write(1);
+  spawnSyncAndAssert(process.execPath, ['main.js'], { env: env(dir), cwd: tmpdir.path }, {
+    stderr: /wrote pack .*: 21 entries/,
+  });
+  // One edited module: loose.
+  fs.writeFileSync(tmpdir.resolve('m0.js'), 'module.exports = 100;\n');
+  spawnSyncAndAssert(process.execPath, ['main.js'], { env: env(dir), cwd: tmpdir.path }, {
+    stderr: /writing loose/,
+  });
+  const pack = packOf(dir, true);
+  assert.strictEqual(fs.readdirSync(path.dirname(pack)).length, 2);
+  // 16 more edited (m1..m16) + m0 read from its loose file >= 16: fold, and
+  // m0's loose file is removed.
+  for (let i = 1; i <= 16; i++) {
+    fs.writeFileSync(tmpdir.resolve(`m${i}.js`), `module.exports = 2 + ${i};\n`);
+  }
+  spawnSyncAndAssert(process.execPath, ['main.js'], { env: env(dir), cwd: tmpdir.path }, {
+    stderr: /wrote pack .*: 21 entries/,
+  });
+  packOf(dir);
+  spawnSyncAndAssert(process.execPath, ['main.js'], { env: env(dir), cwd: tmpdir.path }, {
+    stderr(output) {
+      assert.match(output, /looking up .*m0\.js in pack\.\.\. success/);
+      assert.match(output, /pack unchanged, not writing/);
+      return true;
+    },
+  });
+}
+
+{
+  // `node -e` scripts share one cache key: alternating two of them must not
+  // rewrite the pack on every run.
+  tmpdir.refresh();
+  const dir = tmpdir.resolve('.compile_cache_dir');
+  const script = fixtures.path('snapshot', 'typescript.js');
+  const evalA = `require(${JSON.stringify(script)})`;
+  for (const code of [evalA, '0', evalA, '0']) {
+    spawnSyncAndAssert(process.execPath, ['-e', code], { env: env(dir), cwd: tmpdir.path }, {});
+  }
+  for (const code of [evalA, '0']) {
+    spawnSyncAndAssert(process.execPath, ['-e', code], { env: env(dir), cwd: tmpdir.path }, {
+      stderr(output) {
+        assert.doesNotMatch(output, /wrote pack/);
+        return true;
+      },
+    });
+  }
+}
+
+{
+  // Without the variable, the per-file layout is unchanged.
+  tmpdir.refresh();
+  const dir = tmpdir.resolve('.compile_cache_dir');
+  const e = env(dir);
+  delete e.NODE_COMPILE_CACHE_PACKED;
+  spawnSyncAndAssert(process.execPath, [fixtures.path('snapshot', 'typescript.js')],
+                     { env: e, cwd: tmpdir.path }, { stderr: /writing cache for .*typescript\.js.*success/ });
+  const sub = fs.readdirSync(dir);
+  assert.notDeepStrictEqual(fs.readdirSync(path.join(dir, sub[0])), ['pack']);
+}