Enable incremental caching for faster builds

action.yml

@@ -48,6 +48,14 @@ inputs:
     description: "Check if a cache entry exists without downloading the cache"
     required: false
     default: "false"
+  incremental:
+    description: "Determines whether to cache incremental builds - speeding up builds for more disk usage. Defaults to false."
+    required: false
+    default: "false"
+  incremental-key:
+    description: "The key to use for incremental builds. Used on a per-branch basis"
+    required: false
+    default: ${{ github.ref }}
 outputs:
   cache-hit:
     description: "A boolean value that indicates an exact match was found."

dist/restore/index.js

@@ -86711,16 +86711,26 @@ class CacheConfig {
     constructor() {
         /** All the paths we want to cache */
         this.cachePaths = [];
+        /** All the paths we want to cache for incremental builds */
+        // public incrementalPaths: Array<string> = [];
         /** The primary cache key */
         this.cacheKey = "";
-        /** The secondary (restore) key that only contains the prefix and environment */
+        /** The primary cache key for incremental builds */
+        this.incrementalKey = "";
+        /**
+         *  The secondary (restore) key that only contains the prefix and environment
+         *  This should be used if the primary cacheKey is not available - IE pulling from main on a branch
+         *  instead of the branch itself
+         * */
         this.restoreKey = "";
         /** Whether to cache CARGO_HOME/.bin */
         this.cacheBin = true;
         /** The workspace configurations */
         this.workspaces = [];
         /** The cargo binaries present during main step */
         this.cargoBins = [];
+        /** Whether to cache incremental builds */
+        this.incremental = false;
         /** The prefix portion of the cache key */
         this.keyPrefix = "";
         /** The rust version considered for the cache key */
@@ -86787,6 +86797,9 @@ class CacheConfig {
             }
         }
         self.keyEnvs = keyEnvs;
+        // Make sure we consider incremental builds
+        self.incremental = lib_core.getInput("incremental").toLowerCase() == "true";
+        hasher.update(`incremental=${self.incremental}`);
         key += `-${digest(hasher)}`;
         self.restoreKey = key;
         // Construct the lockfiles portion of the key:
@@ -86908,6 +86921,14 @@ class CacheConfig {
         }
         const bins = await getCargoBins();
         self.cargoBins = Array.from(bins.values());
+        if (self.incremental) {
+            // wire the incremental key to be just for this branch
+            const branchName = lib_core.getInput("incremental-key") || "-shared";
+            const incrementalKey = key + `-incremental--` + branchName;
+            self.incrementalKey = incrementalKey;
+            // Add the incremental cache to the cachePaths so we can restore it
+            self.cachePaths.push(external_path_default().join(config_CARGO_HOME, "incremental-restore.json"));
+        }
         return self;
     }
     /**
@@ -86958,6 +86979,7 @@ class CacheConfig {
         for (const file of this.keyFiles) {
             lib_core.info(`  - ${file}`);
         }
+        lib_core.info(`.. Incremental: ${this.incremental}`);
         lib_core.endGroup();
     }
     /**
@@ -87314,6 +87336,9 @@ async function rmRF(dirName) {
 
 
 
+// import { saveMtimes } from "./incremental";
+
+
 process.on("uncaughtException", (e) => {
     lib_core.error(e.message);
     if (e.stack) {
@@ -87333,10 +87358,12 @@ async function run() {
         }
         var lookupOnly = lib_core.getInput("lookup-only").toLowerCase() === "true";
         lib_core.exportVariable("CACHE_ON_FAILURE", cacheOnFailure);
-        lib_core.exportVariable("CARGO_INCREMENTAL", 0);
         const config = await CacheConfig.new();
         config.printInfo(cacheProvider);
         lib_core.info("");
+        if (!config.incremental) {
+            lib_core.exportVariable("CARGO_INCREMENTAL", 0);
+        }
         lib_core.info(`... ${lookupOnly ? "Checking" : "Restoring"} cache ...`);
         const key = config.cacheKey;
         // Pass a copy of cachePaths to avoid mutating the original array as reported by:
@@ -87346,7 +87373,7 @@ async function run() {
             lookupOnly,
         });
         if (restoreKey) {
-            const match = restoreKey === key;
+            let match = restoreKey === key;
             lib_core.info(`${lookupOnly ? "Found" : "Restored from"} cache key "${restoreKey}" full match: ${match}.`);
             if (!match) {
                 // pre-clean the target directory on cache mismatch
@@ -87359,10 +87386,34 @@ async function run() {
                 // We restored the cache but it is not a full match.
                 config.saveState();
             }
+            // Restore the incremental-restore.json file and write the mtimes to all the files in the list
+            if (config.incremental) {
+                try {
+                    const restoreJson = external_path_default().join(config_CARGO_HOME, "incremental-restore.json");
+                    const restoreString = await external_fs_default().promises.readFile(restoreJson, "utf8");
+                    const restoreData = JSON.parse(restoreString);
+                    lib_core.debug(`restoreData: ${JSON.stringify(restoreData)}`);
+                    if (restoreData.roots.length == 0) {
+                        throw new Error("No incremental roots found");
+                    }
+                    const incrementalKey = await cacheProvider.cache.restoreCache(restoreData.roots, config.incrementalKey, [config.restoreKey], { lookupOnly });
+                    lib_core.debug(`restoring incremental builds from ${incrementalKey}`);
+                    for (const [file, mtime] of Object.entries(restoreData.times)) {
+                        lib_core.debug(`restoring ${file} with mtime ${mtime}`);
+                        await external_fs_default().promises.utimes(file, new Date(mtime), new Date(mtime));
+                    }
+                }
+                catch (err) {
+                    lib_core.debug(`Could not restore incremental cache - ${err}`);
+                    lib_core.debug(`${err.stack}`);
+                    match = false;
+                }
+                config.saveState();
+            }
             setCacheHitOutput(match);
         }
         else {
-            lib_core.info("No cache found.");
+            lib_core.info(`No cache found for ${config.cacheKey} - this key was found ${restoreKey}`);
             config.saveState();
             setCacheHitOutput(false);
         }

dist/save/index.js

@@ -86711,16 +86711,26 @@ class CacheConfig {
     constructor() {
         /** All the paths we want to cache */
         this.cachePaths = [];
+        /** All the paths we want to cache for incremental builds */
+        // public incrementalPaths: Array<string> = [];
         /** The primary cache key */
         this.cacheKey = "";
-        /** The secondary (restore) key that only contains the prefix and environment */
+        /** The primary cache key for incremental builds */
+        this.incrementalKey = "";
+        /**
+         *  The secondary (restore) key that only contains the prefix and environment
+         *  This should be used if the primary cacheKey is not available - IE pulling from main on a branch
+         *  instead of the branch itself
+         * */
         this.restoreKey = "";
         /** Whether to cache CARGO_HOME/.bin */
         this.cacheBin = true;
         /** The workspace configurations */
         this.workspaces = [];
         /** The cargo binaries present during main step */
         this.cargoBins = [];
+        /** Whether to cache incremental builds */
+        this.incremental = false;
         /** The prefix portion of the cache key */
         this.keyPrefix = "";
         /** The rust version considered for the cache key */
@@ -86787,6 +86797,9 @@ class CacheConfig {
             }
         }
         self.keyEnvs = keyEnvs;
+        // Make sure we consider incremental builds
+        self.incremental = core.getInput("incremental").toLowerCase() == "true";
+        hasher.update(`incremental=${self.incremental}`);
         key += `-${digest(hasher)}`;
         self.restoreKey = key;
         // Construct the lockfiles portion of the key:
@@ -86908,6 +86921,14 @@ class CacheConfig {
         }
         const bins = await getCargoBins();
         self.cargoBins = Array.from(bins.values());
+        if (self.incremental) {
+            // wire the incremental key to be just for this branch
+            const branchName = core.getInput("incremental-key") || "-shared";
+            const incrementalKey = key + `-incremental--` + branchName;
+            self.incrementalKey = incrementalKey;
+            // Add the incremental cache to the cachePaths so we can restore it
+            self.cachePaths.push(external_path_default().join(CARGO_HOME, "incremental-restore.json"));
+        }
         return self;
     }
     /**
@@ -86958,6 +86979,7 @@ class CacheConfig {
         for (const file of this.keyFiles) {
             core.info(`  - ${file}`);
         }
+        core.info(`.. Incremental: ${this.incremental}`);
         core.endGroup();
     }
     /**
@@ -87309,12 +87331,59 @@ async function rmRF(dirName) {
     await io.rmRF(dirName);
 }
 
+;// CONCATENATED MODULE: ./src/incremental.ts
+// import * as core from "@actions/core";
+// import * as io from "@actions/io";
+// import { CARGO_HOME } from "./config";
+// import { exists } from "./utils";
+// import { Packages } from "./workspace";
+
+
+async function saveMtimes(targetDirs) {
+    let data = {
+        roots: [],
+        times: {},
+    };
+    let stack = [];
+    // Collect all the incremental files
+    for (const dir of targetDirs) {
+        for (const maybeProfile of await external_fs_default().promises.readdir(dir)) {
+            const profileDir = external_path_default().join(dir, maybeProfile);
+            const incrementalDir = external_path_default().join(profileDir, "incremental");
+            if (external_fs_default().existsSync(incrementalDir)) {
+                stack.push(incrementalDir);
+            }
+        }
+    }
+    // Save the stack as the roots - we cache these directly
+    data.roots = stack.slice();
+    while (stack.length > 0) {
+        const dirName = stack.pop();
+        const dir = await external_fs_default().promises.opendir(dirName);
+        for await (const dirent of dir) {
+            if (dirent.isDirectory()) {
+                stack.push(external_path_default().join(dirName, dirent.name));
+            }
+            else {
+                const fileName = external_path_default().join(dirName, dirent.name);
+                const { mtime } = await external_fs_default().promises.stat(fileName);
+                data.times[fileName] = mtime.getTime();
+            }
+        }
+    }
+    return data;
+}
+
 ;// CONCATENATED MODULE: ./src/save.ts
 
 
 
 
 
+
+
+
+
 process.on("uncaughtException", (e) => {
     core.error(e.message);
     if (e.stack) {
@@ -87339,6 +87408,28 @@ async function run() {
         if (process.env["RUNNER_OS"] == "macOS") {
             await macOsWorkaround();
         }
+        // Save the incremental cache before we delete it
+        if (config.incremental) {
+            core.info(`... Saving incremental cache ...`);
+            try {
+                const targetDirs = config.workspaces.map((ws) => ws.target);
+                const cache = await saveMtimes(targetDirs);
+                const saved = await cacheProvider.cache.saveCache(cache.roots, config.incrementalKey);
+                core.debug(`saved incremental cache with key ${saved} with contents ${cache.roots}, ${cache.times}`);
+                // write the incremental-restore.json file
+                const serialized = JSON.stringify(cache);
+                await external_fs_default().promises.writeFile(external_path_default().join(CARGO_HOME, "incremental-restore.json"), serialized);
+                // Delete the incremental cache before proceeding
+                for (const [path, _mtime] of cache.roots) {
+                    core.debug(`  deleting ${path}`);
+                    await (0,promises_.rm)(path);
+                }
+            }
+            catch (e) {
+                core.debug(`Failed to save incremental cache`);
+                core.debug(`${e.stack}`);
+            }
+        }
         const allPackages = [];
         for (const workspace of config.workspaces) {
             const packages = await workspace.getPackagesOutsideWorkspaceRoot();
@@ -87375,7 +87466,7 @@ async function run() {
         catch (e) {
             core.debug(`${e.stack}`);
         }
-        core.info(`... Saving cache ...`);
+        core.info(`... Saving cache with key ${config.cacheKey}`);
         // Pass a copy of cachePaths to avoid mutating the original array as reported by:
         // https://github.com/actions/toolkit/pull/1378
         // TODO: remove this once the underlying bug is fixed.

src/cleanup.ts

@@ -25,7 +25,7 @@ export async function cleanTargetDir(targetDir: string, packages: Packages, chec
         } else {
           await cleanProfileTarget(dirName, packages, checkTimestamp);
         }
-      } catch {}
+      } catch { }
     } else if (dirent.name !== "CACHEDIR.TAG") {
       await rm(dir.path, dirent);
     }
@@ -43,11 +43,11 @@ async function cleanProfileTarget(profileDir: string, packages: Packages, checkT
       // https://github.com/vertexclique/kaos/blob/9876f6c890339741cc5be4b7cb9df72baa5a6d79/src/cargo.rs#L25
       // https://github.com/eupn/macrotest/blob/c4151a5f9f545942f4971980b5d264ebcd0b1d11/src/cargo.rs#L27
       cleanTargetDir(path.join(profileDir, "target"), packages, checkTimestamp);
-    } catch {}
+    } catch { }
     try {
       // https://github.com/dtolnay/trybuild/blob/eec8ca6cb9b8f53d0caf1aa499d99df52cae8b40/src/cargo.rs#L50
       cleanTargetDir(path.join(profileDir, "trybuild"), packages, checkTimestamp);
-    } catch {}
+    } catch { }
 
     // Delete everything else.
     await rmExcept(profileDir, new Set(["target", "trybuild"]), checkTimestamp);
@@ -86,7 +86,7 @@ export async function getCargoBins(): Promise<Set<string>> {
         bins.add(bin);
       }
     }
-  } catch {}
+  } catch { }
   return bins;
 }
 
@@ -117,7 +117,7 @@ export async function cleanRegistry(packages: Packages, crates = true) {
     const credentials = path.join(CARGO_HOME, ".cargo", "credentials.toml");
     core.debug(`deleting "${credentials}"`);
     await fs.promises.unlink(credentials);
-  } catch {}
+  } catch { }
 
   // `.cargo/registry/index`
   let pkgSet = new Set(packages.map((p) => p.name));
@@ -229,7 +229,7 @@ export async function cleanGit(packages: Packages) {
         await rm(dir.path, dirent);
       }
     }
-  } catch {}
+  } catch { }
 
   // clean the checkouts
   try {
@@ -250,7 +250,7 @@ export async function cleanGit(packages: Packages) {
         }
       }
     }
-  } catch {}
+  } catch { }
 }
 
 const ONE_WEEK = 7 * 24 * 3600 * 1000;
@@ -302,7 +302,7 @@ async function rm(parent: string, dirent: fs.Dirent) {
     } else if (dirent.isDirectory()) {
       await io.rmRF(fileName);
     }
-  } catch {}
+  } catch { }
 }
 
 async function rmRF(dirName: string) {