Use zstd instead of gzip if available

Add zstd to cache versioning

src/cacheHttpClient.ts

@@ -11,9 +11,10 @@ import * as fs from "fs";
 import * as stream from "stream";
 import * as util from "util";
 
-import { Inputs, SocketTimeout } from "./constants";
+import { CompressionMethod, Inputs, SocketTimeout } from "./constants";
 import {
     ArtifactCacheEntry,
+    CacheOptions,
     CommitCacheRequest,
     ReserveCacheRequest,
     ReserveCacheResponse
@@ -84,12 +85,13 @@ function createHttpClient(): HttpClient {
     );
 }
 
-export function getCacheVersion(): string {
+export function getCacheVersion(compressionMethod?: CompressionMethod): string {
     // Add salt to cache version to support breaking changes in cache entry
-    const components = [
-        core.getInput(Inputs.Path, { required: true }),
-        versionSalt
-    ];
+    const components = [core.getInput(Inputs.Path, { required: true })].concat(
+        compressionMethod == CompressionMethod.Zstd
+            ? [compressionMethod, versionSalt]
+            : versionSalt
+    );
 
     return crypto
         .createHash("sha256")
@@ -98,10 +100,11 @@ export function getCacheVersion(): string {
 }
 
 export async function getCacheEntry(
-    keys: string[]
+    keys: string[],
+    options?: CacheOptions
 ): Promise<ArtifactCacheEntry | null> {
     const httpClient = createHttpClient();
-    const version = getCacheVersion();
+    const version = getCacheVersion(options?.compressionMethod);
     const resource = `cache?keys=${encodeURIComponent(
         keys.join(",")
     )}&version=${version}`;
@@ -173,9 +176,12 @@ export async function downloadCache(
 }
 
 // Reserve Cache
-export async function reserveCache(key: string): Promise<number> {
+export async function reserveCache(
+    key: string,
+    options?: CacheOptions
+): Promise<number> {
     const httpClient = createHttpClient();
-    const version = getCacheVersion();
+    const version = getCacheVersion(options?.compressionMethod);
 
     const reserveCacheRequest: ReserveCacheRequest = {
         key,

src/constants.ts

@@ -19,7 +19,15 @@ export enum Events {
     PullRequest = "pull_request"
 }
 
-export const CacheFilename = "cache.tgz";
+export enum CacheFilename {
+    Gzip = "cache.tgz",
+    Zstd = "cache.tzst"
+}
+
+export enum CompressionMethod {
+    Gzip = "gzip",
+    Zstd = "zstd"
+}
 
 // Socket timeout in milliseconds during download.  If no traffic is received
 // over the socket during this period, the socket is destroyed and the download

src/contracts.d.ts

@@ -1,3 +1,5 @@
+import { CompressionMethod } from "./constants";
+
 export interface ArtifactCacheEntry {
     cacheKey?: string;
     scope?: string;
@@ -17,3 +19,7 @@ export interface ReserveCacheRequest {
 export interface ReserveCacheResponse {
     cacheId: number;
 }
+
+export interface CacheOptions {
+    compressionMethod?: CompressionMethod;
+}

src/restore.ts

@@ -54,8 +54,12 @@ async function run(): Promise<void> {
             }
         }
 
+        const compressionMethod = await utils.getCompressionMethod();
+
         try {
-            const cacheEntry = await cacheHttpClient.getCacheEntry(keys);
+            const cacheEntry = await cacheHttpClient.getCacheEntry(keys, {
+                compressionMethod: compressionMethod
+            });
             if (!cacheEntry?.archiveLocation) {
                 core.info(`Cache not found for input keys: ${keys.join(", ")}`);
                 return;
@@ -63,7 +67,7 @@ async function run(): Promise<void> {
 
             const archivePath = path.join(
                 await utils.createTempDirectory(),
-                "cache.tgz"
+                utils.getCacheFileName(compressionMethod)
             );
             core.debug(`Archive Path: ${archivePath}`);
 
@@ -84,7 +88,7 @@ async function run(): Promise<void> {
                     )} MB (${archiveFileSize} B)`
                 );
 
-                await extractTar(archivePath);
+                await extractTar(archivePath, compressionMethod);
             } finally {
                 // Try to delete the archive to save space
                 try {

src/save.ts

@@ -2,7 +2,7 @@ import * as core from "@actions/core";
 import * as path from "path";
 
 import * as cacheHttpClient from "./cacheHttpClient";
-import { CacheFilename, Events, Inputs, State } from "./constants";
+import { Events, Inputs, State } from "./constants";
 import { createTar } from "./tar";
 import * as utils from "./utils/actionUtils";
 
@@ -35,8 +35,12 @@ async function run(): Promise<void> {
             return;
         }
 
+        const compressionMethod = await utils.getCompressionMethod();
+
         core.debug("Reserving Cache");
-        const cacheId = await cacheHttpClient.reserveCache(primaryKey);
+        const cacheId = await cacheHttpClient.reserveCache(primaryKey, {
+            compressionMethod: compressionMethod
+        });
         if (cacheId == -1) {
             core.info(
                 `Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`
@@ -55,10 +59,14 @@ async function run(): Promise<void> {
         core.debug(`${JSON.stringify(cachePaths)}`);
 
         const archiveFolder = await utils.createTempDirectory();
-        const archivePath = path.join(archiveFolder, CacheFilename);
+        const archivePath = path.join(
+            archiveFolder,
+            utils.getCacheFileName(compressionMethod)
+        );
+
         core.debug(`Archive Path: ${archivePath}`);
 
-        await createTar(archiveFolder, cachePaths);
+        await createTar(archiveFolder, cachePaths, compressionMethod);
 
         const fileSizeLimit = 5 * 1024 * 1024 * 1024; // 5GB per repo limit
         const archiveFileSize = utils.getArchiveFileSize(archivePath);

src/tar.ts

@@ -1,27 +1,10 @@
-import * as core from "@actions/core";
 import { exec } from "@actions/exec";
 import * as io from "@actions/io";
 import { existsSync, writeFileSync } from "fs";
 import * as path from "path";
 
-import { CacheFilename } from "./constants";
-
-export async function isGnuTar(): Promise<boolean> {
-    core.debug("Checking tar --version");
-    let versionOutput = "";
-    await exec("tar --version", [], {
-        ignoreReturnCode: true,
-        silent: true,
-        listeners: {
-            stdout: (data: Buffer): string =>
-                (versionOutput += data.toString()),
-            stderr: (data: Buffer): string => (versionOutput += data.toString())
-        }
-    });
-
-    core.debug(versionOutput.trim());
-    return versionOutput.toUpperCase().includes("GNU TAR");
-}
+import { CompressionMethod } from "./constants";
+import * as utils from "./utils/actionUtils";
 
 async function getTarPath(args: string[]): Promise<string> {
     // Explicitly use BSD Tar on Windows
@@ -30,7 +13,7 @@ async function getTarPath(args: string[]): Promise<string> {
         const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
         if (existsSync(systemTar)) {
             return systemTar;
-        } else if (isGnuTar()) {
+        } else if (await utils.useGnuTar()) {
             args.push("--force-local");
         }
     }
@@ -39,7 +22,7 @@ async function getTarPath(args: string[]): Promise<string> {
 
 async function execTar(args: string[], cwd?: string): Promise<void> {
     try {
-        await exec(`"${await getTarPath(args)}"`, args, { cwd: cwd });
+        await exec(`${await getTarPath(args)}`, args, { cwd: cwd });
     } catch (error) {
         throw new Error(`Tar failed with error: ${error?.message}`);
     }
@@ -49,13 +32,18 @@ function getWorkingDirectory(): string {
     return process.env["GITHUB_WORKSPACE"] ?? process.cwd();
 }
 
-export async function extractTar(archivePath: string): Promise<void> {
+export async function extractTar(
+    archivePath: string,
+    compressionMethod: CompressionMethod
+): Promise<void> {
     // Create directory to extract tar into
     const workingDirectory = getWorkingDirectory();
     await io.mkdirP(workingDirectory);
     const args = [
-        "-xz",
-        "-f",
+        ...(compressionMethod == CompressionMethod.Zstd
+            ? ["--use-compress-program", "zstd -d"]
+            : ["-z"]),
+        "-xf",
         archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
         "-P",
         "-C",
@@ -66,20 +54,24 @@ export async function extractTar(archivePath: string): Promise<void> {
 
 export async function createTar(
     archiveFolder: string,
-    sourceDirectories: string[]
+    sourceDirectories: string[],
+    compressionMethod: CompressionMethod
 ): Promise<void> {
     // Write source directories to manifest.txt to avoid command length limits
     const manifestFilename = "manifest.txt";
+    const cacheFileName = utils.getCacheFileName(compressionMethod);
     writeFileSync(
         path.join(archiveFolder, manifestFilename),
         sourceDirectories.join("\n")
     );
-
+    // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
     const workingDirectory = getWorkingDirectory();
     const args = [
-        "-cz",
-        "-f",
-        CacheFilename.replace(new RegExp("\\" + path.sep, "g"), "/"),
+        ...(compressionMethod == CompressionMethod.Zstd
+            ? ["--use-compress-program", "zstd -T0"]
+            : ["-z"]),
+        "-cf",
+        cacheFileName.replace(new RegExp("\\" + path.sep, "g"), "/"),
         "-P",
         "-C",
         workingDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"),

src/utils/actionUtils.ts

@@ -1,4 +1,5 @@
 import * as core from "@actions/core";
+import * as exec from "@actions/exec";
 import * as glob from "@actions/glob";
 import * as io from "@actions/io";
 import * as fs from "fs";
@@ -6,7 +7,13 @@ import * as path from "path";
 import * as util from "util";
 import * as uuidV4 from "uuid/v4";
 
-import { Events, Outputs, State } from "../constants";
+import {
+    CacheFilename,
+    CompressionMethod,
+    Events,
+    Outputs,
+    State
+} from "../constants";
 import { ArtifactCacheEntry } from "../contracts";
 
 // From https://github.com/actions/toolkit/blob/master/packages/tool-cache/src/tool-cache.ts#L23
@@ -116,3 +123,44 @@ export function isValidEvent(): boolean {
 export function unlinkFile(path: fs.PathLike): Promise<void> {
     return util.promisify(fs.unlink)(path);
 }
+
+async function checkVersion(app: string): Promise<string> {
+    core.debug(`Checking ${app} --version`);
+    let versionOutput = "";
+    try {
+        await exec.exec(`${app} --version`, [], {
+            ignoreReturnCode: true,
+            silent: true,
+            listeners: {
+                stdout: (data: Buffer): string =>
+                    (versionOutput += data.toString()),
+                stderr: (data: Buffer): string =>
+                    (versionOutput += data.toString())
+            }
+        });
+    } catch (err) {
+        core.debug(err.message);
+    }
+
+    versionOutput = versionOutput.trim();
+    core.debug(versionOutput);
+    return versionOutput;
+}
+
+export async function getCompressionMethod(): Promise<CompressionMethod> {
+    const versionOutput = await checkVersion("zstd");
+    return versionOutput.toLowerCase().includes("zstd command line interface")
+        ? CompressionMethod.Zstd
+        : CompressionMethod.Gzip;
+}
+
+export function getCacheFileName(compressionMethod: CompressionMethod): string {
+    return compressionMethod == CompressionMethod.Zstd
+        ? CacheFilename.Zstd
+        : CacheFilename.Gzip;
+}
+
+export async function useGnuTar(): Promise<boolean> {
+    const versionOutput = await checkVersion("tar");
+    return versionOutput.toLowerCase().includes("gnu tar");
+}