Merge pull request #306 from actions/with-retries

Add retries to all API calls

src/cacheHttpClient.ts

@@ -30,6 +30,13 @@ function isSuccessStatusCode(statusCode?: number): boolean {
     return statusCode >= 200 && statusCode < 300;
 }
 
+function isServerErrorStatusCode(statusCode?: number): boolean {
+    if (!statusCode) {
+        return true;
+    }
+    return statusCode >= 500;
+}
+
 function isRetryableStatusCode(statusCode?: number): boolean {
     if (!statusCode) {
         return false;
@@ -99,6 +106,75 @@ export function getCacheVersion(compressionMethod?: CompressionMethod): string {
         .digest("hex");
 }
 
+export async function retry<T>(
+    name: string,
+    method: () => Promise<T>,
+    getStatusCode: (T) => number | undefined,
+    maxAttempts = 2
+): Promise<T> {
+    let response: T | undefined = undefined;
+    let statusCode: number | undefined = undefined;
+    let isRetryable = false;
+    let errorMessage = "";
+    let attempt = 1;
+
+    while (attempt <= maxAttempts) {
+        try {
+            response = await method();
+            statusCode = getStatusCode(response);
+
+            if (!isServerErrorStatusCode(statusCode)) {
+                return response;
+            }
+
+            isRetryable = isRetryableStatusCode(statusCode);
+            errorMessage = `Cache service responded with ${statusCode}`;
+        } catch (error) {
+            isRetryable = true;
+            errorMessage = error.message;
+        }
+
+        core.debug(
+            `${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`
+        );
+
+        if (!isRetryable) {
+            core.debug(`${name} - Error is not retryable`);
+            break;
+        }
+
+        attempt++;
+    }
+
+    throw Error(`${name} failed: ${errorMessage}`);
+}
+
+export async function retryTypedResponse<T>(
+    name: string,
+    method: () => Promise<ITypedResponse<T>>,
+    maxAttempts = 2
+): Promise<ITypedResponse<T>> {
+    return await retry(
+        name,
+        method,
+        (response: ITypedResponse<T>) => response.statusCode,
+        maxAttempts
+    );
+}
+
+export async function retryHttpClientResponse<T>(
+    name: string,
+    method: () => Promise<IHttpClientResponse>,
+    maxAttempts = 2
+): Promise<IHttpClientResponse> {
+    return await retry(
+        name,
+        method,
+        (response: IHttpClientResponse) => response.message.statusCode,
+        maxAttempts
+    );
+}
+
 export async function getCacheEntry(
     keys: string[],
     options?: CacheOptions
@@ -109,8 +185,8 @@ export async function getCacheEntry(
         keys.join(",")
     )}&version=${version}`;
 
-    const response = await httpClient.getJson<ArtifactCacheEntry>(
-        getCacheApiUrl(resource)
+    const response = await retryTypedResponse("getCacheEntry", () =>
+        httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource))
     );
     if (response.statusCode === 204) {
         return null;
@@ -145,7 +221,10 @@ export async function downloadCache(
 ): Promise<void> {
     const stream = fs.createWriteStream(archivePath);
     const httpClient = new HttpClient("actions/cache");
-    const downloadResponse = await httpClient.get(archiveLocation);
+    const downloadResponse = await retryHttpClientResponse(
+        "downloadCache",
+        () => httpClient.get(archiveLocation)
+    );
 
     // Abort download if no traffic received over the socket.
     downloadResponse.message.socket.setTimeout(SocketTimeout, () => {
@@ -187,9 +266,11 @@ export async function reserveCache(
         key,
         version
     };
-    const response = await httpClient.postJson<ReserveCacheResponse>(
-        getCacheApiUrl("caches"),
-        reserveCacheRequest
+    const response = await retryTypedResponse("reserveCache", () =>
+        httpClient.postJson<ReserveCacheResponse>(
+            getCacheApiUrl("caches"),
+            reserveCacheRequest
+        )
     );
     return response?.result?.cacheId ?? -1;
 }
@@ -223,32 +304,15 @@ async function uploadChunk(
         "Content-Range": getContentRange(start, end)
     };
 
-    const uploadChunkRequest = async (): Promise<IHttpClientResponse> => {
-        return await httpClient.sendStream(
-            "PATCH",
-            resourceUrl,
-            openStream(),
-            additionalHeaders
-        );
-    };
-
-    const response = await uploadChunkRequest();
-    if (isSuccessStatusCode(response.message.statusCode)) {
-        return;
-    }
-
-    if (isRetryableStatusCode(response.message.statusCode)) {
-        core.debug(
-            `Received ${response.message.statusCode}, retrying chunk at offset ${start}.`
-        );
-        const retryResponse = await uploadChunkRequest();
-        if (isSuccessStatusCode(retryResponse.message.statusCode)) {
-            return;
-        }
-    }
-
-    throw new Error(
-        `Cache service responded with ${response.message.statusCode} during chunk upload.`
+    await retryHttpClientResponse(
+        `uploadChunk (start: ${start}, end: ${end})`,
+        () =>
+            httpClient.sendStream(
+                "PATCH",
+                resourceUrl,
+                openStream(),
+                additionalHeaders
+            )
     );
 }
 
@@ -325,9 +389,11 @@ async function commitCache(
     filesize: number
 ): Promise<ITypedResponse<null>> {
     const commitCacheRequest: CommitCacheRequest = { size: filesize };
-    return await httpClient.postJson<null>(
-        getCacheApiUrl(`caches/${cacheId.toString()}`),
-        commitCacheRequest
+    return await retryTypedResponse("commitCache", () =>
+        httpClient.postJson<null>(
+            getCacheApiUrl(`caches/${cacheId.toString()}`),
+            commitCacheRequest
+        )
     );
 }