Repository navigation
Expand file tree
/
Copy pathstatus.ts
More file actions
226 lines (208 loc) · 7.21 KB
/
Copy pathstatus.ts
File metadata and controls
226 lines (208 loc) · 7.21 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
/*
* We perform enablement checks for overlay analysis to avoid using it on runners that are too small
* to support it. However these checks cannot avoid every potential issue without being overly
* conservative. Therefore, if our enablement checks enable overlay analysis for a runner that is
* too small, we want to remember that, so that we will not try to use overlay analysis until
* something changes (e.g. a larger runner is provisioned, or a new CodeQL version is released).
*
* We use the Actions cache as a lightweight way of providing this functionality.
*/
import * as fs from "fs";
import * as path from "path";
import * as actionsCache from "@actions/cache";
import {
getTemporaryDirectory,
getWorkflowRunAttempt,
getWorkflowRunID,
} from "../actions-util";
import { type CodeQL } from "../codeql";
import * as json from "../json";
import { Logger } from "../logging";
import {
DiskUsage,
getErrorMessage,
getRequiredEnvParam,
waitForResultWithTimeLimit,
} from "../util";
/** The maximum time to wait for a cache operation to complete. */
const MAX_CACHE_OPERATION_MS = 30_000;
/** File name for the serialized overlay status. */
const STATUS_FILE_NAME = "overlay-status.json";
/** Path to the local overlay status file. */
function getStatusFilePath(languages: string[]): string {
return path.join(
getTemporaryDirectory(),
"overlay-status",
[...languages].sort().join("+"),
STATUS_FILE_NAME,
);
}
/** Details of the job that recorded an overlay status. */
interface JobInfo {
/** The check run ID. This is optional since it is not always available. */
checkRunId?: number;
/** The workflow run ID. */
workflowRunId: number;
/** The workflow run attempt number. */
workflowRunAttempt: number;
/** The name of the job (from GITHUB_JOB). */
name: string;
}
/** Status of an overlay analysis for a group of languages. */
export interface OverlayStatus {
/** Whether the job attempted to build an overlay base database. */
attemptedToBuildOverlayBaseDatabase: boolean;
/** Whether the job successfully built an overlay base database. */
builtOverlayBaseDatabase: boolean;
/** Details of the job that recorded this status. */
job?: JobInfo;
}
/** Creates an `OverlayStatus` populated with the details of the current job. */
export function createOverlayStatus(
attributes: Omit<OverlayStatus, "job">,
checkRunId?: number,
): OverlayStatus {
const job: JobInfo = {
workflowRunId: getWorkflowRunID(),
workflowRunAttempt: getWorkflowRunAttempt(),
name: getRequiredEnvParam("GITHUB_JOB"),
checkRunId,
};
return {
...attributes,
job,
};
}
/**
* Whether overlay analysis should be skipped, based on the cached status for the given languages and disk usage.
*/
export async function shouldSkipOverlayAnalysis(
codeql: CodeQL,
languages: string[],
diskUsage: DiskUsage,
logger: Logger,
): Promise<boolean> {
const status = await getOverlayStatus(codeql, languages, diskUsage, logger);
if (status === undefined) {
return false;
}
if (
status.attemptedToBuildOverlayBaseDatabase &&
!status.builtOverlayBaseDatabase
) {
logger.debug(
"Cached overlay status indicates that building an overlay base database was unsuccessful.",
);
return true;
}
logger.debug(
"Cached overlay status does not indicate a previous unsuccessful attempt to build an overlay base database.",
);
return false;
}
/**
* Retrieve overlay status from the Actions cache, if available.
*
* @returns `undefined` if no status was found in the cache (e.g. first run with
* this cache key) or if the cache operation fails.
*/
export async function getOverlayStatus(
codeql: CodeQL,
languages: string[],
diskUsage: DiskUsage,
logger: Logger,
): Promise<OverlayStatus | undefined> {
const cacheKey = await getCacheKey(codeql, languages, diskUsage);
const statusFile = getStatusFilePath(languages);
try {
await fs.promises.mkdir(path.dirname(statusFile), { recursive: true });
const foundKey = await waitForResultWithTimeLimit(
MAX_CACHE_OPERATION_MS,
actionsCache.restoreCache([statusFile], cacheKey, [cacheKey]),
() => {
logger.warning("Timed out restoring overlay status from cache.");
},
);
if (foundKey === undefined) {
logger.debug("No overlay status found in Actions cache.");
return undefined;
}
if (!fs.existsSync(statusFile)) {
logger.debug(
"Overlay status cache entry found but status file is missing.",
);
return undefined;
}
const contents = await fs.promises.readFile(statusFile, "utf-8");
const parsed: unknown = JSON.parse(contents);
if (
!json.isObject<OverlayStatus>(parsed) ||
typeof parsed["attemptedToBuildOverlayBaseDatabase"] !== "boolean" ||
typeof parsed["builtOverlayBaseDatabase"] !== "boolean"
) {
logger.debug(
"Ignoring overlay status cache entry with unexpected format.",
);
return undefined;
}
return parsed as OverlayStatus;
} catch (error) {
logger.warning(
`Failed to restore overlay status from cache: ${getErrorMessage(error)}`,
);
return undefined;
}
}
/**
* Save overlay status to the Actions cache.
*
* @returns `true` if the status was saved successfully, `false` otherwise.
*/
export async function saveOverlayStatus(
codeql: CodeQL,
languages: string[],
diskUsage: DiskUsage,
status: OverlayStatus,
logger: Logger,
): Promise<boolean> {
const cacheKey = await getCacheKey(codeql, languages, diskUsage);
const statusFile = getStatusFilePath(languages);
try {
await fs.promises.mkdir(path.dirname(statusFile), { recursive: true });
await fs.promises.writeFile(statusFile, JSON.stringify(status));
const cacheId = await waitForResultWithTimeLimit(
MAX_CACHE_OPERATION_MS,
actionsCache.saveCache([statusFile], cacheKey),
() => {
logger.warning("Timed out saving overlay status to cache.");
},
);
if (cacheId === undefined) {
return false;
}
logger.debug(`Saved overlay status to Actions cache with key ${cacheKey}`);
return true;
} catch (error) {
logger.warning(
`Failed to save overlay status to cache: ${getErrorMessage(error)}`,
);
return false;
}
}
export async function getCacheKey(
codeql: CodeQL,
languages: string[],
diskUsage: DiskUsage,
): Promise<string> {
// Total disk space, rounded to the nearest 10 GB. This is included in the cache key so that if a
// customer upgrades their runner, we will try again to use overlay analysis, even if the CodeQL
// version has not changed. We round to the nearest 10 GB to work around small differences in disk
// space.
//
// Limitation: this can still flip from "too small" to "large enough" and back again if the disk
// space fluctuates above and below a multiple of 10 GB.
const diskSpaceToNearest10Gb = `${10 * Math.floor(diskUsage.numTotalBytes / (10 * 1024 * 1024 * 1024))}GB`;
// Include the CodeQL version in the cache key so we will try again to use overlay analysis when
// new queries and libraries that may be more efficient are released.
return `codeql-overlay-status-${[...languages].sort().join("+")}-${(await codeql.getVersion()).version}-runner-${diskSpaceToNearest10Gb}`;
}