From 170e87dca9bb85a4cb2c68996a84923b73498b65 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 13 Jan 2022 09:48:02 -0800 Subject: [PATCH] Remove database-related functionality from perf tool (#378) * [perf] Remove all database-related functionality * Throw error on comparison attempt for new package * Update tests --- packages/perf/package.json | 1 - packages/perf/src/analysis/metrics.ts | 54 ++-- packages/perf/src/analysis/overallChange.ts | 8 +- packages/perf/src/cli/benchmark.ts | 17 -- packages/perf/src/cli/compare.ts | 106 ++----- packages/perf/src/cli/compareTypeScript.ts | 272 ------------------ .../perf/src/cli/getPackagesToBenchmark.ts | 108 ------- packages/perf/src/cli/index.ts | 53 ---- packages/perf/src/common/config.ts | 20 -- packages/perf/src/common/db.ts | 49 ---- packages/perf/src/common/index.ts | 2 - packages/perf/src/common/support.ts | 5 - packages/perf/src/common/types.ts | 32 --- packages/perf/src/common/utils.ts | 41 +-- packages/perf/src/github/comment.ts | 3 - packages/perf/src/github/createTable.ts | 86 ++---- .../github/createTablesWithAnalysesMessage.ts | 45 ++- .../github/postDependentsComparisonResults.ts | 5 +- .../github/postInitialComparisonResults.ts | 7 +- .../github/postTypeScriptComparisonResult.ts | 20 -- packages/perf/src/query/index.ts | 38 --- packages/perf/src/write/index.ts | 11 - packages/perf/test/analysis/metrics.test.ts | 18 +- yarn.lock | 36 --- 24 files changed, 110 insertions(+), 927 deletions(-) delete mode 100644 packages/perf/src/cli/compareTypeScript.ts delete mode 100644 packages/perf/src/cli/getPackagesToBenchmark.ts delete mode 100644 packages/perf/src/common/db.ts delete mode 100644 packages/perf/src/common/support.ts delete mode 100644 packages/perf/src/github/postTypeScriptComparisonResult.ts delete mode 100644 packages/perf/src/query/index.ts delete mode 100644 packages/perf/src/write/index.ts diff --git a/packages/perf/package.json b/packages/perf/package.json index 6c138934..45e491b0 100644 --- a/packages/perf/package.json +++ b/packages/perf/package.json @@ -15,7 +15,6 @@ "url": "git://github.com/andrewbranch/definitely-not-slow.git" }, "dependencies": { - "@azure/cosmos": "^3.6.3", "@definitelytyped/definitions-parser": "^0.0.101-next.7", "@definitelytyped/header-parser": "^0.0.100", "@definitelytyped/utils": "^0.0.100", diff --git a/packages/perf/src/analysis/metrics.ts b/packages/perf/src/analysis/metrics.ts index 754933db..a86f875a 100644 --- a/packages/perf/src/analysis/metrics.ts +++ b/packages/perf/src/analysis/metrics.ts @@ -1,5 +1,5 @@ import { mean } from "../measure/utils"; -import { PackageBenchmarkSummary, Document, config, getPercentDiff, supportsMemoryUsage, not } from "../common"; +import { PackageBenchmarkSummary, config, getPercentDiff } from "../common"; import { assertNever } from "@definitelytyped/utils"; export interface FormatOptions { @@ -19,8 +19,8 @@ export type GetSignificance = ( percentDiff: number, beforeValue: number, afterValue: number, - beforeDoc: Document, - afterDoc: Document + beforeDoc: PackageBenchmarkSummary, + afterDoc: PackageBenchmarkSummary ) => SignificanceLevel | undefined; export type CreateGetSignificance = (getSignificance: GetSignificance) => GetSignificance; @@ -29,7 +29,7 @@ export interface Metric { columnName: string; sentenceName: string; formatOptions?: FormatOptions; - getValue: (x: Document) => number | undefined; + getValue: (x: PackageBenchmarkSummary) => number | undefined; getSignificance: GetSignificance; } @@ -139,21 +139,6 @@ function withThreshold(fineIf: FineIf, threshold: number) { }; } -function ignoreIfEitherBenchmark(predicate: (document: Document) => boolean) { - return (getSignificance: GetSignificance): GetSignificance => ( - percentDiff, - beforeValue, - afterValue, - beforeDoc, - afterDoc - ) => { - if (predicate(beforeDoc) || predicate(afterDoc)) { - return undefined; - } - return getSignificance(percentDiff, beforeValue, afterValue, beforeDoc, afterDoc); - }; -} - function compose(x: CreateGetSignificance, ...xs: CreateGetSignificance[]): CreateGetSignificance { return getSignificance => { let current = x(getSignificance); @@ -169,7 +154,7 @@ export const metrics: { [K in MetricName]: Metric } = { columnName: "Type count", sentenceName: "type count", formatOptions: { precision: 0 }, - getValue: x => x.body.typeCount, + getValue: x => x.typeCount, getSignificance: compose( proportionalTo("identifierCount"), withThreshold(FineIf.LessThan, 5000) @@ -178,18 +163,17 @@ export const metrics: { [K in MetricName]: Metric } = { memoryUsage: { columnName: "Memory usage (MiB)", sentenceName: "memory usage", - getValue: x => x.body.memoryUsage / 2 ** 20, + getValue: x => x.memoryUsage / 2 ** 20, getSignificance: compose( proportionalTo("identifierCount"), withThreshold(FineIf.LessThan, 65), - ignoreIfEitherBenchmark(not(supportsMemoryUsage)) )(getOrderOfMagnitudeSignificance) }, assignabilityCacheSize: { columnName: "Assignability cache size", sentenceName: "assignability cache size", formatOptions: { precision: 0 }, - getValue: x => x.body.relationCacheSizes && x.body.relationCacheSizes.assignable, + getValue: x => x.relationCacheSizes && x.relationCacheSizes.assignable, getSignificance: compose( proportionalTo("identifierCount"), withThreshold(FineIf.LessThan, 1000) @@ -199,64 +183,64 @@ export const metrics: { [K in MetricName]: Metric } = { columnName: "Samples taken", sentenceName: "number of samples taken", formatOptions: { precision: 0 }, - getValue: x => Math.max(x.body.completions.trials, x.body.quickInfo.trials), + getValue: x => Math.max(x.completions.trials, x.quickInfo.trials), getSignificance: getInsignificant }, identifierCount: { columnName: "Identifiers in tests", sentenceName: "number of identifiers present in test files", formatOptions: { precision: 0 }, - getValue: x => x.body.testIdentifierCount, + getValue: x => x.testIdentifierCount, getSignificance: getInsignificant }, completionsMean: { columnName: "Mean duration (ms)", sentenceName: "mean duration for getting completions at a position", - getValue: x => x.body.completions.mean, + getValue: x => x.completions.mean, getSignificance: withThreshold(FineIf.LessThan, 150)(getDefaultSignificance) }, completionsStdDev: { columnName: "Std. deviation (ms)", sentenceName: "standard deviation of the durations for getting completions at a position", - getValue: x => x.body.completions.standardDeviation, + getValue: x => x.completions.standardDeviation, getSignificance: getInsignificant }, completionsAvgCV: { columnName: "Mean [CV](https://en.wikipedia.org/wiki/Coefficient_of_variation)", sentenceName: "mean coefficient of variation of samples measured for completions time", - getValue: x => x.body.completions.meanCoefficientOfVariation, + getValue: x => x.completions.meanCoefficientOfVariation, getSignificance: getInsignificant, formatOptions: { percentage: true, noDiff: true } }, completionsWorstMean: { columnName: "Worst duration (ms)", sentenceName: "worst-case duration for getting completions at a position", - getValue: x => mean(x.body.completions.worst.completionsDurations), + getValue: x => mean(x.completions.worst.completionsDurations), getSignificance: withThreshold(FineIf.LessThan, 200)(getDefaultSignificance) }, quickInfoMean: { columnName: "Mean duration (ms)", sentenceName: "mean duration for getting quick info at a position", - getValue: x => x.body.quickInfo.mean, + getValue: x => x.quickInfo.mean, getSignificance: withThreshold(FineIf.LessThan, 150)(getDefaultSignificance) }, quickInfoStdDev: { columnName: "Std. deviation (ms)", sentenceName: "standard deviation of the durations for getting quick info at a position", - getValue: x => x.body.quickInfo.standardDeviation, + getValue: x => x.quickInfo.standardDeviation, getSignificance: getInsignificant }, quickInfoAvgCV: { columnName: "Mean [CV](https://en.wikipedia.org/wiki/Coefficient_of_variation)", sentenceName: "mean coefficient of variation of samples measured for quick info time", - getValue: x => x.body.quickInfo.meanCoefficientOfVariation, + getValue: x => x.quickInfo.meanCoefficientOfVariation, getSignificance: getInsignificant, formatOptions: { percentage: true, noDiff: true } }, quickInfoWorstMean: { columnName: "Worst duration (ms)", sentenceName: "worst-case duration for getting quick info at a position", - getValue: x => mean(x.body.quickInfo.worst.quickInfoDurations), + getValue: x => mean(x.quickInfo.worst.quickInfoDurations), getSignificance: withThreshold(FineIf.LessThan, 200)(getDefaultSignificance) } }; @@ -268,8 +252,8 @@ export interface ComparedMetric { } export function getInterestingMetrics( - before: Document, - after: Document + before: PackageBenchmarkSummary, + after: PackageBenchmarkSummary ): ComparedMetric[] { return Object.values(metrics).reduce( (acc: { metric: Metric; percentDiff: number; significance: SignificanceLevel }[], metric) => { diff --git a/packages/perf/src/analysis/overallChange.ts b/packages/perf/src/analysis/overallChange.ts index 7bac24e3..2176dc4f 100644 --- a/packages/perf/src/analysis/overallChange.ts +++ b/packages/perf/src/analysis/overallChange.ts @@ -1,8 +1,8 @@ -import { PackageBenchmarkSummary, Document } from "../common"; +import { PackageBenchmarkSummary } from "../common"; import { getInterestingMetrics, SignificanceLevel } from "./metrics"; import { assertNever } from "@definitelytyped/utils"; -type BeforeAndAfter = [Document | undefined, Document]; +type BeforeAndAfter = [PackageBenchmarkSummary, PackageBenchmarkSummary]; export enum OverallChange { Same = 0, @@ -12,8 +12,8 @@ export enum OverallChange { } export function getOverallChangeForSingleComparison( - before: Document, - after: Document + before: PackageBenchmarkSummary, + after: PackageBenchmarkSummary ) { let change = OverallChange.Same; for (const { significance } of getInterestingMetrics(before, after)) { diff --git a/packages/perf/src/cli/benchmark.ts b/packages/perf/src/cli/benchmark.ts index 1d3f46fd..3a4357c9 100644 --- a/packages/perf/src/cli/benchmark.ts +++ b/packages/perf/src/cli/benchmark.ts @@ -1,16 +1,12 @@ import * as os from "os"; import * as path from "path"; import { - getDatabase, - DatabaseAccessLevel, - config, getParsedPackages, assertString, assertNumber, getSystemInfo } from "../common"; import { getTypeScript } from "../measure/getTypeScript"; -import { insertDocument } from "../write"; import { printSummary, measurePerf } from "../measure"; import { summarize } from "../analysis"; import { PackageId, formatDependencyVersion, tryParsePackageVersion } from "@definitelytyped/definitions-parser"; @@ -21,7 +17,6 @@ export interface BenchmarkPackageOptions { groups?: PackageId[][]; agentIndex?: number; package?: string; - upload: boolean; tsVersion: string; progress: boolean; iterations: number; @@ -87,7 +82,6 @@ export async function benchmarkPackage( options: BenchmarkPackageOptions ) { const { - upload, progress, iterations, nProcesses, @@ -129,16 +123,5 @@ export async function benchmarkPackage( printSummary([summary]); } - if (upload) { - const { packageBenchmarks } = await getDatabase(DatabaseAccessLevel.Write); - const item = await insertDocument( - summary, - config.database.packageBenchmarksDocumentSchemaVersion, - packageBenchmarks - ); - - return { benchmark, summary, id: item.id }; - } - return { benchmark, summary, id: undefined }; } diff --git a/packages/perf/src/cli/compare.ts b/packages/perf/src/cli/compare.ts index 2f23f219..2a3757a1 100644 --- a/packages/perf/src/cli/compare.ts +++ b/packages/perf/src/cli/compare.ts @@ -1,20 +1,12 @@ import * as os from "os"; import { - getDatabase, - DatabaseAccessLevel, config, getChangedPackages, - packageIdsAreEqual, PackageBenchmarkSummary, getParsedPackages, - Document, - createDocument, shuffle, - systemsAreCloseEnough, - getSystemInfo, forEachWithTimeLimit } from "../common"; -import { getLatestBenchmark } from "../query"; import { benchmarkPackage } from "./benchmark"; import { printSummary } from "../measure"; import { getTypeScript } from "../measure/getTypeScript"; @@ -23,12 +15,9 @@ import { postDependentsComparisonResult } from "../github/postDependentsComparis import { AllPackages, DependencyVersion, - formatDependencyVersion, getAffectedPackages, - PackageId } from "@definitelytyped/definitions-parser"; import { execAndThrowErrors } from "@definitelytyped/utils"; -const currentSystem = getSystemInfo(); export interface CompareOptions { allPackages: AllPackages; @@ -67,7 +56,7 @@ export async function compare({ await getTypeScript(tsVersion); const affectedPackages = getAffectedPackages(allPackages, changedPackages); - const comparisons: [Document | undefined, Document][] = []; + const comparisons: [PackageBenchmarkSummary, PackageBenchmarkSummary][] = []; const maxRunMs = (maxRunSeconds ?? Infinity) * 1000; const { overtime } = await forEachWithTimeLimit( maxRunMs, @@ -104,7 +93,7 @@ export async function compare({ } } - const dependentComparisons: [Document | undefined, Document][] = []; + const dependentComparisons: [PackageBenchmarkSummary, PackageBenchmarkSummary][] = []; await forEachWithTimeLimit( maxRunMs, dependentsToTest, @@ -136,79 +125,35 @@ export async function compare({ } export async function compareBenchmarks({ - allPackages, definitelyTypedPath, typeScriptVersionMajorMinor, packageName, packageVersion, maxRunSeconds, - upload = true -}: CompareOptions): Promise<[Document | undefined, Document]> { - const typings = allPackages.getTypingsData({ name: packageName, version: packageVersion }); - const { packageBenchmarks: container } = await getDatabase(DatabaseAccessLevel.Read); - const latestBenchmarkDocument = await getLatestBenchmark({ - container, - typeScriptVersionMajorMinor, - packageName, - packageVersion: typings.id.version, - matchMinor: allPackages.hasSeparateMinorVersions(packageName) - }); +}: CompareOptions): Promise<[PackageBenchmarkSummary, PackageBenchmarkSummary]> { + await execAndThrowErrors("git checkout -f origin/master && git clean -xdf types", definitelyTypedPath); + const baseBenchmark = (await benchmarkPackage(packageName, packageVersion.toString(), new Date(), { + definitelyTypedPath, + printSummary: false, + iterations: config.benchmarks.languageServiceIterations, + progress: false, + tsVersion: typeScriptVersionMajorMinor, + nProcesses: os.cpus().length, + failOnErrors: true, + installTypeScript: false, + maxRunSeconds + }))?.summary; - let latestBenchmark: PackageBenchmarkSummary | undefined = latestBenchmarkDocument && latestBenchmarkDocument.body; - const packageId: PackageId = { - name: packageName, - version: packageVersion - }; - - const changedPackagesBetweenLastRunAndMaster = - latestBenchmark && - (await getChangedPackages({ - diffFrom: "origin/master", - diffTo: latestBenchmark.sourceVersion, - definitelyTypedPath - })); - - if (latestBenchmarkDocument && !systemsAreCloseEnough(currentSystem, latestBenchmarkDocument.system)) { - latestBenchmark = undefined; + if (!baseBenchmark) { + throw new Error(`Package ${packageName} does not exist in master so cannot be compared.`); } - - if (changedPackagesBetweenLastRunAndMaster || !latestBenchmark) { - let needsRerun = !latestBenchmark; - if (changedPackagesBetweenLastRunAndMaster) { - const affectedPackages = getAffectedPackages(allPackages, changedPackagesBetweenLastRunAndMaster); - const affected = [...affectedPackages.changedPackages, ...affectedPackages.dependentPackages]; - needsRerun = affected.some(affectedPackage => packageIdsAreEqual(packageId, affectedPackage.id)); - } - if (needsRerun) { - console.log( - `No comparable benchmark for ${packageName}/v${formatDependencyVersion( - packageVersion - )}. Checking out master and running one...` - ); - await execAndThrowErrors("git checkout -f origin/master && git clean -xdf types", definitelyTypedPath); - const latest = await benchmarkPackage(packageName, packageVersion.toString(), new Date(), { - definitelyTypedPath, - printSummary: false, - iterations: config.benchmarks.languageServiceIterations, - progress: false, - upload, - tsVersion: typeScriptVersionMajorMinor, - nProcesses: os.cpus().length, - failOnErrors: true, - installTypeScript: false, - maxRunSeconds - }); - await execAndThrowErrors(`git checkout -f . && git checkout - && git clean -xdf types`, definitelyTypedPath); - latestBenchmark = latest && latest.summary; - } - } - - const currentBenchmark = (await benchmarkPackage(packageName, packageVersion.toString(), new Date(), { + + await execAndThrowErrors(`git checkout -f . && git checkout - && git clean -xdf types`, definitelyTypedPath); + const headBenchmark = (await benchmarkPackage(packageName, packageVersion.toString(), new Date(), { definitelyTypedPath, printSummary: true, iterations: config.benchmarks.languageServiceIterations, progress: false, - upload: false, tsVersion: typeScriptVersionMajorMinor, nProcesses: os.cpus().length, failOnErrors: true, @@ -216,18 +161,17 @@ export async function compareBenchmarks({ maxRunSeconds }))!.summary; - if (latestBenchmark) { + if (baseBenchmark) { console.log("\nmaster"); console.log("======"); - console.log(printSummary([latestBenchmark])); + console.log(printSummary([baseBenchmark])); } console.log("\nHEAD"); console.log("===="); - console.log(printSummary([currentBenchmark])); + console.log(printSummary([headBenchmark])); return [ - latestBenchmarkDocument || - (latestBenchmark && createDocument(latestBenchmark, config.database.packageBenchmarksDocumentSchemaVersion)), - createDocument(currentBenchmark, config.database.packageBenchmarksDocumentSchemaVersion) + baseBenchmark, + headBenchmark, ]; } diff --git a/packages/perf/src/cli/compareTypeScript.ts b/packages/perf/src/cli/compareTypeScript.ts deleted file mode 100644 index 272893ac..00000000 --- a/packages/perf/src/cli/compareTypeScript.ts +++ /dev/null @@ -1,272 +0,0 @@ -import * as os from "os"; -import * as fs from "fs"; -import * as path from "path"; -import { promisify } from "util"; -import { - getDatabase, - DatabaseAccessLevel, - Document, - PackageBenchmarkSummary, - getChangedPackages, - packageIdsAreEqual, - getParsedPackages, - config, - toPackageKey, - parsePackageKey, - createDocument, - assertString, - getSystemInfo, - systemsAreCloseEnough, - JSONDocument, - deserializeSummary, - QueryResult, - TypeScriptComparisonRun, - getSourceVersion -} from "../common"; -import { Container, FeedResponse } from "@azure/cosmos"; -import { benchmarkPackage } from "./benchmark"; -import { getTypeScript } from "../measure/getTypeScript"; -import { postTypeScriptComparisonResults } from "../github/postTypeScriptComparisonResult"; -import { insertDocument } from "../write"; -import { PackageId, AllPackages, formatDependencyVersion } from "@definitelytyped/definitions-parser"; -import { assertDefined } from "@definitelytyped/utils"; -const writeFile = promisify(fs.writeFile); -const currentSystem = getSystemInfo(); - -export interface CompareTypeScriptOptions { - compareAgainstVersion: string; - definitelyTypedPath: string; - packages?: PackageId[]; - maxRunSeconds?: number; - localTypeScriptPath: string; - outFile?: string; - groups?: { [key: string]: QueryResult> }[]; - agentCount?: number; - agentIndex?: number; - upload?: boolean; -} - -export interface CompareTypeScriptCLIArgs { - file?: string; - compareAgainstVersion: string; - definitelyTypedPath: string; - localTypeScriptPath: string; - maxRunSeconds?: number; - outFile?: string; - upload?: boolean; - packages?: PackageId[]; -} - -function convertArgs({ file, ...args }: CompareTypeScriptCLIArgs): CompareTypeScriptOptions { - if (file) { - // tslint:disable-next-line:non-literal-require -- filename comes from Azure artifact - const jsonContent = require(path.resolve(assertString(file, "file"))); - return { - ...args, - ...jsonContent.options, - groups: jsonContent.groups - }; - } - - return args; -} - -export function compareTypeScriptCLI(args: CompareTypeScriptCLIArgs) { - return compareTypeScript(convertArgs(args)); -} - -export async function compareTypeScript({ - compareAgainstVersion, - definitelyTypedPath, - packages, - maxRunSeconds, - localTypeScriptPath: typeScriptPath, - upload, - outFile, - groups, - ...opts -}: CompareTypeScriptOptions) { - const { packageBenchmarks, typeScriptComparisons } = await getDatabase( - upload && !outFile ? DatabaseAccessLevel.Write : DatabaseAccessLevel.Read - ); - const agentIndex = groups && assertDefined(opts.agentIndex, "agentIndex"); - const priorResults = groups - ? new Map>>( - Object.keys(groups[agentIndex!]).map(key => [key, deserializeSummary(groups[agentIndex!][key])] as const) - ) - : await getPackagesToTestAndPriorResults( - packageBenchmarks, - compareAgainstVersion, - definitelyTypedPath, - (await getParsedPackages(definitelyTypedPath)).allPackages, - packages - ); - - if (outFile) { - const agentCount = assertDefined(opts.agentCount, "agentCount"); - const fileContent = JSON.stringify( - { - options: { - compareAgainstVersion, - definitelyTypedPath, - maxRunSeconds, - typeScriptPath, - upload - }, - groups: Array.from(priorResults.keys()).reduce((groups, key, index) => { - const agentIndex = index % agentCount; - if (groups[agentIndex]) { - groups[agentIndex][key] = priorResults.get(key)!; - } else { - groups[agentIndex] = { [key]: priorResults.get(key)! }; - } - return groups; - }, [] as { [key: string]: Document }[]) - }, - undefined, - 2 - ); - - return writeFile(outFile, fileContent, "utf8"); - } - - await getTypeScript(compareAgainstVersion); - const packagesToTest = packages - ? packages.map(p => `${p.name}/v${formatDependencyVersion(p.version)}`) - : Array.from(priorResults.keys()); - const now = new Date(); - const comparisons: [Document, Document][] = []; - for (const packageKey of packagesToTest) { - const { name, version } = parsePackageKey(packageKey); - let priorResult: - | Document - | QueryResult> - | undefined = priorResults.get(packageKey); - const priorResultId = priorResult && "id" in priorResult && priorResult.id; - if (priorResult) { - if (!systemsAreCloseEnough(currentSystem, priorResult.system)) { - console.log(`Skipping ${packageKey} because the system is too different`); - continue; - } - } else { - const benchmark = await benchmarkPackage(name, formatDependencyVersion(version), now, { - definitelyTypedPath, - iterations: config.benchmarks.languageServiceIterations, - tsVersion: compareAgainstVersion, - nProcesses: os.cpus().length, - printSummary: true, - progress: false, - upload: !!upload, - installTypeScript: false, - failOnErrors: false, - maxRunSeconds - }); - - if (!benchmark) { - console.log(`Skipping ${packageKey} because it was deleted`); - continue; - } - - priorResult = { - id: benchmark.id, - ...createDocument(benchmark.summary, config.database.packageBenchmarksDocumentSchemaVersion) - }; - } - - const currentResult = createDocument( - (await benchmarkPackage(name, formatDependencyVersion(version), now, { - definitelyTypedPath, - iterations: config.benchmarks.languageServiceIterations, - tsVersion: "local", - localTypeScriptPath: typeScriptPath, - nProcesses: os.cpus().length, - printSummary: true, - progress: false, - upload: false, - installTypeScript: false, - failOnErrors: false, - maxRunSeconds - }))!.summary, - config.database.packageBenchmarksDocumentSchemaVersion - ); - - comparisons.push([priorResult!, currentResult]); - if (upload && priorResultId) { - const comparison: TypeScriptComparisonRun = { - sourceVersion: getSourceVersion(typeScriptPath), - compareAgainstPackageBenchmarkId: priorResultId, - headBenchmark: currentResult.body - }; - await insertDocument( - comparison, - config.database.typeScriptComparisonsDocumentSchemaVersion, - typeScriptComparisons - ); - } - } - - const comment = await postTypeScriptComparisonResults({ comparisons, dryRun: true }); - console.log(comment); -} - -async function getPackagesToTestAndPriorResults( - container: Container, - typeScriptVersion: string, - definitelyTypedPath: string, - allPackages: AllPackages, - packageList?: PackageId[] -) { - const iterator: AsyncIterable>>> = await container.items - .query({ - query: - `SELECT * FROM ${config.database.packageBenchmarksContainerId} b` + - ` WHERE b.body.typeScriptVersionMajorMinor = @typeScriptVersion` + - (packageList - ? ` AND b.body.packageName IN (${packageList!.map(({ name }) => `"${name}"`).join(", ")})` // Couldn’t figure out how to do this with parameters - : ""), - parameters: [{ name: "@typeScriptVersion", value: typeScriptVersion }] - }) - .getAsyncIterator(); - - const packageKeys = packageList && packageList.map(id => toPackageKey(allPackages.resolve(id))); - const packages = new Map>>(); - - for await (const x of iterator) { - for (const result of x.resources) { - if (!result) continue; - const packageKey = toPackageKey(result.body); - if (packageKeys && !packageKeys.includes(packageKey)) { - continue; - } - - const candidate = packages.get(packageKey); - if (candidate && candidate.createdAt > result.createdAt) { - continue; - } - - const packageId: PackageId = { - name: result.body.packageName, - version: { major: result.body.packageVersionMajor, minor: result.body.packageVersionMinor } - }; - const changedPackages = await getChangedPackages({ diffTo: result.body.sourceVersion, definitelyTypedPath }); - if (changedPackages && changedPackages.some(packageIdsAreEqual(packageId))) { - console.log(`Skipping ${packageKey} because it changed`); - continue; - } else if (changedPackages) { - const dependencies = allPackages.getTypingsData(packageId).dependencies; - if ( - Object.entries(dependencies).some(([name, version]) => - changedPackages.some(packageIdsAreEqual({ name, version })) - ) - ) { - console.log(`Skipping ${packageKey} because one or more of its dependencies changed`); - continue; - } - } - - packages.set(packageKey, result); - } - } - - return packages; -} diff --git a/packages/perf/src/cli/getPackagesToBenchmark.ts b/packages/perf/src/cli/getPackagesToBenchmark.ts deleted file mode 100644 index deaa9f2c..00000000 --- a/packages/perf/src/cli/getPackagesToBenchmark.ts +++ /dev/null @@ -1,108 +0,0 @@ -import * as fs from "fs"; -import { promisify } from "util"; -import { - getDatabase, - getParsedPackages, - DatabaseAccessLevel, - compact, - getSystemInfo, - getChangedPackages, - packageIdsAreEqual, - systemsAreCloseEnough -} from "../common"; -import { BenchmarkPackageOptions } from "./benchmark"; -import { getLatestBenchmark } from "../query"; -import { nAtATime } from "@definitelytyped/utils"; -import { getAffectedPackages, PackageId, formatDependencyVersion } from "@definitelytyped/definitions-parser"; -const writeFile = promisify(fs.writeFile); -const currentSystem = getSystemInfo(); - -export interface GetPackagesToBenchmarkOptions { - definitelyTypedPath: string; - tsVersion: string; - agentCount: number; - outFile: string; -} - -export async function getPackagesToBenchmark({ - definitelyTypedPath, - agentCount, - tsVersion, - outFile -}: GetPackagesToBenchmarkOptions) { - if (tsVersion.split(".").length !== 2) { - throw new Error(`Argument 'typeScriptVersion' must be in format 'major.minor' (e.g. '3.1')`); - } - - const { allPackages } = await getParsedPackages(definitelyTypedPath); - const { packageBenchmarks: container } = await getDatabase(DatabaseAccessLevel.Read); - const changedPackages = await nAtATime(10, allPackages.allTypings(), async typingsData => { - const result = await getLatestBenchmark({ - container, - typeScriptVersionMajorMinor: tsVersion, - packageName: typingsData.id.name, - packageVersion: typingsData.id.version, - matchMinor: allPackages.hasSeparateMinorVersions(typingsData.name) - }); - - // No previous run exists; run one - if (!result) { - return typingsData.id; - } - - // System specs are different; run it - if (!systemsAreCloseEnough(result.system, currentSystem)) { - console.log( - `Queueing ${typingsData.id.name}/v${formatDependencyVersion(typingsData.id.version)} due to system change` - ); - return typingsData.id; - } - - const changedPackages = await getChangedPackages({ diffTo: result.body.sourceVersion, definitelyTypedPath }); - if (!changedPackages) { - return undefined; - } - - if (changedPackages.some(packageIdsAreEqual(typingsData.id))) { - // Package has changed; run it - return typingsData.id; - } - - return undefined; - }); - - const affectedPackages = getAffectedPackages(allPackages, compact(changedPackages)); - const packagesToBenchmark = [...affectedPackages.changedPackages, ...affectedPackages.dependentPackages]; - const groups = packagesToBenchmark.reduce((groups: PackageId[][], typingsData, index) => { - const agentIndex = index % agentCount; - if (groups[agentIndex]) { - groups[agentIndex].push(typingsData.id); - } else { - groups[agentIndex] = [typingsData.id]; - } - return groups; - }, []); - - const benchmarkOptions: Partial = { - definitelyTypedPath, - tsVersion, - upload: true - }; - - await writeFile( - outFile, - JSON.stringify( - { - changedPackageCount: affectedPackages.changedPackages.length, - dependentPackageCount: affectedPackages.dependentPackages.length, - totalPackageCount: packagesToBenchmark.length, - system: currentSystem, - options: benchmarkOptions, - groups - }, - undefined, - 2 - ), - "utf8" - ); -} diff --git a/packages/perf/src/cli/index.ts b/packages/perf/src/cli/index.ts index 2e7f0c82..b9c2f753 100644 --- a/packages/perf/src/cli/index.ts +++ b/packages/perf/src/cli/index.ts @@ -3,10 +3,7 @@ import path from "path"; import process from "process"; import yargs from "yargs"; import { benchmark } from "./benchmark"; -import { getPackagesToBenchmark } from "./getPackagesToBenchmark"; import { compare } from "./compare"; -import { compareTypeScriptCLI } from "./compareTypeScript"; -import { parsePackageKey } from "../common"; const maxRunSeconds = { type: "number", @@ -91,33 +88,6 @@ void yargs }, benchmark ) - .command( - "getPackagesToBenchmark", - "generates a benchmark manifest file to be run by multiple agents", - { - definitelyTypedPath: { - ...definitelyTypedPath, - coerce: (dtPath: string) => (path.isAbsolute(dtPath) ? dtPath : path.resolve(process.cwd(), dtPath)) - }, - agentCount: { - type: "number", - requiresArg: true, - description: "the number of agents that will run the benchmarks", - demandOption: true - }, - tsVersion: { - ...tsVersion, - demandOption: true - }, - outFile: { - type: "string", - requiresArg: true, - description: "the path to the manifest file to be written", - demandOption: true - } - }, - getPackagesToBenchmark - ) .command( "compare", "compare packages modified in a PR to those packages in the main branch", @@ -147,29 +117,6 @@ void yargs }, compare ) - .command( - "compareTypeScript", - "compare packages’ performance between different TypeScript versions", - { - file, - compareAgainstVersion: { - type: "string", - requiresArg: true, - description: "the TypeScript major/minor version to compare against", - demandOption: true - }, - packages: { - type: ("array" as unknown) as undefined, // yargs seems to have a problem with "array" + `coerce` - description: "list of packages to benchmark", - requiresArg: true, - coerce: (packages: string[]) => packages.map(parsePackageKey) - }, - maxRunSeconds, - definitelyTypedPath, - localTypeScriptPath - }, - compareTypeScriptCLI - ) .help().argv; process.on("unhandledRejection", err => { diff --git a/packages/perf/src/common/config.ts b/packages/perf/src/common/config.ts index 759aeb28..2b1a7299 100644 --- a/packages/perf/src/common/config.ts +++ b/packages/perf/src/common/config.ts @@ -4,26 +4,6 @@ export const config = { benchmarks: { languageServiceIterations: 5 }, - database: { - benchmarksDatabaseId: "benchmarks", - packageBenchmarksContainerId: "packageBenchmarks", - packageBenchmarksDocumentSchemaVersion: 5, - typeScriptComparisonsContainerId: "typeScriptComparisons", - typeScriptComparisonsDocumentSchemaVersion: 1, - endpoint: "https://dt-perf.documents.azure.com:443/", - get writeKey() { - return assertDefined( - process.env.DATABASE_WRITE_KEY, - `Required environment variable 'DATABASE_WRITE_KEY' was not set` - ); - }, - get readKey() { - return assertDefined( - process.env.DATABASE_READ_KEY, - `Required environment variable 'DATABASE_READ_KEY' was not set` - ); - } - }, github: { userAgent: "definitely-not-slow", typeScriptBotUsername: "typescript-bot", diff --git a/packages/perf/src/common/db.ts b/packages/perf/src/common/db.ts deleted file mode 100644 index c01dc275..00000000 --- a/packages/perf/src/common/db.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { CosmosClient, Database, Container } from "@azure/cosmos"; -import { config } from "./config"; -import { assertNever } from "@definitelytyped/utils"; - -export const enum DatabaseAccessLevel { - Read = "read", - Write = "write" -} - -function getKey(accessLevel: DatabaseAccessLevel) { - switch (accessLevel) { - case DatabaseAccessLevel.Read: - return config.database.readKey; - case DatabaseAccessLevel.Write: - return config.database.writeKey; - default: - assertNever(accessLevel); - } -} - -export async function getDatabase( - accessLevel: DatabaseAccessLevel -): Promise<{ - database: Database; - packageBenchmarks: Container; - typeScriptComparisons: Container; -}> { - const client = new CosmosClient({ - endpoint: config.database.endpoint, - key: getKey(accessLevel) - }); - - const { database } = await client.databases.createIfNotExists({ - id: config.database.benchmarksDatabaseId - }); - - const { container: packageBenchmarks } = await database.containers.createIfNotExists({ - id: config.database.packageBenchmarksContainerId, - partitionKey: { - paths: ["/body/packageName"] - } - }); - - const { container: typeScriptComparisons } = await database.containers.createIfNotExists({ - id: config.database.typeScriptComparisonsContainerId - }); - - return { database, packageBenchmarks, typeScriptComparisons }; -} diff --git a/packages/perf/src/common/index.ts b/packages/perf/src/common/index.ts index df1b4582..5e13a031 100644 --- a/packages/perf/src/common/index.ts +++ b/packages/perf/src/common/index.ts @@ -1,6 +1,4 @@ export * from "./config"; -export * from "./db"; export * from "./getParsedPackages"; export * from "./types"; export * from "./utils"; -export * from "./support"; diff --git a/packages/perf/src/common/support.ts b/packages/perf/src/common/support.ts deleted file mode 100644 index bcc2e397..00000000 --- a/packages/perf/src/common/support.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { Document, PackageBenchmarkSummary } from "./types"; - -export function supportsMemoryUsage(doc: Document) { - return doc.version >= 4; -} diff --git a/packages/perf/src/common/types.ts b/packages/perf/src/common/types.ts index 3354b400..665958d7 100644 --- a/packages/perf/src/common/types.ts +++ b/packages/perf/src/common/types.ts @@ -87,35 +87,3 @@ export interface SystemInfo { hash: string; nodeVersion: string; } - -export interface Document { - version: number; - createdAt: Date; - system: SystemInfo; - body: T; -} - -type Serialized = { - [K in keyof T]: T[K] extends string - ? string - : T[K] extends number - ? number - : T[K] extends boolean - ? boolean - : T[K] extends Date - ? string - : T[K] extends readonly (infer U)[] - ? Serialized[] - : T[K] extends {} - ? Serialized - : never; -}; - -export interface JSONDocument { - version: number; - createdAt: string; - system: SystemInfo; - body: Serialized; -} - -export type QueryResult = T & { id: string }; diff --git a/packages/perf/src/common/utils.ts b/packages/perf/src/common/utils.ts index 757c4660..f041da4e 100644 --- a/packages/perf/src/common/utils.ts +++ b/packages/perf/src/common/utils.ts @@ -3,7 +3,7 @@ import * as fs from "fs"; import { randomBytes, createHash } from "crypto"; import { promisify } from "util"; import { execSync } from "child_process"; -import { SystemInfo, Document, JSONDocument, PackageBenchmarkSummary, QueryResult, PackageBenchmark } from "./types"; +import { SystemInfo, PackageBenchmarkSummary, PackageBenchmark } from "./types"; import { assertDefined, execAndThrowErrors } from "@definitelytyped/utils"; import { PackageId, @@ -104,31 +104,6 @@ export function isWithin(actual: number, expected: number, tolerance: number): b return Math.abs(getPercentDiff(actual, expected)) <= tolerance; } -export function systemsAreCloseEnough(a: SystemInfo, b: SystemInfo, cpuSpeedTolerance = 0.1): boolean { - if (a.hash === b.hash) { - return true; - } - return ( - a.arch === b.arch && - a.platform === b.platform && - a.nodeVersion === b.nodeVersion && - a.cpus.length === b.cpus.length && - a.cpus.every((cpu, index) => { - const otherCPU = b.cpus[index]; - return cpu.model === otherCPU.model && isWithin(cpu.speed, otherCPU.speed, cpuSpeedTolerance); - }) - ); -} - -export function createDocument(body: T, version: number): Document { - return { - version, - createdAt: new Date(), - system: getSystemInfo(), - body - }; -} - export function isVersionedBenchmark( obj: any ): obj is Pick { @@ -176,20 +151,6 @@ export function toPackageKey( return `${packageId.name}/${formatDependencyVersion(packageId.version)}`; } -export function deserializeSummary( - doc: QueryResult> -): QueryResult>; -export function deserializeSummary(doc: JSONDocument): Document { - return { - ...doc, - createdAt: new Date(doc.createdAt), - body: { - ...doc.body, - batchRunStart: new Date(doc.body.batchRunStart) - } - }; -} - export function getSourceVersion(cwd: string) { return execSync("git rev-parse HEAD", { cwd, encoding: "utf8" }).trim(); } diff --git a/packages/perf/src/github/comment.ts b/packages/perf/src/github/comment.ts index 51afdfa0..35ee8694 100644 --- a/packages/perf/src/github/comment.ts +++ b/packages/perf/src/github/comment.ts @@ -14,9 +14,6 @@ function assertCommentSafe(str: string) { export interface CommentData { overallChange?: OverallChange; - benchmarks: { - createdAt: Date; - }[]; } export function createPerfCommentBody(data: CommentData, body: string): string { diff --git a/packages/perf/src/github/createTable.ts b/packages/perf/src/github/createTable.ts index b7fb0bca..54f25ce8 100644 --- a/packages/perf/src/github/createTable.ts +++ b/packages/perf/src/github/createTable.ts @@ -1,11 +1,11 @@ import table from "markdown-table"; -import { PackageBenchmarkSummary, Document, config, getPercentDiff, compact, supportsMemoryUsage } from "../common"; +import { PackageBenchmarkSummary, config, getPercentDiff, compact } from "../common"; import { metrics, Metric, FormatOptions, SignificanceLevel } from "../analysis"; import { assertNever } from "@definitelytyped/utils"; export function createComparisonTable( - before: Document, - after: Document, + before: PackageBenchmarkSummary, + after: PackageBenchmarkSummary, beforeTitle: string, afterTitle: string ) { @@ -13,9 +13,7 @@ export function createComparisonTable( compact([ ["", beforeTitle, afterTitle, "diff"], ["**Batch compilation**"], - supportsMemoryUsage(before) && supportsMemoryUsage(after) - ? createComparisonRowFromMetric(metrics.memoryUsage, before, after) - : undefined, + createComparisonRowFromMetric(metrics.memoryUsage, before, after), createComparisonRowFromMetric(metrics.typeCount, before, after), createComparisonRowFromMetric(metrics.assignabilityCacheSize, before, after), [], @@ -32,10 +30,10 @@ export function createComparisonTable( after, x => sourceLink( - x.body.completions.worst.identifierText, - x.body.sourceVersion, - x.body.completions.worst.fileName, - x.body.completions.worst.line + x.completions.worst.identifierText, + x.sourceVersion, + x.completions.worst.fileName, + x.completions.worst.line ), undefined, { indent: 1 } @@ -50,35 +48,19 @@ export function createComparisonTable( after, x => sourceLink( - x.body.quickInfo.worst.identifierText, - x.body.sourceVersion, - x.body.quickInfo.worst.fileName, - x.body.quickInfo.worst.line + x.quickInfo.worst.identifierText, + x.sourceVersion, + x.quickInfo.worst.fileName, + x.quickInfo.worst.line ), undefined, { indent: 1 } ), - - // Only show system info if they’re not identical - ...(before.system.hash === after.system.hash - ? [] - : [ - [], - ["**System information**"], - createComparisonRow("Node version", before, after, x => x.system.nodeVersion), - createComparisonRow("CPU count", before, after, x => x.system.cpus.length, undefined, { precision: 0 }), - createComparisonRow("CPU speed", before, after, x => `${x.system.cpus[0].speed / 1000} GHz`), - createComparisonRow("CPU model", before, after, x => x.system.cpus[0].model), - createComparisonRow("CPU Architecture", before, after, x => x.system.arch), - createComparisonRow("Memory", before, after, x => `${format(x.system.totalmem / 2 ** 30)} GiB`), - createComparisonRow("Platform", before, after, x => x.system.platform), - createComparisonRow("Release", before, after, x => x.system.release) - ]) ]) ); } -export function createSingleRunTable(benchmark: Document) { +export function createSingleRunTable(benchmark: PackageBenchmarkSummary) { return table([ ["**Batch compilation**"], // createSingleRunRowFromMetric(metrics.memoryUsage, benchmark), @@ -97,10 +79,10 @@ export function createSingleRunTable(benchmark: Document sourceLink( - x.body.completions.worst.identifierText, - x.body.sourceVersion, - x.body.completions.worst.fileName, - x.body.completions.worst.line + x.completions.worst.identifierText, + x.sourceVersion, + x.completions.worst.fileName, + x.completions.worst.line ), { indent: 1 } ), @@ -113,23 +95,13 @@ export function createSingleRunTable(benchmark: Document sourceLink( - x.body.quickInfo.worst.identifierText, - x.body.sourceVersion, - x.body.quickInfo.worst.fileName, - x.body.quickInfo.worst.line + x.quickInfo.worst.identifierText, + x.sourceVersion, + x.quickInfo.worst.fileName, + x.quickInfo.worst.line ), { indent: 1 } ), - [], - ["**System information**"], - createSingleRunRow("Node version", benchmark, x => x.system.nodeVersion), - createSingleRunRow("CPU count", benchmark, x => x.system.cpus.length, { precision: 0 }), - createSingleRunRow("CPU speed", benchmark, x => `${x.system.cpus[0].speed / 1000} GHz`), - createSingleRunRow("CPU model", benchmark, x => x.system.cpus[0].model), - createSingleRunRow("CPU Architecture", benchmark, x => x.system.arch), - createSingleRunRow("Memory", benchmark, x => `${format(x.system.totalmem / 2 ** 30)} GiB`), - createSingleRunRow("Platform", benchmark, x => x.system.platform), - createSingleRunRow("Release", benchmark, x => x.system.release) ]); } @@ -141,8 +113,8 @@ function sourceLink(text: string, sourceVersion: string, fileName: string, line: function createComparisonRowFromMetric( metric: Metric, - before: Document, - after: Document, + before: PackageBenchmarkSummary, + after: PackageBenchmarkSummary, formatOptions: FormatOptions = {} ) { const beforeValue = metric.getValue(before); @@ -169,7 +141,7 @@ function createComparisonRowFromMetric( function createSingleRunRowFromMetric( metric: Metric, - benchmark: Document, + benchmark: PackageBenchmarkSummary, formatOptions?: FormatOptions ) { return createSingleRunRow(metric.columnName, benchmark, metric.getValue, { @@ -180,9 +152,9 @@ function createSingleRunRowFromMetric( function createComparisonRow( title: string, - a: Document, - b: Document, - getValue: (x: Document) => number | string | undefined, + a: PackageBenchmarkSummary, + b: PackageBenchmarkSummary, + getValue: (x: PackageBenchmarkSummary) => number | string | undefined, diff?: string, formatOptions: FormatOptions = {} ): string[] { @@ -199,8 +171,8 @@ function createComparisonRow( function createSingleRunRow( title: string, - benchmark: Document, - getValue: (x: Document) => number | string | undefined, + benchmark: PackageBenchmarkSummary, + getValue: (x: PackageBenchmarkSummary) => number | string | undefined, formatOptions: FormatOptions = {} ): string[] { const value = getValue(benchmark); diff --git a/packages/perf/src/github/createTablesWithAnalysesMessage.ts b/packages/perf/src/github/createTablesWithAnalysesMessage.ts index d7e434a2..d9a5f813 100644 --- a/packages/perf/src/github/createTablesWithAnalysesMessage.ts +++ b/packages/perf/src/github/createTablesWithAnalysesMessage.ts @@ -1,8 +1,6 @@ import { createComparisonTable, createSingleRunTable } from "./createTable"; import { PackageBenchmarkSummary, - systemsAreCloseEnough, - Document, compact, toPackageKey, packageVersionsAreEqual @@ -10,7 +8,7 @@ import { import { getInterestingMetrics, SignificanceLevel, ComparedMetric } from "../analysis"; export function createTablesWithAnalysesMessage( - pairs: [Document | undefined, Document][], + pairs: [PackageBenchmarkSummary, PackageBenchmarkSummary][], prNumber: number, alwaysWriteHeading = false, alwaysCollapseDetails = false @@ -24,12 +22,11 @@ export function createTablesWithAnalysesMessage( ? createComparisonTable(before, after, getBeforeTitle(before, after), getAfterTitle(before, after, prNumber)) : createSingleRunTable(after), ``, - before && getSystemMismatchMessage(before, after) ].join("\n"); return compact([ pairs.length > 1 || alwaysWriteHeading - ? `### ${after.body.packageName}/v${after.body.packageVersionMajor}.${after.body.packageVersionMinor}` + ? `### ${after.packageName}/v${after.packageVersionMajor}.${after.packageVersionMinor}` : undefined, getIntroMessage(before, after), ``, @@ -43,47 +40,47 @@ export function createTablesWithAnalysesMessage( .join("\n\n"); } -function getDetailsSummaryTitle(comparisonsCount: number, benchmark: Document) { +function getDetailsSummaryTitle(comparisonsCount: number, benchmark: PackageBenchmarkSummary) { let titleStart = "Comparison details"; if (comparisonsCount > 1) { - titleStart += ` for ${toPackageKey(benchmark.body)}`; + titleStart += ` for ${toPackageKey(benchmark)}`; } return titleStart + " 📊"; } -function getBeforeTitle(before: Document, after: Document) { - if (packageVersionsAreEqual(before.body, after.body)) { +function getBeforeTitle(before: PackageBenchmarkSummary, after: PackageBenchmarkSummary) { + if (packageVersionsAreEqual(before, after)) { return "master"; } - return `${before.body.packageVersionMajor}.${before.body.packageVersionMinor}@master`; + return `${before.packageVersionMajor}.${before.packageVersionMinor}@master`; } function getAfterTitle( - before: Document, - after: Document, + before: PackageBenchmarkSummary, + after: PackageBenchmarkSummary, prNumber: number ) { - if (packageVersionsAreEqual(before.body, after.body)) { + if (packageVersionsAreEqual(before, after)) { return `#${prNumber}`; } - return `${after.body.packageVersionMajor}.${after.body.packageVersionMinor} in #${prNumber}`; + return `${after.packageVersionMajor}.${after.packageVersionMinor} in #${prNumber}`; } function getIntroMessage( - before: Document | undefined, - after: Document + before: PackageBenchmarkSummary | undefined, + after: PackageBenchmarkSummary ) { - if (before && packageVersionsAreEqual(before.body, after.body)) { + if (before && packageVersionsAreEqual(before, after)) { return; } if (before) { - return `These typings are for a version of ${before.body.packageName} that doesn’t yet exist on master, so I’ve compared them with v${before.body.packageVersionMajor}.${before.body.packageVersionMinor}.`; + return `These typings are for a version of ${before.packageName} that doesn’t yet exist on master, so I’ve compared them with v${before.packageVersionMajor}.${before.packageVersionMinor}.`; } - return `These typings are for a package that doesn’t yet exist on master, so I don’t have anything to compare against yet! In the future, I’ll be able to compare PRs to ${after.body.packageName} with its source on master.`; + return `These typings are for a package that doesn’t yet exist on master, so I don’t have anything to compare against yet! In the future, I’ll be able to compare PRs to ${after.packageName} with its source on master.`; } -function getLanguageServiceCrashMessage(benchmark: Document) { - if (benchmark.body.languageServiceCrashed) { +function getLanguageServiceCrashMessage(benchmark: PackageBenchmarkSummary) { + if (benchmark.languageServiceCrashed) { return ( `Before we get into it, I need to mention that **the language service crashed** while taking these measurements. ` + `This isn’t your fault—on the contrary, you helped us find a probably TypeScript bug! But, be aware that these results ` + @@ -94,12 +91,6 @@ function getLanguageServiceCrashMessage(benchmark: Document, b: Document) { - return !systemsAreCloseEnough(a.system, b.system) - ? `First off, note that the system varied slightly between these two runs, so you’ll have to take these measurements with a grain of salt.` - : undefined; -} - function getInterestingMetricsMessage(interestingMetrics: readonly ComparedMetric[]) { if (!interestingMetrics.length) { return `It looks like nothing changed too much. I won’t post performance data again unless it gets worse.`; diff --git a/packages/perf/src/github/postDependentsComparisonResults.ts b/packages/perf/src/github/postDependentsComparisonResults.ts index 1774d93a..f3134b1e 100644 --- a/packages/perf/src/github/postDependentsComparisonResults.ts +++ b/packages/perf/src/github/postDependentsComparisonResults.ts @@ -1,11 +1,11 @@ -import { PackageBenchmarkSummary, Document, config, compact } from "../common"; +import { PackageBenchmarkSummary, config, compact } from "../common"; import { getOctokit } from "./getOctokit"; import { createTablesWithAnalysesMessage } from "./createTablesWithAnalysesMessage"; import { createPerfCommentBody } from "./comment"; import { getOverallChangeForComparisons } from "../analysis"; import { assertDefined } from "@definitelytyped/utils"; -type BeforeAndAfter = [Document | undefined, Document]; +type BeforeAndAfter = [PackageBenchmarkSummary, PackageBenchmarkSummary]; export interface PostDependentsComparisonResultOptions { comparisons: BeforeAndAfter[]; @@ -37,7 +37,6 @@ export async function postDependentsComparisonResult({ comparisons, dryRun }: Po body: createPerfCommentBody( { overallChange: getOverallChangeForComparisons(comparisons), - benchmarks: comparisons.map(([, b]) => ({ createdAt: b.createdAt })) }, message ) diff --git a/packages/perf/src/github/postInitialComparisonResults.ts b/packages/perf/src/github/postInitialComparisonResults.ts index d0a59b1f..d3926030 100644 --- a/packages/perf/src/github/postInitialComparisonResults.ts +++ b/packages/perf/src/github/postInitialComparisonResults.ts @@ -1,4 +1,4 @@ -import { PackageBenchmarkSummary, Document, config, compact, findLast } from "../common"; +import { PackageBenchmarkSummary, config, compact, findLast } from "../common"; import { getOctokit } from "./getOctokit"; import { createTablesWithAnalysesMessage } from "./createTablesWithAnalysesMessage"; import { isPerfComment, createPerfCommentBody, getCommentData, CommentData } from "./comment"; @@ -6,7 +6,7 @@ import { OverallChange, getOverallChangeForComparisons } from "../analysis"; import { setLabels } from "./setLabels"; import { assertDefined } from "@definitelytyped/utils"; -type BeforeAndAfter = [Document | undefined, Document]; +type BeforeAndAfter = [PackageBenchmarkSummary, PackageBenchmarkSummary]; export interface PostInitialComparisonResultsOptions { comparisons: BeforeAndAfter[]; @@ -45,7 +45,6 @@ export async function postInitialComparisonResults({ const mostRecentComment = findLast(comments.data, isPerfComment); const commentData: CommentData = { overallChange: currentOverallChange, - benchmarks: comparisons.map(([, b]) => ({ createdAt: b.createdAt })) }; if (mostRecentComment) { const lastOverallChange = getCommentData(mostRecentComment)?.overallChange; @@ -64,7 +63,7 @@ export async function postInitialComparisonResults({ /*alwaysWriteHeader*/ false, /*alwaysCollapseDetails*/ true ), - comparisons[0][1].body.sourceVersion + comparisons[0][1].sourceVersion ); await octokit.issues.createComment({ diff --git a/packages/perf/src/github/postTypeScriptComparisonResult.ts b/packages/perf/src/github/postTypeScriptComparisonResult.ts deleted file mode 100644 index 6de2e5eb..00000000 --- a/packages/perf/src/github/postTypeScriptComparisonResult.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { PackageBenchmarkSummary, Document, toPackageKey } from "../common"; -import { createComparisonTable } from "./createTable"; - -type BeforeAndAfter = [Document, Document]; - -export interface PostTypeScriptComparisonResultsOptions { - comparisons: BeforeAndAfter[]; - dryRun: boolean; -} - -export async function postTypeScriptComparisonResults({ comparisons }: PostTypeScriptComparisonResultsOptions) { - return comparisons - .map(([baseline, head]) => - [ - `### ${toPackageKey(baseline.body)}`, - createComparisonTable(baseline, head, baseline.body.typeScriptVersion, "HEAD") - ].join("\n") - ) - .join("\n\n"); -} diff --git a/packages/perf/src/query/index.ts b/packages/perf/src/query/index.ts deleted file mode 100644 index 76346acd..00000000 --- a/packages/perf/src/query/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Container } from "@azure/cosmos"; -import { DirectoryParsedTypingVersion } from "@definitelytyped/definitions-parser"; -import { config, PackageBenchmarkSummary, Document, QueryResult } from "../common"; - -export interface GetLatestBenchmarkOptions { - container: Container; - packageName: string; - packageVersion: DirectoryParsedTypingVersion; - matchMinor: boolean; - typeScriptVersionMajorMinor: string; -} - -export async function getLatestBenchmark({ - container, - packageName, - packageVersion, - matchMinor, - typeScriptVersionMajorMinor -}: GetLatestBenchmarkOptions): Promise> | undefined> { - const response = await container.items - .query>>({ - query: - `SELECT TOP 1 * FROM ${config.database.packageBenchmarksContainerId} b` + - ` WHERE b.body.packageName = @packageName` + - ` AND b.body.packageVersionMajor = @packageVersionMajor` + - (matchMinor ? ` AND b.body.packageVersionMinor = @packageVersionMinor` : "") + - ` AND b.body.typeScriptVersionMajorMinor = @tsVersion` + - ` ORDER BY b.createdAt DESC`, - parameters: [ - { name: "@packageName", value: packageName }, - { name: "@packageVersionMajor", value: packageVersion.major }, - { name: "@packageVersionMinor", value: packageVersion.minor ?? "" }, - { name: "@tsVersion", value: typeScriptVersionMajorMinor } - ] - }) - .fetchNext(); - return response.resources[0]; -} diff --git a/packages/perf/src/write/index.ts b/packages/perf/src/write/index.ts deleted file mode 100644 index 22c19ae2..00000000 --- a/packages/perf/src/write/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Container, Item } from "@azure/cosmos"; -import { PackageBenchmarkSummary, createDocument, TypeScriptComparisonRun } from "../common"; - -export async function insertDocument( - benchmark: PackageBenchmarkSummary | TypeScriptComparisonRun, - version: number, - container: Container -): Promise { - const response = await container.items.create(createDocument(benchmark, version)); - return response.item; -} diff --git a/packages/perf/test/analysis/metrics.test.ts b/packages/perf/test/analysis/metrics.test.ts index b9426a12..7ec52878 100644 --- a/packages/perf/test/analysis/metrics.test.ts +++ b/packages/perf/test/analysis/metrics.test.ts @@ -1,5 +1,5 @@ import { metrics, SignificanceLevel } from "../../src/analysis"; -import { Document, PackageBenchmarkSummary, config } from "../../src/common"; +import { PackageBenchmarkSummary, config } from "../../src/common"; describe("analysis", () => { describe("metrics", () => { @@ -8,8 +8,8 @@ describe("analysis", () => { 6, 1000, 6000, - { body: { testIdentifierCount: 1000, typeCount: 1000 } } as Document, - { body: { testIdentifierCount: 6000, typeCount: 6000 } } as Document + { testIdentifierCount: 1000, typeCount: 1000 } as PackageBenchmarkSummary, + { testIdentifierCount: 6000, typeCount: 6000 } as PackageBenchmarkSummary ); expect(significance1).toBe(undefined); @@ -18,8 +18,8 @@ describe("analysis", () => { 6, 1000, 6000, - { body: { testIdentifierCount: 1000, typeCount: 1000 } } as Document, - { body: { testIdentifierCount: 1000, typeCount: 6000 } } as Document + { testIdentifierCount: 1000, typeCount: 1000 } as PackageBenchmarkSummary, + { testIdentifierCount: 1000, typeCount: 6000 } as PackageBenchmarkSummary ); expect(significance2).toBe(SignificanceLevel.Warning); @@ -28,8 +28,8 @@ describe("analysis", () => { config.comparison.percentDiffWarningThreshold + 0.01, 1000, 200, - { body: { testIdentifierCount: 1000, typeCount: 1000 } } as Document, - { body: { testIdentifierCount: 5000, typeCount: 2000 } } as Document + { testIdentifierCount: 1000, typeCount: 1000 } as PackageBenchmarkSummary, + { testIdentifierCount: 5000, typeCount: 2000 } as PackageBenchmarkSummary ); expect(significance3).toBe(undefined); @@ -40,8 +40,8 @@ describe("analysis", () => { 6, 100, 600, - { body: { testIdentifierCount: 100, typeCount: 100 } } as Document, - { body: { testIdentifierCount: 600, typeCount: 600 } } as Document + { testIdentifierCount: 100, typeCount: 100 } as PackageBenchmarkSummary, + { testIdentifierCount: 600, typeCount: 600 } as PackageBenchmarkSummary ); expect(significance1).toBe(undefined); diff --git a/yarn.lock b/yarn.lock index 3110f904..9abf05ba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -104,22 +104,6 @@ "@opentelemetry/api" "^1.0.1" tslib "^2.2.0" -"@azure/cosmos@^3.6.3": - version "3.6.3" - resolved "https://registry.yarnpkg.com/@azure/cosmos/-/cosmos-3.6.3.tgz#bb53bde35fbf338160691d2b6a62f47b4a9a1cbb" - integrity sha512-JoCDxl0TnL6EHL4xD3KC9r2bMivK13q1jl7h69wd/YFLlt3aBTTCehtAX+y4alNSENpL53XdRdw/cna0mI2XDw== - dependencies: - "@types/debug" "^4.1.4" - debug "^4.1.1" - fast-json-stable-stringify "^2.0.0" - node-abort-controller "^1.0.4" - node-fetch "^2.6.0" - os-name "^3.1.0" - priorityqueuejs "^1.0.0" - semaphore "^1.0.5" - tslib "^1.10.0" - uuid "^3.3.2" - "@azure/functions@^1.2.3": version "1.2.3" resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-1.2.3.tgz#65765837e7319eedffbf8a971cb2f78d4e043d54" @@ -1773,11 +1757,6 @@ resolved "https://registry.yarnpkg.com/@types/command-exists/-/command-exists-1.2.0.tgz#d97e0ed10097090e4ab0367ed425b0312fad86f3" integrity sha512-ugsxEJfsCuqMLSuCD4PIJkp5Uk2z6TCMRCgYVuhRo5cYQY3+1xXTQkSlPtkpGHuvWMjS2KTeVQXxkXRACMbM6A== -"@types/debug@^4.1.4": - version "4.1.5" - resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd" - integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ== - "@types/fs-extra@^5.0.2": version "5.1.0" resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-5.1.0.tgz#2a325ef97901504a3828718c390d34b8426a10a1" @@ -6367,11 +6346,6 @@ node-abi@^2.21.0: dependencies: semver "^5.4.1" -node-abort-controller@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-1.0.4.tgz#4095e41d58b2fae169d2f9892904d603e11c7a39" - integrity sha512-7cNtLKTAg0LrW3ViS2C7UfIzbL3rZd8L0++5MidbKqQVJ8yrH6+1VRSHl33P0ZjBTbOJd37d9EYekvHyKkB0QQ== - node-addon-api@^3.0.0: version "3.2.1" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" @@ -7120,11 +7094,6 @@ pretty-format@^25.2.1, pretty-format@^25.5.0: ansi-styles "^4.0.0" react-is "^16.12.0" -priorityqueuejs@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/priorityqueuejs/-/priorityqueuejs-1.0.0.tgz#2ee4f23c2560913e08c07ce5ccdd6de3df2c5af8" - integrity sha1-LuTyPCVgkT4IwHzlzN1t498sWvg= - process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" @@ -7622,11 +7591,6 @@ saxes@^3.1.9: dependencies: xmlchars "^2.1.1" -semaphore@^1.0.5: - version "1.1.0" - resolved "https://registry.yarnpkg.com/semaphore/-/semaphore-1.1.0.tgz#aaad8b86b20fe8e9b32b16dc2ee682a8cd26a8aa" - integrity sha512-O4OZEaNtkMd/K0i6js9SL+gqy0ZCBMgUvlSqHKi4IBdjhe7wB8pwztUk1BbZ1fmrvpwFrPbHzqd2w5pTcJH6LA== - "semver@2 >=2.2.1 || 3.x || 4 || 5 || 7", semver@^7.1.1, semver@^7.1.3, semver@^7.2.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5: version "7.3.5" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7"