casraf/DefinitelyTyped-tools
1
0
Fork 0
mirror of https://github.com/chenasraf/DefinitelyTyped-tools.git synced 2026-05-17 17:48:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update imports for publisher

Browse Source
This commit is contained in:
Andrew Branch
2020-03-04 17:22:59 -08:00
parent c8a7c5dace
commit ed92171619
102 changed files with 572 additions and 5514 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -1,4 +1,5 @@
node_modules
.DS_Store
*.log
dist
dist
*.tsbuildinfo

3
jest.config.js
View File

@@ -3,7 +3,8 @@ module.exports = {
testEnvironment: "node",
globals: {
"ts-jest": {
tsConfig: "./tsconfig.test.json"
tsConfig: "./tsconfig.test.json",
diagnostics: false
}
}
};

17
packages/definitions-parser/.vscode/launch.json vendored Normal file
View File

@@ -0,0 +1,17 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "node",
"request": "launch",
"name": "parse-definitions.js",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/bin/parse-definitions.js",
"args": [],
"sourceMaps": true
}
]
}

16
packages/definitions-parser/src/dataFile.ts Normal file
View File

@@ -0,0 +1,16 @@
import { ensureDir } from "fs-extra";
import { writeJson, joinPaths, readFileAndWarn } from "@definitelytyped/utils";
import { dataDirPath } from "./lib/settings";
export function readDataFile(generatedBy: string, fileName: string): Promise<object> {
return readFileAndWarn(generatedBy, dataFilePath(fileName));
}
export async function writeDataFile(filename: string, content: {}, formatted = true): Promise<void> {
await ensureDir(dataDirPath);
await writeJson(dataFilePath(filename), content, formatted);
}
export function dataFilePath(filename: string): string {
return joinPaths(dataDirPath, filename);
}

16
packages/definitions-parser/src/getDefinitelyTyped.ts
View File

@@ -8,7 +8,20 @@ import {
} from "@definitelytyped/utils";
import { dataDirPath, definitelyTypedZipUrl } from "./lib/settings";
import { ParseDefinitionsOptions } from "./lib/common";
/** Settings that may be determined dynamically. */
export interface ParseDefinitionsOptions {
/**
* e.g. '../DefinitelyTyped'
* This is overridden to `cwd` when running the tester, as that is run from within DefinitelyTyped.
* If undefined, downloads instead.
*/
readonly definitelyTypedPath: string | undefined;
/** Whether to show progress bars. Good when running locally, bad when running in CI. */
readonly progress: boolean;
/** Disabled in CI since it has problems logging errors from other processes. */
readonly parseInParallel: boolean;
}
export async function getDefinitelyTyped(options: ParseDefinitionsOptions, log: LoggerWithErrors): Promise<FS> {
if (options.definitelyTypedPath === undefined) {
@@ -27,3 +40,4 @@ export async function getDefinitelyTyped(options: ParseDefinitionsOptions, log:
export function getLocallyInstalledDefinitelyTyped(path: string): FS {
return new DiskFS(`${path}/`);
}

2
packages/definitions-parser/src/index.ts
View File

@@ -1,2 +1,4 @@
export * from "./dataFile";
export * from "./getDefinitelyTyped";
export * from "./parseDefinitions";
export * from "./packages";

44
packages/definitions-parser/src/lib/common.ts
View File

@@ -1,44 +0,0 @@
import { ensureDir, readJson } from "fs-extra";
import { writeJson, joinPaths } from "@definitelytyped/utils";
import { dataDirPath } from "./settings";
/** Settings that may be determined dynamically. */
export interface ParseDefinitionsOptions {
/**
* e.g. '../DefinitelyTyped'
* This is overridden to `cwd` when running the tester, as that is run from within DefinitelyTyped.
* If undefined, downloads instead.
*/
readonly definitelyTypedPath: string | undefined;
/** Whether to show progress bars. Good when running locally, bad when running in CI. */
readonly progress: boolean;
/** Disabled in CI since it has problems logging errors from other processes. */
readonly parseInParallel: boolean;
}
/** Options for running locally. */
export const defaultLocalOptions: ParseDefinitionsOptions = { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true };
export const defaultCIOptions: ParseDefinitionsOptions = { definitelyTypedPath: undefined, progress: false, parseInParallel: false };
export function readDataFile(generatedBy: string, fileName: string): Promise<object> {
return readFileAndWarn(generatedBy, dataFilePath(fileName));
}
/** If a file doesn't exist, warn and tell the step it should have been generated by. */
export async function readFileAndWarn(generatedBy: string, filePath: string): Promise<object> {
try {
return await readJson(filePath);
} catch (e) {
console.error(`Run ${generatedBy} first!`);
throw e;
}
}
export async function writeDataFile(filename: string, content: {}, formatted = true): Promise<void> {
await ensureDir(dataDirPath);
await writeJson(dataFilePath(filename), content, formatted);
}
export function dataFilePath(filename: string): string {
return joinPaths(dataDirPath, filename);
}

2
packages/definitions-parser/src/lib/definition-parser.ts
View File

@@ -10,7 +10,7 @@ import {
TypingsDataRaw,
TypingsVersionsRaw,
TypingVersion,
} from "./packages";
} from "../packages";
import { dependenciesWhitelist } from "./settings";
import { FS, split, mapDefined, filter, sort, withoutStart, computeHash, hasWindowsSlashes, join, flatMap, unique, unmangleScopedPackage } from "@definitelytyped/utils";

3
packages/definitions-parser/src/lib/settings.ts
View File

@@ -11,3 +11,6 @@ export const definitelyTypedZipUrl = "https://codeload.github.com/DefinitelyType
export const dependenciesWhitelist: ReadonlySet<string> =
new Set(readFileSync(joinPaths(root, "dependenciesWhitelist.txt")).split(/\r?\n/));
/** Note: this is 'types' and not '@types' */
export const scopeName = "types";

14
packages/definitions-parser/src/lib/packages.ts → packages/definitions-parser/src/packages.ts
View File

@@ -1,12 +1,8 @@
import assert = require("assert");
import { AllTypeScriptVersion, Author, TypeScriptVersion } from "definitelytyped-header-parser";
import { FS } from "../getDefinitelyTyped";
import { assertSorted, joinPaths, mapValues, unmangleScopedPackage } from "../util/util";
import { readDataFile } from "./common";
import { outputDirPath, scopeName } from "./settings";
import { compare as compareSemver, Semver } from "./versions";
import { AllTypeScriptVersion, Author, TypeScriptVersion } from "@definitelytyped/header-parser";
import { FS, mapValues, assertSorted, unmangleScopedPackage, joinPaths, Semver } from "@definitelytyped/utils";
import { readDataFile } from "./dataFile";
import { outputDirPath, scopeName } from "./lib/settings";
export class AllPackages {
static async read(dt: FS): Promise<AllPackages> {
@@ -451,7 +447,7 @@ export class TypingsVersions {
* Sorted from latest to oldest so that we publish the current version first.
* This is important because older versions repeatedly reset the "latest" tag to the current version.
*/
this.versions = Array.from(versionMappings.keys()).sort(compareSemver).reverse();
this.versions = Array.from(versionMappings.keys()).sort(Semver.compare).reverse();
this.map = new Map(this.versions.map(version => {
const dataKey = versionMappings.get(version)!;

6
packages/definitions-parser/src/parseDefinitions.ts
View File

@@ -1,15 +1,15 @@
import { FS, LoggerWithErrors, filterNAtATimeOrdered, runWithChildProcesses } from "@definitelytyped/utils";
import { writeDataFile } from "./lib/common";
import { writeDataFile } from "./dataFile";
import { getTypingInfo } from "./lib/definition-parser";
import { definitionParserWorkerFilename } from "./lib/definition-parser-worker";
import { AllPackages, readNotNeededPackages, typesDataFilename, TypingsVersionsRaw } from "./lib/packages";
import { AllPackages, readNotNeededPackages, typesDataFilename, TypingsVersionsRaw } from "./packages";
export interface ParallelOptions {
readonly nProcesses: number;
readonly definitelyTypedPath: string;
}
export default async function parseDefinitions(dt: FS, parallel: ParallelOptions | undefined, log: LoggerWithErrors): Promise<AllPackages> {
export async function parseDefinitions(dt: FS, parallel: ParallelOptions | undefined, log: LoggerWithErrors): Promise<AllPackages> {
log.info("Parsing definitions...");
const typesFS = dt.subDir("types");
const packageNames = await filterNAtATimeOrdered(parallel ? parallel.nProcesses : 1, typesFS.readdir(), name => typesFS.isDirectory(name));

10
packages/definitions-parser/test/tsconfig.json Normal file
View File

@@ -0,0 +1,10 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"types": ["node", "jest"]
},
"references": [
{ "path": ".." },
{ "path": "../../definitions-parser" }
]
}

2
packages/header-parser/test/tsconfig.json
View File

@@ -1,5 +1,5 @@
{
"extends": "../tsconfig.json",
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"types": ["node", "jest"]
},

3
packages/header-parser/tsconfig.json
View File

@@ -2,8 +2,7 @@
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src",
"noFallthroughCasesInSwitch": true
"rootDir": "src"
}
}

1570
packages/header-parser/tsconfig.tsbuildinfo
View File

File diff suppressed because it is too large Load Diff

0
packages/types-publisher/.editorconfig → packages/publisher/.editorconfig
View File

0
packages/types-publisher/.gitignore → packages/publisher/.gitignore vendored
View File

0
packages/types-publisher/.travis.yml → packages/publisher/.travis.yml
View File

11
packages/types-publisher/.vscode/launch.json → packages/publisher/.vscode/launch.json vendored
View File

@@ -4,15 +4,6 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "node",
"request": "launch",
"name": "parse-definitions.js",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/bin/parse-definitions.js",
"args": [],
"sourceMaps": true
},
{
"type": "node",
"request": "launch",
@@ -32,4 +23,4 @@
"sourceMaps": true
}
]
}
}

0
packages/types-publisher/CODEOWNERS → packages/publisher/CODEOWNERS
View File

0
packages/types-publisher/DEBUGGING.md → packages/publisher/DEBUGGING.md
View File

0
packages/types-publisher/IISNode.yml → packages/publisher/IISNode.yml
View File

0
packages/types-publisher/README.md → packages/publisher/README.md
View File

48
packages/types-publisher/package-lock.json → packages/publisher/package-lock.json generated
View File

@@ -595,15 +595,6 @@
"@babel/types": "^7.3.0"
}
},
"@types/charm": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@types/charm/-/charm-1.0.1.tgz",
"integrity": "sha512-F9OalGhk60p/DnACfa1SWtmVTMni0+w9t/qfb5Bu7CsurkEjZFN7Z+ii/VGmYpaViPz7o3tBahRQae9O7skFlQ==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/color-name": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
@@ -1485,14 +1476,6 @@
"supports-color": "^5.3.0"
}
},
"charm": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/charm/-/charm-1.0.2.tgz",
"integrity": "sha1-it02cVOm2aWBMxBSxAkJkdqZXjU=",
"requires": {
"inherits": "^2.0.1"
}
},
"ci-info": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
@@ -2046,9 +2029,9 @@
}
},
"typescript": {
"version": "3.8.2",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.2.tgz",
"integrity": "sha512-EgOVgL/4xfVrCMbhYKUQTdF37SQn4Iw73H5BgCrF1Abdun7Kwy/QZsE/ssAy0y4LxBbvua3PIbFsbRczWWnDdQ=="
"version": "3.8.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz",
"integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w=="
},
"yargs": {
"version": "12.0.5",
@@ -2081,17 +2064,17 @@
}
},
"dtslint": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/dtslint/-/dtslint-3.1.0.tgz",
"integrity": "sha512-L1j0+keWRAMPYTk0986UVhCnvwLvgnpSU/ECKzY7UCwiirqJLi3v2KpSlPbqW7Xl8G3D5/64oICNp5EZuvfTvg==",
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/dtslint/-/dtslint-3.3.0.tgz",
"integrity": "sha512-fQ1Q8Rvnz8ejiUe081qjYYeXi8XuNw8cR8dKv57FwZ5HG3KG541eOE3MeyBFbkZZAIZutl7KHcqhRXj0eaKg0g==",
"requires": {
"definitelytyped-header-parser": "3.8.2",
"definitelytyped-header-parser": "3.9.0",
"dts-critic": "^3.0.0",
"fs-extra": "^6.0.1",
"json-stable-stringify": "^1.0.1",
"strip-json-comments": "^2.0.1",
"tslint": "5.14.0",
"typescript": "^3.9.0-dev.20200226",
"typescript": "^3.9.0-dev.20200304",
"yargs": "^15.1.0"
},
"dependencies": {
@@ -2137,6 +2120,15 @@
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"definitelytyped-header-parser": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/definitelytyped-header-parser/-/definitelytyped-header-parser-3.9.0.tgz",
"integrity": "sha512-slbwZ5h5lasB12t+9EAGYr060aCMqEXp6cwD7CoTriK40HNDYU56/XQ6S4sbjBK8ReGRMnB/uDx0elKkb4kuQA==",
"requires": {
"@types/parsimmon": "^1.3.0",
"parsimmon": "^1.2.0"
}
},
"diff": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
@@ -9931,9 +9923,9 @@
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
},
"typescript": {
"version": "3.9.0-dev.20200226",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.0-dev.20200226.tgz",
"integrity": "sha512-IvsxGC/DfR3cUsTbcZzS/15zTEk7G3Wau+mjqwwo0jXmayMzPrRmN+N8rY/8ddxn3wHxuD/YG0gSrBam3rBkZg=="
"version": "3.9.0-dev.20200304",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.0-dev.20200304.tgz",
"integrity": "sha512-eUip/GgJmjp4qtHiJDxVhE5SDDiPzBUg7KBAFUgb7HgL/tv10JAHej7fnS1i+7xrq1eDtbkJyPaYOVnhL9db7Q=="
},
"underscore": {
"version": "1.9.2",

7
packages/types-publisher/package.json → packages/publisher/package.json
View File

@@ -6,19 +6,19 @@
"types": "./bin/index.d.ts",
"description": "Publish DefinitelyTyped definitions to NPM",
"dependencies": {
"@definitelytyped/definitions-parser": "^1.0.0",
"@definitelytyped/header-parser": "^0.0.0",
"@definitelytyped/utils": "^1.0.0",
"@octokit/rest": "^16.1.0",
"@types/tar-stream": "^1.6.0",
"adal-node": "^0.1.22",
"applicationinsights": "^1.0.7",
"azure-keyvault": "^3.0.4",
"azure-storage": "^2.0.0",
"charm": "^1.0.2",
"definitelytyped-header-parser": "3.8.2",
"dtslint": "latest",
"fs-extra": "4.0.0",
"fstream": "^1.0.12",
"longjohn": "^0.2.11",
"moment": "^2.18.1",
"npm": "^6.13.4",
"npm-registry-client": "^8.1.0",
"oboe": "^2.1.3",
@@ -30,7 +30,6 @@
"yargs": "^8.0.2"
},
"devDependencies": {
"@types/charm": "^1.0.0",
"@types/fs-extra": "4.0.0",
"@types/jest": "^23.3.9",
"@types/mz": "^0.0.31",

0
packages/types-publisher/server.js → packages/publisher/server.js
View File

0
packages/types-publisher/src/.vscode/launch.json → packages/publisher/src/.vscode/launch.json vendored
View File

13
packages/types-publisher/src/calculate-versions.ts → packages/publisher/src/calculate-versions.ts
View File

@@ -1,15 +1,14 @@
import assert = require("assert");
import { FS, getDefinitelyTyped } from "./get-definitely-typed";
import { Options, writeDataFile } from "./lib/common";
import { defaultLocalOptions } from "./lib/common";
import { CachedNpmInfoClient, NpmInfoVersion, UncachedNpmInfoClient, withNpmCache } from "./lib/npm-client";
import { AllPackages, NotNeededPackage, TypingsData } from "./lib/packages";
import { ChangedPackages, ChangedPackagesJson, ChangedTypingJson, Semver, versionsFilename } from "./lib/versions";
import { loggerWithErrors, LoggerWithErrors } from "./util/logging";
import { assertDefined, best, logUncaughtErrors, mapDefined, mapDefinedAsync } from "./util/util";
import { ChangedPackages, ChangedPackagesJson, ChangedTypingJson, versionsFilename } from "./lib/versions";
import { getDefinitelyTyped, AllPackages, TypingsData, NotNeededPackage, writeDataFile } from "@definitelytyped/definitions-parser";
import { assertDefined, best, logUncaughtErrors, mapDefined, mapDefinedAsync, loggerWithErrors, FS, LoggerWithErrors, Semver } from "@definitelytyped/utils";
if (!module.parent) {
const log = loggerWithErrors()[0];
logUncaughtErrors(async () => calculateVersions(await getDefinitelyTyped(Options.defaults, log), new UncachedNpmInfoClient(), log));
logUncaughtErrors(async () => calculateVersions(await getDefinitelyTyped(defaultLocalOptions, log), new UncachedNpmInfoClient(), log));
}
export default async function calculateVersions(

18
packages/types-publisher/src/check-parse-results.ts → packages/publisher/src/check-parse-results.ts
View File

@@ -1,18 +1,15 @@
import { FS, getDefinitelyTyped } from "./get-definitely-typed";
import { Options } from "./lib/common";
import { defaultLocalOptions } from "./lib/common";
import { NpmInfoRawVersions, NpmInfoVersion, UncachedNpmInfoClient } from "./lib/npm-client";
import { AllPackages, formatTypingVersion, TypingsData, TypingVersion } from "./lib/packages";
import { Semver } from "./lib/versions";
import { Logger, logger, loggerWithErrors, writeLog } from "./util/logging";
import { assertDefined, best, logUncaughtErrors, mapDefined, nAtATime } from "./util/util";
import { getDefinitelyTyped, AllPackages, formatTypingVersion, TypingsData, TypingVersion, ParseDefinitionsOptions } from "@definitelytyped/definitions-parser";
import { assertDefined, best, logUncaughtErrors, mapDefined, nAtATime, loggerWithErrors, FS, logger, writeLog, Logger, Semver } from "@definitelytyped/utils";
if (!module.parent) {
const log = loggerWithErrors()[0];
logUncaughtErrors(
async () => checkParseResults(true, await getDefinitelyTyped(Options.defaults, log), Options.defaults, new UncachedNpmInfoClient()));
async () => checkParseResults(true, await getDefinitelyTyped(defaultLocalOptions, log), defaultLocalOptions, new UncachedNpmInfoClient()));
}
export default async function checkParseResults(includeNpmChecks: boolean, dt: FS, options: Options, client: UncachedNpmInfoClient): Promise<void> {
export default async function checkParseResults(includeNpmChecks: boolean, dt: FS, options: ParseDefinitionsOptions, client: UncachedNpmInfoClient): Promise<void> {
const allPackages = await AllPackages.read(dt);
const [log, logResult] = logger();
@@ -34,11 +31,10 @@ export default async function checkParseResults(includeNpmChecks: boolean, dt: F
}
if (includeNpmChecks) {
await nAtATime(10, allPackages.allTypings(), pkg => checkNpm(pkg, log, dependedOn, client), {
await nAtATime(10, allPackages.allTypings(), pkg => checkNpm(pkg, log, dependedOn, client), options.progress ? {
name: "Checking for typed packages...",
flavor: pkg => pkg.desc,
options,
});
} : undefined);
}
await writeLog("conflicts.md", logResult());

0
packages/types-publisher/src/clean.ts → packages/publisher/src/clean.ts
View File

15
packages/types-publisher/src/crawl-npm.ts → packages/publisher/src/crawl-npm.ts
View File

@@ -2,26 +2,25 @@ import assert = require("assert");
import oboe = require("oboe");
import { packageHasTypes } from "./check-parse-results";
import { Options, writeDataFile } from "./lib/common";
import { UncachedNpmInfoClient } from "./lib/npm-client";
import { npmRegistry } from "./lib/settings";
import ProgressBar, { strProgress } from "./util/progress";
import { filterNAtATimeOrdered, logUncaughtErrors } from "./util/util";
import { filterNAtATimeOrdered, logUncaughtErrors, ProgressBar, strProgress } from "@definitelytyped/utils";
import { defaultLocalOptions } from "./lib/common";
import { ParseDefinitionsOptions, writeDataFile } from "@definitelytyped/definitions-parser";
if (!module.parent) {
logUncaughtErrors(main(Options.defaults));
logUncaughtErrors(main(defaultLocalOptions));
}
/** Prints out every package on NPM with 'types'. */
async function main(options: Options): Promise<void> {
async function main(options: ParseDefinitionsOptions): Promise<void> {
const all = await allNpmPackages();
await writeDataFile("all-npm-packages.json", all);
const client = new UncachedNpmInfoClient();
const allTyped = await filterNAtATimeOrdered(10, all, pkg => packageHasTypes(pkg, client), {
const allTyped = await filterNAtATimeOrdered(10, all, pkg => packageHasTypes(pkg, client), options.progress ? {
name: "Checking for types...",
flavor: (name, isTyped) => isTyped ? name : undefined,
options,
});
} : undefined);
await writeDataFile("all-typed-packages.json", allTyped);
console.log(allTyped.join("\n"));
console.log(`Found ${allTyped.length} typed packages.`);

10
packages/types-publisher/src/create-search-index.ts → packages/publisher/src/create-search-index.ts
View File

@@ -1,11 +1,9 @@
import * as yargs from "yargs";
import { getDefinitelyTyped } from "./get-definitely-typed";
import { Options, writeDataFile } from "./lib/common";
import { getDefinitelyTyped, AllPackages, writeDataFile, TypingsData } from "@definitelytyped/definitions-parser";
import { loggerWithErrors, logUncaughtErrors } from "@definitelytyped/utils";
import { UncachedNpmInfoClient } from "./lib/npm-client";
import { AllPackages, TypingsData } from "./lib/packages";
import { loggerWithErrors } from "./util/logging";
import { logUncaughtErrors } from "./util/util";
import { defaultLocalOptions } from "./lib/common";
if (!module.parent) {
const log = loggerWithErrors()[0];
@@ -14,7 +12,7 @@ if (!module.parent) {
logUncaughtErrors(doSingle(single, new UncachedNpmInfoClient()));
} else {
logUncaughtErrors(
async () => createSearchIndex(await AllPackages.read(await getDefinitelyTyped(Options.defaults, log)), new UncachedNpmInfoClient()));
async () => createSearchIndex(await AllPackages.read(await getDefinitelyTyped(defaultLocalOptions, log)), new UncachedNpmInfoClient()));
}
}

14
packages/types-publisher/src/full.ts → packages/publisher/src/full.ts
View File

@@ -5,17 +5,15 @@ import calculateVersions from "./calculate-versions";
import { clean } from "./clean";
import createSearchIndex from "./create-search-index";
import generatePackages from "./generate-packages";
import { getDefinitelyTyped } from "./get-definitely-typed";
import { Options } from "./lib/common";
import { UncachedNpmInfoClient } from "./lib/npm-client";
import parseDefinitions from "./parse-definitions";
import publishPackages from "./publish-packages";
import publishRegistry from "./publish-registry";
import uploadBlobsAndUpdateIssue from "./upload-blobs";
import { Fetcher } from "./util/io";
import { LoggerWithErrors, loggerWithErrors } from "./util/logging";
import { assertDefined, currentTimeStamp, logUncaughtErrors, numberOfOsProcesses } from "./util/util";
import { getDefinitelyTyped, parseDefinitions, ParseDefinitionsOptions } from "@definitelytyped/definitions-parser";
import { Fetcher, logUncaughtErrors, loggerWithErrors, LoggerWithErrors, assertDefined } from "@definitelytyped/utils";
import { currentTimeStamp, numberOfOsProcesses } from "./util/util";
import validate from "./validate";
import { defaultLocalOptions } from "./lib/common";
if (!module.parent) {
if (process.env.APPINSIGHTS_INSTRUMENTATIONKEY) {
@@ -23,7 +21,7 @@ if (!module.parent) {
appInsights.start();
}
const dry = !!yargs.argv.dry;
logUncaughtErrors(full(dry, currentTimeStamp(), process.env.GH_API_TOKEN || "", new Fetcher(), Options.defaults, loggerWithErrors()[0]));
logUncaughtErrors(full(dry, currentTimeStamp(), process.env.GH_API_TOKEN || "", new Fetcher(), defaultLocalOptions, loggerWithErrors()[0]));
}
export default async function full(
@@ -31,7 +29,7 @@ export default async function full(
timeStamp: string,
githubAccessToken: string,
fetcher: Fetcher,
options: Options,
options: ParseDefinitionsOptions,
log: LoggerWithErrors): Promise<void> {
const infoClient = new UncachedNpmInfoClient();
clean();

19
packages/types-publisher/src/generate-packages.ts → packages/publisher/src/generate-packages.ts
View File

@@ -1,20 +1,15 @@
import { makeTypesVersionsForPackageJson } from "definitelytyped-header-parser";
import { makeTypesVersionsForPackageJson } from "@definitelytyped/header-parser";
import { emptyDir, mkdir, mkdirp, readFileSync } from "fs-extra";
import * as path from "path";
import * as yargs from "yargs";
import { FS, getDefinitelyTyped } from "./get-definitely-typed";
import { Options, Registry } from "./lib/common";
import { CachedNpmInfoClient, UncachedNpmInfoClient, withNpmCache } from "./lib/npm-client";
import {
AllPackages, AnyPackage, DependencyVersion, getFullNpmName, License, NotNeededPackage, PackageJsonDependency, TypingsData,
} from "./lib/packages";
import { defaultLocalOptions, Registry } from "./lib/common";
import { CachedNpmInfoClient, UncachedNpmInfoClient, withNpmCache, skipBadPublishes } from "./lib/npm-client";
import { outputDirPath, sourceBranch } from "./lib/settings";
import { ChangedPackages, readChangedPackages, skipBadPublishes } from "./lib/versions";
import { writeFile } from "./util/io";
import { logger, Logger, loggerWithErrors, writeLog } from "./util/logging";
import { writeTgz } from "./util/tgz";
import { assertNever, joinPaths, logUncaughtErrors, sortObjectKeys } from "./util/util";
import { assertNever, joinPaths, logUncaughtErrors, sortObjectKeys, loggerWithErrors, FS, logger, writeLog, writeFile, Logger } from "@definitelytyped/utils";
import { getDefinitelyTyped, AllPackages, TypingsData, NotNeededPackage, AnyPackage, PackageJsonDependency, getFullNpmName, DependencyVersion, License } from "@definitelytyped/definitions-parser";
import { readChangedPackages, ChangedPackages } from "./lib/versions";
const mitLicense = readFileSync(joinPaths(__dirname, "..", "LICENSE"), "utf-8");
@@ -22,7 +17,7 @@ if (!module.parent) {
const tgz = !!yargs.argv.tgz;
logUncaughtErrors(async () => {
const log = loggerWithErrors()[0];
const dt = await getDefinitelyTyped(Options.defaults, log);
const dt = await getDefinitelyTyped(defaultLocalOptions, log);
const allPackages = await AllPackages.read(dt);
await generatePackages(dt, allPackages, await readChangedPackages(allPackages), tgz);
});

7
packages/types-publisher/src/index.ts → packages/publisher/src/index.ts
View File

@@ -1,12 +1,5 @@
export { getDefinitelyTyped } from "./get-definitely-typed";
export { withNpmCache, CachedNpmInfoClient, NpmPublishClient, UncachedNpmInfoClient } from "./lib/npm-client";
export { AllPackages } from "./lib/packages";
export { clean } from "./clean";
export { getLatestTypingVersion } from "./calculate-versions";
export { default as parseDefinitions } from "./parse-definitions";
export { parseNProcesses } from "./tester/test-runner";
export { consoleLogger, loggerWithErrors } from "./util/logging";
export { logUncaughtErrors, nAtATime } from "./util/util";
export { updateLatestTag, updateTypeScriptVersionTags } from "./lib/package-publisher";

5
packages/types-publisher/src/lib/azure-container.ts → packages/publisher/src/lib/azure-container.ts
View File

@@ -1,13 +1,10 @@
import { BlobService, common, createBlobService, ErrorOrResponse, ErrorOrResult } from "azure-storage";
import * as fs from "fs";
import * as https from "https";
import { streamDone, streamOfString, stringOfStream } from "../util/io";
import { gzip, unGzip } from "../util/tgz";
import { parseJson } from "../util/util";
import { getSecret, Secret } from "./secrets";
import { azureContainer, azureStorageAccount } from "./settings";
import { streamOfString, streamDone, stringOfStream, parseJson } from "@definitelytyped/utils";
export default class BlobWriter {
static async create(): Promise<BlobWriter> {

5
packages/types-publisher/src/lib/blob-uploader.ts → packages/publisher/src/lib/blob-uploader.ts
View File

@@ -1,10 +1,7 @@
import assert = require("assert");
import { readdir } from "fs-extra";
import * as path from "path";
import { Logger, logger, logPath, writeLog } from "../util/logging";
import { joinPaths, unique } from "../util/util";
import { logger, writeLog, logPath, Logger, joinPaths, unique } from "@definitelytyped/utils";
import BlobWriter, { urlOfBlob } from "./azure-container";
const maxNumberOfOldLogsDirectories = 5;

22
packages/publisher/src/lib/common.ts Normal file
View File

@@ -0,0 +1,22 @@
import { ParseDefinitionsOptions } from "@definitelytyped/definitions-parser";
if (process.env.LONGJOHN) {
console.log("=== USING LONGJOHN ===");
const longjohn = require("longjohn") as { async_trace_limit: number }; // tslint:disable-line no-var-requires
longjohn.async_trace_limit = -1; // unlimited
}
/** Which registry to publish to */
export enum Registry {
/** types-registry and @types/* on NPM */
NPM,
/** @definitelytyped/types-registry and @types/* on Github */
Github,
}
export interface TesterOptions extends ParseDefinitionsOptions {
// Tester can only run on files stored on-disk.
readonly definitelyTypedPath: string;
}
export const defaultLocalOptions: TesterOptions = { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true };

42
packages/types-publisher/src/lib/npm-client.ts → packages/publisher/src/lib/npm-client.ts
View File

@@ -2,15 +2,13 @@ import assert = require("assert");
import { ensureFile, pathExists } from "fs-extra";
import RegClient = require("npm-registry-client");
import { resolve as resolveUrl } from "url";
import { Fetcher, readFile, readJson, sleep, writeJson } from "../util/io";
import { Logger, loggerWithErrors } from "../util/logging";
import { createTgz } from "../util/tgz";
import { assertNever, identity, joinPaths, mapToRecord, recordToMap } from "../util/util";
import { Registry } from "./common";
import { getSecret, Secret } from "./secrets";
import { githubRegistry, npmApi, npmRegistry, npmRegistryHostName } from "./settings";
import { joinPaths, loggerWithErrors, readJson, recordToMap, writeJson, mapToRecord, Fetcher, sleep, assertNever, Logger, readFile, identity, best, assertDefined, Semver } from "@definitelytyped/utils";
import { NotNeededPackage } from "@definitelytyped/definitions-parser";
const cacheFile = joinPaths(__dirname, "..", "..", "cache", "npmInfo.json");
@@ -215,3 +213,39 @@ function promisifyVoid(callsBack: (cb: (error: Error | undefined) => void) => vo
});
});
}
/**
* When we fail to publish a deprecated package, it leaves behind an entry in the time property.
* So the keys of 'time' give the actual 'latest'.
* If that's not equal to the expected latest, try again by bumping the patch version of the last attempt by 1.
*/
export function skipBadPublishes(pkg: NotNeededPackage, client: CachedNpmInfoClient, log: Logger) {
// because this is called right after isAlreadyDeprecated, we can rely on the cache being up-to-date
const info = assertDefined(client.getNpmInfoFromCache(pkg.fullEscapedNpmName));
const notNeeded = pkg.version;
const latest = Semver.parse(findActualLatest(info.time));
if (latest.equals(notNeeded) || latest.greaterThan(notNeeded) ||
info.versions.has(notNeeded.versionString) && !assertDefined(info.versions.get(notNeeded.versionString)).deprecated) {
const plusOne = new Semver(latest.major, latest.minor, latest.patch + 1);
log(`Deprecation of ${notNeeded.versionString} failed, instead using ${plusOne.versionString}.`);
return new NotNeededPackage({
asOfVersion: plusOne.versionString,
libraryName: pkg.libraryName,
sourceRepoURL: pkg.sourceRepoURL,
typingsPackageName: pkg.name,
});
}
return pkg;
}
function findActualLatest(times: Map<string, string>) {
const actual = best(
times, ([k, v], [bestK, bestV]) =>
(bestK === "modified" || bestK === "created") ? true :
(k === "modified" || k === "created") ? false :
new Date(v).getTime() > new Date(bestV).getTime());
if (!actual) {
throw new Error("failed to find actual latest");
}
return actual[0];
}

13
packages/types-publisher/src/lib/package-publisher.ts → packages/publisher/src/lib/package-publisher.ts
View File

@@ -1,13 +1,10 @@
import assert = require("assert");
import { TypeScriptVersion } from "definitelytyped-header-parser";
import { Logger } from "../util/logging";
import { joinPaths } from "../util/util";
import { readFileAndWarn, Registry } from "./common";
import { Logger, joinPaths, readFileAndWarn } from "@definitelytyped/utils";
import { Registry } from "./common";
import { NpmPublishClient } from "./npm-client";
import { AnyPackage, NotNeededPackage } from "./packages";
import { ChangedTyping } from "./versions";
import { ChangedTyping } from "@definitelytyped/definitions-parser/src/lib/versions";
import { NotNeededPackage, AnyPackage } from "@definitelytyped/definitions-parser";
import { TypeScriptVersion } from "../../../header-parser/src";
export async function publishTypingsPackage(
client: NpmPublishClient,

0
packages/types-publisher/src/lib/rolling-logs.ts → packages/publisher/src/lib/rolling-logs.ts
View File

4
packages/types-publisher/src/lib/secrets.ts → packages/publisher/src/lib/secrets.ts
View File

@@ -1,8 +1,6 @@
import { AuthenticationContext } from "adal-node";
import { KeyVaultClient, KeyVaultCredentials } from "azure-keyvault";
import { mapDefined } from "../util/util";
import { mapDefined } from "@definitelytyped/utils";
import { azureKeyvault } from "./settings";
export enum Secret {

7
packages/types-publisher/src/lib/settings.ts → packages/publisher/src/lib/settings.ts
View File

@@ -1,4 +1,3 @@
import { readFileSync } from "fs";
import { join as joinPaths } from "path";
/** URL of the NPM registry to upload to. */
@@ -7,8 +6,7 @@ export const githubRegistryHostName = "npm.pkg.github.com";
export const npmRegistry = `https://${npmRegistryHostName}/`;
export const githubRegistry = `https://${githubRegistryHostName}/`;
export const npmApi = "api.npmjs.org";
/** Note: this is 'types' and not '@types' */
export const scopeName = "types";
const root = joinPaths(__dirname, "..", "..");
export const dataDirPath = joinPaths(root, "data");
export const outputDirPath = joinPaths(root, "output");
@@ -30,6 +28,3 @@ export const azureKeyvault = "https://types-publisher-keys.vault.azure.net";
export const errorsIssue = "Microsoft/types-publisher/issues/40";
export const typesDirectoryName = "types";
export const dependenciesWhitelist: ReadonlySet<string> =
new Set(readFileSync(joinPaths(root, "dependenciesWhitelist.txt"), "utf-8").split(/\r?\n/));

38
packages/publisher/src/lib/versions.ts Normal file
View File

@@ -0,0 +1,38 @@
import { assertDefined } from "@definitelytyped/utils";
import { AllPackages, NotNeededPackage, PackageId, TypingsData } from "@definitelytyped/definitions-parser/src/packages";
import { readDataFile } from "@definitelytyped/definitions-parser";
export const versionsFilename = "versions.json";
export interface ChangedTyping {
readonly pkg: TypingsData;
/** This is the version to be published, meaning it's the version that doesn't exist yet. */
readonly version: string;
/** For a non-latest version, this is the latest version; publishing an old version updates the 'latest' tag and we want to change it back. */
readonly latestVersion: string | undefined;
}
export interface ChangedPackagesJson {
readonly changedTypings: ReadonlyArray<ChangedTypingJson>;
readonly changedNotNeededPackages: ReadonlyArray<string>;
}
export interface ChangedTypingJson {
readonly id: PackageId;
readonly version: string;
readonly latestVersion?: string;
}
export interface ChangedPackages {
readonly changedTypings: ReadonlyArray<ChangedTyping>;
readonly changedNotNeededPackages: ReadonlyArray<NotNeededPackage>;
}
export async function readChangedPackages(allPackages: AllPackages): Promise<ChangedPackages> {
const json = await readDataFile("calculate-versions", versionsFilename) as ChangedPackagesJson;
return {
changedTypings: json.changedTypings.map(({ id, version, latestVersion }): ChangedTyping =>
({ pkg: allPackages.getTypingsData(id), version, latestVersion })),
changedNotNeededPackages: json.changedNotNeededPackages.map(id => assertDefined(allPackages.getNotNeededPackage(id))),
};
}

10
packages/types-publisher/src/lib/webhook-server.ts → packages/publisher/src/lib/webhook-server.ts
View File

@@ -3,11 +3,9 @@ import { createServer, IncomingMessage, Server, ServerResponse } from "http";
import { setInterval } from "timers";
import full from "../full";
import { Fetcher, stringOfStream } from "../util/io";
import { joinLogWithErrors, LoggerWithErrors, loggerWithErrors, LogWithErrors } from "../util/logging";
import { currentTimeStamp, parseJson } from "../util/util";
import { Options } from "./common";
import { ParseDefinitionsOptions } from "@definitelytyped/definitions-parser"
import { Fetcher, stringOfStream, loggerWithErrors, LogWithErrors, joinLogWithErrors, LoggerWithErrors, parseJson } from "@definitelytyped/utils";
import { currentTimeStamp } from "../util/util";
import RollingLogs from "./rolling-logs";
import { sourceBranch } from "./settings";
@@ -16,7 +14,7 @@ export default async function webhookServer(
githubAccessToken: string,
dry: boolean,
fetcher: Fetcher,
options: Options,
options: ParseDefinitionsOptions,
): Promise<Server> {
const fullOne = updateOneAtATime(async (log, timestamp) => {
log.info(""); log.info("");

4
packages/types-publisher/src/make-server-run.ts → packages/publisher/src/make-server-run.ts
View File

@@ -1,10 +1,8 @@
import * as yargs from "yargs";
import { logUncaughtErrors, makeHttpRequest } from "@definitelytyped/utils";
import { getSecret, Secret } from "./lib/secrets";
import { sourceBranch } from "./lib/settings";
import { expectedSignature } from "./lib/webhook-server";
import { makeHttpRequest } from "./util/io";
import { logUncaughtErrors } from "./util/util";
if (!module.parent) {
const remote = yargs.argv.remote;

8
packages/types-publisher/src/mocks.ts → packages/publisher/src/mocks.ts
View File

@@ -1,7 +1,5 @@
import { parseHeaderOrFail } from "definitelytyped-header-parser";
import { Dir, FS, InMemoryDT } from "./get-definitely-typed";
import { Semver } from "./lib/versions";
import { parseHeaderOrFail } from "@definitelytyped/header-parser";
import { Dir, FS, InMemoryFS, Semver } from "@definitelytyped/utils";
class DTMock {
public readonly fs: FS;
@@ -17,7 +15,7 @@ class DTMock {
"sourceRepoURL": "https://github.com/angular/angular2"
}]
}`);
this.fs = new InMemoryDT(this.root, "DefinitelyTyped");
this.fs = new InMemoryFS(this.root, "DefinitelyTyped");
}
public pkgDir(packageName: string): Dir {

15
packages/types-publisher/src/publish-packages.ts → packages/publisher/src/publish-packages.ts
View File

@@ -1,21 +1,18 @@
import appInsights = require("applicationinsights");
import * as yargs from "yargs";
import { getDefinitelyTyped } from "./get-definitely-typed";
import { Options, Registry } from "./lib/common";
import { NpmPublishClient, UncachedNpmInfoClient, withNpmCache } from "./lib/npm-client";
import { defaultLocalOptions, Registry } from "./lib/common";
import { NpmPublishClient, UncachedNpmInfoClient, withNpmCache, skipBadPublishes } from "./lib/npm-client";
import { deprecateNotNeededPackage, publishNotNeededPackage, publishTypingsPackage } from "./lib/package-publisher";
import { AllPackages } from "./lib/packages";
import { ChangedPackages, readChangedPackages, skipBadPublishes } from "./lib/versions";
import { Fetcher } from "./util/io";
import { logger, loggerWithErrors, writeLog } from "./util/logging";
import { logUncaughtErrors } from "./util/util";
import { getDefinitelyTyped, AllPackages } from "@definitelytyped/definitions-parser";
import { loggerWithErrors, logUncaughtErrors, logger, Fetcher, writeLog } from "@definitelytyped/utils";
import { readChangedPackages, ChangedPackages } from "./lib/versions";
if (!module.parent) {
const dry = !!yargs.argv.dry;
const deprecateName = yargs.argv.deprecate as string | undefined;
logUncaughtErrors(async () => {
const dt = await getDefinitelyTyped(Options.defaults, loggerWithErrors()[0]);
const dt = await getDefinitelyTyped(defaultLocalOptions, loggerWithErrors()[0]);
if (deprecateName !== undefined) {
// A '--deprecate' command is available in case types-publisher got stuck *while* trying to deprecate a package.
// Normally this should not be needed.

12
packages/types-publisher/src/publish-registry.ts → packages/publisher/src/publish-registry.ts
View File

@@ -2,15 +2,11 @@ import assert = require("assert");
import { emptyDir } from "fs-extra";
import * as yargs from "yargs";
import { FS, getDefinitelyTyped } from "./get-definitely-typed";
import { Options, Registry as RegistryName } from "./lib/common";
import { defaultLocalOptions, Registry as RegistryName } from "./lib/common";
import { CachedNpmInfoClient, NpmPublishClient, UncachedNpmInfoClient, withNpmCache } from "./lib/npm-client";
import { AllPackages, NotNeededPackage, readNotNeededPackages, TypingsData } from "./lib/packages";
import { outputDirPath, validateOutputPath } from "./lib/settings";
import { Semver } from "./lib/versions";
import { npmInstallFlags, readJson, sleep, writeFile, writeJson } from "./util/io";
import { logger, Logger, loggerWithErrors, writeLog } from "./util/logging";
import { assertDefined, best, computeHash, execAndThrowErrors, joinPaths, logUncaughtErrors, mapDefined } from "./util/util";
import { getDefinitelyTyped, AllPackages, readNotNeededPackages, NotNeededPackage, TypingsData } from "@definitelytyped/definitions-parser";
import { assertDefined, best, computeHash, execAndThrowErrors, joinPaths, logUncaughtErrors, mapDefined, loggerWithErrors, FS, logger, writeLog, writeJson, writeFile, Logger, sleep, npmInstallFlags, readJson, Semver } from "@definitelytyped/utils";
const typesRegistry = "types-registry";
const registryOutputPath = joinPaths(outputDirPath, typesRegistry);
@@ -21,7 +17,7 @@ Generated by [types-publisher](https://github.com/Microsoft/types-publisher).`;
if (!module.parent) {
const dry = !!yargs.argv.dry;
logUncaughtErrors(async () => {
const dt = await getDefinitelyTyped(Options.defaults, loggerWithErrors()[0]);
const dt = await getDefinitelyTyped(defaultLocalOptions, loggerWithErrors()[0]);
await publishRegistry(dt, await AllPackages.read(dt), dry, new UncachedNpmInfoClient());
});
}

2
packages/types-publisher/src/test-get-secrets.ts → packages/publisher/src/test-get-secrets.ts
View File

@@ -1,7 +1,7 @@
// Run `node ./bin/test-get-secrets.js` to test that we can fetch secrets from Azure Keyvault
import { allSecrets, getSecret, Secret } from "./lib/secrets";
import { logUncaughtErrors } from "./util/util";
import { logUncaughtErrors } from "@definitelytyped/utils";
if (!module.parent) {
logUncaughtErrors(main());

6
packages/types-publisher/src/tester/get-affected-packages.ts → packages/publisher/src/tester/get-affected-packages.ts
View File

@@ -1,5 +1,5 @@
import { AllPackages, formatDependencyVersion, getMangledNameForScopedPackage, PackageBase, PackageId, TypingsData } from "../lib/packages";
import { mapDefined, mapIter, sort } from "../util/util";
import { AllPackages, formatDependencyVersion, getMangledNameForScopedPackage, PackageBase, PackageId, TypingsData } from "@definitelytyped/definitions-parser";
import { mapDefined, mapIterable, sort } from "@definitelytyped/utils";
export interface Affected {
readonly changedPackages: ReadonlyArray<TypingsData>;
@@ -12,7 +12,7 @@ export function getAffectedPackages(allPackages: AllPackages, changedPackageIds:
const resolved = changedPackageIds.map(id => allPackages.tryResolve(id));
// If a package doesn't exist, that's because it was deleted.
const changed = mapDefined(resolved, id => allPackages.tryGetTypingsData(id));
const dependent = mapIter(collectDependers(resolved, getReverseDependencies(allPackages, resolved)), p => allPackages.getTypingsData(p));
const dependent = mapIterable(collectDependers(resolved, getReverseDependencies(allPackages, resolved)), p => allPackages.getTypingsData(p));
return { changedPackages: changed, dependentPackages: sortPackages(dependent), allPackages };
}

44
packages/types-publisher/src/tester/test-runner.ts → packages/publisher/src/tester/test-runner.ts
View File

@@ -4,37 +4,47 @@ import { pathExists, remove } from "fs-extra";
import os = require("os");
import * as fold from "travis-fold";
import * as yargs from "yargs";
import { FS, getDefinitelyTyped } from "../get-definitely-typed";
import { Options, TesterOptions } from "../lib/common";
import { parseVersionFromDirectoryName } from "../lib/definition-parser";
import { TesterOptions, defaultLocalOptions } from "../lib/common";
import { NpmInfo, UncachedNpmInfoClient } from "../lib/npm-client";
import { AllPackages, DependencyVersion, formatDependencyVersion, NotNeededPackage, PackageId, TypingsData } from "../lib/packages";
import { sourceBranch, typesDirectoryName } from "../lib/settings";
import { Semver } from "../lib/versions";
import { npmInstallFlags } from "../util/io";
import { consoleLogger, Logger, LoggerWithErrors, loggerWithErrors } from "../util/logging";
import {
getDefinitelyTyped,
AllPackages,
DependencyVersion,
formatDependencyVersion,
NotNeededPackage,
PackageId,
TypingsData,
} from "@definitelytyped/definitions-parser"
import {
assertDefined,
CrashRecoveryState,
exec,
execAndThrowErrors,
flatMap,
joinPaths,
logUncaughtErrors,
mapIter,
numberOfOsProcesses,
mapIterable,
runWithListeningChildProcesses,
} from "../util/util";
loggerWithErrors,
consoleLogger,
FS,
Semver,
npmInstallFlags,
LoggerWithErrors,
Logger,
flatMapIterable,
} from "@definitelytyped/utils";
import { allDependencies, getAffectedPackages } from "./get-affected-packages";
import { numberOfOsProcesses } from "../util/util";
import { parseVersionFromDirectoryName } from "@definitelytyped/definitions-parser/src/lib/definition-parser";
const perfDir = joinPaths(os.homedir(), ".dts", "perf");
const suggestionsDir = joinPaths(os.homedir(), ".dts", "suggestions");
if (!module.parent) {
if (yargs.argv.affected) {
logUncaughtErrors(testAffectedOnly(Options.defaults));
logUncaughtErrors(testAffectedOnly(defaultLocalOptions));
} else {
const selection = yargs.argv.all ? "all" : yargs.argv._[0] ? new RegExp(yargs.argv._[0]) : "affected";
const options = testerOptions(!!yargs.argv.runFromDefinitelyTyped);
@@ -70,7 +80,7 @@ export function parseNProcesses(): number {
export function testerOptions(runFromDefinitelyTyped: boolean): TesterOptions {
return runFromDefinitelyTyped
? { definitelyTypedPath: process.cwd(), progress: false, parseInParallel: true }
: Options.defaults;
: defaultLocalOptions;
}
export default async function runTests(
@@ -121,7 +131,7 @@ export function getNotNeededPackages(allPackages: AllPackages, diffs: GitDiff[])
`Unexpected file deleted: ${d.file}
When removing packages, you should only delete files that are a part of removed packages.`)
.name));
return mapIter(deletedPackages, p => {
return mapIterable(deletedPackages, p => {
if (allPackages.hasTypingFor({ name: p, version: "*" })) {
throw new Error(`Please delete all files in ${p} when adding it to notNeededPackages.json.`);
}
@@ -310,8 +320,8 @@ export function gitChanges(diffs: GitDiff[]): PackageId[] {
}
}
return Array.from(flatMap(changedPackages, ([name, versions]) =>
mapIter(versions, ([_, version]) => ({ name, version }))));
return Array.from(flatMapIterable(changedPackages, ([name, versions]) =>
mapIterable(versions, ([_, version]) => ({ name, version }))));
}
/*

7
packages/types-publisher/src/tester/test.ts → packages/publisher/src/tester/test.ts
View File

@@ -2,14 +2,13 @@ import yargs = require("yargs");
import checkParseResults from "../check-parse-results";
import { clean } from "../clean";
import { getDefinitelyTyped } from "../get-definitely-typed";
import { getDefinitelyTyped } from "@definitelytyped/definitions-parser";
import { logUncaughtErrors, loggerWithErrors } from "@definitelytyped/utils";
import { TesterOptions } from "../lib/common";
import { UncachedNpmInfoClient } from "../lib/npm-client";
import parseDefinitions from "../parse-definitions";
import { loggerWithErrors } from "../util/logging";
import { logUncaughtErrors } from "../util/util";
import runTests, { getAffectedPackagesFromDiff, parseNProcesses, testerOptions } from "./test-runner";
import parseDefinitions from "@definitelytyped/definitions-parser/src/parseDefinitions";
if (!module.parent) {
const options = testerOptions(!!yargs.argv.runFromDefinitelyTyped);

0
packages/types-publisher/src/types/adal-node.d.ts → packages/publisher/src/types/adal-node.d.ts vendored
View File

0
packages/types-publisher/src/types/azure-keyvault.d.ts → packages/publisher/src/types/azure-keyvault.d.ts vendored
View File

0
packages/types-publisher/src/types/fstream.d.ts → packages/publisher/src/types/fstream.d.ts vendored
View File

0
packages/types-publisher/src/types/npm-registry-client.d.ts → packages/publisher/src/types/npm-registry-client.d.ts vendored
View File

3
packages/types-publisher/src/upload-blobs.ts → packages/publisher/src/upload-blobs.ts
View File

@@ -1,5 +1,6 @@
import uploadBlobsAndUpdateIssue from "./lib/blob-uploader";
import { currentTimeStamp, logUncaughtErrors } from "./util/util";
import { currentTimeStamp } from "./util/util";
import { logUncaughtErrors } from "@definitelytyped/utils";
if (!module.parent) {
logUncaughtErrors(uploadBlobsAndUpdateIssue(currentTimeStamp()));

3
packages/types-publisher/src/util/tgz.ts → packages/publisher/src/util/tgz.ts
View File

@@ -2,8 +2,7 @@ import { createWriteStream } from "fs";
import { FStreamEntry, Reader } from "fstream";
import { Pack } from "tar";
import * as zlib from "zlib";
import { streamDone } from "./io";
import { streamDone } from "@definitelytyped/utils";
export function gzip(input: NodeJS.ReadableStream): NodeJS.ReadableStream {
return input.pipe(zlib.createGzip());

8
packages/publisher/src/util/util.ts Normal file
View File

@@ -0,0 +1,8 @@
import * as os from "os";
export function currentTimeStamp(): string {
return new Date().toISOString();
}
export const numberOfOsProcesses = process.env.TRAVIS === "true" ? 2 : os.cpus().length;

11
packages/types-publisher/src/validate.ts → packages/publisher/src/validate.ts
View File

@@ -1,14 +1,11 @@
import { mkdirp, remove } from "fs-extra";
import * as yargs from "yargs";
import { FS, getDefinitelyTyped } from "./get-definitely-typed";
import { Options } from "./lib/common";
import { AllPackages, getFullNpmName } from "./lib/packages";
import { getDefinitelyTyped, AllPackages, getFullNpmName } from "@definitelytyped/definitions-parser";
import { defaultLocalOptions } from "./lib/common";
import { validateOutputPath } from "./lib/settings";
import { readChangedPackages } from "./lib/versions";
import { writeFile, writeJson } from "./util/io";
import { LoggerWithErrors, loggerWithErrors, moveLogsWithErrors, quietLoggerWithErrors, writeLog } from "./util/logging";
import { exec, joinPaths, logUncaughtErrors, nAtATime } from "./util/util";
import { exec, joinPaths, logUncaughtErrors, nAtATime, loggerWithErrors, FS, writeLog, LoggerWithErrors, quietLoggerWithErrors, moveLogsWithErrors, writeJson, writeFile } from "@definitelytyped/utils";
if (!module.parent) {
const all = !!yargs.argv.all;
@@ -25,7 +22,7 @@ if (!module.parent) {
logUncaughtErrors(doValidate(packageNames));
} else {
const log = loggerWithErrors()[0];
logUncaughtErrors(getDefinitelyTyped(Options.defaults, log).then(validate));
logUncaughtErrors(getDefinitelyTyped(defaultLocalOptions, log).then(validate));
}
}

11
packages/types-publisher/src/webhook.ts → packages/publisher/src/webhook.ts
View File

@@ -1,11 +1,8 @@
import appInsights = require("applicationinsights");
import * as yargs from "yargs";
import { Options } from "./lib/common";
import { getSecret, Secret } from "./lib/secrets";
import webhookServer from "./lib/webhook-server";
import { Fetcher } from "./util/io";
import { logUncaughtErrors } from "./util/util";
import { Fetcher, logUncaughtErrors } from "@definitelytyped/utils";
if (!module.parent) {
logUncaughtErrors(main());
@@ -27,7 +24,11 @@ export default async function main(): Promise<void> {
}
const fetcher = new Fetcher();
try {
const s = await webhookServer(key, githubAccessToken, dry, fetcher, Options.azure);
const s = await webhookServer(key, githubAccessToken, dry, fetcher, {
definitelyTypedPath: undefined,
progress: false,
parseInParallel: false
});
console.log(`Listening on port ${port}`);
s.listen(port);
} catch (e) {

10
packages/types-publisher/src/generate-packages.test.ts → packages/publisher/test/generate-packages.test.ts
View File

@@ -1,8 +1,8 @@
import { createNotNeededPackageJSON, createPackageJSON, createReadme, getLicenseFileText } from "./generate-packages";
import { Registry } from "./lib/common";
import { AllPackages, License, NotNeededPackage, readNotNeededPackages, TypesDataFile, TypingsData, TypingsDataRaw } from "./lib/packages";
import { createMockDT } from "./mocks";
import { testo } from "./util/test";
import { createNotNeededPackageJSON, createPackageJSON, createReadme, getLicenseFileText } from "../src/generate-packages";
import { Registry } from "../src/lib/common";
import { AllPackages, License, NotNeededPackage, readNotNeededPackages, TypesDataFile, TypingsData, TypingsDataRaw } from "@definitelytyped/definitions-parser";
import { createMockDT } from "../src/mocks";
import { testo } from "./utils";
function createRawPackage(license: License): TypingsDataRaw {
return {

6
packages/types-publisher/src/tester/get-affected-packages.test.ts → packages/publisher/test/tester/get-affected-packages.test.ts
View File

@@ -1,7 +1,7 @@
import { AllPackages, NotNeededPackage, TypesDataFile } from "../lib/packages";
import { createTypingsVersionRaw, testo } from "../util/test";
import { AllPackages, NotNeededPackage, TypesDataFile } from "@definitelytyped/definitions-parser";
import { createTypingsVersionRaw, testo } from "../utils";
import { getAffectedPackages } from "./get-affected-packages";
import { getAffectedPackages } from "../../src/tester/get-affected-packages";
const typesData: TypesDataFile = {
jquery: createTypingsVersionRaw("jquery", [], []),
known: createTypingsVersionRaw("known", [{ name: "jquery", version: { major: 1 }}], []),

8
packages/types-publisher/src/tester/test-runner.test.ts → packages/publisher/test/tester/test-runner.test.ts
View File

@@ -1,8 +1,8 @@
import { NpmInfo } from "../lib/npm-client";
import { AllPackages, NotNeededPackage, TypesDataFile } from "../lib/packages";
import { createTypingsVersionRaw, testo } from "../util/test";
import { NpmInfo } from "../../src/lib/npm-client";
import { AllPackages, NotNeededPackage, TypesDataFile } from "@definitelytyped/definitions-parser";
import { createTypingsVersionRaw, testo } from "../utils";
import { checkNotNeededPackage, getNotNeededPackages, GitDiff } from "./test-runner";
import { checkNotNeededPackage, getNotNeededPackages, GitDiff } from "../../src/tester/test-runner";
const typesData: TypesDataFile = {
jquery: createTypingsVersionRaw("jquery", [], []),

10
packages/publisher/test/tsconfig.json Normal file
View File

@@ -0,0 +1,10 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"types": ["node", "jest"]
},
"references": [
{ "path": ".." },
{ "path": "../../definitions-parser" }
]
}

2
packages/types-publisher/src/util/test.ts → packages/publisher/test/utils.ts
View File

@@ -1,4 +1,4 @@
import { License, PackageId, TypingsVersionsRaw } from "../lib/packages";
import { PackageId, TypingsVersionsRaw, License } from "@definitelytyped/definitions-parser";
export function testo(o: { [s: string]: () => void }) {
for (const k of Object.keys(o)) {

15
packages/publisher/tsconfig.json Normal file
View File

@@ -0,0 +1,15 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src",
"baseUrl": ".",
"paths": {
"*": ["src/types/*"]
}
},
"references": [
{ "path": "../definitions-parser" },
{ "path": "../utils" }
]
}

21
packages/types-publisher/LICENSE
View File

@@ -1,21 +0,0 @@
MIT License
Copyright (c) Microsoft Corporation. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE

199
packages/types-publisher/dependenciesWhitelist.txt
View File

@@ -1,199 +0,0 @@
@babel/code-frame
@babel/core
@babel/generator
@babel/parser
@babel/template
@babel/traverse
@babel/types
@electron/get
@emotion/core
@emotion/serialize
@emotion/styled
@hapi/boom
@hapi/iron
@hapi/wreck
@jest/environment
@jest/fake-timers
@jest/transform
@jest/types
@loadable/component
@material-ui/core
@material-ui/types
@sentry/browser
@storybook/addons
@storybook/react
@types/base-x
@types/expect
@types/firebase
@types/highcharts
@types/hoist-non-react-statics
@types/mkdirp-promise
@types/next-redux-wrapper
@types/ink
@types/js-data
@types/react-native-tab-view
@types/react-navigation
@types/three
@types/rsmq
@types/vue
@types/webdriverio
@types/winston
@types/wonder-commonlib
@types/wonder-frp
@uirouter/angularjs
@vue/test-utils
abort-controller
actions-on-google
activex-helpers
adal-node
ajv
anydb-sql
aphrodite
apollo-client
apollo-link
apollo-link-http-common
ast-types
aws-sdk
axe-core
axios
base-x
bignumber.js
bookshelf
broadcast-channel
cac
cassandra-driver
chalk
chokidar
commander
connect-mongo
cordova-plugin-camera
cordova-plugin-file
cordova-plugin-file-transfer
cosmiconfig
cropperjs
csstype
cypress
date-fns
decimal.js
del
dexie
dnd-core
dotenv
egg
electron
electron-notarize
electron-osx-sign
electron-store
ethers
eventemitter2
eventemitter3
expect
express-graphql
express-rate-limit
fast-glob
fastify
final-form
flatpickr
form-data
google-auth-library
google-gax
graphql
graphql-tools
grpc
handlebars
hoist-non-react-statics
i18next
immutable
indefinite-observable
inversify
jest-diff
jest-environment-node
jest-environment-jsdom
jest-mock
jest-snapshot
jointjs
levelup
lit-element
localforage
log4js
logform
loglevel
logrocket
knex
knockout
magic-string
mali
meteor-typings
moment
monaco-editor
moment-range
mqtt
next
nock
normalize-url
objection
opentracing
parchment
parse5
path-to-regexp
pkcs11js
popper.js
postcss
polished
preact
pretty-format
probot
prom-client
protobufjs
protractor
query-string
quill-delta
raven-js
re-resizable
react-autosize-textarea
react-dnd
react-dnd-touch-backend
react-intl
react-native-fs
react-native-maps
react-native-modal
react-native-svg
react-native-tab-view
react-navigation
recast
redux
redux-observable
redux-persist
redux-saga
redux-thunk
rollup
rrule
rxjs
safe-buffer
should
smooth-scrollbar
scroll-into-view-if-needed
source-map
styled-components
sw-toolbox
swagger-parser
terser
three
tslint
ts-toolbelt
tweetnacl
typescript
utility-types
vega-typings
vfile
vfile-message
vue
vue-router
vuex
webpack-chain
winston
winston-transport
xmlbuilder
xpath
zipkin
zipkin-transport-http

15
packages/types-publisher/jest.config.js
View File

@@ -1,15 +0,0 @@
module.exports = {
"roots": [
"src"
],
"preset": "ts-jest",
"testRegex": "(/__tests__/.*|\\.(test|spec))\\.tsx?$",
"moduleFileExtensions": [
"ts",
"tsx",
"js",
"jsx",
"json",
"node"
],
}

28
packages/types-publisher/src/get-definitely-typed.test.ts
View File

@@ -1,28 +0,0 @@
import { Dir, FS, getDefinitelyTyped, InMemoryDT } from "./get-definitely-typed";
import { Options } from "./lib/common";
import { loggerWithErrors } from "./util/logging";
import { testo } from "./util/test";
testo({
async downloadDefinitelyTyped() {
const dt = await getDefinitelyTyped(Options.azure, loggerWithErrors()[0]);
expect(dt.exists("types")).toBe(true);
expect(dt.exists("buncho")).toBe(false);
},
createDirs() {
const root = new Dir(undefined);
root.set("file1.txt", "ok");
expect(root.has("file1.txt")).toBe(true);
expect(root.get("file1.txt")).toBe("ok");
},
simpleMemoryFS() {
const root = new Dir(undefined);
root.set("file1.txt", "ok");
const dir = root.subdir("sub1");
dir.set("file2.txt", "x");
const fs: FS = new InMemoryDT(root, "test/");
expect(fs.exists("file1.txt")).toBe(true);
expect(fs.readFile("file1.txt")).toBe("ok");
expect(fs.readFile("sub1/file2.txt")).toBe("x");
},
});

265
packages/types-publisher/src/get-definitely-typed.ts
View File

@@ -1,265 +0,0 @@
import appInsights = require("applicationinsights");
import assert = require("assert");
import { ensureDir, pathExistsSync, readdirSync, statSync } from "fs-extra";
import https = require("https");
import tarStream = require("tar-stream");
import * as yargs from "yargs";
import * as zlib from "zlib";
import { Options } from "./lib/common";
import { dataDirPath, definitelyTypedZipUrl } from "./lib/settings";
import { readFileSync, readJsonSync, stringOfStream } from "./util/io";
import { LoggerWithErrors, loggerWithErrors } from "./util/logging";
import { assertDefined, exec, joinPaths, logUncaughtErrors, withoutStart } from "./util/util";
/**
* Readonly filesystem.
* Paths provided to these methods should be relative to the FS object's root but not start with '/' or './'.
*/
export interface FS {
/**
* Alphabetically sorted list of files and subdirectories.
* If dirPath is missing, reads the root.
*/
readdir(dirPath?: string): ReadonlyArray<string>;
readJson(path: string): unknown;
readFile(path: string): string;
isDirectory(dirPath: string): boolean;
exists(path: string): boolean;
/** FileSystem rooted at a child directory. */
subDir(path: string): FS;
/** Representation of current location, for debugging. */
debugPath(): string;
}
if (!module.parent) {
if (process.env.APPINSIGHTS_INSTRUMENTATIONKEY) {
appInsights.setup();
appInsights.start();
}
const dry = !!yargs.argv.dry;
console.log("gettingDefinitelyTyped: " + (dry ? "from github" : "locally"));
logUncaughtErrors(async () => {
const dt = await getDefinitelyTyped(dry ? Options.azure : Options.defaults, loggerWithErrors()[0]);
assert(dt.exists("types"));
assert(!(dt.exists("buncho")));
});
}
export async function getDefinitelyTyped(options: Options, log: LoggerWithErrors): Promise<FS> {
if (options.definitelyTypedPath === undefined) {
log.info("Downloading Definitely Typed ...");
await ensureDir(dataDirPath);
return downloadAndExtractFile(definitelyTypedZipUrl);
}
const { error, stderr, stdout } = await exec("git diff --name-only", options.definitelyTypedPath);
if (error) { throw error; }
if (stderr) { throw new Error(stderr); }
if (stdout) { throw new Error(`'git diff' should be empty. Following files changed:\n${stdout}`); }
log.info(`Using local Definitely Typed at ${options.definitelyTypedPath}.`);
return new DiskFS(`${options.definitelyTypedPath}/`);
}
export function getLocallyInstalledDefinitelyTyped(path: string): FS {
return new DiskFS(`${path}/`);
}
function downloadAndExtractFile(url: string): Promise<FS> {
return new Promise<FS>((resolve, reject) => {
const root = new Dir(undefined);
function insertFile(path: string, content: string): void {
const components = path.split("/");
const baseName = assertDefined(components.pop());
let dir = root;
for (const component of components) {
dir = dir.subdir(component);
}
dir.set(baseName, content);
}
https.get(url, response => {
const extract = tarStream.extract();
response.pipe(zlib.createGunzip()).pipe(extract);
interface Header {
readonly name: string;
readonly type: "file" | "directory";
}
extract.on("entry", (header: Header, stream: NodeJS.ReadableStream, next: () => void) => {
const name = assertDefined(withoutStart(header.name, "DefinitelyTyped-master/"));
switch (header.type) {
case "file":
stringOfStream(stream, name).then(s => {
insertFile(name, s);
next();
}).catch(reject);
break;
case "directory":
next();
break;
default:
throw new Error(`Unexpected file system entry kind ${header.type}`);
}
});
extract.on("error", reject);
extract.on("finish", () => { resolve(new InMemoryDT(root.finish(), "")); });
});
});
}
interface ReadonlyDir extends ReadonlyMap<string, ReadonlyDir | string> {
readonly parent: Dir | undefined;
}
// Map entries are Dir for directory and string for file.
export class Dir extends Map<string, Dir | string> implements ReadonlyDir {
constructor(readonly parent: Dir | undefined) { super(); }
subdir(name: string): Dir {
const x = this.get(name);
if (x !== undefined) {
if (typeof x === "string") {
throw new Error(`File ${name} has same name as a directory?`);
}
return x;
}
const res = new Dir(this);
this.set(name, res);
return res;
}
finish(): Dir {
const out = new Dir(this.parent);
for (const key of Array.from(this.keys()).sort()) {
const subDirOrFile = this.get(key)!;
out.set(key, typeof subDirOrFile === "string" ? subDirOrFile : subDirOrFile.finish());
}
return out;
}
}
export class InMemoryDT implements FS {
/** pathToRoot is just for debugging */
constructor(readonly curDir: ReadonlyDir, readonly pathToRoot: string) {}
private tryGetEntry(path: string): ReadonlyDir | string | undefined {
validatePath(path);
if (path === "") {
return this.curDir;
}
const components = path.split("/");
const baseName = assertDefined(components.pop());
let dir = this.curDir;
for (const component of components) {
const entry = component === ".." ? dir.parent : dir.get(component);
if (entry === undefined) {
return undefined;
}
if (!(entry instanceof Dir)) {
throw new Error(`No file system entry at ${this.pathToRoot}/${path}. Siblings are: ${Array.from(dir.keys()).toString()}`);
}
dir = entry;
}
return dir.get(baseName);
}
private getEntry(path: string): ReadonlyDir | string {
const entry = this.tryGetEntry(path);
if (entry === undefined) { throw new Error(`No file system entry at ${this.pathToRoot}/${path}`); }
return entry;
}
private getDir(dirPath: string): Dir {
const res = this.getEntry(dirPath);
if (!(res instanceof Dir)) {
throw new Error(`${this.pathToRoot}/${dirPath} is a file, not a directory.`);
}
return res;
}
readFile(filePath: string): string {
const res = this.getEntry(filePath);
if (typeof res !== "string") {
throw new Error(`${this.pathToRoot}/${filePath} is a directory, not a file.`);
}
return res;
}
readdir(dirPath?: string): ReadonlyArray<string> {
return Array.from((dirPath === undefined ? this.curDir : this.getDir(dirPath)).keys());
}
readJson(path: string): unknown {
return JSON.parse(this.readFile(path)) as unknown;
}
isDirectory(path: string): boolean {
return typeof this.getEntry(path) !== "string";
}
exists(path: string): boolean {
return this.tryGetEntry(path) !== undefined;
}
subDir(path: string): FS {
return new InMemoryDT(this.getDir(path), joinPaths(this.pathToRoot, path));
}
debugPath(): string {
return this.pathToRoot;
}
}
class DiskFS implements FS {
constructor(private readonly rootPrefix: string) {
assert(rootPrefix.endsWith("/"));
}
private getPath(path: string | undefined): string {
if (path === undefined) {
return this.rootPrefix;
}
validatePath(path);
return this.rootPrefix + path;
}
readdir(dirPath?: string): ReadonlyArray<string> {
return readdirSync(this.getPath(dirPath)).sort().filter(name => name !== ".DS_Store");
}
isDirectory(dirPath: string): boolean {
return statSync(this.getPath(dirPath)).isDirectory();
}
readJson(path: string): unknown {
return readJsonSync(this.getPath(path));
}
readFile(path: string): string {
return readFileSync(this.getPath(path));
}
exists(path: string): boolean {
return pathExistsSync(this.getPath(path));
}
subDir(path: string): FS {
return new DiskFS(`${this.rootPrefix}${path}/`);
}
debugPath(): string {
return this.rootPrefix.slice(0, this.rootPrefix.length - 1); // remove trailing '/'
}
}
/** FS only handles simple paths like `foo/bar` or `../foo`. No `./foo` or `/foo`. */
function validatePath(path: string): void {
if (path.startsWith(".") && path !== ".editorconfig" && !path.startsWith("../")) {
throw new Error(`${path}: filesystem doesn't support paths of the form './x'.`);
}
if (path.startsWith("/")) {
throw new Error(`${path}: filesystem doesn't support paths of the form '/xxx'.`);
}
if (path.endsWith("/")) {
throw new Error(`${path}: filesystem doesn't support paths of the form 'xxx/'.`);
}
}

66
packages/types-publisher/src/lib/common.ts
View File

@@ -1,66 +0,0 @@
import { ensureDir } from "fs-extra";
import { readJson, writeJson } from "../util/io";
import { joinPaths } from "../util/util";
import { dataDirPath } from "./settings";
if (process.env.LONGJOHN) {
console.log("=== USING LONGJOHN ===");
const longjohn = require("longjohn") as { async_trace_limit: number }; // tslint:disable-line no-var-requires
longjohn.async_trace_limit = -1; // unlimited
}
/** Which registry to publish to */
export enum Registry {
/** types-registry and @types/* on NPM */
NPM,
/** @definitelytyped/types-registry and @types/* on Github */
Github,
}
/** Settings that may be determined dynamically. */
export interface Options {
/**
* e.g. '../DefinitelyTyped'
* This is overridden to `cwd` when running the tester, as that is run from within DefinitelyTyped.
* If undefined, downloads instead.
*/
readonly definitelyTypedPath: string | undefined;
/** Whether to show progress bars. Good when running locally, bad when running on travis / azure. */
readonly progress: boolean;
/** Disabled on azure since it has problems logging errors from other processes. */
readonly parseInParallel: boolean;
}
export namespace Options {
/** Options for running locally. */
export const defaults: TesterOptions = { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true };
export const azure: Options = { definitelyTypedPath: undefined, progress: false, parseInParallel: false };
}
export interface TesterOptions extends Options {
// Tester can only run on files stored on-disk.
readonly definitelyTypedPath: string;
}
export function readDataFile(generatedBy: string, fileName: string): Promise<object> {
return readFileAndWarn(generatedBy, dataFilePath(fileName));
}
/** If a file doesn't exist, warn and tell the step it should have been generated by. */
export async function readFileAndWarn(generatedBy: string, filePath: string): Promise<object> {
try {
return await readJson(filePath);
} catch (e) {
console.error(`Run ${generatedBy} first!`);
throw e;
}
}
export async function writeDataFile(filename: string, content: {}, formatted = true): Promise<void> {
await ensureDir(dataDirPath);
await writeJson(dataFilePath(filename), content, formatted);
}
export function dataFilePath(filename: string): string {
return joinPaths(dataDirPath, filename);
}

24
packages/types-publisher/src/lib/definition-parser-worker.ts
View File

@@ -1,24 +0,0 @@
import assert = require("assert");
import process = require("process");
import { getLocallyInstalledDefinitelyTyped } from "../get-definitely-typed";
import { logUncaughtErrors } from "../util/util";
import { getTypingInfo } from "./definition-parser";
// This file is "called" by runWithChildProcesses from parse-definition.ts
export const definitionParserWorkerFilename = __filename;
if (!module.parent) {
process.on("message", message => {
assert(process.argv.length === 3);
const typesPath = process.argv[2];
// tslint:disable-next-line no-async-without-await
logUncaughtErrors(async () => {
for (const packageName of message as ReadonlyArray<string>) {
const data = getTypingInfo(packageName, getLocallyInstalledDefinitelyTyped(typesPath).subDir(packageName));
process.send!({ data, packageName });
}
});
});
}

98
packages/types-publisher/src/lib/definition-parser.test.ts
View File

@@ -1,98 +0,0 @@
// tslint:disable:object-literal-key-quotes
import { createMockDT } from "../mocks";
import { getTypingInfo } from "./definition-parser";
describe(getTypingInfo, () => {
it("keys data by major.minor version", () => {
const dt = createMockDT();
dt.addOldVersionOfPackage("jquery", "1.42");
dt.addOldVersionOfPackage("jquery", "2");
const info = getTypingInfo("jquery", dt.pkgFS("jquery"));
expect(Object.keys(info).sort()).toEqual(["1.42", "2.0", "3.3"]);
});
describe("concerning multiple versions", () => {
it("records what the version directory looks like on disk", () => {
const dt = createMockDT();
dt.addOldVersionOfPackage("jquery", "2");
dt.addOldVersionOfPackage("jquery", "1.5");
const info = getTypingInfo("jquery", dt.pkgFS("jquery"));
expect(info).toEqual({
"1.5": expect.objectContaining({
libraryVersionDirectoryName: "1.5",
}),
"2.0": expect.objectContaining({
libraryVersionDirectoryName: "2",
}),
"3.3": expect.objectContaining({
// The latest version does not have its own version directory
libraryVersionDirectoryName: undefined,
}),
});
});
it("records a path mapping to the version directory", () => {
const dt = createMockDT();
dt.addOldVersionOfPackage("jquery", "2");
dt.addOldVersionOfPackage("jquery", "1.5");
const info = getTypingInfo("jquery", dt.pkgFS("jquery"));
expect(info).toEqual({
"1.5": expect.objectContaining({
pathMappings: [{
packageName: "jquery",
version: { major: 1, minor: 5 },
}],
}),
"2.0": expect.objectContaining({
pathMappings: [{
packageName: "jquery",
version: { major: 2, minor: undefined },
}],
}),
"3.3": expect.objectContaining({
// The latest version does not have path mappings of its own
pathMappings: [],
}),
});
});
describe("validation thereof", () => {
it("throws if a directory exists for the latest major version", () => {
const dt = createMockDT();
dt.addOldVersionOfPackage("jquery", "3");
expect(() => {
getTypingInfo("jquery", dt.pkgFS("jquery"));
}).toThrow(
"The latest version is 3.3, so the subdirectory 'v3' is not allowed; " +
"since it applies to any 3.* version, up to and including 3.3.",
);
});
it("throws if a directory exists for the latest minor version", () => {
const dt = createMockDT();
dt.addOldVersionOfPackage("jquery", "3.3");
expect(() => {
getTypingInfo("jquery", dt.pkgFS("jquery"));
}).toThrow(
"The latest version is 3.3, so the subdirectory 'v3.3' is not allowed.",
);
});
it("does not throw when a minor version is older than the latest", () => {
const dt = createMockDT();
dt.addOldVersionOfPackage("jquery", "3.0");
expect(() => {
getTypingInfo("jquery", dt.pkgFS("jquery"));
}).not.toThrow();
});
});
});
});

503
packages/types-publisher/src/lib/definition-parser.ts
View File

@@ -1,503 +0,0 @@
import { isTypeScriptVersion, parseHeaderOrFail, TypeScriptVersion } from "definitelytyped-header-parser";
import * as ts from "typescript";
import { FS } from "../get-definitely-typed";
import {
computeHash, filter, flatMap, hasWindowsSlashes, join, mapDefined, sort, split, unique, unmangleScopedPackage, withoutStart,
} from "../util/util";
import { allReferencedFiles, createSourceFile, getModuleInfo, getTestDependencies } from "./module-info";
import {
formatTypingVersion,
getLicenseFromPackageJson,
PackageId,
PackageJsonDependency,
PathMapping,
TypingsDataRaw,
TypingsVersionsRaw,
TypingVersion,
} from "./packages";
import { dependenciesWhitelist } from "./settings";
function matchesVersion(typingsDataRaw: TypingsDataRaw, version: TypingVersion, considerLibraryMinorVersion: boolean) {
return typingsDataRaw.libraryMajorVersion === version.major
&& (considerLibraryMinorVersion ?
(version.minor === undefined || typingsDataRaw.libraryMinorVersion === version.minor)
: true);
}
function formattedLibraryVersion(typingsDataRaw: TypingsDataRaw) {
return `${typingsDataRaw.libraryMajorVersion}.${typingsDataRaw.libraryMinorVersion}`;
}
/** @param fs Rooted at the package's directory, e.g. `DefinitelyTyped/types/abs` */
export function getTypingInfo(packageName: string, fs: FS): TypingsVersionsRaw {
if (packageName !== packageName.toLowerCase()) {
throw new Error(`Package name \`${packageName}\` should be strictly lowercase`);
}
interface OlderVersionDir { readonly directoryName: string; readonly version: TypingVersion; }
const [rootDirectoryLs, olderVersionDirectories] = split<string, OlderVersionDir>(fs.readdir(), fileOrDirectoryName => {
const version = parseVersionFromDirectoryName(fileOrDirectoryName);
return version === undefined ? undefined : { directoryName: fileOrDirectoryName, version };
});
const considerLibraryMinorVersion = olderVersionDirectories.some(({ version }) => version.minor !== undefined);
const latestData: TypingsDataRaw = {
libraryVersionDirectoryName: undefined,
...combineDataForAllTypesVersions(packageName, rootDirectoryLs, fs, undefined),
};
const older = olderVersionDirectories.map(({ directoryName, version: directoryVersion }) => {
if (matchesVersion(latestData, directoryVersion, considerLibraryMinorVersion)) {
const latest = `${latestData.libraryMajorVersion}.${latestData.libraryMinorVersion}`;
throw new Error(
`The latest version is ${latest}, so the subdirectory '${directoryName}' is not allowed` +
(`v${latest}` === directoryName ?
"." : `; since it applies to any ${latestData.libraryMajorVersion}.* version, up to and including ${latest}.`),
);
}
const ls = fs.readdir(directoryName);
const data: TypingsDataRaw = {
libraryVersionDirectoryName: formatTypingVersion(directoryVersion),
...combineDataForAllTypesVersions(packageName, ls, fs.subDir(directoryName), directoryVersion),
};
if (!matchesVersion(data, directoryVersion, considerLibraryMinorVersion)) {
if (considerLibraryMinorVersion) {
throw new Error(
`Directory ${directoryName} indicates major.minor version ${directoryVersion.major}.${directoryVersion.minor}, ` +
`but header indicates major.minor version ${data.libraryMajorVersion}.${data.libraryMinorVersion}`,
);
}
throw new Error(
`Directory ${directoryName} indicates major version ${directoryVersion.major}, but header indicates major version ` +
data.libraryMajorVersion.toString(),
);
}
return data;
});
const res: TypingsVersionsRaw = {};
res[formattedLibraryVersion(latestData)] = latestData;
for (const o of older) {
res[formattedLibraryVersion(o)] = o;
}
return res;
}
const packageJsonName = "package.json";
interface LsMinusTypesVersionsAndPackageJson {
readonly remainingLs: ReadonlyArray<string>;
readonly typesVersions: ReadonlyArray<TypeScriptVersion>;
readonly hasPackageJson: boolean;
}
function getTypesVersionsAndPackageJson(ls: ReadonlyArray<string>): LsMinusTypesVersionsAndPackageJson {
const withoutPackageJson = ls.filter(name => name !== packageJsonName);
const [remainingLs, typesVersions] = split(withoutPackageJson, fileOrDirectoryName => {
const match = /^ts(\d+\.\d+)$/.exec(fileOrDirectoryName);
if (match === null) { return undefined; }
const version = match[1];
if (!isTypeScriptVersion(version)) {
throw new Error(`Directory name starting with 'ts' should be a valid TypeScript version. Got: ${version}`);
}
return version;
});
return { remainingLs, typesVersions, hasPackageJson: withoutPackageJson.length !== ls.length };
}
/**
* Parses a directory name into a version that either holds a single major version or a major and minor version.
*
* @example
*
* ```ts
* parseVersionFromDirectoryName("v1") // { major: 1 }
* parseVersionFromDirectoryName("v0.61") // { major: 0, minor: 61 }
* ```
*/
export function parseVersionFromDirectoryName(directoryName: string): TypingVersion | undefined {
const match = /^v(\d+)(\.(\d+))?$/.exec(directoryName);
if (match === null) {
return undefined;
}
return {
major: Number(match[1]),
minor: match[3] !== undefined ? Number(match[3]) : undefined, // tslint:disable-line strict-type-predicates (false positive)
};
}
function combineDataForAllTypesVersions(
typingsPackageName: string,
ls: ReadonlyArray<string>,
fs: FS,
directoryVersion: TypingVersion | undefined,
): Omit<TypingsDataRaw, "libraryVersionDirectoryName"> {
const { remainingLs, typesVersions, hasPackageJson } = getTypesVersionsAndPackageJson(ls);
// Every typesVersion has an index.d.ts, but only the root index.d.ts should have a header.
const { contributors, libraryMajorVersion, libraryMinorVersion, typeScriptVersion: minTsVersion, libraryName, projects } =
parseHeaderOrFail(readFileAndThrowOnBOM("index.d.ts", fs));
const dataForRoot = getTypingDataForSingleTypesVersion(undefined, typingsPackageName, fs.debugPath(), remainingLs, fs, directoryVersion);
const dataForOtherTypesVersions = typesVersions.map(tsVersion => {
const subFs = fs.subDir(`ts${tsVersion}`);
return getTypingDataForSingleTypesVersion(tsVersion, typingsPackageName, fs.debugPath(), subFs.readdir(), subFs, directoryVersion);
});
const allTypesVersions = [dataForRoot, ...dataForOtherTypesVersions];
const packageJson = hasPackageJson ? fs.readJson(packageJsonName) as { readonly license?: unknown, readonly dependencies?: unknown } : {};
const license = getLicenseFromPackageJson(packageJson.license);
const packageJsonDependencies = checkPackageJsonDependencies(packageJson.dependencies, packageJsonName);
const files = Array.from(flatMap(allTypesVersions, ({ typescriptVersion, declFiles }) =>
declFiles.map(file =>
typescriptVersion === undefined ? file : `ts${typescriptVersion}/${file}`)));
return {
libraryName,
typingsPackageName,
projectName: projects[0], // TODO: collect multiple project names
contributors,
libraryMajorVersion,
libraryMinorVersion,
minTsVersion,
typesVersions,
files,
license,
// TODO: Explicit type arguments shouldn't be necessary. https://github.com/Microsoft/TypeScript/issues/27507
dependencies: getAllUniqueValues<"dependencies", PackageId>(allTypesVersions, "dependencies"),
testDependencies: getAllUniqueValues<"testDependencies", string>(allTypesVersions, "testDependencies"),
pathMappings: getAllUniqueValues<"pathMappings", PathMapping>(allTypesVersions, "pathMappings"),
packageJsonDependencies,
contentHash: hash(hasPackageJson ? [...files, packageJsonName] : files, mapDefined(allTypesVersions, a => a.tsconfigPathsForHash), fs),
globals: getAllUniqueValues<"globals", string>(allTypesVersions, "globals"),
declaredModules: getAllUniqueValues<"declaredModules", string>(allTypesVersions, "declaredModules"),
};
}
function getAllUniqueValues<K extends string, T>(records: ReadonlyArray<Record<K, ReadonlyArray<T>>>, key: K): ReadonlyArray<T> {
return unique(flatMap(records, x => x[key]));
}
interface TypingDataFromIndividualTypeScriptVersion {
/** Undefined for root (which uses `// TypeScript Version: ` comment instead) */
readonly typescriptVersion: TypeScriptVersion | undefined;
readonly dependencies: ReadonlyArray<PackageId>;
readonly testDependencies: ReadonlyArray<string>;
readonly pathMappings: ReadonlyArray<PathMapping>;
readonly declFiles: ReadonlyArray<string>;
readonly tsconfigPathsForHash: string | undefined;
readonly globals: ReadonlyArray<string>;
readonly declaredModules: ReadonlyArray<string>;
}
/**
* @param typescriptVersion Set if this is in e.g. a `ts3.1` directory.
* @param packageName Name of the outermost directory; e.g. for "node/v4" this is just "node".
* @param ls All file/directory names in `directory`.
* @param fs FS rooted at the directory for this particular TS version, e.g. `types/abs/ts3.1` or `types/abs` when typescriptVersion is undefined.
*/
function getTypingDataForSingleTypesVersion(
typescriptVersion: TypeScriptVersion | undefined,
packageName: string,
packageDirectory: string,
ls: ReadonlyArray<string>,
fs: FS,
directoryVersion: TypingVersion | undefined,
): TypingDataFromIndividualTypeScriptVersion {
const tsconfig = fs.readJson("tsconfig.json") as TsConfig;
checkFilesFromTsConfig(packageName, tsconfig, fs.debugPath());
const { types, tests } = allReferencedFiles(tsconfig.files!, fs, packageName, packageDirectory);
const usedFiles = new Set([...types.keys(), ...tests.keys(), "tsconfig.json", "tslint.json"]);
const otherFiles = ls.indexOf(unusedFilesName) > -1 ? (fs.readFile(unusedFilesName)).split(/\r?\n/g).filter(Boolean) : [];
checkAllFilesUsed(ls, usedFiles, otherFiles, packageName, fs);
for (const untestedTypeFile of filter(otherFiles, name => name.endsWith(".d.ts"))) {
// add d.ts files from OTHER_FILES.txt in order get their dependencies
types.set(untestedTypeFile, createSourceFile(untestedTypeFile, fs.readFile(untestedTypeFile)));
}
const { dependencies: dependenciesWithDeclaredModules, globals, declaredModules } = getModuleInfo(packageName, types);
const declaredModulesSet = new Set(declaredModules);
// Don't count an import of "x" as a dependency if we saw `declare module "x"` somewhere.
const dependenciesSet = new Set(filter(dependenciesWithDeclaredModules, m => !declaredModulesSet.has(m)));
const testDependencies = Array.from(
filter(
getTestDependencies(packageName, types, tests.keys(), dependenciesSet, fs),
m => !declaredModulesSet.has(m),
),
);
const { dependencies, pathMappings } = calculateDependencies(packageName, tsconfig, dependenciesSet, directoryVersion);
const tsconfigPathsForHash = JSON.stringify(tsconfig.compilerOptions.paths);
return {
typescriptVersion,
dependencies,
testDependencies,
pathMappings,
globals,
declaredModules,
declFiles: sort(types.keys()),
tsconfigPathsForHash,
};
}
function checkPackageJsonDependencies(dependencies: unknown, path: string): ReadonlyArray<PackageJsonDependency> {
if (dependencies === undefined) { // tslint:disable-line strict-type-predicates (false positive)
return [];
}
if (dependencies === null || typeof dependencies !== "object") { // tslint:disable-line strict-type-predicates
throw new Error(`${path} should contain "dependencies" or not exist.`);
}
const deps: PackageJsonDependency[] = [];
for (const dependencyName of Object.keys(dependencies!)) { // `dependencies` cannot be null because of check above.
if (!dependenciesWhitelist.has(dependencyName)) {
const msg = dependencyName.startsWith("@types/")
? `Dependency ${dependencyName} not in whitelist.
Don't use a 'package.json' for @types dependencies unless this package relies on
an old version of types that have since been moved to the source repo.
For example, if package *P* used to have types on Definitely Typed at @types/P,
but now has its own types, a dependent package *D* will need to use package.json
to refer to @types/P if it relies on old versions of P's types.
In this case, please make a pull request to types-publisher adding @types/P to \`dependenciesWhitelist.txt\`.`
: `Dependency ${dependencyName} not in whitelist.
If you are depending on another \`@types\` package, do *not* add it to a \`package.json\`. Path mapping should make the import work.
For namespaced dependencies you then have to add a \`paths\` mapping from \`@namespace/library\` to \`namespace__library\` in \`tsconfig.json\`.
If this is an external library that provides typings, please make a pull request to types-publisher adding it to \`dependenciesWhitelist.txt\`.`;
throw new Error(`In ${path}: ${msg}`);
}
const version = (dependencies as { [key: string]: unknown })[dependencyName];
if (typeof version !== "string") { // tslint:disable-line strict-type-predicates
throw new Error(`In ${path}: Dependency version for ${dependencyName} should be a string.`);
}
deps.push({ name: dependencyName, version });
}
return deps;
}
function checkFilesFromTsConfig(packageName: string, tsconfig: TsConfig, directoryPath: string): void {
const tsconfigPath = `${directoryPath}/tsconfig.json`;
if (tsconfig.include) {
throw new Error(`In tsconfig, don't use "include", must use "files"`);
}
const files = tsconfig.files;
if (!files) {
throw new Error(`${tsconfigPath} needs to specify "files"`);
}
for (const file of files) {
if (file.startsWith("./")) {
throw new Error(`In ${tsconfigPath}: Unnecessary "./" at the start of ${file}`);
}
if (file.endsWith(".d.ts") && file !== "index.d.ts") {
throw new Error(`${packageName}: Only index.d.ts may be listed explicitly in tsconfig's "files" entry.
Other d.ts files must either be referenced through index.d.ts, tests, or added to OTHER_FILES.txt.`);
}
if (!file.endsWith(".d.ts") && !file.startsWith("test/")) {
const expectedName = `${packageName}-tests.ts`;
if (file !== expectedName && file !== `${expectedName}x`) {
const message = file.endsWith(".ts") || file.endsWith(".tsx")
? `Expected file '${file}' to be named '${expectedName}' or to be inside a '${directoryPath}/test/' directory`
: (`Unexpected file extension for '${file}' -- expected '.ts' or '.tsx' (maybe this should not be in "files", but ` +
"OTHER_FILES.txt)");
throw new Error(message);
}
}
}
}
interface TsConfig {
include?: ReadonlyArray<string>;
files?: ReadonlyArray<string>;
compilerOptions: ts.CompilerOptions;
}
/** In addition to dependencies found in source code, also get dependencies from tsconfig. */
interface DependenciesAndPathMappings { readonly dependencies: ReadonlyArray<PackageId>; readonly pathMappings: ReadonlyArray<PathMapping>; }
function calculateDependencies(
packageName: string,
tsconfig: TsConfig,
dependencyNames: ReadonlySet<string>,
directoryVersion: TypingVersion | undefined,
): DependenciesAndPathMappings {
const paths = tsconfig.compilerOptions && tsconfig.compilerOptions.paths || {};
const dependencies: PackageId[] = [];
const pathMappings: PathMapping[] = [];
for (const dependencyName of Object.keys(paths)) {
// Might have a path mapping for "foo/*" to support subdirectories
const rootDirectory = withoutEnd(dependencyName, "/*");
if (rootDirectory !== undefined) {
if (!(rootDirectory in paths)) {
throw new Error(`In ${packageName}: found path mapping for ${dependencyName} but not for ${rootDirectory}`);
}
continue;
}
const pathMappingList = paths[dependencyName];
if (pathMappingList.length !== 1) {
throw new Error(`In ${packageName}: Path mapping for ${dependencyName} may only have 1 entry.`);
}
const pathMapping = pathMappingList[0];
// Path mapping may be for "@foo/bar" -> "foo__bar".
const scopedPackageName = unmangleScopedPackage(pathMapping);
if (scopedPackageName !== undefined) {
if (dependencyName !== scopedPackageName) {
throw new Error(`Expected directory ${pathMapping} to be the path mapping for ${dependencyName}`);
}
continue;
}
const pathMappingVersion = parseDependencyVersionFromPath(dependencyName, dependencyName, pathMapping);
if (dependencyName === packageName) {
if (directoryVersion === undefined) {
throw new Error(`In ${packageName}: Latest version of a package should not have a path mapping for itself.`);
}
if (
directoryVersion.major !== pathMappingVersion.major
|| directoryVersion.minor !== pathMappingVersion.minor
) {
const correctPathMapping = [`${dependencyName}/v${formatTypingVersion(directoryVersion)}`];
throw new Error(`In ${packageName}: Must have a "paths" entry of "${dependencyName}": ${JSON.stringify(correctPathMapping)}`);
}
} else {
if (dependencyNames.has(dependencyName)) {
dependencies.push({ name: dependencyName, version: pathMappingVersion });
}
}
// Else, the path mapping may be necessary if it is for a transitive dependency. We will check this in check-parse-results.
pathMappings.push({ packageName: dependencyName, version: pathMappingVersion });
}
if (directoryVersion !== undefined && !(paths && packageName in paths)) {
const mapping = JSON.stringify([`${packageName}/v${formatTypingVersion(directoryVersion)}`]);
throw new Error(
`${packageName}: Older version ${formatTypingVersion(directoryVersion)} must have a "paths" entry of "${packageName}": ${mapping}`,
);
}
for (const dependency of dependencyNames) {
if (!dependencies.some(d => d.name === dependency) && !nodeBuiltins.has(dependency)) {
dependencies.push({ name: dependency, version: "*" });
}
}
return { dependencies, pathMappings };
}
const nodeBuiltins: ReadonlySet<string> = new Set([
"assert", "async_hooks", "buffer", "child_process", "cluster", "console", "constants", "crypto",
"dgram", "dns", "domain", "events", "fs", "http", "http2", "https", "module", "net", "os",
"path", "perf_hooks", "process", "punycode", "querystring", "readline", "repl", "stream",
"string_decoder", "timers", "tls", "tty", "url", "util", "v8", "vm", "zlib",
]);
function parseDependencyVersionFromPath(packageName: string, dependencyName: string, dependencyPath: string): TypingVersion {
const versionString = withoutStart(dependencyPath, `${dependencyName}/`);
const version = versionString === undefined ? undefined : parseVersionFromDirectoryName(versionString);
if (version === undefined) {
throw new Error(`In ${packageName}, unexpected path mapping for ${dependencyName}: '${dependencyPath}'`);
}
return version;
}
function withoutEnd(s: string, end: string): string | undefined {
if (s.endsWith(end)) {
return s.slice(0, s.length - end.length);
}
return undefined;
}
function hash(files: ReadonlyArray<string>, tsconfigPathsForHash: ReadonlyArray<string>, fs: FS): string {
const fileContents = files.map(f => `${f}**${readFileAndThrowOnBOM(f, fs)}`);
let allContent = fileContents.join("||");
for (const path of tsconfigPathsForHash) {
allContent += path;
}
return computeHash(allContent);
}
export function readFileAndThrowOnBOM(fileName: string, fs: FS): string {
const text = fs.readFile(fileName);
if (text.charCodeAt(0) === 0xFEFF) {
const commands = [
"npm install -g strip-bom-cli",
`strip-bom ${fileName} > fix`,
`mv fix ${fileName}`,
];
throw new Error(`File '${fileName}' has a BOM. Try using:\n${commands.join("\n")}`);
}
return text;
}
const unusedFilesName = "OTHER_FILES.txt";
function checkAllFilesUsed(ls: ReadonlyArray<string>, usedFiles: Set<string>, otherFiles: string[], packageName: string, fs: FS): void {
// Double-check that no windows "\\" broke in.
for (const fileName of usedFiles) {
if (hasWindowsSlashes(fileName)) {
throw new Error(`In ${packageName}: windows slash detected in ${fileName}`);
}
}
checkAllUsedRecur(new Set(ls), usedFiles, new Set(otherFiles), fs);
}
function checkAllUsedRecur(ls: Iterable<string>, usedFiles: Set<string>, unusedFiles: Set<string>, fs: FS): void {
for (const lsEntry of ls) {
if (usedFiles.has(lsEntry)) {
continue;
}
if (unusedFiles.has(lsEntry)) {
unusedFiles.delete(lsEntry);
continue;
}
if (fs.isDirectory(lsEntry)) {
const subdir = fs.subDir(lsEntry);
// We allow a "scripts" directory to be used for scripts.
if (lsEntry === "node_modules" || lsEntry === "scripts") {
continue;
}
const lssubdir = subdir.readdir();
if (lssubdir.length === 0) {
// tslint:disable-next-line strict-string-expressions
throw new Error(`Empty directory ${subdir} (${join(usedFiles)})`);
}
function takeSubdirectoryOutOfSet(originalSet: Set<string>): Set<string> {
const subdirSet = new Set<string>();
for (const file of originalSet) {
const sub = withoutStart(file, `${lsEntry}/`);
if (sub !== undefined) {
originalSet.delete(file);
subdirSet.add(sub);
}
}
return subdirSet;
}
checkAllUsedRecur(lssubdir, takeSubdirectoryOutOfSet(usedFiles), takeSubdirectoryOutOfSet(unusedFiles), subdir);
} else {
if (lsEntry.toLowerCase() !== "readme.md" && lsEntry !== "NOTICE" && lsEntry !== ".editorconfig" && lsEntry !== unusedFilesName) {
throw new Error(`Unused file ${fs.debugPath()}/${lsEntry} (used files: ${JSON.stringify(Array.from(usedFiles))})`);
}
}
}
for (const unusedFile of unusedFiles) {
if (usedFiles.has(unusedFile)) {
throw new Error(`File ${fs.debugPath()}/${unusedFile} listed in ${unusedFilesName} is already reachable from tsconfig.json.`);
}
throw new Error(`File ${fs.debugPath()}/${unusedFile} listed in ${unusedFilesName} does not exist.`);
}
}

79
packages/types-publisher/src/lib/module-info.test.ts
View File

@@ -1,79 +0,0 @@
import * as ts from "typescript";
import { Dir, InMemoryDT } from "../get-definitely-typed";
import { createMockDT } from "../mocks";
import { testo } from "../util/test";
import { allReferencedFiles, getModuleInfo, getTestDependencies } from "./module-info";
const fs = createMockDT().fs;
function getBoringReferences() {
return allReferencedFiles(["index.d.ts", "boring-tests.ts"], fs.subDir("types").subDir("boring"), "boring", "types/boring");
}
testo({
allReferencedFilesFromTsconfigFiles() {
const { types, tests } = getBoringReferences();
expect(Array.from(types.keys())).toEqual(["index.d.ts", "secondary.d.ts", "quaternary.d.ts", "tertiary.d.ts", "commonjs.d.ts", "v1.d.ts"]);
expect(Array.from(tests.keys())).toEqual(["boring-tests.ts"]);
},
allReferencedFilesFromTestIncludesSecondaryInternalFiles() {
const { types, tests } = allReferencedFiles(["boring-tests.ts"], fs.subDir("types").subDir("boring"), "boring", "types/boring");
expect(Array.from(types.keys())).toEqual(["secondary.d.ts", "quaternary.d.ts", "tertiary.d.ts", "commonjs.d.ts", "v1.d.ts"]);
expect(Array.from(tests.keys())).toEqual(["boring-tests.ts"]);
},
allReferencedFilesFromTsconfigGlobal() {
const { types, tests } = allReferencedFiles(["jquery-tests.ts", "index.d.ts"], fs.subDir("types").subDir("jquery"), "jquery", "types/jquery");
expect(Array.from(types.keys())).toEqual(["index.d.ts", "JQuery.d.ts"]);
expect(Array.from(tests.keys())).toEqual(["jquery-tests.ts"]);
},
allReferencedFilesFromTestIncludesSecondaryTripleSlashTypes() {
const { types, tests } = allReferencedFiles(
["globby-tests.ts", "test/other-tests.ts"],
fs.subDir("types").subDir("globby"),
"globby",
"types/globby",
);
expect(Array.from(types.keys())).toEqual(["merges.d.ts"]);
expect(Array.from(tests.keys())).toEqual(["globby-tests.ts", "test/other-tests.ts"]);
},
getModuleInfoWorksWithOtherFiles() {
const { types } = getBoringReferences();
// written as if it were from OTHER_FILES.txt
types.set(
"untested.d.ts",
ts.createSourceFile("untested.d.ts", fs.subDir("types").subDir("boring").readFile("untested.d.ts"), ts.ScriptTarget.Latest, false),
);
const i = getModuleInfo("boring", types);
expect(i.dependencies).toEqual(new Set(["manual", "react", "react-default", "things", "vorticon"]));
},
getModuleInfoForNestedTypeReferences() {
const { types } = allReferencedFiles(
["index.d.ts", "globby-tests.ts", "test/other-tests.ts"],
fs.subDir("types").subDir("globby"),
"globby",
"types/globby",
);
expect(Array.from(types.keys())).toEqual(["index.d.ts", "sneaky.d.ts", "merges.d.ts"]);
const i = getModuleInfo("globby", types);
expect(i.dependencies).toEqual(new Set(["andere"]));
},
versionTypeRefThrows() {
const fail = new Dir(undefined);
const memFS = new InMemoryDT(fail, "typeref-fails");
fail.set("index.d.ts", `// Type definitions for fail 1.0
// Project: https://youtube.com/typeref-fails
// Definitions by: Type Ref Fails <https://github.com/typeref-fails>
// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
/// <reference types="elser/v3" />
`);
const { types } = allReferencedFiles(["index.d.ts"], memFS, "typeref-fails", "types/typeref-fails");
expect(Array.from(types.keys())).toEqual(["index.d.ts"]);
expect(() => getModuleInfo("typeref-fails", types)).toThrow("do not directly import specific versions of another types package");
},
getTestDependenciesWorks() {
const { types, tests } = getBoringReferences();
const i = getModuleInfo("boring", types);
const d = getTestDependencies("boring", types, tests.keys(), i.dependencies, fs.subDir("types").subDir("boring"));
expect(d).toEqual(new Set(["super-big-fun-hus"]));
},
});

407
packages/types-publisher/src/lib/module-info.ts
View File

@@ -1,407 +0,0 @@
import assert = require("assert");
import * as path from "path";
import * as ts from "typescript";
import { FS } from "../get-definitely-typed";
import { hasWindowsSlashes, joinPaths, normalizeSlashes, sort } from "../util/util";
import { readFileAndThrowOnBOM } from "./definition-parser";
export function getModuleInfo(packageName: string, all: Map<string, ts.SourceFile>): ModuleInfo {
const dependencies = new Set<string>();
const declaredModules: string[] = [];
const globals = new Set<string>();
function addDependency(ref: string): void {
if (ref.startsWith(".")) { return; }
const dependency = rootName(ref, all);
if (dependency !== packageName) {
dependencies.add(dependency);
}
// TODO: else throw new Error(`Package ${packageName} references itself. (via ${src.fileName})`);
}
for (const sourceFile of all.values()) {
for (const ref of imports(sourceFile)) {
addDependency(ref);
}
for (const ref of sourceFile.typeReferenceDirectives) {
addDependency(ref.fileName);
}
if (ts.isExternalModule(sourceFile)) {
if (sourceFileExportsSomething(sourceFile)) {
declaredModules.push(properModuleName(packageName, sourceFile.fileName));
const namespaceExport = sourceFile.statements.find(ts.isNamespaceExportDeclaration);
if (namespaceExport) {
globals.add(namespaceExport.name.text);
}
}
} else {
for (const node of sourceFile.statements) {
switch (node.kind) {
case ts.SyntaxKind.ModuleDeclaration: {
const decl = node as ts.ModuleDeclaration;
const name = decl.name.text;
if (decl.name.kind === ts.SyntaxKind.StringLiteral) {
declaredModules.push(assertNoWindowsSlashes(packageName, name));
} else if (isValueNamespace(decl)) {
globals.add(name);
}
break;
}
case ts.SyntaxKind.VariableStatement:
for (const decl of (node as ts.VariableStatement).declarationList.declarations) {
if (decl.name.kind === ts.SyntaxKind.Identifier) {
globals.add(decl.name.text);
}
}
break;
case ts.SyntaxKind.EnumDeclaration:
case ts.SyntaxKind.ClassDeclaration:
case ts.SyntaxKind.FunctionDeclaration: {
// Deliberately not doing this for types, because those won't show up in JS code and can't be used for ATA
const nameNode = (node as ts.EnumDeclaration | ts.ClassDeclaration | ts.FunctionDeclaration).name;
if (nameNode) {
globals.add(nameNode.text);
}
break;
}
case ts.SyntaxKind.ImportEqualsDeclaration:
case ts.SyntaxKind.InterfaceDeclaration:
case ts.SyntaxKind.TypeAliasDeclaration:
break;
default:
throw new Error(`Unexpected node kind ${ts.SyntaxKind[node.kind]}`);
}
}
}
}
return { dependencies, declaredModules, globals: sort(globals) };
}
/**
* A file is a proper module if it is an external module *and* it has at least one export.
* A module with only imports is not a proper module; it likely just augments some other module.
*/
function sourceFileExportsSomething({ statements }: ts.SourceFile): boolean {
return statements.some(statement => {
switch (statement.kind) {
case ts.SyntaxKind.ImportEqualsDeclaration:
case ts.SyntaxKind.ImportDeclaration:
return false;
case ts.SyntaxKind.ModuleDeclaration:
return (statement as ts.ModuleDeclaration).name.kind === ts.SyntaxKind.Identifier;
default:
return true;
}
});
}
interface ModuleInfo {
dependencies: Set<string>;
// Anything from a `declare module "foo"`
declaredModules: string[];
// Every global symbol
globals: string[];
}
/**
* Given a file name, get the name of the module it declares.
* `foo/index.d.ts` declares "foo", `foo/bar.d.ts` declares "foo/bar", "foo/bar/index.d.ts" declares "foo/bar"
*/
function properModuleName(folderName: string, fileName: string): string {
const part = path.basename(fileName) === "index.d.ts" ? path.dirname(fileName) : withoutExtension(fileName, ".d.ts");
return part === "." ? folderName : joinPaths(folderName, part);
}
/**
* "foo/bar/baz" -> "foo"; "@foo/bar/baz" -> "@foo/bar"
* Note: Throws an error for references like
*/
function rootName(importText: string, typeFiles: Map<string, unknown>): string {
let slash = importText.indexOf("/");
// Root of `@foo/bar/baz` is `@foo/bar`
if (importText.startsWith("@")) {
// Use second "/"
slash = importText.indexOf("/", slash + 1);
}
const root = importText.slice(0, slash);
const postImport = importText.slice(slash + 1);
if (slash > -1 && postImport.match(/v\d+$/) && !typeFiles.has(postImport + ".d.ts")) {
throw new Error(`${importText}: do not directly import specific versions of another types package.
You should work with the latest version of ${root} instead.`);
}
return slash === -1 ? importText : root;
}
function withoutExtension(str: string, ext: string): string {
assert(str.endsWith(ext));
return str.slice(0, str.length - ext.length);
}
/** Returns a map from filename (path relative to `directory`) to the SourceFile we parsed for it. */
export function allReferencedFiles(
entryFilenames: ReadonlyArray<string>, fs: FS, packageName: string, baseDirectory: string,
): { types: Map<string, ts.SourceFile>, tests: Map<string, ts.SourceFile> } {
const seenReferences = new Set<string>();
const types = new Map<string, ts.SourceFile>();
const tests = new Map<string, ts.SourceFile>();
entryFilenames.forEach(text => recur({ text, exact: true }));
return { types, tests };
function recur({ text, exact }: Reference): void {
if (seenReferences.has(text)) {
return;
}
seenReferences.add(text);
const resolvedFilename = exact ? text : resolveModule(text, fs);
if (fs.exists(resolvedFilename)) {
const src = createSourceFile(resolvedFilename, readFileAndThrowOnBOM(resolvedFilename, fs));
if (resolvedFilename.endsWith(".d.ts")) {
types.set(resolvedFilename, src);
} else {
tests.set(resolvedFilename, src);
}
const refs = findReferencedFiles(
src,
packageName,
path.dirname(resolvedFilename),
normalizeSlashes(path.relative(baseDirectory, fs.debugPath())),
);
refs.forEach(recur);
}
}
}
function resolveModule(importSpecifier: string, fs: FS): string {
importSpecifier = importSpecifier.endsWith("/") ? importSpecifier.slice(0, importSpecifier.length - 1) : importSpecifier;
if (importSpecifier !== "." && importSpecifier !== "..") {
if (fs.exists(importSpecifier + ".d.ts")) {
return importSpecifier + ".d.ts";
}
if (fs.exists(importSpecifier + ".ts")) {
return importSpecifier + ".ts";
}
if (fs.exists(importSpecifier + ".tsx")) {
return importSpecifier + ".tsx";
}
}
return importSpecifier === "." ? "index.d.ts" : joinPaths(importSpecifier, "index.d.ts");
}
interface Reference {
/** <reference path> includes exact filename, so true. import "foo" may reference "foo.d.ts" or "foo/index.d.ts", so false. */
readonly exact: boolean;
text: string;
}
/**
* @param subDirectory The specific directory within the DefinitelyTyped directory we are in.
* For example, `baseDirectory` may be `react-router` and `subDirectory` may be `react-router/lib`.
* versionsBaseDirectory may be "" when not in typesVersions or ".." when inside `react-router/ts3.1`
*/
function findReferencedFiles(src: ts.SourceFile, packageName: string, subDirectory: string, baseDirectory: string) {
const refs: Reference[] = [];
for (const ref of src.referencedFiles) {
// Any <reference path="foo"> is assumed to be local
addReference({ text: ref.fileName, exact: true });
}
for (const ref of src.typeReferenceDirectives) {
// only <reference types="../packagename/x" /> references are local (or "packagename/x", though in 3.7 that doesn't work in DT).
if (ref.fileName.startsWith("../" + packageName + "/")) {
addReference({ text: ref.fileName, exact: false });
} else if (ref.fileName.startsWith(packageName + "/")) {
addReference({ text: convertToRelativeReference(ref.fileName), exact: false });
}
}
for (const ref of imports(src)) {
if (ref.startsWith(".")) {
addReference({ text: ref, exact: false });
}
if (ref.startsWith(packageName + "/")) {
addReference({ text: convertToRelativeReference(ref), exact: false });
}
}
return refs;
function addReference(ref: Reference): void {
// `path.normalize` may add windows slashes
const full = normalizeSlashes(path.normalize(joinPaths(subDirectory, assertNoWindowsSlashes(src.fileName, ref.text))));
// allow files in typesVersions directories (i.e. 'ts3.1') to reference files in parent directory
if (full.startsWith("../" + packageName + "/")) {
ref.text = full.slice(packageName.length + 4);
refs.push(ref);
return;
}
if (full.startsWith("..")
&& (baseDirectory === "" || path.normalize(joinPaths(baseDirectory, full)).startsWith(".."))) {
throw new Error(
`${src.fileName}: ` +
'Definitions must use global references to other packages, not parent ("../xxx") references.' +
`(Based on reference '${ref.text}')`);
}
ref.text = full;
refs.push(ref);
}
/** boring/foo -> ./foo when subDirectory === '.'; ../foo when it's === 'x'; ../../foo when it's 'x/y' */
function convertToRelativeReference(name: string) {
const relative = "." + "/..".repeat(subDirectory === "." ? 0 : subDirectory.split("/").length);
return relative + name.slice(packageName.length);
}
}
/**
* All strings referenced in `import` statements.
* Does *not* include <reference> directives.
*/
function* imports({ statements }: ts.SourceFile | ts.ModuleBlock): Iterable<string> {
for (const node of statements) {
switch (node.kind) {
case ts.SyntaxKind.ImportDeclaration:
case ts.SyntaxKind.ExportDeclaration: {
const { moduleSpecifier } = node as ts.ImportDeclaration | ts.ExportDeclaration;
if (moduleSpecifier && moduleSpecifier.kind === ts.SyntaxKind.StringLiteral) {
yield (moduleSpecifier as ts.StringLiteral).text;
}
break;
}
case ts.SyntaxKind.ImportEqualsDeclaration: {
const { moduleReference } = node as ts.ImportEqualsDeclaration;
if (moduleReference.kind === ts.SyntaxKind.ExternalModuleReference) {
yield parseRequire(moduleReference);
}
break;
}
case ts.SyntaxKind.ModuleDeclaration: {
const { name, body } = node as ts.ModuleDeclaration;
if (name.kind === ts.SyntaxKind.StringLiteral && body) {
yield* imports(body as ts.ModuleBlock);
}
break;
}
default:
}
}
}
function parseRequire(reference: ts.ExternalModuleReference): string {
const { expression } = reference;
if (!expression || !ts.isStringLiteral(expression)) {
throw new Error(`Bad 'import =' reference: ${reference.getText()}`);
}
return expression.text;
}
function isValueNamespace(ns: ts.ModuleDeclaration): boolean {
if (!ns.body) {
throw new Error("@types should not use shorthand ambient modules");
}
return ns.body.kind === ts.SyntaxKind.ModuleDeclaration
? isValueNamespace(ns.body as ts.ModuleDeclaration)
: (ns.body as ts.ModuleBlock).statements.some(statementDeclaresValue);
}
function statementDeclaresValue(statement: ts.Statement): boolean {
switch (statement.kind) {
case ts.SyntaxKind.VariableStatement:
case ts.SyntaxKind.ClassDeclaration:
case ts.SyntaxKind.FunctionDeclaration:
case ts.SyntaxKind.EnumDeclaration:
return true;
case ts.SyntaxKind.ModuleDeclaration:
return isValueNamespace(statement as ts.ModuleDeclaration);
case ts.SyntaxKind.InterfaceDeclaration:
case ts.SyntaxKind.TypeAliasDeclaration:
case ts.SyntaxKind.ImportEqualsDeclaration:
return false;
default:
throw new Error(`Forgot to implement ambient namespace statement ${ts.SyntaxKind[statement.kind]}`);
}
}
function assertNoWindowsSlashes(packageName: string, fileName: string): string {
if (hasWindowsSlashes(fileName)) {
throw new Error(`In ${packageName}: Use forward slash instead when referencing ${fileName}`);
}
return fileName;
}
export function getTestDependencies(
packageName: string,
typeFiles: Map<string, unknown>,
testFiles: Iterable<string>,
dependencies: ReadonlySet<string>,
fs: FS,
): Iterable<string> {
const testDependencies = new Set<string>();
for (const filename of testFiles) {
const content = readFileAndThrowOnBOM(filename, fs);
const sourceFile = createSourceFile(filename, content);
const { fileName, referencedFiles, typeReferenceDirectives } = sourceFile;
const filePath = () => path.join(packageName, fileName);
let hasImports = false;
let isModule = false;
let referencesSelf = false;
for (const { fileName: ref } of referencedFiles) {
throw new Error(`Test files should not use '<reference path="" />'. '${filePath()}' references '${ref}'.`);
}
for (const { fileName: referencedPackage } of typeReferenceDirectives) {
if (dependencies.has(referencedPackage)) {
throw new Error(
`'${filePath()}' unnecessarily references '${referencedPackage}', which is already referenced in the type definition.`);
}
if (referencedPackage === packageName) {
referencesSelf = true;
}
testDependencies.add(referencedPackage);
}
for (const imported of imports(sourceFile)) {
hasImports = true;
if (!imported.startsWith(".")) {
const dep = rootName(imported, typeFiles);
if (!dependencies.has(dep) && dep !== packageName) {
testDependencies.add(dep);
}
}
}
isModule = hasImports || (() => {
// FIXME: This results in files without imports to be walked twice,
// once in the `imports(...)` function, and once more here:
for (const node of sourceFile.statements) {
if (
node.kind === ts.SyntaxKind.ExportAssignment ||
node.kind === ts.SyntaxKind.ExportDeclaration
) {
return true;
}
}
return false;
})();
if (isModule && referencesSelf) {
throw new Error(`'${filePath()}' unnecessarily references the package. This can be removed.`);
}
}
return testDependencies;
}
export function createSourceFile(filename: string, content: string): ts.SourceFile {
return ts.createSourceFile(filename, content, ts.ScriptTarget.Latest, /*setParentNodes*/false);
}

48
packages/types-publisher/src/lib/packages.test.ts
View File

@@ -1,48 +0,0 @@
import { createMockDT } from "../mocks";
import { getTypingInfo } from "./definition-parser";
import { TypingsVersions } from "./packages";
describe(TypingsVersions, () => {
let versions: TypingsVersions;
beforeAll(() => {
const dt = createMockDT();
dt.addOldVersionOfPackage("jquery", "1");
dt.addOldVersionOfPackage("jquery", "2");
dt.addOldVersionOfPackage("jquery", "2.5");
versions = new TypingsVersions(getTypingInfo("jquery", dt.pkgFS("jquery")));
});
it("sorts the data from latest to oldest version", () => {
expect(Array.from(versions.getAll()).map(v => v.major)).toEqual([3, 2, 2, 1]);
});
it("returns the latest version", () => {
expect(versions.getLatest().major).toEqual(3);
});
it("finds the latest version when any version is wanted", () => {
expect(versions.get("*").major).toEqual(3);
});
it("finds the latest minor version for the given major version", () => {
expect(versions.get({ major: 2 }).major).toEqual(2);
expect(versions.get({ major: 2 }).minor).toEqual(5);
});
it("finds a specific version", () => {
expect(versions.get({ major: 2, minor: 0 }).major).toEqual(2);
expect(versions.get({ major: 2, minor: 0 }).minor).toEqual(0);
});
it("formats a version directory names", () => {
expect(versions.get({ major: 2, minor: 0 }).versionDirectoryName).toEqual("v2");
expect(versions.get({ major: 2, minor: 0 }).subDirectoryPath).toEqual("jquery/v2");
});
it("formats missing version error nicely", () => {
expect(() => versions.get({ major: 111, minor: 1001 })).toThrow("Could not find version 111.1001");
expect(() => versions.get({ major: 111 })).toThrow("Could not find version 111.*");
});
});

546
packages/types-publisher/src/lib/packages.ts
View File

@@ -1,546 +0,0 @@
import assert = require("assert");
import { AllTypeScriptVersion, Author, TypeScriptVersion } from "definitelytyped-header-parser";
import { FS } from "../get-definitely-typed";
import { assertSorted, joinPaths, mapValues, unmangleScopedPackage } from "../util/util";
import { readDataFile } from "./common";
import { outputDirPath, scopeName } from "./settings";
import { compare as compareSemver, Semver } from "./versions";
export class AllPackages {
static async read(dt: FS): Promise<AllPackages> {
return AllPackages.from(await readTypesDataFile(), readNotNeededPackages(dt));
}
static from(data: TypesDataFile, notNeeded: ReadonlyArray<NotNeededPackage>): AllPackages {
return new AllPackages(mapValues(new Map(Object.entries(data)), raw => new TypingsVersions(raw)), notNeeded);
}
static async readTypings(): Promise<ReadonlyArray<TypingsData>> {
return AllPackages.from(await readTypesDataFile(), []).allTypings();
}
static async readLatestTypings(): Promise<ReadonlyArray<TypingsData>> {
return AllPackages.from(await readTypesDataFile(), []).allLatestTypings();
}
/** Use for `--single` tasks only. Do *not* call this in a loop! */
static async readSingle(name: string): Promise<TypingsData> {
const data = await readTypesDataFile();
const raw = data[name];
if (!raw) {
throw new Error(`Can't find package ${name}`);
}
const versions = Object.keys(raw);
if (versions.length > 1) {
throw new Error(`Package ${name} has multiple versions.`);
}
return new TypingsData(raw[versions[0]], /*isLatest*/ true);
}
static readSingleNotNeeded(name: string, dt: FS): NotNeededPackage {
const notNeeded = readNotNeededPackages(dt);
const pkg = notNeeded.find(p => p.name === name);
if (pkg === undefined) {
throw new Error(`Cannot find not-needed package ${name}`);
}
return pkg;
}
private constructor(
private readonly data: ReadonlyMap<string, TypingsVersions>,
private readonly notNeeded: ReadonlyArray<NotNeededPackage>) {}
getNotNeededPackage(name: string): NotNeededPackage | undefined {
return this.notNeeded.find(p => p.name === name);
}
hasTypingFor(dep: PackageId): boolean {
return this.tryGetTypingsData(dep) !== undefined;
}
tryResolve(dep: PackageId): PackageId {
const versions = this.data.get(getMangledNameForScopedPackage(dep.name));
return versions ? versions.get(dep.version).id : dep;
}
/** Gets the latest version of a package. E.g. getLatest(node v6) was node v10 (before node v11 came out). */
getLatest(pkg: TypingsData): TypingsData {
return pkg.isLatest ? pkg : this.getLatestVersion(pkg.name);
}
private getLatestVersion(packageName: string): TypingsData {
const latest = this.tryGetLatestVersion(packageName);
if (!latest) {
throw new Error(`No such package ${packageName}.`);
}
return latest;
}
tryGetLatestVersion(packageName: string): TypingsData | undefined {
const versions = this.data.get(getMangledNameForScopedPackage(packageName));
return versions && versions.getLatest();
}
getTypingsData(id: PackageId): TypingsData {
const pkg = this.tryGetTypingsData(id);
if (!pkg) {
throw new Error(`No typings available for ${JSON.stringify(id)}`);
}
return pkg;
}
tryGetTypingsData({ name, version }: PackageId): TypingsData | undefined {
const versions = this.data.get(getMangledNameForScopedPackage(name));
return versions && versions.tryGet(version);
}
allPackages(): ReadonlyArray<AnyPackage> {
return [ ...this.allTypings(), ...this.allNotNeeded() ];
}
/** Note: this includes older version directories (`foo/v0`) */
allTypings(): ReadonlyArray<TypingsData> {
return assertSorted(Array.from(flattenData(this.data)), t => t.name);
}
allLatestTypings(): ReadonlyArray<TypingsData> {
return assertSorted(Array.from(this.data.values()).map(versions => versions.getLatest()), t => t.name);
}
allNotNeeded(): ReadonlyArray<NotNeededPackage> {
return this.notNeeded;
}
/** Returns all of the dependences *that have typings*, ignoring others, and including test dependencies. */
*allDependencyTypings(pkg: TypingsData): Iterable<TypingsData> {
for (const { name, version } of pkg.dependencies) {
const versions = this.data.get(getMangledNameForScopedPackage(name));
if (versions) {
yield versions.get(version);
}
}
for (const name of pkg.testDependencies) {
const versions = this.data.get(getMangledNameForScopedPackage(name));
if (versions) {
yield versions.getLatest();
}
}
}
}
// Same as the function in moduleNameResolver.ts in typescript
export function getMangledNameForScopedPackage(packageName: string): string {
if (packageName.startsWith("@")) {
const replaceSlash = packageName.replace("/", "__");
if (replaceSlash !== packageName) {
return replaceSlash.slice(1); // Take off the "@"
}
}
return packageName;
}
export const typesDataFilename = "definitions.json";
function* flattenData(data: ReadonlyMap<string, TypingsVersions>): Iterable<TypingsData> {
for (const versions of data.values()) {
yield* versions.getAll();
}
}
export type AnyPackage = NotNeededPackage | TypingsData;
interface BaseRaw {
/**
* The name of the library.
*
* A human readable version, e.g. it might be "Moment.js" even though `packageName` is "moment".
*/
readonly libraryName: string;
/**
* The NPM name to publish this under, e.g. "jquery".
*
* This does not include "@types".
*/
readonly typingsPackageName: string;
}
/** Prefer to use `AnyPackage` instead of this. */
export abstract class PackageBase {
static compare(a: PackageBase, b: PackageBase): number { return a.name.localeCompare(b.name); }
/** Note: for "foo__bar" this is still "foo__bar", not "@foo/bar". */
readonly name: string;
readonly libraryName: string;
get unescapedName(): string {
return unmangleScopedPackage(this.name) || this.name;
}
/** Short description for debug output. */
get desc(): string {
return this.isLatest ? this.name : `${this.name} v${this.major}.${this.minor}`;
}
constructor(data: BaseRaw) {
this.name = data.typingsPackageName;
this.libraryName = data.libraryName;
}
isNotNeeded(): this is NotNeededPackage {
return this instanceof NotNeededPackage;
}
abstract readonly isLatest: boolean;
abstract readonly projectName: string;
abstract readonly declaredModules: ReadonlyArray<string>;
abstract readonly globals: ReadonlyArray<string>;
abstract readonly minTypeScriptVersion: TypeScriptVersion;
/** '@types/foo' for a package 'foo'. */
get fullNpmName(): string {
return getFullNpmName(this.name);
}
/** '@types%2ffoo' for a package 'foo'. */
get fullEscapedNpmName(): string {
return `@${scopeName}%2f${this.name}`;
}
abstract readonly major: number;
abstract readonly minor: number;
get id(): PackageId {
return { name: this.name, version: { major: this.major, minor: this.minor }};
}
get outputDirectory(): string {
return joinPaths(outputDirPath, this.desc);
}
}
export function getFullNpmName(packageName: string): string {
return `@${scopeName}/${getMangledNameForScopedPackage(packageName)}`;
}
interface NotNeededPackageRaw extends BaseRaw {
/**
* If this is available, @types typings are deprecated as of this version.
* This is useful for packages that previously had DefinitelyTyped definitions but which now provide their own.
*/
// This must be "major.minor.patch"
readonly asOfVersion: string;
/** The package's own url, *not* DefinitelyTyped's. */
readonly sourceRepoURL: string;
}
export class NotNeededPackage extends PackageBase {
readonly version: Semver;
get license(): License.MIT { return License.MIT; }
readonly sourceRepoURL: string;
constructor(raw: NotNeededPackageRaw) {
super(raw);
this.sourceRepoURL = raw.sourceRepoURL;
for (const key of Object.keys(raw)) {
if (!["libraryName", "typingsPackageName", "sourceRepoURL", "asOfVersion"].includes(key)) {
throw new Error(`Unexpected key in not-needed package: ${key}`);
}
}
assert(raw.libraryName && raw.typingsPackageName && raw.sourceRepoURL && raw.asOfVersion);
this.version = Semver.parse(raw.asOfVersion);
}
get major(): number { return this.version.major; }
get minor(): number { return this.version.minor; }
// A not-needed package has no other versions. (TODO: allow that?)
get isLatest(): boolean { return true; }
get projectName(): string { return this.sourceRepoURL; }
get declaredModules(): ReadonlyArray<string> { return []; }
get globals(): ReadonlyArray<string> { return this.globals; }
get minTypeScriptVersion(): TypeScriptVersion { return TypeScriptVersion.lowest; }
readme(): string {
return `This is a stub types definition for ${this.libraryName} (${this.sourceRepoURL}).\n
${this.libraryName} provides its own type definitions, so you don't need ${getFullNpmName(this.name)} installed!`;
}
deprecatedMessage(): string {
return `This is a stub types definition. ${this.name} provides its own type definitions, so you do not need this installed.`;
}
}
export interface TypingsVersionsRaw {
[version: string]: TypingsDataRaw;
}
export interface TypingVersion {
major: number;
minor?: number;
}
export function formatTypingVersion(version: TypingVersion) {
return `${version.major}${version.minor === undefined ? "" : `.${version.minor}`}`;
}
/** If no version is specified, uses "*". */
export type DependencyVersion = TypingVersion | "*";
export function formatDependencyVersion(version: DependencyVersion) {
return version === "*" ? "*" : formatTypingVersion(version);
}
export interface PackageJsonDependency {
readonly name: string;
readonly version: string;
}
export interface TypingsDataRaw extends BaseRaw {
/**
* Other definitions, that exist in the same typings repo, that this package depends on.
*
* These will refer to *package names*, not *folder names*.
*/
readonly dependencies: ReadonlyArray<PackageId>;
/**
* Other definitions, that exist in the same typings repo, that the tests, but not the types, of this package depend on.
*
* These are always the latest version and will not include anything already in `dependencies`.
*/
readonly testDependencies: ReadonlyArray<string>;
/**
* External packages, from outside the typings repo, that provide definitions that this package depends on.
*/
readonly packageJsonDependencies: ReadonlyArray<PackageJsonDependency>;
/**
* Represents that there was a path mapping to a package.
*
* Not all path mappings are direct dependencies, they may be necessary for transitive dependencies. However, where `dependencies` and
* `pathMappings` share a key, they *must* share the same value.
*/
readonly pathMappings: ReadonlyArray<PathMapping>;
/**
* List of people that have contributed to the definitions in this package.
*
* These people will be requested for issue/PR review in the https://github.com/DefinitelyTyped/DefinitelyTyped repo.
*/
readonly contributors: ReadonlyArray<Author>;
/**
* The [older] version of the library that this definition package refers to, as represented *on-disk*.
*
* @note The latest version always exists in the root of the package tree and thus does not have a value for this property.
*/
readonly libraryVersionDirectoryName?: string;
/**
* Major version of the library.
*
* This data is parsed from a header comment in the entry point `.d.ts` and will be `0` if the file did not specify a version.
*/
readonly libraryMajorVersion: number;
/**
* Minor version of the library.
*
* This data is parsed from a header comment in the entry point `.d.ts` and will be `0` if the file did not specify a version.
*/
readonly libraryMinorVersion: number;
/**
* Minimum required TypeScript version to consume the definitions from this package.
*/
readonly minTsVersion: AllTypeScriptVersion;
/**
* List of TS versions that have their own directories, and corresponding "typesVersions" in package.json.
* Usually empty.
*/
readonly typesVersions: ReadonlyArray<TypeScriptVersion>;
/**
* Files that should be published with this definition, e.g. ["jquery.d.ts", "jquery-extras.d.ts"]
*
* Does *not* include a partial `package.json` because that will not be copied directly.
*/
readonly files: ReadonlyArray<string>;
/**
* The license that this definition package is released under.
*
* Can be either MIT or Apache v2, defaults to MIT when not explicitly defined in this package’s "package.json".
*/
readonly license: License;
/**
* A hash of the names and contents of the `files` list, used for versioning.
*/
readonly contentHash: string;
/**
* Name or URL of the project, e.g. "http://cordova.apache.org".
*/
readonly projectName: string;
/**
* A list of *values* declared in the global namespace.
*
* @note This does not include *types* declared in the global namespace.
*/
readonly globals: ReadonlyArray<string>;
/**
* External modules declared by this package. Includes the containing folder name when applicable (e.g. proper module).
*/
readonly declaredModules: ReadonlyArray<string>;
}
/**
* @see {TypingsDataRaw.pathMappings}
*/
export interface PathMapping {
readonly packageName: string;
readonly version: TypingVersion;
}
// TODO: support BSD -- but must choose a *particular* BSD license from the list at https://spdx.org/licenses/
export const enum License { MIT = "MIT", Apache20 = "Apache-2.0" }
const allLicenses = [License.MIT, License.Apache20];
export function getLicenseFromPackageJson(packageJsonLicense: unknown): License {
if (packageJsonLicense === undefined) { // tslint:disable-line strict-type-predicates (false positive)
return License.MIT;
}
if (typeof packageJsonLicense === "string" && packageJsonLicense === "MIT") {
throw new Error(`Specifying '"license": "MIT"' is redundant, this is the default.`);
}
if (allLicenses.includes(packageJsonLicense as License)) {
return packageJsonLicense as License;
}
throw new Error(`'package.json' license is ${JSON.stringify(packageJsonLicense)}.\nExpected one of: ${JSON.stringify(allLicenses)}}`);
}
export class TypingsVersions {
private readonly map: ReadonlyMap<Semver, TypingsData>;
/**
* Sorted from latest to oldest.
*/
private readonly versions: Semver[];
constructor(data: TypingsVersionsRaw) {
const versionMappings = new Map(Object.keys(data).map(key => {
const version = Semver.parse(key, true);
if (version) {
return [version, key];
}
throw new Error(`Unable to parse version ${key}`);
}));
/**
* Sorted from latest to oldest so that we publish the current version first.
* This is important because older versions repeatedly reset the "latest" tag to the current version.
*/
this.versions = Array.from(versionMappings.keys()).sort(compareSemver).reverse();
this.map = new Map(this.versions.map(version => {
const dataKey = versionMappings.get(version)!;
return [version, new TypingsData(data[dataKey], version.equals(this.versions[0]))];
}));
}
getAll(): Iterable<TypingsData> {
return this.map.values();
}
get(version: DependencyVersion): TypingsData {
return version === "*" ? this.getLatest() : this.getLatestMatch(version);
}
tryGet(version: DependencyVersion): TypingsData | undefined {
return version === "*" ? this.getLatest() : this.tryGetLatestMatch(version);
}
getLatest(): TypingsData {
return this.map.get(this.versions[0])!;
}
private getLatestMatch(version: TypingVersion): TypingsData {
const data = this.tryGetLatestMatch(version);
if (!data) {
throw new Error(`Could not find version ${version.major}.${version.minor ?? "*"}`);
}
return data;
}
private tryGetLatestMatch(version: TypingVersion): TypingsData | undefined {
const found = this.versions.find(v => v.major === version.major && (version.minor === undefined || v.minor === version.minor));
return found && this.map.get(found);
}
}
export class TypingsData extends PackageBase {
constructor(private readonly data: TypingsDataRaw, readonly isLatest: boolean) {
super(data);
}
get testDependencies(): ReadonlyArray<string> { return this.data.testDependencies; }
get contributors(): ReadonlyArray<Author> { return this.data.contributors; }
get major(): number { return this.data.libraryMajorVersion; }
get minor(): number { return this.data.libraryMinorVersion; }
get minTypeScriptVersion(): TypeScriptVersion {
return TypeScriptVersion.isSupported(this.data.minTsVersion) ? this.data.minTsVersion : TypeScriptVersion.lowest;
}
get typesVersions(): ReadonlyArray<TypeScriptVersion> { return this.data.typesVersions; }
get files(): ReadonlyArray<string> { return this.data.files; }
get license(): License { return this.data.license; }
get packageJsonDependencies(): ReadonlyArray<PackageJsonDependency> { return this.data.packageJsonDependencies; }
get contentHash(): string { return this.data.contentHash; }
get declaredModules(): ReadonlyArray<string> { return this.data.declaredModules; }
get projectName(): string { return this.data.projectName; }
get globals(): ReadonlyArray<string> { return this.data.globals; }
get pathMappings(): ReadonlyArray<PathMapping> { return this.data.pathMappings; }
get dependencies(): ReadonlyArray<PackageId> {
return this.data.dependencies;
}
get versionDirectoryName() {
return this.data.libraryVersionDirectoryName && `v${this.data.libraryVersionDirectoryName}`;
}
/** Path to this package, *relative* to the DefinitelyTyped directory. */
get subDirectoryPath(): string {
return this.isLatest ? this.name : `${this.name}/${this.versionDirectoryName}`;
}
}
/** Uniquely identifies a package. */
export interface PackageId {
readonly name: string;
readonly version: DependencyVersion;
}
export interface TypesDataFile {
readonly [packageName: string]: TypingsVersionsRaw;
}
function readTypesDataFile(): Promise<TypesDataFile> {
return readDataFile("parse-definitions", typesDataFilename) as Promise<TypesDataFile>;
}
export function readNotNeededPackages(dt: FS): ReadonlyArray<NotNeededPackage> {
const rawJson = dt.readJson("notNeededPackages.json"); // tslint:disable-line await-promise (tslint bug)
return (rawJson as { readonly packages: ReadonlyArray<NotNeededPackageRaw> }).packages.map(raw => new NotNeededPackage(raw));
}

31
packages/types-publisher/src/lib/versions.test.ts
View File

@@ -1,31 +0,0 @@
import { Semver } from "./versions";
describe(Semver, () => {
it("returns a formatted description", () => {
expect(new Semver(1, 2, 3).versionString).toEqual("1.2.3");
});
it("parses semver versions", () => {
expect(Semver.parse("0.42.1").versionString).toEqual("0.42.1");
});
it("parses versions that do not strictly adhere to semver", () => {
expect(Semver.parse("1", true).versionString).toEqual("1.0.0");
expect(Semver.parse("0.42", true).versionString).toEqual("0.42.0");
});
it("throws when a version cannot be parsed", () => {
expect(() => Semver.parse("1")).toThrow();
expect(() => Semver.parse("1", false)).toThrow();
});
it("returns whether or not it's equal to another Semver", () => {
expect(Semver.parse("1.2.3").equals(new Semver(1, 2, 3))).toBe(true);
expect(Semver.parse("1.2.3").equals(new Semver(3, 2, 1))).toBe(false);
});
it("returns whether or not it's greater than another Semver", () => {
expect(Semver.parse("1.2.3").greaterThan(new Semver(1, 2, 2))).toBe(true);
expect(Semver.parse("1.2.3").equals(new Semver(1, 2, 4))).toBe(false);
});
});

146
packages/types-publisher/src/lib/versions.ts
View File

@@ -1,146 +0,0 @@
import { Logger } from "../util/logging";
import { assertDefined, best, intOfString } from "../util/util";
import { readDataFile } from "./common";
import { CachedNpmInfoClient } from "./npm-client";
import { AllPackages, NotNeededPackage, PackageId, TypingsData } from "./packages";
export const versionsFilename = "versions.json";
export interface ChangedTyping {
readonly pkg: TypingsData;
/** This is the version to be published, meaning it's the version that doesn't exist yet. */
readonly version: string;
/** For a non-latest version, this is the latest version; publishing an old version updates the 'latest' tag and we want to change it back. */
readonly latestVersion: string | undefined;
}
export interface ChangedPackagesJson {
readonly changedTypings: ReadonlyArray<ChangedTypingJson>;
readonly changedNotNeededPackages: ReadonlyArray<string>;
}
export interface ChangedTypingJson {
readonly id: PackageId;
readonly version: string;
readonly latestVersion?: string;
}
export interface ChangedPackages {
readonly changedTypings: ReadonlyArray<ChangedTyping>;
readonly changedNotNeededPackages: ReadonlyArray<NotNeededPackage>;
}
export async function readChangedPackages(allPackages: AllPackages): Promise<ChangedPackages> {
const json = await readDataFile("calculate-versions", versionsFilename) as ChangedPackagesJson;
return {
changedTypings: json.changedTypings.map(({ id, version, latestVersion }): ChangedTyping =>
({ pkg: allPackages.getTypingsData(id), version, latestVersion })),
changedNotNeededPackages: json.changedNotNeededPackages.map(id => assertDefined(allPackages.getNotNeededPackage(id))),
};
}
/**
* When we fail to publish a deprecated package, it leaves behind an entry in the time property.
* So the keys of 'time' give the actual 'latest'.
* If that's not equal to the expected latest, try again by bumping the patch version of the last attempt by 1.
*/
export function skipBadPublishes(pkg: NotNeededPackage, client: CachedNpmInfoClient, log: Logger) {
// because this is called right after isAlreadyDeprecated, we can rely on the cache being up-to-date
const info = assertDefined(client.getNpmInfoFromCache(pkg.fullEscapedNpmName));
const notNeeded = pkg.version;
const latest = Semver.parse(findActualLatest(info.time));
if (latest.equals(notNeeded) || latest.greaterThan(notNeeded) ||
info.versions.has(notNeeded.versionString) && !assertDefined(info.versions.get(notNeeded.versionString)).deprecated) {
const plusOne = new Semver(latest.major, latest.minor, latest.patch + 1);
log(`Deprecation of ${notNeeded.versionString} failed, instead using ${plusOne.versionString}.`);
return new NotNeededPackage({
asOfVersion: plusOne.versionString,
libraryName: pkg.libraryName,
sourceRepoURL: pkg.sourceRepoURL,
typingsPackageName: pkg.name,
});
}
return pkg;
}
function findActualLatest(times: Map<string, string>) {
const actual = best(
times, ([k, v], [bestK, bestV]) =>
(bestK === "modified" || bestK === "created") ? true :
(k === "modified" || k === "created") ? false :
new Date(v).getTime() > new Date(bestV).getTime());
if (!actual) {
throw new Error("failed to find actual latest");
}
return actual[0];
}
/** Version of a package published to NPM. */
export class Semver {
static parse(semver: string, coerce?: boolean): Semver {
const result = Semver.tryParse(semver, coerce);
if (!result) {
throw new Error(`Unexpected semver: ${semver}`);
}
return result;
}
static fromRaw({ major, minor, patch }: Semver): Semver {
return new Semver(major, minor, patch);
}
/**
* Per the semver spec <http://semver.org/#spec-item-2>:
*
* A normal version number MUST take the form X.Y.Z where X, Y, and Z are non-negative integers, and MUST NOT contain leading zeroes.
*
* @note This must parse the output of `versionString`.
*
* @param semver The version string.
* @param coerce Without this optional parameter the version MUST follow the above semver spec. However, when set to `true` components after the
* major version may be omitted. I.e. `1` equals `1.0` and `1.0.0`.
*/
static tryParse(semver: string, coerce?: boolean): Semver | undefined {
const rgx = /^(\d+)(\.(\d+))?(\.(\d+))?$/;
const match = rgx.exec(semver);
if (match) {
const { 1: major, 3: minor, 5: patch } = match;
if ((minor !== undefined && patch !== undefined) || coerce) { // tslint:disable-line:strict-type-predicates
return new Semver(intOfString(major), intOfString(minor || "0"), intOfString(patch || "0"));
}
}
return undefined;
}
constructor(readonly major: number, readonly minor: number, readonly patch: number) {}
get versionString(): string {
const { major, minor, patch } = this;
return `${major}.${minor}.${patch}`;
}
equals(other: Semver): boolean {
return compare(this, other) === 0;
}
greaterThan(other: Semver): boolean {
return compare(this, other) === 1;
}
}
/**
* Returns 0 if equal, 1 if x > y, -1 if x < y
*/
export function compare(x: Semver, y: Semver) {
const versions: Array<[number, number]> = [[x.major, y.major], [x.minor, y.minor], [x.patch, y.patch]];
for (const [componentX, componentY] of versions) {
if (componentX > componentY) {
return 1;
}
if (componentX < componentY) {
return -1;
}
}
return 0;
}

30
packages/types-publisher/src/parse-definitions.test.ts
View File

@@ -1,30 +0,0 @@
import { createMockDT } from "./mocks";
import parseDefinitions from "./parse-definitions";
import { loggerWithErrors } from "./util/logging";
import { testo } from "./util/test";
testo({
// async parseDefinitions() {
// const log = loggerWithErrors()[0]
// const dt = await getDefinitelyTyped(Options.defaults, log);
// const defs = await parseDefinitions(dt, undefined, log)
// expect(defs.allNotNeeded().length).toBeGreaterThan(0)
// expect(defs.allTypings().length).toBeGreaterThan(5000)
// const j = defs.tryGetLatestVersion("jquery")
// expect(j).toBeDefined()
// expect(j!.fullNpmName).toContain("types")
// expect(j!.fullNpmName).toContain("jquery")
// expect(defs.allPackages().length).toEqual(defs.allTypings().length + defs.allNotNeeded().length)
// },
async mockParse() {
const log = loggerWithErrors()[0];
const defs = await parseDefinitions(createMockDT().fs, undefined, log);
expect(defs.allNotNeeded().length).toBe(1);
expect(defs.allTypings().length).toBe(3);
const j = defs.tryGetLatestVersion("jquery");
expect(j).toBeDefined();
expect(j!.fullNpmName).toContain("types");
expect(j!.fullNpmName).toContain("jquery");
expect(defs.allPackages().length).toEqual(defs.allTypings().length + defs.allNotNeeded().length);
},
});

76
packages/types-publisher/src/parse-definitions.ts
View File

@@ -1,76 +0,0 @@
import * as yargs from "yargs";
import { FS, getDefinitelyTyped } from "./get-definitely-typed";
import { Options, writeDataFile } from "./lib/common";
import { getTypingInfo } from "./lib/definition-parser";
import { definitionParserWorkerFilename } from "./lib/definition-parser-worker";
import { AllPackages, readNotNeededPackages, typesDataFilename, TypingsVersionsRaw } from "./lib/packages";
import { parseNProcesses } from "./tester/test-runner";
import { LoggerWithErrors, loggerWithErrors } from "./util/logging";
import { assertDefined, filterNAtATimeOrdered, logUncaughtErrors, runWithChildProcesses } from "./util/util";
if (!module.parent) {
const singleName = yargs.argv.single as string | undefined;
const options = Options.defaults;
logUncaughtErrors(async () => {
const log = loggerWithErrors()[0];
const dt = await getDefinitelyTyped(options, log);
if (singleName) {
await single(singleName, dt);
} else {
await parseDefinitions(
dt,
options.parseInParallel
? { nProcesses: parseNProcesses(), definitelyTypedPath: assertDefined(options.definitelyTypedPath) }
: undefined,
log);
}
});
}
export interface ParallelOptions { readonly nProcesses: number; readonly definitelyTypedPath: string; }
export default async function parseDefinitions(dt: FS, parallel: ParallelOptions | undefined, log: LoggerWithErrors): Promise<AllPackages> {
log.info("Parsing definitions...");
const typesFS = dt.subDir("types");
const packageNames = await filterNAtATimeOrdered(parallel ? parallel.nProcesses : 1, typesFS.readdir(), name => typesFS.isDirectory(name));
log.info(`Found ${packageNames.length} packages.`);
const typings: { [name: string]: TypingsVersionsRaw } = {};
const start = Date.now();
if (parallel) {
log.info("Parsing in parallel...");
await runWithChildProcesses({
inputs: packageNames,
commandLineArgs: [`${parallel.definitelyTypedPath}/types`],
workerFile: definitionParserWorkerFilename,
nProcesses: parallel.nProcesses,
handleOutput({ data, packageName}: { data: TypingsVersionsRaw, packageName: string }) {
typings[packageName] = data;
},
});
} else {
log.info("Parsing non-parallel...");
for (const packageName of packageNames) {
typings[packageName] = getTypingInfo(packageName, typesFS.subDir(packageName));
}
}
log.info("Parsing took " + ((Date.now() - start) / 1000) + " s");
await writeDataFile(typesDataFilename, sorted(typings));
return AllPackages.from(typings, readNotNeededPackages(dt));
}
function sorted<T>(obj: { [name: string]: T }): { [name: string]: T } {
const out: { [name: string]: T } = {};
for (const key of Object.keys(obj).sort()) {
out[key] = obj[key];
}
return out;
}
async function single(singleName: string, dt: FS): Promise<void> {
const data = getTypingInfo(singleName, dt.subDir("types").subDir(singleName));
const typings = { [singleName]: data };
await writeDataFile(typesDataFilename, typings);
console.log(JSON.stringify(data, undefined, 4));
}

153
packages/types-publisher/src/util/io.ts
View File

@@ -1,153 +0,0 @@
import {
readFile as readFileWithEncoding,
readFileSync as readFileWithEncodingSync,
stat,
writeFile as writeFileWithEncoding,
writeJson as writeJsonRaw,
} from "fs-extra";
import { request as httpRequest } from "http";
import { Agent, request } from "https";
import { Readable as ReadableStream } from "stream";
import { StringDecoder } from "string_decoder";
import { parseJson } from "./util";
export async function readFile(path: string): Promise<string> {
const res = await readFileWithEncoding(path, { encoding: "utf8" });
if (res.includes("<22>")) {
throw new Error(`Bad character in ${path}`);
}
return res;
}
export function readFileSync(path: string): string {
const res = readFileWithEncodingSync(path, { encoding: "utf8" });
if (res.includes("<22>")) {
throw new Error(`Bad character in ${path}`);
}
return res;
}
export function readJsonSync(path: string): object {
return parseJson(readFileSync(path));
}
export async function readJson(path: string): Promise<object> {
return parseJson(await readFile(path));
}
export function writeFile(path: string, content: string): Promise<void> {
return writeFileWithEncoding(path, content, { encoding: "utf8" });
}
export function writeJson(path: string, content: unknown, formatted = true): Promise<void> {
return writeJsonRaw(path, content, { spaces: formatted ? 4 : 0 });
}
export function streamOfString(text: string): NodeJS.ReadableStream {
const s = new ReadableStream();
s.push(text);
s.push(null); // tslint:disable-line no-null-keyword
return s;
}
export function stringOfStream(stream: NodeJS.ReadableStream, description: string): Promise<string> {
const decoder = new StringDecoder("utf8");
let body = "";
stream.on("data", (data: Buffer) => {
body += decoder.write(data);
});
return new Promise<string>((resolve, reject) => {
stream.on("error", reject);
stream.on("end", () => {
body += decoder.end();
if (body.includes("<22>")) {
reject(`Bad character decode in ${description}`);
} else {
resolve(body);
}
});
});
}
export function streamDone(stream: NodeJS.WritableStream): Promise<void> {
return new Promise<void>((resolve, reject) => {
stream.on("error", reject).on("finish", resolve);
});
}
export interface FetchOptions {
readonly hostname: string;
readonly port?: number;
readonly path: string;
readonly retries?: boolean | number;
readonly body?: string;
readonly method?: "GET" | "PATCH" | "POST";
readonly headers?: {};
}
export class Fetcher {
private readonly agent = new Agent({ keepAlive: true });
async fetchJson(options: FetchOptions): Promise<unknown> {
const text = await this.fetch(options);
try {
return JSON.parse(text) as unknown;
} catch (e) {
throw new Error(`Bad response from server:\noptions: ${JSON.stringify(options)}\n\n${text}`);
}
}
async fetch(options: FetchOptions): Promise<string> {
const maxRetries = options.retries === false || options.retries === undefined ? 0 : options.retries === true ? 10 : options.retries;
for (let retries = maxRetries; retries > 1; retries--) {
try {
return await doRequest(options, request, this.agent);
} catch (err) {
if (!/EAI_AGAIN|ETIMEDOUT|ECONNRESET/.test((err as Error).message)) {
throw err;
}
}
await sleep(1);
}
return doRequest(options, request, this.agent);
}
}
/** Only used for testing. */
export function makeHttpRequest(options: FetchOptions): Promise<string> {
return doRequest(options, httpRequest);
}
function doRequest(options: FetchOptions, makeRequest: typeof request, agent?: Agent): Promise<string> {
return new Promise((resolve, reject) => {
const req = makeRequest(
{
hostname: options.hostname,
port: options.port,
path: `/${options.path}`,
agent,
method: options.method || "GET",
headers: options.headers,
},
res => {
let text = "";
res.on("data", (d: string) => { text += d; });
res.on("error", reject);
res.on("end", () => { resolve(text); });
});
if (options.body !== undefined) {
req.write(options.body);
}
req.end();
});
}
export async function sleep(seconds: number): Promise<void> {
return new Promise<void>(resolve => setTimeout(resolve, seconds * 1000));
}
export async function isDirectory(path: string): Promise<boolean> {
return (await stat(path)).isDirectory();
}
export const npmInstallFlags = "--ignore-scripts --no-shrinkwrap --no-package-lock --no-bin-links --no-save";

102
packages/types-publisher/src/util/logging.ts
View File

@@ -1,102 +0,0 @@
import { ensureDir } from "fs-extra";
import { logDir } from "../lib/settings";
import { joinPaths } from "../util/util";
import { writeFile } from "./io";
/** Anything capable of receiving messages is a logger. */
export type Logger = (message: string) => void;
/** Recording of every message sent to a Logger. */
export type Log = string[];
/** Stores two separate loggers. */
export interface LoggerWithErrors {
info: Logger;
error: Logger;
}
/** Recording of every message sent to a LoggerWithErrors. */
export interface LogWithErrors {
infos: Log;
errors: Log;
}
/** Logger that *just* outputs to the console and does not save anything. */
export const consoleLogger: LoggerWithErrors = {
info: console.log, // tslint:disable-line no-unbound-method
error: console.error, // tslint:disable-line no-unbound-method
};
/** Logger that *just* records writes and does not output to console. */
export function quietLogger(): [Logger, () => Log] {
const logged: Log = [];
return [ (message: string) => logged.push(message), () => logged ];
}
/** Performs a side-effect and also records all logs. */
function alsoConsoleLogger(consoleLog: Logger): [Logger, () => Log] {
const [log, logResult] = quietLogger();
return [
(message: string) => {
consoleLog(message);
log(message);
},
logResult,
];
}
/** Logger that writes to console in addition to recording a result. */
export function logger(): [Logger, () => Log] {
return alsoConsoleLogger(consoleLogger.info);
}
/** Helper for creating `info` and `error` loggers together. */
function loggerWithErrorsHelper(loggerOrQuietLogger: () => [Logger, () => Log]): [LoggerWithErrors, () => LogWithErrors] {
const [info, infoResult] = loggerOrQuietLogger();
const [error, errorResult] = loggerOrQuietLogger();
return [
{ info, error },
() => ({ infos: infoResult(), errors: errorResult() }),
];
}
/** Records `info` and `error` messages without writing to console. */
export function quietLoggerWithErrors(): [LoggerWithErrors, () => LogWithErrors] {
return loggerWithErrorsHelper(quietLogger);
}
/** Records `info` and `error` messages, calling appropriate console methods as well. */
export function loggerWithErrors(): [LoggerWithErrors, () => LogWithErrors] {
return loggerWithErrorsHelper(logger);
}
/**
* Move everything from one Log to another logger.
* This is useful for performing several tasks in parallel, but outputting their logs in sequence.
*/
export function moveLogs(dest: Logger, src: Log, mapper?: (message: string) => string): void {
for (const line of src) {
dest(mapper ? mapper(line) : line);
}
}
/** Perform `moveLogs` for both parts of a LogWithErrors. */
export function moveLogsWithErrors(dest: LoggerWithErrors, {infos, errors}: LogWithErrors, mapper?: (message: string) => string): void {
moveLogs(dest.info, infos, mapper);
moveLogs(dest.error, errors, mapper);
}
export function logPath(logName: string): string {
return joinPaths(logDir, logName);
}
export async function writeLog(logName: string, contents: ReadonlyArray<string>): Promise<void> {
await ensureDir(logDir);
await writeFile(logPath(logName), contents.join("\r\n"));
}
export function joinLogWithErrors({infos, errors}: LogWithErrors): Log {
return errors.length ? infos.concat(["", "=== ERRORS ===", ""], errors) : infos;
}

100
packages/types-publisher/src/util/progress.ts
View File

@@ -1,100 +0,0 @@
import charm = require("charm");
export interface Options {
/** Text to display in front of the progress bar. */
name: string;
/** Length of the progress bar. */
width?: number;
/** Only render an update if this many milliseconds have passed. */
updateMinTime?: number;
}
export default class ProgressBar {
private readonly console = new UpdatableConsole();
private readonly name: string;
private readonly width: number;
private readonly updateMinTime: number;
/** Most recent flavor text. */
private flavor = "";
private lastUpdateMillis = 0;
constructor(options: Options) {
this.name = options.name;
this.width = options.width === undefined ? 20 : options.width;
this.updateMinTime = options.updateMinTime === undefined ? 250 : options.updateMinTime;
}
update(current: number, flavor?: string): void {
if (flavor !== undefined) {
this.flavor = flavor;
}
const now = +(new Date());
const diff = now - this.lastUpdateMillis;
if (diff > this.updateMinTime) {
this.lastUpdateMillis = now;
this.doUpdate(current);
}
}
private doUpdate(current: number): void {
const nCellsFilled = Math.ceil(this.width * Math.min(1, Math.max(0, current)));
this.console.update(c => {
c.write(this.name);
c.write(" [");
c.write("█".repeat(nCellsFilled));
if (nCellsFilled < this.width) {
c.right(this.width - nCellsFilled);
}
c.write("]");
if (this.flavor.length) {
c.write(` ${this.flavor}`);
}
});
}
done(): void {
this.flavor = "Done!";
this.doUpdate(1);
this.console.end();
}
}
/** A mutable line of text on the console. */
class UpdatableConsole {
private readonly charm = charm(process.stdout);
update(action: (charm: charm.CharmInstance) => void): void {
this.charm.push();
this.charm.erase("line");
action(this.charm);
this.charm.pop();
}
end(): void {
this.charm.write("\n");
this.charm.end();
}
}
const firstLetter = "a".charCodeAt(0);
const lastLetter = "z".charCodeAt(0);
const charWidth = lastLetter - firstLetter;
const strProgressTotal = charWidth * charWidth; // 2 characters
/** Tracks a string's progress through the alphabet. */
export function strProgress(str: string): number {
const x = charProgress(str.charCodeAt(0)) * charWidth + charProgress(str.charCodeAt(1));
return x / strProgressTotal;
function charProgress(ch: number): number {
if (Number.isNaN(ch) || ch <= firstLetter) {
return 0;
}
if (ch >= lastLetter) {
return charWidth;
}
return ch - firstLetter;
}
}

644
packages/types-publisher/src/util/util.ts
View File

@@ -1,644 +0,0 @@
import assert = require("assert");
import { ChildProcess, exec as node_exec, fork } from "child_process";
import * as crypto from "crypto";
import moment = require("moment");
import * as os from "os";
import { Options } from "../lib/common";
import ProgressBar from "./progress";
export function assertDefined<T>(x: T | undefined, message?: string | Error | undefined): T {
assert(x !== undefined, message);
return x!;
}
const DEFAULT_CRASH_RECOVERY_MAX_OLD_SPACE_SIZE = 4096;
export function parseJson(text: string): object {
try {
return JSON.parse(text) as object;
} catch (err) {
throw new Error(`${(err as Error).message} due to JSON: ${text}`);
}
}
export function currentTimeStamp(): string {
return moment().format("YYYY-MM-DDTHH:mm:ss.SSSZZ");
}
export const numberOfOsProcesses = process.env.TRAVIS === "true" ? 2 : os.cpus().length;
/** Progress options needed for `nAtATime`. Other options will be inferred. */
interface ProgressOptions<T, U> {
readonly name: string;
flavor(input: T, output: U): string | undefined;
readonly options: Options;
}
export async function nAtATime<T, U>(
n: number,
inputs: ReadonlyArray<T>,
use: (t: T) => Awaitable<U>,
progressOptions?: ProgressOptions<T, U>): Promise<U[]> {
const progress = progressOptions && progressOptions.options.progress ? new ProgressBar({ name: progressOptions.name }) : undefined;
const results = new Array(inputs.length);
// We have n "threads" which each run `continuouslyWork`.
// They all share `nextIndex`, so each work item is done only once.
let nextIndex = 0;
await Promise.all(initArray(n, async () => {
while (nextIndex !== inputs.length) {
const index = nextIndex;
nextIndex++;
const input = inputs[index];
const output = await use(input);
results[index] = output;
if (progress) {
progress!.update(index / inputs.length, progressOptions!.flavor(input, output));
}
}
}));
if (progress) {
progress.done();
}
return results;
}
export function filter<T>(iterable: Iterable<T>, predicate: (value: T) => boolean): IterableIterator<T> {
const iter = iterable[Symbol.iterator]();
return {
[Symbol.iterator](): IterableIterator<T> { return this; },
next(): IteratorResult<T> {
while (true) {
const res = iter.next();
if (res.done || predicate(res.value)) {
return res;
}
}
},
};
}
export type Awaitable<T> = T | Promise<T>;
export async function filterNAtATimeOrdered<T>(
n: number, inputs: ReadonlyArray<T>, shouldKeep: (input: T) => Awaitable<boolean>, progress?: ProgressOptions<T, boolean>): Promise<T[]> {
const shouldKeeps: boolean[] = await nAtATime(n, inputs, shouldKeep, progress);
return inputs.filter((_, idx) => shouldKeeps[idx]);
}
export function unique<T>(arr: Iterable<T>): T[] {
return [...new Set(arr)];
}
export function logUncaughtErrors(promise: Promise<unknown> | (() => Promise<unknown>)): void {
(typeof promise === "function" ? promise() : promise).catch(error => {
console.error(error);
process.exit(1);
});
}
function initArray<T>(length: number, makeElement: (i: number) => T): T[] {
const arr = new Array(length);
for (let i = 0; i < length; i++) {
arr[i] = makeElement(i);
}
return arr;
}
/** Always use "/" for consistency. (This affects package content hash.) */
export function joinPaths(...paths: string[]): string {
return paths.join("/");
}
/** Convert a path to use "/" instead of "\\" for consistency. (This affects content hash.) */
export function normalizeSlashes(path: string): string {
return path.replace(/\\/g, "/");
}
export function hasWindowsSlashes(path: string): boolean {
return path.includes("\\");
}
export function intOfString(str: string): number {
const n = Number.parseInt(str, 10);
if (Number.isNaN(n)) {
throw new Error(`Error in parseInt(${JSON.stringify(str)})`);
}
return n;
}
export function sortObjectKeys<T extends { [key: string]: unknown }>(data: T): T {
const out = {} as T; // tslint:disable-line no-object-literal-type-assertion
for (const key of Object.keys(data).sort()) {
out[key as keyof T] = data[key as keyof T];
}
return out;
}
/** Run a command and return the error, stdout, and stderr. (Never throws.) */
export function exec(cmd: string, cwd?: string): Promise<{ error: Error | undefined, stdout: string, stderr: string }> {
return new Promise<{ error: Error | undefined, stdout: string, stderr: string }>(resolve => {
// Fix "stdout maxBuffer exceeded" error
// See https://github.com/DefinitelyTyped/DefinitelyTyped/pull/26545#issuecomment-402274021
const maxBuffer = 1024 * 1024 * 1; // Max = 1 MiB, default is 200 KiB
node_exec(cmd, { encoding: "utf8", cwd, maxBuffer }, (error, stdout, stderr) => {
resolve({ error: error === null ? undefined : error, stdout: stdout.trim(), stderr: stderr.trim() });
});
});
}
/** Run a command and return the stdout, or if there was an error, throw. */
export async function execAndThrowErrors(cmd: string, cwd?: string): Promise<string> {
const { error, stdout, stderr } = await exec(cmd, cwd);
if (error) {
throw new Error(`${error.stack}\n${stderr}`);
}
return stdout + stderr;
}
/**
* Returns the input that is better than all others, or `undefined` if there are no inputs.
* @param isBetter Returns true if `a` should be preferred over `b`.
*/
export function best<T>(inputs: Iterable<T>, isBetter: (a: T, b: T) => boolean): T | undefined {
const iter = inputs[Symbol.iterator]();
const first = iter.next();
if (first.done) {
return undefined;
}
let res = first.value;
while (true) {
const { value, done } = iter.next();
if (done) {
break;
}
if (isBetter(value, res)) {
res = value;
}
}
return res;
}
export function computeHash(content: string): string {
// Normalize line endings
const normalContent = content.replace(/\r\n?/g, "\n");
const h = crypto.createHash("sha256");
h.update(normalContent, "utf8");
return h.digest("hex");
}
export function mapValues<K, V1, V2>(map: Map<K, V1>, valueMapper: (value: V1) => V2): Map<K, V2> {
const out = new Map<K, V2>();
map.forEach((value, key) => {
out.set(key, valueMapper(value));
});
return out;
}
export function mapDefined<T, U>(arr: Iterable<T>, mapper: (t: T) => U | undefined): U[] {
const out = [];
for (const a of arr) {
const res = mapper(a);
if (res !== undefined) {
out.push(res);
}
}
return out;
}
export async function mapDefinedAsync<T, U>(arr: Iterable<T>, mapper: (t: T) => Promise<U | undefined>): Promise<U[]> {
const out = [];
for (const a of arr) {
const res = await mapper(a);
if (res !== undefined) {
out.push(res);
}
}
return out;
}
export function* mapIter<T, U>(inputs: Iterable<T>, mapper: (t: T) => U): Iterable<U> {
for (const input of inputs) {
yield mapper(input);
}
}
export function* flatMap<T, U>(inputs: Iterable<T>, mapper: (t: T) => Iterable<U>): Iterable<U> {
for (const input of inputs) {
yield* mapper(input);
}
}
export function sort<T>(values: Iterable<T>, comparer?: (a: T, b: T) => number): T[] {
return Array.from(values).sort(comparer);
}
export function join<T>(values: Iterable<T>, joiner = ", "): string {
let s = "";
for (const v of values) {
// tslint:disable-next-line strict-string-expressions
s += `${v}${joiner}`;
}
return s.slice(0, s.length - joiner.length);
}
export interface RunWithChildProcessesOptions<In> {
readonly inputs: ReadonlyArray<In>;
readonly commandLineArgs: string[];
readonly workerFile: string;
readonly nProcesses: number;
handleOutput(output: unknown): void;
}
export function runWithChildProcesses<In>(
{ inputs, commandLineArgs, workerFile, nProcesses, handleOutput }: RunWithChildProcessesOptions<In>,
): Promise<void> {
return new Promise((resolve, reject) => {
const nPerProcess = Math.floor(inputs.length / nProcesses);
let processesLeft = nProcesses;
let rejected = false;
const allChildren: ChildProcess[] = [];
for (let i = 0; i < nProcesses; i++) {
const lo = nPerProcess * i;
const hi = i === nProcesses - 1 ? inputs.length : lo + nPerProcess;
let outputsLeft = hi - lo; // Expect one output per input
if (outputsLeft === 0) {
// No work for this process to do, so don't launch it
processesLeft--;
continue;
}
const child = fork(workerFile, commandLineArgs);
allChildren.push(child);
child.send(inputs.slice(lo, hi));
child.on("message", outputMessage => {
handleOutput(outputMessage as unknown);
assert(outputsLeft > 0);
outputsLeft--;
if (outputsLeft === 0) {
assert(processesLeft > 0);
processesLeft--;
if (processesLeft === 0) {
resolve();
}
child.kill();
}
});
child.on("disconnect", () => {
if (outputsLeft !== 0) {
fail();
}
});
child.on("close", () => { assert(rejected || outputsLeft === 0); });
child.on("error", fail);
}
function fail(): void {
rejected = true;
for (const child of allChildren) {
child.kill();
}
reject(new Error("Parsing failed."));
}
});
}
export const enum CrashRecoveryState {
Normal,
Retry,
RetryWithMoreMemory,
Crashed,
}
interface RunWithListeningChildProcessesOptions<In> {
readonly inputs: ReadonlyArray<In>;
readonly commandLineArgs: string[];
readonly workerFile: string;
readonly nProcesses: number;
readonly cwd: string;
readonly crashRecovery?: boolean;
readonly crashRecoveryMaxOldSpaceSize?: number;
readonly softTimeoutMs?: number;
handleOutput(output: unknown, processIndex: number | undefined): void;
handleStart?(input: In, processIndex: number | undefined): void;
handleCrash?(input: In, state: CrashRecoveryState, processIndex: number | undefined): void;
}
export function runWithListeningChildProcesses<In>(
{ inputs, commandLineArgs, workerFile, nProcesses, cwd, handleOutput, crashRecovery,
crashRecoveryMaxOldSpaceSize = DEFAULT_CRASH_RECOVERY_MAX_OLD_SPACE_SIZE,
handleStart, handleCrash, softTimeoutMs = Infinity }: RunWithListeningChildProcessesOptions<In>,
): Promise<void> {
return new Promise((resolve, reject) => {
let inputIndex = 0;
let processesLeft = nProcesses;
let rejected = false;
const runningChildren = new Set<ChildProcess>();
const maxOldSpaceSize = getMaxOldSpaceSize(process.execArgv) || 0;
const startTime = Date.now();
for (let i = 0; i < nProcesses; i++) {
if (inputIndex === inputs.length) {
processesLeft--;
continue;
}
const processIndex = nProcesses > 1 ? i + 1 : undefined;
let child: ChildProcess;
let crashRecoveryState = CrashRecoveryState.Normal;
let currentInput: In;
const onMessage = (outputMessage: unknown) => {
try {
const oldCrashRecoveryState = crashRecoveryState;
crashRecoveryState = CrashRecoveryState.Normal;
handleOutput(outputMessage as {}, processIndex);
if (inputIndex === inputs.length || Date.now() - startTime > softTimeoutMs) {
stopChild(/*done*/ true);
} else {
if (oldCrashRecoveryState !== CrashRecoveryState.Normal) {
// retry attempt succeeded, restart the child for further tests.
console.log(`${processIndex}> Restarting...`);
restartChild(nextTask, process.execArgv);
} else {
nextTask();
}
}
} catch (e) {
onError(e);
}
};
const onClose = () => {
if (rejected || !runningChildren.has(child)) {
return;
}
try {
// treat any unhandled closures of the child as a crash
if (crashRecovery) {
switch (crashRecoveryState) {
case CrashRecoveryState.Normal:
crashRecoveryState = CrashRecoveryState.Retry;
break;
case CrashRecoveryState.Retry:
// skip crash recovery if we're already passing a value for --max_old_space_size that
// is >= crashRecoveryMaxOldSpaceSize
crashRecoveryState = maxOldSpaceSize < crashRecoveryMaxOldSpaceSize
? CrashRecoveryState.RetryWithMoreMemory
: crashRecoveryState = CrashRecoveryState.Crashed;
break;
default:
crashRecoveryState = CrashRecoveryState.Crashed;
}
} else {
crashRecoveryState = CrashRecoveryState.Crashed;
}
if (handleCrash) {
handleCrash(currentInput, crashRecoveryState, processIndex);
}
switch (crashRecoveryState) {
case CrashRecoveryState.Retry:
restartChild(resumeTask, process.execArgv);
break;
case CrashRecoveryState.RetryWithMoreMemory:
restartChild(resumeTask, [
...getExecArgvWithoutMaxOldSpaceSize(),
`--max_old_space_size=${crashRecoveryMaxOldSpaceSize}`,
]);
break;
case CrashRecoveryState.Crashed:
crashRecoveryState = CrashRecoveryState.Normal;
if (inputIndex === inputs.length || Date.now() - startTime > softTimeoutMs) {
stopChild(/*done*/ true);
} else {
restartChild(nextTask, process.execArgv);
}
break;
default:
assert.fail(`${processIndex}> Unexpected crashRecoveryState: ${crashRecoveryState}`);
}
} catch (e) {
onError(e);
}
};
const onError = (err?: Error) => {
child.removeAllListeners();
runningChildren.delete(child);
fail(err);
};
const startChild = (taskAction: () => void, execArgv: string[]) => {
try {
child = fork(workerFile, commandLineArgs, { cwd, execArgv });
runningChildren.add(child);
} catch (e) {
fail(e);
return;
}
try {
let closed = false;
const thisChild = child;
const onChildClosed = () => {
// Don't invoke `onClose` more than once for a single child.
if (!closed && child === thisChild) {
closed = true;
onClose();
}
};
const onChildDisconnectedOrExited = () => {
if (!closed && thisChild === child) {
// Invoke `onClose` after enough time has elapsed to allow `close` to be triggered.
// This is to ensure our `onClose` logic gets called in some conditions
const timeout = 1000;
setTimeout(onChildClosed, timeout);
}
};
child.on("message", onMessage);
child.on("close", onChildClosed);
child.on("disconnect", onChildDisconnectedOrExited);
child.on("exit", onChildDisconnectedOrExited);
child.on("error", onError);
taskAction();
} catch (e) {
onError(e);
}
};
const stopChild = (done: boolean) => {
try {
assert(runningChildren.has(child), `${processIndex}> Child not running`);
if (done) {
processesLeft--;
if (processesLeft === 0) {
resolve();
}
}
runningChildren.delete(child);
child.removeAllListeners();
child.kill();
} catch (e) {
onError(e);
}
};
const restartChild = (taskAction: () => void, execArgv: string[]) => {
try {
assert(runningChildren.has(child), `${processIndex}> Child not running`);
console.log(`${processIndex}> Restarting...`);
stopChild(/*done*/ false);
startChild(taskAction, execArgv);
} catch (e) {
onError(e);
}
};
const resumeTask = () => {
try {
assert(runningChildren.has(child), `${processIndex}> Child not running`);
child.send(currentInput);
} catch (e) {
onError(e);
}
};
const nextTask = () => {
try {
assert(runningChildren.has(child), `${processIndex}> Child not running`);
currentInput = inputs[inputIndex];
inputIndex++;
if (handleStart) {
handleStart(currentInput, processIndex);
}
child.send(currentInput);
} catch (e) {
onError(e);
}
};
startChild(nextTask, process.execArgv);
}
function fail(err?: Error): void {
if (!rejected) {
rejected = true;
for (const child of runningChildren) {
try {
child.removeAllListeners();
child.kill();
} catch {
// do nothing
}
}
const message = err ? `: ${err.message}` : "";
reject(new Error(`Something went wrong in ${runWithListeningChildProcesses.name}${message}`));
}
}
});
}
const maxOldSpaceSizeRegExp = /^--max[-_]old[-_]space[-_]size(?:$|=(\d+))/;
interface MaxOldSpaceSizeArgument {
index: number;
size: number;
value: number | undefined;
}
function getMaxOldSpaceSizeArg(argv: ReadonlyArray<string>): MaxOldSpaceSizeArgument | undefined {
for (let index = 0; index < argv.length; index++) {
const match = maxOldSpaceSizeRegExp.exec(argv[index]);
if (match) {
const value = match[1] ? parseInt(match[1], 10) :
argv[index + 1] ? parseInt(argv[index + 1], 10) :
undefined;
const size = match[1] ? 1 : 2; // tslint:disable-line:no-magic-numbers
return { index, size, value };
}
}
return undefined;
}
function getMaxOldSpaceSize(argv: ReadonlyArray<string>): number | undefined {
const arg = getMaxOldSpaceSizeArg(argv);
return arg && arg.value;
}
let execArgvWithoutMaxOldSpaceSize: ReadonlyArray<string> | undefined;
function getExecArgvWithoutMaxOldSpaceSize(): ReadonlyArray<string> {
if (!execArgvWithoutMaxOldSpaceSize) {
// remove --max_old_space_size from execArgv
const execArgv = process.execArgv.slice();
let maxOldSpaceSizeArg = getMaxOldSpaceSizeArg(execArgv);
while (maxOldSpaceSizeArg) {
execArgv.splice(maxOldSpaceSizeArg.index, maxOldSpaceSizeArg.size);
maxOldSpaceSizeArg = getMaxOldSpaceSizeArg(execArgv);
}
execArgvWithoutMaxOldSpaceSize = execArgv;
}
return execArgvWithoutMaxOldSpaceSize;
}
export function assertNever(_: never): never {
throw new Error();
}
export function recordToMap<T>(record: Record<string, T>): Map<string, T>;
export function recordToMap<T, U>(record: Record<string, T>, cb: (t: T) => U): Map<string, U>;
export function recordToMap<T, U>(record: Record<string, T>, cb?: (t: T) => U): Map<string, T | U> {
const m = new Map<string, T | U>();
for (const key of Object.keys(record)) {
m.set(key, cb ? cb(record[key]) : record[key]);
}
return m;
}
export function mapToRecord<T>(map: Map<string, T>): Record<string, T>;
export function mapToRecord<T, U>(map: Map<string, T>, cb: (t: T) => U): Record<string, U>;
export function mapToRecord<T, U>(map: Map<string, T>, cb?: (t: T) => U): Record<string, T | U> {
const o: Record<string, T | U> = {};
map.forEach((value, key) => { o[key] = cb ? cb(value) : value; });
return o;
}
export function identity<T>(t: T): T { return t; }
export function withoutStart(s: string, start: string): string | undefined {
return s.startsWith(start) ? s.slice(start.length) : undefined;
}
// Based on `getPackageNameFromAtTypesDirectory` in TypeScript.
export function unmangleScopedPackage(packageName: string): string | undefined {
const separator = "__";
return packageName.includes(separator) ? `@${packageName.replace(separator, "/")}` : undefined;
}
/** Returns [values that cb returned undefined for, defined results of cb]. */
export function split<T, U>(inputs: ReadonlyArray<T>, cb: (t: T) => U | undefined): [ReadonlyArray<T>, ReadonlyArray<U>] {
const keep: T[] = [];
const splitOut: U[] = [];
for (const input of inputs) {
const res = cb(input);
if (res === undefined) { keep.push(input); } else { splitOut.push(res); }
}
return [keep, splitOut];
}
export function assertSorted(a: ReadonlyArray<string>): ReadonlyArray<string>;
export function assertSorted<T>(a: ReadonlyArray<T>, cb: (t: T) => string): ReadonlyArray<T>;
export function assertSorted<T>(a: ReadonlyArray<T>, cb: (t: T) => string = (t: T) => t as unknown as string): ReadonlyArray<T> {
let prev = a[0];
for (let i = 1; i < a.length; i++) {
const x = a[i];
assert(cb(x) >= cb(prev), `${JSON.stringify(x)} >= ${JSON.stringify(prev)}`);
prev = x;
}
return a;
}

23
packages/types-publisher/tsconfig.json
View File

@@ -1,23 +0,0 @@
{
"include": ["src"],
"compilerOptions": {
"module": "commonjs",
"strict": true,
"declaration": true,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"allowUnreachableCode": false,
"allowUnusedLabels": false,
"outDir": "bin",
"sourceMap": true,
"target": "es2017",
"newLine": "crlf",
"baseUrl": ".",
"paths": {
"*": ["src/types/*"]
}
}
}

59
packages/types-publisher/tslint.json
View File

@@ -1,59 +0,0 @@
{
"extends": "tslint:all",
"rules": {
"arrow-parens": [true, "ban-single-arg-parens"],
"comment-format": [true, "check-space"],
"indent": [true, "spaces"],
"interface-name": [true, "never-prefix"],
"max-line-length": [true, 150],
"object-literal-key-quotes": [true, "as-needed"],
"only-arrow-functions": [true, "allow-declarations"],
"variable-name": [true, "check-format", "allow-leading-underscore"],
"strict-comparisons": [
true,
{
"allow-object-equal-comparison": true,
"allow-string-order-comparison": true
}
],
// TODO
"strict-boolean-expressions": false,
"no-unnecessary-type-assertion": false, // false positive
"no-this-assignment": [true, { "allow-destructuring": true }],
"typedef": false,
"no-any": false,
"no-inferred-empty-object-type": false,
"no-unsafe-any": false,
"prefer-template": false,
"restrict-plus-operands": false,
"completed-docs": false,
"file-name-casing": false,
"forin": false,
"increment-decrement": false,
"linebreak-style": false,
"max-classes-per-file": false,
"member-access": false,
"member-ordering": false,
"newline-before-return": false,
"newline-per-chained-call": false,
"no-console": false,
"no-default-export": false,
"no-default-import": false,
"no-unused-variable": false,
"no-magic-numbers": false,
"no-namespace": false,
"no-non-null-assertion": false,
"no-object-literal-type-assertion": false,
"no-use-before-declare": false,
"no-parameter-properties": false,
"no-parameter-reassignment": false,
"no-require-imports": false,
"no-void-expression": false,
"object-literal-sort-keys": false,
"promise-function-async": false,
"type-literal-delimiter": false
}
}

27
packages/utils/src/assertions.ts
View File

@@ -4,3 +4,30 @@ export function assertDefined<T>(x: T | undefined, message?: string | Error | un
assert(x !== undefined, message);
return x!;
}
export function assertNever(member: never, message = "Illegal value:", stackCrawlMark?: AnyFunction): never {
const detail = JSON.stringify(member);
return fail(`${message} ${detail}`, stackCrawlMark || assertNever);
}
type AnyFunction = (...args: never[]) => void;
export function fail(message?: string, stackCrawlMark?: AnyFunction): never {
debugger;
const e = new Error(message ? `Debug Failure. ${message}` : "Debug Failure.");
if ((<any>Error).captureStackTrace) {
(<any>Error).captureStackTrace(e, stackCrawlMark || fail);
}
throw e;
}
export function assertSorted(a: ReadonlyArray<string>): ReadonlyArray<string>;
export function assertSorted<T>(a: ReadonlyArray<T>, cb: (t: T) => string): ReadonlyArray<T>;
export function assertSorted<T>(a: ReadonlyArray<T>, cb: (t: T) => string = (t: T) => t as unknown as string): ReadonlyArray<T> {
let prev = a[0];
for (let i = 1; i < a.length; i++) {
const x = a[i];
assert(cb(x) >= cb(prev), `${JSON.stringify(x)} >= ${JSON.stringify(prev)}`);
prev = x;
}
return a;
}

2
packages/utils/src/async.ts
View File

@@ -1,4 +1,4 @@
import { ProgressBar } from "./lib/progress";
import { ProgressBar } from "./progress";
import { initArray } from "./collections";
/** Progress options needed for `nAtATime`. Other options will be inferred. */

55
packages/utils/src/collections.ts
View File

@@ -37,13 +37,13 @@ export async function mapDefinedAsync<T, U>(arr: Iterable<T>, mapper: (t: T) =>
return out;
}
export function* mapIterator<T, U>(inputs: Iterable<T>, mapper: (t: T) => U): Iterable<U> {
export function* mapIterable<T, U>(inputs: Iterable<T>, mapper: (t: T) => U): Iterable<U> {
for (const input of inputs) {
yield mapper(input);
}
}
export function* flatMapIterator<T, U>(inputs: Iterable<T>, mapper: (t: T) => Iterable<U>): Iterable<U> {
export function* flatMapIterable<T, U>(inputs: Iterable<T>, mapper: (t: T) => Iterable<U>): Iterable<U> {
for (const input of inputs) {
yield* mapper(input);
}
@@ -158,3 +158,54 @@ export function flatMap<T, U>(array: readonly T[] | undefined, mapfn: (x: T, i:
export function unique<T>(arr: Iterable<T>): T[] {
return [...new Set(arr)];
}
export function sortObjectKeys<T extends { [key: string]: unknown }>(data: T): T {
const out = {} as T;
for (const key of Object.keys(data).sort()) {
out[key as keyof T] = data[key as keyof T];
}
return out;
}
export function recordToMap<T>(record: Record<string, T>): Map<string, T>;
export function recordToMap<T, U>(record: Record<string, T>, cb: (t: T) => U): Map<string, U>;
export function recordToMap<T, U>(record: Record<string, T>, cb?: (t: T) => U): Map<string, T | U> {
const m = new Map<string, T | U>();
for (const key of Object.keys(record)) {
m.set(key, cb ? cb(record[key]) : record[key]);
}
return m;
}
export function mapToRecord<T>(map: Map<string, T>): Record<string, T>;
export function mapToRecord<T, U>(map: Map<string, T>, cb: (t: T) => U): Record<string, U>;
export function mapToRecord<T, U>(map: Map<string, T>, cb?: (t: T) => U): Record<string, T | U> {
const o: Record<string, T | U> = {};
map.forEach((value, key) => { o[key] = cb ? cb(value) : value; });
return o;
}
/**
* Returns the input that is better than all others, or `undefined` if there are no inputs.
* @param isBetter Returns true if `a` should be preferred over `b`.
*/
export function best<T>(inputs: Iterable<T>, isBetter: (a: T, b: T) => boolean): T | undefined {
const iter = inputs[Symbol.iterator]();
const first = iter.next();
if (first.done) {
return undefined;
}
let res = first.value;
while (true) {
const { value, done } = iter.next();
if (done) {
break;
}
if (isBetter(value, res)) {
res = value;
}
}
return res;
}

2
packages/utils/src/index.ts
View File

@@ -6,3 +6,5 @@ export * from "./io";
export * from "./logging";
export * from "./miscellany";
export * from "./process";
export * from "./progress";
export * from "./semver";

10
packages/utils/src/io.ts
View File

@@ -32,6 +32,16 @@ export function readFileSync(path: string): string {
return res;
}
/** If a file doesn't exist, warn and tell the step it should have been generated by. */
export async function readFileAndWarn(generatedBy: string, filePath: string): Promise<object> {
try {
return await readJson(filePath);
} catch (e) {
console.error(`Run ${generatedBy} first!`);
throw e;
}
}
export function readJsonSync(path: string): object {
return parseJson(readFileSync(path));
}

0
packages/utils/src/lib/progress.ts → packages/utils/src/progress.ts
View File

76
packages/utils/src/semver.ts Normal file
View File

@@ -0,0 +1,76 @@
/** Version of a package published to NPM. */
export class Semver {
static parse(semver: string, coerce?: boolean): Semver {
const result = Semver.tryParse(semver, coerce);
if (!result) {
throw new Error(`Unexpected semver: ${semver}`);
}
return result;
}
static fromRaw({ major, minor, patch }: Semver): Semver {
return new Semver(major, minor, patch);
}
/**
* Returns 0 if equal, 1 if x > y, -1 if x < y
*/
static compare(x: Semver, y: Semver) {
const versions: Array<[number, number]> = [[x.major, y.major], [x.minor, y.minor], [x.patch, y.patch]];
for (const [componentX, componentY] of versions) {
if (componentX > componentY) {
return 1;
}
if (componentX < componentY) {
return -1;
}
}
return 0;
}
/**
* Per the semver spec <http://semver.org/#spec-item-2>:
*
* A normal version number MUST take the form X.Y.Z where X, Y, and Z are non-negative integers, and MUST NOT contain leading zeroes.
*
* @note This must parse the output of `versionString`.
*
* @param semver The version string.
* @param coerce Without this optional parameter the version MUST follow the above semver spec. However, when set to `true` components after the
* major version may be omitted. I.e. `1` equals `1.0` and `1.0.0`.
*/
static tryParse(semver: string, coerce?: boolean): Semver | undefined {
const rgx = /^(\d+)(\.(\d+))?(\.(\d+))?$/;
const match = rgx.exec(semver);
if (match) {
const { 1: major, 3: minor, 5: patch } = match;
if ((minor !== undefined && patch !== undefined) || coerce) { // tslint:disable-line:strict-type-predicates
return new Semver(intOfString(major), intOfString(minor || "0"), intOfString(patch || "0"));
}
}
return undefined;
}
constructor(readonly major: number, readonly minor: number, readonly patch: number) { }
get versionString(): string {
const { major, minor, patch } = this;
return `${major}.${minor}.${patch}`;
}
equals(other: Semver): boolean {
return Semver.compare(this, other) === 0;
}
greaterThan(other: Semver): boolean {
return Semver.compare(this, other) === 1;
}
}
function intOfString(str: string): number {
const n = Number.parseInt(str, 10);
if (Number.isNaN(n)) {
throw new Error(`Error in parseInt(${JSON.stringify(str)})`);
}
return n;
}

29
packages/utils/test/semver.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { Semver } from "../src/semver";
it("returns a formatted description", () => {
expect(new Semver(1, 2, 3).versionString).toEqual("1.2.3");
});
it("parses semver versions", () => {
expect(Semver.parse("0.42.1").versionString).toEqual("0.42.1");
});
it("parses versions that do not strictly adhere to semver", () => {
expect(Semver.parse("1", true).versionString).toEqual("1.0.0");
expect(Semver.parse("0.42", true).versionString).toEqual("0.42.0");
});
it("throws when a version cannot be parsed", () => {
expect(() => Semver.parse("1")).toThrow();
expect(() => Semver.parse("1", false)).toThrow();
});
it("returns whether or not it's equal to another Semver", () => {
expect(Semver.parse("1.2.3").equals(new Semver(1, 2, 3))).toBe(true);
expect(Semver.parse("1.2.3").equals(new Semver(3, 2, 1))).toBe(false);
});
it("returns whether or not it's greater than another Semver", () => {
expect(Semver.parse("1.2.3").greaterThan(new Semver(1, 2, 2))).toBe(true);
expect(Semver.parse("1.2.3").equals(new Semver(1, 2, 4))).toBe(false);
});

9
packages/utils/test/tsconfig.json Normal file
View File

@@ -0,0 +1,9 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"types": ["node", "jest"]
},
"references": [
{ "path": ".." }
]
}

Some files were not shown because too many files have changed in this diff Show More