casraf/DefinitelyTyped-tools
1
0
Fork 0
mirror of https://github.com/chenasraf/DefinitelyTyped-tools.git synced 2026-05-18 01:49:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Mostly done switching to package.json dep/devDeps

Browse Source 
Deletes a whole bunch of stuff and simplifies or reworks others.

I noticed that we still want to walk the file looking for globals SO I
have to put back in huge chunks of code I took out. Creating a commit so
I can diff on github.
This commit is contained in:
Nathan Shively-Sanders
2023-04-03 08:54:28 -07:00
parent 9ffae5f6f6
commit 55bde30076
11 changed files with 67 additions and 623 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
packages/definitions-parser/src/get-affected-packages.ts
View File

@@ -7,6 +7,7 @@ import {
getMangledNameForScopedPackage,
formatDependencyVersion,
} from "./packages";
import { parsePackageSemver } from './lib/definition-parser';
export interface Affected {
readonly changedPackages: readonly TypingsData[];
@@ -87,15 +88,8 @@ function getReverseDependencies(
}
}
for (const typing of allPackages.allTypings()) {
for (const [name, version] of Object.entries(typing.dependencies)) {
const dependencies = map.get(packageIdToKey(allPackages.tryResolve({ name, version })));
if (dependencies) {
dependencies[1].add(typing.id);
}
}
for (const dependencyName of typing.testDependencies) {
const version = typing.pathMappings[dependencyName] || "*";
const dependencies = map.get(packageIdToKey(allPackages.tryResolve({ name: dependencyName, version })));
for (const [ name, version ] of typing.allPackageJsonDependencies()) {
const dependencies = map.get(packageIdToKey(allPackages.tryResolve({ name, version: parsePackageSemver(version) })));
if (dependencies) {
dependencies[1].add(typing.id);
}

270
packages/definitions-parser/src/lib/definition-parser.ts
View File

@@ -1,33 +1,29 @@
import * as ts from "typescript";
import { parseHeaderOrFail } from "@definitelytyped/header-parser";
import { allReferencedFiles, createSourceFile, getModuleInfo, getTestDependencies } from "./module-info";
import { allReferencedFiles, createSourceFile } from "./module-info";
import {
DependencyVersion,
formatTypingVersion,
getLicenseFromPackageJson,
PackageJsonDependency,
TypingsDataRaw,
TypingsVersionsRaw,
DirectoryParsedTypingVersion,
getMangledNameForScopedPackage,
} from "../packages";
import * as semver from "semver";
import { getAllowedPackageJsonDependencies } from "./settings";
import {
FS,
split,
hasWindowsSlashes,
mapDefined,
filter,
sort,
withoutStart,
computeHash,
hasWindowsSlashes,
join,
flatMap,
unique,
unmangleScopedPackage,
removeVersionFromPackageName,
hasVersionNumberInMapping,
mangleScopedPackage,
createModuleResolutionHost,
} from "@definitelytyped/utils";
import { TypeScriptVersion } from "@definitelytyped/typescript-versions";
@@ -200,6 +196,16 @@ export function parsePackageVersion(versionString: string): DirectoryParsedTypin
return version;
}
export function parsePackageSemver(version: string): DependencyVersion {
const start = new semver.Range(version).set[0][0].semver
if (start === (semver.Comparator as any).ANY) {
return "*"
}
else {
return { major: start.major, minor: start.minor }
}
}
async function combineDataForAllTypesVersions(
typingsPackageName: string,
ls: readonly string[],
@@ -212,6 +218,7 @@ async function combineDataForAllTypesVersions(
? (fs.readJson(packageJsonName) as {
readonly license?: unknown;
readonly dependencies?: unknown;
readonly devDependencies?: unknown;
readonly imports?: unknown;
readonly exports?: unknown;
readonly type?: unknown;
@@ -250,7 +257,9 @@ async function combineDataForAllTypesVersions(
});
const allTypesVersions = [dataForRoot, ...dataForOtherTypesVersions];
const license = getLicenseFromPackageJson(packageJson.license);
const packageJsonDependencies = await checkPackageJsonDependencies(packageJson.dependencies, packageJsonName);
const allowedDependencies = await getAllowedPackageJsonDependencies()
checkPackageJsonDependencies(packageJson.dependencies, packageJsonName, allowedDependencies);
checkPackageJsonDependencies(packageJson.devDependencies, packageJsonName, allowedDependencies);
const files = Array.from(
flatMap(allTypesVersions, ({ typescriptVersion, declFiles }) =>
@@ -270,37 +279,25 @@ async function combineDataForAllTypesVersions(
typesVersions,
files,
license,
dependencies: Object.assign({}, ...allTypesVersions.map((v) => v.dependencies)), // TODO: Get these from packageJsonDependencies instead
testDependencies: getAllUniqueValues<"testDependencies", string>(allTypesVersions, "testDependencies"), // TODO: Get these from packageJsonDevDependencies instead
pathMappings: Object.assign({}, ...allTypesVersions.map((v) => v.pathMappings)),
packageJsonDependencies,
packageJsonDependencies: packageJson.dependencies,
packageJsonDevDependencies: packageJson.devDependencies,
// TODO: Add devDependencies here (aka testDependencies)
contentHash: hash(
hasPackageJson ? [...files, packageJsonName] : files,
mapDefined(allTypesVersions, (a) => a.tsconfigPathsForHash),
fs
),
globals: getAllUniqueValues<"globals", string>(allTypesVersions, "globals"),
declaredModules: getAllUniqueValues<"declaredModules", string>(allTypesVersions, "declaredModules"),
imports: checkPackageJsonImports(packageJson.imports, packageJsonName),
exports: checkPackageJsonExportsAndAddPJsonEntry(packageJson.exports, packageJsonName),
type: packageJsonType,
};
}
function getAllUniqueValues<K extends string, T>(records: readonly Record<K, readonly T[]>[], key: K): readonly T[] {
return unique(flatMap(records, (x) => x[key]));
}
interface TypingDataFromIndividualTypeScriptVersion {
/** Undefined for root (which uses `// TypeScript Version: ` comment instead) */
readonly typescriptVersion: TypeScriptVersion | undefined;
readonly dependencies: { readonly [name: string]: DependencyVersion };
readonly testDependencies: readonly string[];
readonly pathMappings: { readonly [packageName: string]: DirectoryParsedTypingVersion };
readonly declFiles: readonly string[];
readonly declFiles: readonly string[]; // TODO: Used to map file.d.ts to ts4.1/file.d.ts -- not sure why this is needed
readonly tsconfigPathsForHash: string | undefined;
readonly globals: readonly string[];
readonly declaredModules: readonly string[];
}
/**
@@ -372,22 +369,6 @@ function getTypingDataForSingleTypesVersion(
createSourceFile(untestedTypeFile, fs.readFile(untestedTypeFile), moduleResolutionHost, compilerOptions)
);
}
// TODO: All this should be pulled from package.json instead
const { dependencies: dependenciesWithDeclaredModules, globals, declaredModules } = getModuleInfo(packageName, types);
const declaredModulesSet = new Set(declaredModules);
// Don't count an import of "x" as a dependency if we saw `declare module "x"` somewhere.
const dependenciesSet = new Set(
[...dependenciesWithDeclaredModules]
.filter((m) => !declaredModulesSet.has(m))
.map((m) => rootName(m, types, packageName))
.filter((dependency) => dependency !== packageName)
);
const testDependencies = [
...getTestDependencies(packageName, tests.keys(), dependenciesSet, fs, moduleResolutionHost, compilerOptions),
]
.filter((m) => !declaredModulesSet.has(m))
.map((m) => rootName(m, types, packageName))
.filter((dependency) => dependency !== packageName);
const { paths } = tsconfig.compilerOptions;
const hydratedPackageName = unmangleScopedPackage(packageName) ?? packageName;
@@ -400,20 +381,9 @@ function getTypingDataForSingleTypesVersion(
);
}
const { dependencies, pathMappings } = calculateDependencies(
packageName,
tsconfig,
dependenciesSet,
directoryVersion
);
const tsconfigPathsForHash = JSON.stringify(tsconfig.compilerOptions.paths);
return {
typescriptVersion,
dependencies,
testDependencies,
pathMappings,
globals,
declaredModules,
declFiles: sort(types.keys()),
tsconfigPathsForHash,
};
@@ -423,27 +393,6 @@ function slicePrefix(s: string, prefix: string): string {
return s.startsWith(prefix) ? s.slice(prefix.length) : s;
}
/**
* "foo/bar/baz" -> "foo"; "@foo/bar/baz" -> "@foo/bar"
* Note: Throws an error for references like
* "bar/v3" because referencing old versions of *other* packages is illegal;
* those directories won't exist in the published @types package.
*/
function rootName(importText: string, typeFiles: Map<string, unknown>, packageName: string): string {
let slash = importText.indexOf("/");
// Root of `@foo/bar/baz` is `@foo/bar`
if (importText.startsWith("@")) {
// Use second "/"
slash = importText.indexOf("/", slash + 1);
}
const root = importText.slice(0, slash);
const postImport = importText.slice(slash + 1);
if (slash > -1 && postImport.match(/v\d+$/) && !typeFiles.has(postImport + ".d.ts") && root !== packageName) {
throw new Error(`${importText}: do not directly import specific versions of another types package.
You should work with the latest version of ${root} instead.`);
}
return slash === -1 ? importText : root;
}
// TODO: Expand these checks too, adding name and version just like dtslint
function checkPackageJsonExportsAndAddPJsonEntry(exports: unknown, path: string) {
if (exports === undefined) return exports;
@@ -481,38 +430,31 @@ function checkPackageJsonType(type: unknown, path: string) {
return type;
}
async function checkPackageJsonDependencies(
function checkPackageJsonDependencies(
dependencies: unknown,
path: string
): Promise<readonly PackageJsonDependency[]> {
path: string,
allowedDependencies: ReadonlySet<string>
): asserts dependencies is Record<string, string> {
if (dependencies === undefined) {
// tslint:disable-line strict-type-predicates (false positive)
return [];
return;
}
if (dependencies === null || typeof dependencies !== "object") {
// tslint:disable-line strict-type-predicates
throw new Error(`${path} should contain "dependencies" or not exist.`);
}
const deps: PackageJsonDependency[] = [];
for (const dependencyName of Object.keys(dependencies!)) {
// `dependencies` cannot be null because of check above.
if (!dependencyName.startsWith("@types/") && !(await getAllowedPackageJsonDependencies()).has(dependencyName)) {
if (!dependencyName.startsWith("@types/") && !allowedDependencies.has(dependencyName)) {
const msg = `Dependency ${dependencyName} not in the allowed dependencies list.
Please make a pull request to microsoft/DefinitelyTyped-tools adding it to \`packages/definitions-parser/allowedPackageJsonDependencies.txt\`.`;
throw new Error(`In ${path}: ${msg}`);
}
const version = (dependencies as { [key: string]: unknown })[dependencyName];
if (typeof version !== "string") {
// tslint:disable-line strict-type-predicates
throw new Error(`In ${path}: Dependency version for ${dependencyName} should be a string.`);
}
deps.push({ name: dependencyName, version });
}
return deps;
}
function checkFilesFromTsConfig(packageName: string, tsconfig: TsConfig, directoryPath: string): void {
@@ -561,164 +503,6 @@ interface TsConfig {
compilerOptions: ts.CompilerOptions;
}
/** In addition to dependencies found in source code, also get dependencies from tsconfig. */
// So...there's nothing to do here, but to move the path mappings requirements into dtslint/checks.ts
// TODO: Stop calculating dependencies
// TODO: Figure out if path mappings are still needed for xxx/v12 folders (or anything else?)
// do this by comparing the published package.json with the currently generated one, as well as the new ones that Jake has created in DT
interface DependenciesAndPathMappings {
readonly dependencies: { readonly [name: string]: DependencyVersion };
readonly pathMappings: { readonly [packageName: string]: DirectoryParsedTypingVersion };
}
function calculateDependencies(
packageName: string,
tsconfig: TsConfig,
dependencyNames: ReadonlySet<string>,
directoryVersion: DirectoryParsedTypingVersion | undefined
): DependenciesAndPathMappings {
const paths = (tsconfig.compilerOptions && tsconfig.compilerOptions.paths) || {};
const dependencies: { [name: string]: DependencyVersion } = {};
const pathMappings: { [packageName: string]: DirectoryParsedTypingVersion } = {};
const scopedPackageName = unmangleScopedPackage(packageName) ?? packageName;
for (const dependencyName of Object.keys(paths)) {
const pathMappingList = paths[dependencyName];
if (pathMappingList.length !== 1) {
throw new Error(`In ${packageName}: Path mapping for ${dependencyName} may only have 1 entry.`);
}
const pathMapping = pathMappingList[0];
if (pathMapping === "./node_modules/" + dependencyName) {
// allow passthrough remappings for packages like webpack that have shipped their own types,
// but have some dependents on DT that depend on the new types and some that depend on the old types
continue;
}
// Path mapping may be for "@foo/*" -> "foo__*".
const unversionedScopedPackageName = removeVersionFromPackageName(unmangleScopedPackage(pathMapping));
if (unversionedScopedPackageName !== undefined) {
if (dependencyName !== unversionedScopedPackageName) {
throw new Error(`Expected directory ${pathMapping} to be the path mapping for ${dependencyName}`);
}
if (!hasVersionNumberInMapping(pathMapping)) {
continue;
}
}
// Might have a path mapping for "foo/*" to support subdirectories
const rootDirectory = withoutEnd(dependencyName, "/*");
if (rootDirectory !== undefined) {
if (!(rootDirectory in paths)) {
throw new Error(`In ${packageName}: found path mapping for ${dependencyName} but not for ${rootDirectory}`);
}
continue;
}
// buffer -> node/buffer may be required because of the non-node 'buffer' package on npm
// which DT infrastructure depends on, and which resolves before node's ambient module 'buffer'
if (dependencyName === "buffer" && pathMapping === "node/buffer") {
dependencies.node = "*";
continue;
}
const pathMappingVersion = parseDependencyVersionFromPath(dependencyName, dependencyName, pathMapping);
if (dependencyName === packageName) {
if (directoryVersion === undefined) {
throw new Error(`In ${packageName}: Latest version of a package should not have a path mapping for itself.`);
}
if (directoryVersion.major !== pathMappingVersion.major || directoryVersion.minor !== pathMappingVersion.minor) {
const correctPathMapping = [`${dependencyName}/v${formatTypingVersion(directoryVersion)}`];
throw new Error(
`In ${packageName}: Must have a "paths" entry of "${dependencyName}": ${JSON.stringify(correctPathMapping)}`
);
}
} else {
if (dependencyNames.has(dependencyName)) {
dependencies[dependencyName] = pathMappingVersion;
}
}
// Else, the path mapping may be necessary if it is for a transitive dependency. We will check this in check-parse-results.
pathMappings[dependencyName] = pathMappingVersion;
}
if (directoryVersion !== undefined && !(paths && scopedPackageName in paths)) {
const mapping = JSON.stringify([`${packageName}/v${formatTypingVersion(directoryVersion)}`]);
throw new Error(
`${scopedPackageName}: Older version ${formatTypingVersion(
directoryVersion
)} must have a "paths" entry of "${scopedPackageName}": ${mapping}`
);
}
for (const dependency of dependencyNames) {
if (!dependencies[dependency] && !nodeBuiltins.has(dependency)) {
dependencies[dependency] = "*";
}
}
return { dependencies, pathMappings };
}
const nodeBuiltins: ReadonlySet<string> = new Set([
"assert",
"async_hooks",
"buffer",
"child_process",
"cluster",
"console",
"constants",
"crypto",
"dgram",
"dns",
"domain",
"events",
"fs",
"http",
"http2",
"https",
"module",
"net",
"os",
"path",
"perf_hooks",
"process",
"punycode",
"querystring",
"readline",
"repl",
"stream",
"string_decoder",
"timers",
"tls",
"tty",
"url",
"util",
"v8",
"vm",
"zlib",
]);
function parseDependencyVersionFromPath(
packageName: string,
dependencyName: string,
dependencyPath: string
): DirectoryParsedTypingVersion {
const versionString = withoutStart(dependencyPath, `${mangleScopedPackage(dependencyName)}/`);
const version = versionString === undefined ? undefined : parseVersionFromDirectoryName(versionString);
if (version === undefined) {
// TODO: This sounds like it reads path mapping, so needs to be undone
throw new Error(`In ${packageName}, unexpected path mapping for ${dependencyName}: '${dependencyPath}'`);
}
return version;
}
function withoutEnd(s: string, end: string): string | undefined {
if (s.endsWith(end)) {
return s.slice(0, s.length - end.length);
}
return undefined;
}
function hash(files: readonly string[], tsconfigPathsForHash: readonly string[], fs: FS): string {
const fileContents = files.map((f) => `${f}**${readFileAndThrowOnBOM(f, fs)}`);
let allContent = fileContents.join("||");

184
packages/definitions-parser/src/lib/module-info.ts
View File

@@ -1,131 +1,15 @@
import assert = require("assert");
import * as path from "path";
import * as ts from "typescript";
import { sort, joinPaths, FS, hasWindowsSlashes, assertDefined } from "@definitelytyped/utils";
import { FS, assertDefined } from "@definitelytyped/utils";
import { readFileAndThrowOnBOM } from "./definition-parser";
import { getMangledNameForScopedPackage } from "../packages";
export function getModuleInfo(packageName: string, all: Map<string, ts.SourceFile>): ModuleInfo {
const dependencies = new Set<string>();
const declaredModules: string[] = [];
const globals = new Set<string>();
function addDependency(ref: string): void {
if (!ref.startsWith(".")) {
dependencies.add(ref);
}
}
for (const sourceFile of all.values()) {
for (const ref of imports(sourceFile)) {
addDependency(ref.text);
}
for (const ref of sourceFile.typeReferenceDirectives) {
addDependency(ref.fileName);
}
if (ts.isExternalModule(sourceFile)) {
if (sourceFileExportsSomething(sourceFile)) {
declaredModules.push(properModuleName(packageName, sourceFile.fileName));
const namespaceExport = sourceFile.statements.find(ts.isNamespaceExportDeclaration);
if (namespaceExport) {
globals.add(namespaceExport.name.text);
}
}
} else {
for (const node of sourceFile.statements) {
switch (node.kind) {
case ts.SyntaxKind.ModuleDeclaration: {
const decl = node as ts.ModuleDeclaration;
const name = decl.name.text;
if (decl.name.kind === ts.SyntaxKind.StringLiteral) {
declaredModules.push(assertNoWindowsSlashes(packageName, name));
} else if (isValueNamespace(decl)) {
globals.add(name);
}
break;
}
case ts.SyntaxKind.VariableStatement:
for (const decl of (node as ts.VariableStatement).declarationList.declarations) {
if (decl.name.kind === ts.SyntaxKind.Identifier) {
globals.add(decl.name.text);
}
}
break;
case ts.SyntaxKind.EnumDeclaration:
case ts.SyntaxKind.ClassDeclaration:
case ts.SyntaxKind.FunctionDeclaration: {
// Deliberately not doing this for types, because those won't show up in JS code and can't be used for ATA
const nameNode = (node as ts.EnumDeclaration | ts.ClassDeclaration | ts.FunctionDeclaration).name;
if (nameNode) {
globals.add(nameNode.text);
}
break;
}
case ts.SyntaxKind.ImportEqualsDeclaration:
case ts.SyntaxKind.InterfaceDeclaration:
case ts.SyntaxKind.TypeAliasDeclaration:
case ts.SyntaxKind.EmptyStatement:
break;
default:
throw new Error(`Unexpected node kind ${ts.SyntaxKind[node.kind]}`);
}
}
}
}
return { dependencies, declaredModules, globals: sort(globals) };
}
/**
* A file is a proper module if it is an external module *and* it has at least one export.
* A module with only imports is not a proper module; it likely just augments some other module.
*/
function sourceFileExportsSomething({ statements }: ts.SourceFile): boolean {
return statements.some((statement) => {
switch (statement.kind) {
case ts.SyntaxKind.ImportEqualsDeclaration:
case ts.SyntaxKind.ImportDeclaration:
return false;
case ts.SyntaxKind.ModuleDeclaration:
return (statement as ts.ModuleDeclaration).name.kind === ts.SyntaxKind.Identifier;
default:
return true;
}
});
}
interface ModuleInfo {
/** Full (possibly deep) module specifiers of dependencies (imports, type references, etc.). */
dependencies: Set<string>;
/** Anything from a `declare module "foo"` */
declaredModules: string[];
/** Every global symbol */
globals: string[];
}
const extensions: Map<string, string> = new Map();
extensions.set(".d.ts", ""); // TODO: Inaccurate?
extensions.set(".d.mts", ".mjs");
extensions.set(".d.cts", ".cjs");
/**
* Given a file name, get the name of the module it declares.
* `foo/index.d.ts` declares "foo", `foo/bar.d.ts` declares "foo/bar", "foo/bar/index.d.ts" declares "foo/bar"
*/
function properModuleName(folderName: string, fileName: string): string {
const part =
path.basename(fileName) === "index.d.ts" ? path.dirname(fileName) : withoutExtensions(fileName, extensions);
return part === "." ? folderName : joinPaths(folderName, part);
}
function withoutExtensions(str: string, exts: typeof extensions): string {
const entries = Array.from(exts.entries());
const ext = entries.find(([e, _]) => str.endsWith(e));
assert(ext, `file "${str}" should end with extension ${entries.map(([e, _]) => `"${e}"`).join(", ")}`);
return str.slice(0, str.length - ext[0].length) + ext[1];
}
/** Returns a map from filename (path relative to `directory`) to the SourceFile we parsed for it. */
export function allReferencedFiles(
entryFilenames: readonly string[],
@@ -355,72 +239,6 @@ function statementDeclaresValue(statement: ts.Statement): boolean {
}
}
function assertNoWindowsSlashes(packageName: string, fileName: string): string {
if (hasWindowsSlashes(fileName)) {
throw new Error(`In ${packageName}: Use forward slash instead when referencing ${fileName}`);
}
return fileName;
}
export function getTestDependencies(
packageName: string,
testFiles: Iterable<string>,
dependencies: ReadonlySet<string>,
fs: FS,
moduleResolutionHost: ts.ModuleResolutionHost,
compilerOptions: ts.CompilerOptions
): Iterable<string> {
const testDependencies = new Set<string>();
for (const filename of testFiles) {
const content = readFileAndThrowOnBOM(filename, fs);
const sourceFile = createSourceFile(filename, content, moduleResolutionHost, compilerOptions);
const { fileName, referencedFiles, typeReferenceDirectives } = sourceFile;
const filePath = () => path.join(packageName, fileName);
let hasImports = false;
let isModule = false;
let referencesSelf = false;
for (const { fileName: ref } of referencedFiles) {
throw new Error(`Test files should not use '<reference path="" />'. '${filePath()}' references '${ref}'.`);
}
for (const { fileName: referencedPackage } of typeReferenceDirectives) {
if (dependencies.has(referencedPackage)) {
throw new Error(
`'${filePath()}' unnecessarily references '${referencedPackage}', which is already referenced in the type definition.`
);
}
if (referencedPackage === packageName) {
referencesSelf = true;
}
testDependencies.add(referencedPackage);
}
for (const imported of imports(sourceFile)) {
hasImports = true;
if (!imported.text.startsWith(".") && !dependencies.has(imported.text)) {
testDependencies.add(imported.text);
}
}
isModule =
hasImports ||
(() => {
// Note that this results in files without imports to be walked twice,
// once in the `imports(...)` function, and once more here:
for (const node of sourceFile.statements) {
if (node.kind === ts.SyntaxKind.ExportAssignment || node.kind === ts.SyntaxKind.ExportDeclaration) {
return true;
}
}
return false;
})();
if (isModule && referencesSelf) {
throw new Error(`'${filePath()}' unnecessarily references the package. This can be removed.`);
}
}
return testDependencies;
}
export function createSourceFile(
filename: string,
content: string,

98
packages/definitions-parser/src/packages.ts
View File

@@ -5,7 +5,7 @@ import { AllTypeScriptVersion, TypeScriptVersion } from "@definitelytyped/typesc
import * as semver from "semver";
import { readDataFile } from "./data-file";
import { scopeName, typesDirectoryName } from "./lib/settings";
import { parseVersionFromDirectoryName } from "./lib/definition-parser";
import { parseVersionFromDirectoryName, parsePackageSemver } from "./lib/definition-parser";
export class AllPackages {
static async read(dt: FS): Promise<AllPackages> {
@@ -136,25 +136,13 @@ export class AllPackages {
return this.notNeeded;
}
/** Returns all of the dependences *that have typings*, ignoring others, and including test dependencies. */
/** Returns all of the dependences *that have typings*, ignoring others, and including test dependencies.
* I have NO idea why it's an iterator. Surely not for efficiency. */
*allDependencyTypings(pkg: TypingsData): Iterable<TypingsData> {
for (const [name, version] of Object.entries(pkg.dependencies)) {
for (const [ name, version ] of pkg.allPackageJsonDependencies()) {
const versions = this.data.get(getMangledNameForScopedPackage(name));
if (versions) {
yield versions.get(
version,
pkg.pathMappings[name]
? `${pkg.name} references this version of ${name} in its path mappings in tsconfig.json. If you are deleting this version, update ${pkg.name}’s path mappings accordingly.\n`
: undefined
);
}
}
for (const name of pkg.testDependencies) {
const versions = this.data.get(getMangledNameForScopedPackage(name));
if (versions) {
const version = pkg.pathMappings[name];
yield version ? versions.get(version) : versions.getLatest();
yield versions.get(parsePackageSemver(version), undefined);
}
}
}
@@ -218,8 +206,6 @@ export abstract class PackageBase {
}
abstract readonly isLatest: boolean;
abstract readonly declaredModules: readonly string[];
abstract readonly globals: readonly string[];
abstract readonly minTypeScriptVersion: TypeScriptVersion;
/** '@types/foo' for a package 'foo'. */
@@ -330,10 +316,8 @@ export function formatDependencyVersion(version: DependencyVersion) {
return version === "*" ? "*" : formatTypingVersion(version);
}
export interface PackageJsonDependency {
readonly name: string;
readonly version: string;
}
/** Maps name to version */
export type PackageJsonDependencies = Record<string, string>
export interface TypingsDataRaw extends BaseRaw {
/**
@@ -343,13 +327,6 @@ export interface TypingsDataRaw extends BaseRaw {
*/
readonly typingsPackageName: string;
/**
* Other definitions, that exist in the same typings repo, that this package depends on.
*
* These will refer to *package names*, not *folder names*.
*/
readonly dependencies: { readonly [name: string]: DependencyVersion };
/**
* Package `imports`, as read in the `package.json` file
*/
@@ -366,24 +343,15 @@ export interface TypingsDataRaw extends BaseRaw {
readonly type?: string;
/**
* Other definitions, that exist in the same typings repo, that the tests, but not the types, of this package depend on.
*
* These are always the latest version and will not include anything already in `dependencies`.
* Packages that provide definitions that this package depends on.
* NOTE: Includes `@types/` packages.
*/
readonly testDependencies: readonly string[];
readonly packageJsonDependencies: PackageJsonDependencies;
/**
* External packages, from outside the typings repo, that provide definitions that this package depends on.
* Packages that this package's tests or other development depends on.
*/
readonly packageJsonDependencies: readonly PackageJsonDependency[];
/**
* Represents that there was a path mapping to a package.
*
* Not all path mappings are direct dependencies, they may be necessary for transitive dependencies. However, where `dependencies` and
* `pathMappings` share a key, they *must* share the same value.
*/
readonly pathMappings: { readonly [packageName: string]: DirectoryParsedTypingVersion };
readonly packageJsonDevDependencies: PackageJsonDependencies;
/**
* List of people that have contributed to the definitions in this package.
@@ -428,6 +396,7 @@ export interface TypingsDataRaw extends BaseRaw {
* Files that should be published with this definition, e.g. ["jquery.d.ts", "jquery-extras.d.ts"]
*
* Does *not* include a partial `package.json` because that will not be copied directly.
* TODO: THis might change!
*/
readonly files: readonly string[];
@@ -447,18 +416,6 @@ export interface TypingsDataRaw extends BaseRaw {
* Name or URL of the project, e.g. "http://cordova.apache.org".
*/
readonly projectName: string;
/**
* A list of *values* declared in the global namespace.
*
* @note This does not include *types* declared in the global namespace.
*/
readonly globals: readonly string[];
/**
* External modules declared by this package. Includes the containing folder name when applicable (e.g. proper module).
*/
readonly declaredModules: readonly string[];
}
// Note that BSD is not supported -- for that, we'd have to choose a *particular* BSD license from the list at https://spdx.org/licenses/
@@ -545,9 +502,6 @@ export class TypingsData extends PackageBase {
return this.data.typingsPackageName;
}
get testDependencies(): readonly string[] {
return this.data.testDependencies;
}
get contributors(): readonly Author[] {
return this.data.contributors;
}
@@ -574,28 +528,26 @@ export class TypingsData extends PackageBase {
get license(): License {
return this.data.license;
}
get packageJsonDependencies(): readonly PackageJsonDependency[] {
get packageJsonDependencies(): PackageJsonDependencies {
return this.data.packageJsonDependencies;
}
get packageJsonDevDependencies(): PackageJsonDependencies {
return this.data.packageJsonDevDependencies;
}
*allPackageJsonDependencies(): Iterable<[string, string]> {
for (const [name, version] of Object.entries(this.packageJsonDependencies)) {
yield [name, version]
}
for (const [name, version] of Object.entries(this.packageJsonDevDependencies)) {
yield [name, version]
}
}
get contentHash(): string {
return this.data.contentHash;
}
get declaredModules(): readonly string[] {
return this.data.declaredModules;
}
get projectName(): string {
return this.data.projectName;
}
get globals(): readonly string[] {
return this.data.globals;
}
get pathMappings(): { readonly [packageName: string]: DirectoryParsedTypingVersion } {
return this.data.pathMappings;
}
get dependencies(): { readonly [name: string]: DependencyVersion } {
return this.data.dependencies;
}
get type() {
return this.data.type;

47
packages/dtslint-runner/src/check-parse-results.ts
View File

@@ -3,7 +3,6 @@ import {
ParseDefinitionsOptions,
TypingsData,
AllPackages,
formatTypingVersion,
getDefinitelyTyped,
} from "@definitelytyped/definitions-parser";
import { mapDefined, FS, logger, writeLog, Logger, ProgressBar, cacheDir, max, min } from "@definitelytyped/utils";
@@ -33,17 +32,12 @@ export async function checkParseResults(
checkTypeScriptVersions(allPackages);
checkPathMappings(allPackages);
const dependedOn = new Set<string>();
const packages = allPackages.allPackages();
for (const pkg of packages) {
if (pkg instanceof TypingsData) {
for (const dep of Object.keys(pkg.dependencies)) {
dependedOn.add(dep);
}
for (const dep of pkg.testDependencies) {
dependedOn.add(dep);
for (const [ name ] of pkg.allPackageJsonDependencies()) {
dependedOn.add(name);
}
}
}
@@ -75,43 +69,6 @@ function checkTypeScriptVersions(allPackages: AllPackages): void {
}
}
export function checkPathMappings(allPackages: AllPackages): void {
for (const pkg of allPackages.allTypings()) {
const unusedPathMappings = new Set(
Object.keys(pkg.pathMappings).filter((m) => m !== pkg.name && m !== pkg.unescapedName)
);
// If A depends on B, and B has path mappings, A must have the same mappings.
for (const dependency of allPackages.allDependencyTypings(pkg)) {
for (const [transitiveDependencyName, transitiveDependencyVersion] of Object.entries(dependency.pathMappings)) {
const pathMappingVersion = pkg.pathMappings[transitiveDependencyName];
if (
pathMappingVersion &&
(pathMappingVersion.major !== transitiveDependencyVersion.major ||
pathMappingVersion.minor !== transitiveDependencyVersion.minor)
) {
const expectedPathMapping = `${transitiveDependencyName}/v${formatTypingVersion(
transitiveDependencyVersion
)}`;
throw new Error(
`${pkg.desc} depends on ${dependency.desc}, which has a path mapping for ${expectedPathMapping}. ` +
`${pkg.desc} must have the same path mappings as its dependencies.`
);
}
unusedPathMappings.delete(transitiveDependencyName);
}
unusedPathMappings.delete(dependency.name);
}
if (unusedPathMappings.size > 0) {
throw new Error(`${pkg.desc} has unused path mappings for [${Array.from(unusedPathMappings).join(", ")}].
If these mappings are actually used, they could be missing in a dependency's tsconfig.json instead.
Check the path mappings for [${Array.from(allPackages.allDependencyTypings(pkg))
.map((d) => d.name)
.join(", ")}].`);
}
}
}
async function checkNpm(
{ major, minor, name, libraryName, projectName, contributors }: TypingsData,
log: Logger,

2
packages/dtslint-runner/src/main.ts
View File

@@ -153,7 +153,7 @@ export async function runDTSLint({
suggestionLines.push(`"${packageName}": [${suggestions.join(",")}]`);
}
}
console.log(`{${suggestionLines.join(",")}}`);
// console.log(`{${suggestionLines.join(",")}}`);
logPerformance();

2
packages/publisher/src/calculate-versions.ts
View File

@@ -51,7 +51,7 @@ async function computeChangedPackages(allPackages: AllPackages, log: LoggerWithE
const { version, needsPublish } = await fetchTypesPackageVersionInfo(pkg, /*publish*/ true, log);
if (needsPublish) {
log.info(`Need to publish: ${pkg.desc}@${version}`);
for (const { name } of pkg.packageJsonDependencies) {
for (const [ name ] of Object.keys(pkg.packageJsonDependencies)) {
// Assert that dependencies exist on npm.
// Also checked when we install the dependencies, in dtslint-runner.
await pacote.manifest(name, { cache: cacheDir }).catch((reason) => {

9
packages/publisher/src/full.ts
View File

@@ -30,14 +30,7 @@ export default async function full(
): Promise<void> {
clean();
const dt = await getDefinitelyTyped(options, log);
const allPackages = await parseDefinitions(
dt,
options.parseInParallel
? { nProcesses: numberOfOsProcesses, definitelyTypedPath: assertDefined(options.definitelyTypedPath) }
: undefined,
log
);
const changedPackages = await calculateVersions(dt, log);
await generatePackages(dt, allPackages, changedPackages);
await generatePackages(dt, changedPackages);
await publishPackages(changedPackages, dry, githubAccessToken, fetcher);
}

50
packages/publisher/src/generate-packages.ts
View File

@@ -9,7 +9,6 @@ import {
assertNever,
joinPaths,
logUncaughtErrors,
sortObjectKeys,
loggerWithErrors,
FS,
logger,
@@ -25,11 +24,8 @@ import {
TypingsData,
NotNeededPackage,
AnyPackage,
PackageJsonDependency,
getFullNpmName,
DependencyVersion,
License,
formatTypingVersion,
} from "@definitelytyped/definitions-parser";
import * as pacote from "pacote";
import { readChangedPackages, ChangedPackages } from "./lib/versions";
@@ -44,13 +40,12 @@ if (!module.parent) {
const log = loggerWithErrors()[0];
const dt = await getDefinitelyTyped(defaultLocalOptions, log);
const allPackages = await AllPackages.read(dt);
await generatePackages(dt, allPackages, await readChangedPackages(allPackages), tgz);
await generatePackages(dt, await readChangedPackages(allPackages), tgz);
});
}
export default async function generatePackages(
dt: FS,
allPackages: AllPackages,
changedPackages: ChangedPackages,
tgz = false
): Promise<void> {
@@ -61,7 +56,7 @@ export default async function generatePackages(
await emptyDir(outputDirPath);
for (const { pkg, version } of changedPackages.changedTypings) {
await generateTypingPackage(pkg, allPackages, version, dt);
await generateTypingPackage(pkg, version, dt);
if (tgz) {
await writeTgz(outputDirectory(pkg), `${outputDirectory(pkg)}.tgz`);
}
@@ -76,7 +71,6 @@ export default async function generatePackages(
}
async function generateTypingPackage(
typing: TypingsData,
packages: AllPackages,
version: string,
dt: FS
): Promise<void> {
@@ -86,7 +80,7 @@ async function generateTypingPackage(
? typesDirectory
: typesDirectory.subDir(typing.versionDirectoryName);
await writeCommonOutputs(typing, createPackageJSON(typing, version, packages), createReadme(typing, packageFS));
await writeCommonOutputs(typing, createPackageJSON(typing, version), createReadme(typing, packageFS));
await Promise.all(
typing.files.map(async (file) => writeFile(await outputFilePath(typing, file), packageFS.readFile(file)))
);
@@ -123,11 +117,7 @@ async function outputFilePath(pkg: AnyPackage, filename: string): Promise<string
return full;
}
interface Dependencies {
[name: string]: string;
}
export function createPackageJSON(typing: TypingsData, version: string, packages: AllPackages): string {
export function createPackageJSON(typing: TypingsData, version: string): string {
// Use the ordering of fields from https://docs.npmjs.com/files/package.json
const out: {} = {
name: typing.fullNpmName,
@@ -147,7 +137,7 @@ export function createPackageJSON(typing: TypingsData, version: string, packages
directory: `types/${typing.name}`,
},
scripts: {},
dependencies: getDependencies(typing.packageJsonDependencies, typing, packages),
dependencies: typing.packageJsonDependencies,
typesPublisherContentHash: typing.contentHash,
typeScriptVersion: typing.minTypeScriptVersion,
};
@@ -169,34 +159,6 @@ export function createPackageJSON(typing: TypingsData, version: string, packages
const definitelyTypedURL = "https://github.com/DefinitelyTyped/DefinitelyTyped";
/** Adds inferred dependencies to `dependencies`, if they are not already specified in either `dependencies` or `peerDependencies`. */
function getDependencies(
packageJsonDependencies: readonly PackageJsonDependency[],
typing: TypingsData,
allPackages: AllPackages
): Dependencies {
const dependencies: Dependencies = {};
for (const { name, version } of packageJsonDependencies) {
dependencies[name] = version;
}
for (const [name, version] of Object.entries(typing.dependencies)) {
const typesDependency = getFullNpmName(name);
// A dependency "foo" is already handled if we already have a dependency on the package "foo" or "@types/foo".
if (
!packageJsonDependencies.some((d) => d.name === name || d.name === typesDependency) &&
allPackages.hasTypingFor({ name, version })
) {
dependencies[typesDependency] = dependencySemver(version);
}
}
return sortObjectKeys(dependencies);
}
function dependencySemver(dependency: DependencyVersion): string {
return dependency === "*" ? dependency : "^" + formatTypingVersion(dependency);
}
export function createNotNeededPackageJSON(pkg: NotNeededPackage): string {
const out = {
name: pkg.fullNpmName,
@@ -243,7 +205,7 @@ export function createReadme(typing: TypingsData, packageFS: FS): string {
lines.push("");
lines.push("### Additional Details");
lines.push(` * Last updated: ${new Date().toUTCString()}`);
const dependencies = Object.keys(typing.dependencies).map(getFullNpmName).sort();
const dependencies = Object.keys(typing.packageJsonDependencies).map(getFullNpmName).sort();
lines.push(
` * Dependencies: ${
dependencies.length ? dependencies.map((d) => `[${d}](https://npmjs.com/package/${d})`).join(", ") : "none"

8
packages/utils/src/collections.ts
View File

@@ -167,14 +167,6 @@ export function unique<T>(arr: Iterable<T>): T[] {
return [...new Set(arr)];
}
export function sortObjectKeys<T extends { [key: string]: unknown }>(data: T): T {
const out = {} as T;
for (const key of Object.keys(data).sort()) {
out[key as keyof T] = data[key as keyof T];
}
return out;
}
export function min<T>(array: readonly [T, ...(T | undefined)[]]): T;
export function min<T>(array: readonly T[], compare?: (a: T, b: T) => number): T | undefined;
export function min<T>(array: readonly T[], compare?: (a: T, b: T) => number) {

8
packages/utils/src/miscellany.ts
View File

@@ -60,14 +60,6 @@ export function mangleScopedPackage(packageName: string): string {
return isScopedPackage(packageName) ? packageName.replace(/\//, "__").replace("@", "") : packageName;
}
export function removeVersionFromPackageName(packageName: string | undefined): string | undefined {
return packageName?.replace(/\/v\d+(\.\d+)?(\/\*)?$/, "$2");
}
export function hasVersionNumberInMapping(packageName: string): boolean {
return /\/v\d+(\.\d+)?(\/\*)?$/.test(packageName);
}
export async function sleep(seconds: number): Promise<void> {
return new Promise<void>((resolve) => setTimeout(resolve, seconds * 1000));
}