1. Remove npm consistency checks in dtslint-runner. They are unused.

2. Simplify and reduce JSON parsing in dtslint-runner. Pass around AllPackages object instead of writing it to disk and reading it over and over. 3. Sort AllPackages object before returning it in addition to before writing it to disk.
2026-05-17 17:48:07 +00:00 · 2023-04-17 11:53:02 -07:00
parent 05fcf435a7
commit dfa99bd36c
8 changed files with 20 additions and 176 deletions
--- a/packages/definitions-parser/src/git.ts
+++ b/packages/definitions-parser/src/git.ts
@@ -14,7 +14,6 @@ import {
  flatMapIterable,
  mapIterable,
  mapDefined,
-  FS,
  consoleLogger,
  assertDefined,
  cacheDir,
@@ -89,11 +88,10 @@ export function gitChanges(diffs: GitDiff[]): PackageId[] {
 }

 export async function getAffectedPackagesFromDiff(
-  dt: FS,
+  allPackages: AllPackages,
  definitelyTypedPath: string,
  selection: "all" | "affected" | RegExp
 ) {
-  const allPackages = await AllPackages.read(dt);
  const diffs = await gitDiff(consoleLogger.info, definitelyTypedPath);
  if (diffs.find((d) => d.file === "notNeededPackages.json")) {
    for (const deleted of getNotNeededPackages(allPackages, diffs)) {
--- a/packages/definitions-parser/src/parse-definitions.ts
+++ b/packages/definitions-parser/src/parse-definitions.ts
@@ -60,8 +60,9 @@ export async function parseDefinitions(
    throw new Error(errors.join("\n"));
  }
  log.info("Parsing took " + (Date.now() - start) / 1000 + " s");
-  await writeDataFile(typesDataFilename, sorted(typings));
-  return AllPackages.from(typings, readNotNeededPackages(dt));
+  const sortedTypings = sorted(typings);
+  await writeDataFile(typesDataFilename, sortedTypings);
+  return AllPackages.from(sortedTypings, readNotNeededPackages(dt));
 }

 function sorted<T>(obj: { [name: string]: T }): { [name: string]: T } {
--- a/packages/dtslint-runner/package.json
+++ b/packages/dtslint-runner/package.json
@@ -29,15 +29,11 @@
    "@octokit/rest": "^16.43.2",
    "fs-extra": "^9.0.0",
    "glob": "^7.2.3",
-    "pacote": "^13.6.1",
-    "semver": "^7.3.7",
    "stats-lite": "^2.2.0",
    "yargs": "^15.3.1"
  },
  "devDependencies": {
    "@types/fs-extra": "^8.1.0",
-    "@types/pacote": "^11.1.5",
-    "@types/semver": "^7.3.10",
    "@types/stats-lite": "^2.2.0",
    "source-map-support": "^0.5.21"
  }
--- a/packages/dtslint-runner/src/check-parse-results.ts
+++ b/packages/dtslint-runner/src/check-parse-results.ts
@@ -1,174 +1,25 @@
 import console from "console";
 import {
-  ParseDefinitionsOptions,
-  TypingsData,
  AllPackages,
  getDefinitelyTyped,
 } from "@definitelytyped/definitions-parser";
-import { mapDefined, FS, logger, writeLog, Logger, ProgressBar, cacheDir, max, min } from "@definitelytyped/utils";
-import * as pacote from "pacote";
-import * as semver from "semver";
-
 if (!module.parent) {
  const options = { definitelyTypedPath: undefined, progress: false, parseInParallel: false };
  getDefinitelyTyped(options, console).then((dt) => {
-    checkParseResults(/*includeNpmChecks*/ false, dt);
+    AllPackages.read(dt).then(checkParseResults)
  });
 }

-export async function checkParseResults(includeNpmChecks: false, dt: FS): Promise<void>;
-export async function checkParseResults(
-  includeNpmChecks: true,
-  dt: FS,
-  options: ParseDefinitionsOptions
-): Promise<void>;
-export async function checkParseResults(
-  includeNpmChecks: boolean,
-  dt: FS,
-  options?: ParseDefinitionsOptions
-): Promise<void> {
-  const allPackages = await AllPackages.read(dt);
-  const [log, logResult] = logger();
-
-  checkTypeScriptVersions(allPackages);
-
-  const dependedOn = new Set<string>();
-  const packages = allPackages.allPackages();
-  for (const pkg of packages) {
-    if (pkg instanceof TypingsData) {
-      for (const [ name ] of pkg.allPackageJsonDependencies()) {
-        dependedOn.add(name);
-      }
-    }
-  }
-
-  if (includeNpmChecks) {
-    const allTypings = allPackages.allTypings();
-    const progress = options!.progress && new ProgressBar({ name: "Checking for typed packages..." });
-    let i = 0;
-    await Promise.all(
-      allTypings.map((pkg) =>
-        checkNpm(pkg, log, dependedOn).then(() => {
-          if (progress) progress.update(++i / allTypings.length, pkg.desc);
-        })
-      )
-    );
-    if (progress) progress.done();
-  }
-
-  await writeLog("conflicts.md", logResult());
-}
-
-function checkTypeScriptVersions(allPackages: AllPackages): void {
+export function checkParseResults(allPackages: AllPackages): void {
+  const errors = []
  for (const pkg of allPackages.allTypings()) {
    for (const dep of allPackages.allDependencyTypings(pkg)) {
      if (dep.minTypeScriptVersion > pkg.minTypeScriptVersion) {
-        throw new Error(`${pkg.desc} depends on ${dep.desc} but has a lower required TypeScript version.`);
+        errors.push(`${pkg.desc} depends on ${dep.desc} but has a lower required TypeScript version.`);
      }
    }
  }
-}
-
-async function checkNpm(
-  { major, minor, name, libraryName, projectName, contributors }: TypingsData,
-  log: Logger,
-  dependedOn: ReadonlySet<string>
-): Promise<void> {
-  if (notNeededExceptions.has(name)) {
-    return;
-  }
-
-  const info = await pacote.packument(name, { cache: cacheDir, fullMetadata: true }).catch((reason) => {
-    if (reason.code !== "E404") throw reason;
-    return undefined;
-  }); // Gets info for the real package, not the @types package
-  if (!info) {
-    return;
-  }
-
-  const versions = getRegularVersions(info.versions);
-  const firstTypedVersion = min(
-    mapDefined(versions, ({ hasTypes, version }) => (hasTypes ? version : undefined)),
-    semver.compare
-  );
-  // A package might have added types but removed them later, so check the latest version too
-  if (firstTypedVersion === undefined || !max(versions, (a, b) => semver.compare(a.version, b.version))!.hasTypes) {
-    return;
-  }
-
-  const ourVersion = `${major}.${minor}`;
-
-  log("");
-  log(`Typings already defined for ${name} (${libraryName}) as of ${firstTypedVersion} (our version: ${ourVersion})`);
-  const contributorUrls = contributors
-    .map((c) => {
-      const gh = "https://github.com/";
-      return c.url.startsWith(gh) ? `@${c.url.slice(gh.length)}` : `${c.name} (${c.url})`;
-    })
-    .join(", ");
-  log("  To fix this:");
-  log(`  git checkout -b not-needed-${name}`);
-  const yarnargs = [name, firstTypedVersion, projectName];
-  if (libraryName !== name) {
-    yarnargs.push(JSON.stringify(libraryName));
-  }
-  log("  yarn not-needed " + yarnargs.join(" "));
-  log(`  git add --all && git commit -m "${name}: Provides its own types" && git push -u origin not-needed-${name}`);
-  log(`  And comment PR: This will deprecate \`@types/${name}\` in favor of just \`${name}\`. CC ${contributorUrls}`);
-  if (semver.gt(`${major}.${minor}.0`, firstTypedVersion)) {
-    log("  WARNING: our version is greater!");
-  }
-  if (dependedOn.has(name)) {
-    log("  WARNING: other packages depend on this!");
+  if (errors.length) {
+    throw new Error(errors.join("\n"));
  }
 }
-
-function getRegularVersions(
-  versions: pacote.Packument["versions"]
-): readonly { readonly version: semver.SemVer; readonly hasTypes: boolean }[] {
-  return Object.entries(versions).map(([versionString, info]) => ({
-    version: new semver.SemVer(versionString),
-    hasTypes: versionHasTypes(info),
-  }));
-}
-
-function versionHasTypes(info: pacote.Manifest): boolean {
-  return "types" in info || "typings" in info;
-}
-
-const notNeededExceptions: ReadonlySet<string> = new Set([
-  // https://github.com/DefinitelyTyped/DefinitelyTyped/pull/22306
-  "angular-ui-router",
-  "ui-router-extras",
-  // Declares to bundle types, but they're also in the `.npmignore` (https://github.com/nkovacic/angular-touchspin/issues/21)
-  "angular-touchspin",
-  // "typings" points to the wrong file (https://github.com/Microsoft/Bing-Maps-V8-TypeScript-Definitions/issues/31)
-  "bingmaps",
-  // Types are bundled, but not officially released (https://github.com/DefinitelyTyped/DefinitelyTyped/pull/22313#issuecomment-353225893)
-  "dwt",
-  // Waiting on some typing errors to be fixed (https://github.com/julien-c/epub/issues/30)
-  "epub",
-  // Typings file is not in package.json "files" list (https://github.com/silentmatt/expr-eval/issues/127)
-  "expr-eval",
-  // NPM package "express-serve-static-core" isn't a real package -- express-serve-static-core exists only for the purpose of types
-  "express-serve-static-core",
-  // Has "typings": "index.d.ts" but does not actually bundle typings. https://github.com/kolodny/immutability-helper/issues/79
-  "immutability-helper",
-  // Has `"typings": "compiled/typings/node-mysql-wrapper/node-mysql-wrapper.d.ts",`, but `compiled/typings` doesn't exist.
-  // Package hasn't updated in 2 years and author seems to have deleted their account, so no chance of being fixed.
-  "node-mysql-wrapper",
-  // raspi packages bundle types, but can only be installed on a Raspberry Pi, so they are duplicated to DefinitelyTyped.
-  // See https://github.com/DefinitelyTyped/DefinitelyTyped/pull/21618
-  "raspi",
-  "raspi-board",
-  "raspi-gpio",
-  "raspi-i2c",
-  "raspi-led",
-  "raspi-onewire",
-  "raspi-peripheral",
-  "raspi-pwm",
-  "raspi-serial",
-  "raspi-soft-pwm",
-  // Declare "typings" but don't actually have them yet (https://github.com/stampit-org/stampit/issues/245)
-  "stampit",
-]);
--- a/packages/dtslint-runner/src/prepareAffectedPackages.ts
+++ b/packages/dtslint-runner/src/prepareAffectedPackages.ts
@@ -19,16 +19,16 @@ export async function prepareAffectedPackages({
    parseInParallel: nProcesses > 1,
  };
  const dt = await getDefinitelyTyped(options, log);
-  await parseDefinitions(dt, nProcesses ? { definitelyTypedPath, nProcesses } : undefined, log);
+  const allPackages = await parseDefinitions(dt, nProcesses ? { definitelyTypedPath, nProcesses } : undefined, log);
  try {
-    await checkParseResults(/*includeNpmChecks*/ false, dt);
+    checkParseResults(allPackages);
  } catch (err) {
-    await getAffectedPackagesFromDiff(dt, definitelyTypedPath, "affected");
+    await getAffectedPackagesFromDiff(allPackages, definitelyTypedPath, "affected");
    throw err;
  }

  const { changedPackages, dependentPackages } = await getAffectedPackagesFromDiff(
-    dt,
+    allPackages,
    definitelyTypedPath,
    "affected"
  );
--- a/packages/dtslint-runner/src/prepareAllPackages.ts
+++ b/packages/dtslint-runner/src/prepareAllPackages.ts
@@ -1,4 +1,4 @@
-import { AllPackages, getDefinitelyTyped, parseDefinitions } from "@definitelytyped/definitions-parser";
+import { getDefinitelyTyped, parseDefinitions } from "@definitelytyped/definitions-parser";
 import { loggerWithErrors } from "@definitelytyped/utils";
 import { checkParseResults } from "./check-parse-results";
 import { installDependencies } from "./prepareAffectedPackages";
@@ -16,9 +16,10 @@ export async function prepareAllPackages({
    parseInParallel: nProcesses > 1,
  };
  const dt = await getDefinitelyTyped(options, log);
-  await parseDefinitions(dt, nProcesses ? { definitelyTypedPath, nProcesses } : undefined, log);
-  await checkParseResults(/*includeNpmChecks*/ false, dt);
-  const allPackages = await AllPackages.read(dt);
+  // TODO: Make this work to make it easier on Jake. (Basically, sort before returning AND writing)
+  const allPackages = await parseDefinitions(dt, nProcesses ? { definitelyTypedPath, nProcesses } : undefined, log);
+  checkParseResults(allPackages);
+  // const allPackages = await AllPackages.read(dt);
  if (!noInstall) {
    await installDependencies(definitelyTypedPath);
  }
--- a/packages/header-parser/src/index.ts
+++ b/packages/header-parser/src/index.ts
@@ -279,9 +279,6 @@ export function validatePackageJson(packageName: string, packageJson: Record<str
    if (!packageJson.contributors || !Array.isArray(packageJson.contributors)) {
      errors.push(`${packageName}'s package.json has bad "contributors": must be an array of type Array<{ name: string, url: string, githubUsername: string}>.`);
    }
-    else if (packageJson.contributors.length === 0) {
-      errors.push(`${packageName}'s package.json has bad "contributors": must have at least one contributor.`);
-    }
    else {
      const es = checkPackageJsonContributors(packageName, packageJson.contributors);
      if (es.length) {
--- a/yarn.lock
+++ b/yarn.lock
@@ -2178,7 +2178,7 @@
  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-6.2.3.tgz#5798ecf1bec94eaa64db39ee52808ec0693315aa"
  integrity sha512-KQf+QAMWKMrtBMsB8/24w53tEsxllMj6TuA80TT/5igJalLI/zm0L3oXRbIAl4Ohfc85gyHX/jhMwsVkmhLU4A==

-"@types/semver@^7.3.10", "@types/semver@^7.3.12":
+"@types/semver@^7.3.12":
  version "7.3.13"
  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.13.tgz#da4bfd73f49bd541d28920ab0e2bf0ee80f71c91"
  integrity sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==