Cleanup + search index generation

packages/types-publisher/.gitignore

@@ -1,4 +1,5 @@
 node_modules/*
 output/*
 bin/*
-log.md
+logs/
+data/

packages/types-publisher/src/check-parse-results.ts

@@ -1,22 +1,35 @@
+import { TypingsData, TypesDataFile, typesDataFilename, readDataFile, writeDataFile, writeLogSync } from './lib/common';
 
+const typeData = <TypesDataFile>readDataFile(typesDataFilename);
 
-function detectProjectAndLibraryNameDuplicates() {
-	check(info => info.libraryName, 'Library Name');
-	check(info => info.projectName, 'Project Name');
+if (typeData === undefined) {
+	console.log('Run parse-definitions first!');
+} else {
+	main();
+}
 
-	function check(func: (info: TypingsData) => string, key: string) {
-		const lookup: { [libName: string]: string[] } = {};
-		infos.forEach(info => {
-			const name = func(info);
-			if (name !== undefined) {
-				(lookup[name] || (lookup[name] = [])).push(info.typingsPackageName);
-			}
-		});
-		for (const k of Object.keys(lookup)) {
-			if (lookup[k].length > 1) {
-				warningLog.push(` * Duplicate ${key} descriptions "${k}"`);
-				lookup[k].forEach(n => warningLog.push(`   * ${n}`));
-			}
+function main() {
+	const libConflicts = check(info => info.libraryName, 'Library Name');
+	const projConflicts = check(info => info.projectName, 'Project Name');
+
+	writeLogSync('conflicts.md', libConflicts.concat(projConflicts));
+}
+
+function check(func: (info: TypingsData) => string, key: string) {
+	const lookup: { [libName: string]: string[] } = {};
+	const infos = Object.keys(typeData).map(k => typeData[k]);
+	const result: string[] = [];
+	infos.forEach(info => {
+		const name = func(info);
+		if (name !== undefined) {
+			(lookup[name] || (lookup[name] = [])).push(info.typingsPackageName);
+		}
+	});
+	for (const k of Object.keys(lookup)) {
+		if (lookup[k].length > 1) {
+			result.push(` * Duplicate ${key} descriptions "${k}"`);
+			lookup[k].forEach(n => result.push(`   * ${n}`));
 		}
 	}
+	return result;
 }

packages/types-publisher/src/create-search-index.ts

@@ -1,55 +1,43 @@
-// https://api.npmjs.org/downloads/point/last-month/jquery,express,flarp,react
-
+import { TypingsData, TypesDataFile, typesDataFilename, readDataFile, writeDataFile } from './lib/common';
 import * as fs from 'fs';
 import * as request from 'request';
+import * as generator from './lib/search-index-generator';
 
-const rawData: SearchRecord[] = JSON.parse(fs.readFileSync('search-raw.json', 'utf-8'));
+const typeData = <TypesDataFile>readDataFile(typesDataFilename);
 
-searchData.push({
-	packageName: info.data.projectName,
-	libraryName: info.data.libraryName,
-	globals: info.data.globals,
-	npmPackageName: info.data.typingsPackageName,
-	typePackageName: info.data.typingsPackageName,
-	declaredExternalModules: info.data.declaredModules
-});
-
-interface NpmResult {
-	[packageName: string]: {
-		downloads: number;
-	}
+if (typeData === undefined) {
+	console.log('Run parse-definitions first!');
+} else {
+	main();
 }
 
-function getDownloadCounts(done: () => void) {
-	function next() {
-		const unchecked = rawData.filter(r => (r.npmPackageName !== undefined) && (r.downloads === undefined));
-		if (unchecked.length === 0) {
-			done();
-		} else {
-			// Unknown: How many can we query at once?
-			const nextToCheck = unchecked.slice(0, 200);
-			const url = 'https://api.npmjs.org/downloads/point/last-month/' + nextToCheck.map(r => r.npmPackageName).join(',');
-			request.get(url, (err: any, resp: any, data: string) => {
-				const json = JSON.parse(data);
-				if (err) throw err;
-				nextToCheck.forEach(r => {
-					const result = json[r.npmPackageName];
-					r.downloads = result ? result.downloads : 0;
-				});
-				next();
-			});
-		}
-	}
-
-	next();
-}
-
-
 function main() {
-	getDownloadCounts(() => {
-		rawData.sort((a, b) => a.downloads - b.downloads);
-		fs.writeFileSync('search-with-downloads.json', JSON.stringify(rawData, undefined, 4), 'utf-8');
-	});
-}
+	const packages = Object.keys(typeData);
 
-main();
+	const fullRecords: generator.SearchRecord[] = [];
+	const minRecords: generator.MinifiedSearchRecord[] = [];
+
+	next();
+
+	function next() {
+		if (packages.length === 0) {
+			fullRecords.sort((a, b) => a.downloads - b.downloads);
+			minRecords.sort((a, b) => a.d - b.d);
+
+			writeDataFile('search-index-full.json', fullRecords);
+			writeDataFile('search-index-min.json', minRecords, false);
+			writeDataFile('search-index-head.json', minRecords.slice(0, 100), false);
+
+			return;
+		}
+
+		const packageName = packages.shift();
+		const info = typeData[packageName];
+
+		generator.createSearchRecords(info, (full, min) => {
+			fullRecords.push(full);
+			minRecords.push(min);
+			next();
+		});
+	}
+}

packages/types-publisher/src/lib/common.ts

@@ -118,11 +118,11 @@ export function writeLogSync(logName: string, contents: string[]) {
 	fs.writeFileSync(path.join(logDir, logName), contents.join('\r\n'), 'utf-8');
 }
 
-export function writeDataFile(filename: string, content: {}) {
+export function writeDataFile(filename: string, content: {}, formatted = true) {
 	const dataDir = path.join(home, 'data');
 	mkdir(dataDir);
 	if (typeof content !== 'string') {
-		content = JSON.stringify(content, undefined, 4);
+		content = JSON.stringify(content, undefined, formatted ? 4 : undefined);
 	}
 	fs.writeFileSync(path.join(dataDir, filename), content, 'utf-8');
 }

packages/types-publisher/src/lib/definition-parser.ts

@@ -319,7 +319,8 @@ export function getTypingInfo(directory: string): TypingParseFailResult | Typing
 	const authors = regexMatch(/^\/\/ Definitions by: (.+)$/m, 'Unknown');
 	const libraryMajorVersion = regexMatch(/^\/\/ Type definitions for \D+ v?(\d+)/m, '0');
 	const libraryMinorVersion = regexMatch(/^\/\/ Type definitions for \D+ v?\d+\.(\d+)/m, '0');
-	const libraryName = regexMatch(/^\/\/ Type definitions for ([A-Za-z]+)/m, 'Unknown').trim();
+	// const libraryName = regexMatch(/^\/\/ Type definitions for ([^\s]+)/m, 'Unknown').trim();
+	const libraryName = regexMatch(/^\/\/ Type definitions for (.+)$/m, 'Unknown').trim();
 	const projectName = regexMatch(/^\/\/ Project: (.+)$/m, '');
 	const packageName = path.basename(directory);
 	const sourceRepoURL = 'https://www.github.com/DefinitelyTyped/DefinitelyTyped';

packages/types-publisher/src/lib/search-index-generator.ts

@@ -0,0 +1,83 @@
+import { TypingsData } from './common';
+import * as fs from 'fs';
+import * as request from 'request';
+
+export interface SearchRecord {
+	// types package name
+	typePackageName: string;
+	// globals
+	globals: string[];
+	// modules
+	declaredExternalModules: string[];
+	// project name
+	projectName: string;
+	// library name
+	libraryName: string;
+	// downloads in the last month from NPM
+	downloads: number;
+}
+
+export interface MinifiedSearchRecord {
+	// types package name
+	t: string;
+	// globals
+	g: string[];
+	// modules
+	m: string[];
+	// project name
+	p: string;
+	// library name
+	l: string;
+	// downloads in the last month from NPM
+	d: number;
+}
+
+function createSearchRecord(info: TypingsData, downloads: number): SearchRecord {
+	return ({
+		projectName: info.projectName,
+		libraryName: info.libraryName,
+		globals: info.globals,
+		typePackageName: info.typingsPackageName,
+		declaredExternalModules: info.declaredModules,
+		downloads
+	});
+}
+
+function createMinifiedSearchRecord(data: SearchRecord): MinifiedSearchRecord {
+	return ({
+		t: data.typePackageName,
+		g: data.globals,
+		m: data.declaredExternalModules,
+		p: data.projectName,
+		l: data.libraryName,
+		d: data.downloads
+	});
+}
+
+interface NpmResult {
+	[packageName: string]: {
+		downloads: number;
+	}
+}
+
+export function createSearchRecords(info: TypingsData, done: (full: SearchRecord, min: MinifiedSearchRecord) => void) {
+	const pkg = info.typingsPackageName;
+	const url = 'https://api.npmjs.org/downloads/point/last-month/' + pkg;
+
+	const skipDownloads = process.argv.some(arg => arg === '--skipDownloads');
+
+	if (skipDownloads) {
+		setImmediate(() => {
+			const record = createSearchRecord(info, -1);
+			done(record, createMinifiedSearchRecord(record));
+		});
+	} else {
+		request.get(url, (err: any, resp: any, data: string) => {
+			const json: NpmResult = JSON.parse(data);
+			if (err) throw err;
+			const result = json[pkg];
+			const record = createSearchRecord(info, result ? result.downloads : 0);
+			done(record, createMinifiedSearchRecord(record));
+		});
+	}
+}

packages/types-publisher/src/searchRecord.d.ts

@@ -1,16 +0,0 @@
-interface SearchRecord {
-	// types package name
-	typePackageName: string;
-	// npm package name
-	npmPackageName: string;
-	// globals
-	globals: string[];
-	// modules
-	declaredExternalModules: string[];
-	// project name
-	packageName: string;
-	// library name
-	libraryName: string;
-	// downloads in the last month from NPM
-	downloads?: number;
-}

packages/types-publisher/tsconfig.json

@@ -1,18 +1,21 @@
 {
 	"files": [
 		"src/settings.d.ts",
-		"src/searchRecord.d.ts",
 		"typings/tsd.d.ts",
 
 		"src/lib/definition-parser.ts",
 		"src/lib/package-generator.ts",
 		"src/lib/package-publisher.ts",
+		"src/lib/search-index-generator.ts",
 		"src/lib/common.ts",
 
 		"src/parse-definitions.ts",
 		"src/generate-packages.ts",
-		"src/publish-packages.ts"
+		"src/publish-packages.ts",
+		"src/create-search-index.ts",
+		"src/check-parse-results.ts"
 	],
+
 	"compilerOptions": {
 		"module": "commonjs",
 		"noImplicitAny": true,