From 9ef674fe5cd3b2ad899ce913f614cabbd9a0b42b Mon Sep 17 00:00:00 2001 From: Chen Asraf Date: Fri, 29 Nov 2024 01:47:48 +0200 Subject: [PATCH] refactor: extract and use config options --- README.md | 14 +++++++++++++- package.json | 3 ++- pnpm-lock.yaml | 15 +++++++++++++++ src/github.ts | 20 ++++++++++++-------- src/loader.ts | 4 +++- src/parser.ts | 38 +++++++++++++++++++++++++------------- src/types.ts | 17 +++++++++++++++++ tsconfig.json | 4 ++-- 8 files changed, 89 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 50f4ae0..7e80355 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,21 @@ In your `src/content/config.ts` file, add a new collection and use the loader: import githubReposLoader from 'github-repos-astro-loader' const project = defineCollection({ loader: githubReposLoader({ - username: 'myusername', // The GitHub username you want to fetch the repositories for (required) + // Required + apiToken: GITHUB_TOKEN, // GitHub API token to use for requests + username: 'myusername', // The GitHub username you want to fetch the repositories for + + // Optional orgs: ['myorg'], // A list of GitHub orgs to fetch repositories from debug: true, // Output debug logs during processing + force: false, // Ignore cache and force a full re-fetch + filter: (repo) => // Filter repositories to include in the collection + [ + !repo.fork, + repo.stargazers_count! > 0, + // + ].every(Boolean), + }), }), }) ``` diff --git a/package.json b/package.json index 5e3572f..eb08759 100644 --- a/package.json +++ b/package.json @@ -22,15 +22,16 @@ "devDependencies": { "@astropub/md": "^1.0.0", "@eslint/js": "^9.15.0", + "@octokit/types": "^13.6.2", "@types/node": "^22.10.0", "astro": "^4.16.16", "prettier": "^3.4.1", "typescript": "^5.7.2", "typescript-eslint": "^8.16.0" }, - "dependencies": {}, "peerDependencies": { "@astropub/md": "*", + "@octokit/types": "*", "astro": "*", "date-fns": "*", "yaml": "*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ea52b9e..0bbca48 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ importers: '@eslint/js': specifier: ^9.15.0 version: 9.15.0 + '@octokit/types': + specifier: ^13.6.2 + version: 13.6.2 '@types/node': specifier: ^22.10.0 version: 22.10.0 @@ -481,6 +484,12 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@octokit/openapi-types@22.2.0': + resolution: {integrity: sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==} + + '@octokit/types@13.6.2': + resolution: {integrity: sha512-WpbZfZUcZU77DrSW4wbsSgTPfKcp286q3ItaIgvSbBpZJlu6mnYXAkjZz6LVZPXkEvLIM8McanyZejKTYUHipA==} + '@oslojs/encoding@1.1.0': resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} @@ -2371,6 +2380,12 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.17.1 + '@octokit/openapi-types@22.2.0': {} + + '@octokit/types@13.6.2': + dependencies: + '@octokit/openapi-types': 22.2.0 + '@oslojs/encoding@1.1.0': {} '@rollup/pluginutils@5.1.3(rollup@4.27.4)': diff --git a/src/github.ts b/src/github.ts index 6b0ea98..7960423 100644 --- a/src/github.ts +++ b/src/github.ts @@ -1,10 +1,7 @@ import fs from 'node:fs/promises' import path from 'node:path' import { logger } from './logger.js' - -const GITHUB_TOKEN = process.env.GH_TOKEN || process.env.GITHUB_TOKEN! -export const headers = new Headers() -headers.set('Authorization', `Bearer ${GITHUB_TOKEN}`) +import { InternalLoaderOptions } from './types.js' export const overridesDir = path.join(process.cwd(), 'src', 'content', 'project-overrides') export const projectIgnoreFile = path.join(overridesDir, '.projectignore') @@ -13,21 +10,22 @@ export const projectKeepFile = path.join(overridesDir, '.projectkeep') export let projectIgnore: string[] = [] export let projectKeep: string[] = [] -export async function fetchRepos(endpoint: string) { +export async function fetchRepos(endpoint: string, options: InternalLoaderOptions) { const repos = [] - let response = await fetchReposPage(endpoint) + let response = await fetchReposPage(endpoint, options) repos.push(...response.repos) while (response.url) { - response = await fetchReposPage(response.url) + response = await fetchReposPage(response.url, options) repos.push(...response.repos) } return repos } -async function fetchReposPage(endpoint: string) { +async function fetchReposPage(endpoint: string, options: InternalLoaderOptions) { + const headers = getAuthorization(options) logger.log('Fetching endpoint', endpoint) const response = await fetch(endpoint, { headers }) if (!response.ok) { @@ -61,3 +59,9 @@ export async function reloadOverrides() { projectIgnore = await loadFileList(projectIgnoreFile) projectKeep = await loadFileList(projectKeepFile) } + +export function getAuthorization(options: InternalLoaderOptions): Headers { + const headers = new Headers() + headers.set('Authorization', `Bearer ${options.apiToken}`) + return headers +} diff --git a/src/loader.ts b/src/loader.ts index 5fd827c..02c7c40 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -11,7 +11,9 @@ async function reloadProjects( opts: LoaderOptions, ) { logger.enabled = opts.debug ?? false - const lastUpdated = parseDate(meta.get('lastUpdated') ?? '1970-01-01T00:00:00Z') + const lastUpdated = parseDate( + (opts.force ? undefined : meta.get('lastUpdated')) ?? '1970-01-01T00:00:00Z', + ) await reloadOverrides() const options = InternalLoaderOptions.parse({ diff --git a/src/parser.ts b/src/parser.ts index 450f53e..2e6c3cb 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -3,10 +3,10 @@ import path from 'node:path' import yaml from 'yaml' import { parseJSON as parseDate } from 'date-fns/parseJSON' import { formatISO as formatDate } from 'date-fns/formatISO' -import { GitHubProjectSchema, InternalLoaderOptions } from './types.js' +import { GitHubProjectSchema, GitHubRepositoryAPIResponse, InternalLoaderOptions } from './types.js' import { fileExists, parseMarkdown } from './utils.js' import { logger } from './logger.js' -import { fetchRepos, headers, overridesDir, projectIgnore, projectKeep } from './github.js' +import { fetchRepos, getAuthorization, overridesDir, projectIgnore, projectKeep } from './github.js' export async function getProjectsList( options: InternalLoaderOptions, @@ -15,9 +15,12 @@ export async function getProjectsList( const repos = await fetchRepos( `https://api.github.com/users/${options.username}/repos?per_page=100`, + options, ) for (const org of options.orgs) { - repos.push(...(await fetchRepos(`https://api.github.com/orgs/${org}/repos?per_page=100`))) + repos.push( + ...(await fetchRepos(`https://api.github.com/orgs/${org}/repos?per_page=100`, options)), + ) } logger.log(`Fetched ${repos.length} projects from GitHub`) @@ -26,7 +29,7 @@ export async function getProjectsList( for (const repo of repos) { logger.log(`Processing ${repo.name}`) if (!projectFilter(repo, options)) { - logger.log(`Skipping ${repo.name}`) + logger.log() continue } @@ -38,6 +41,7 @@ export async function getProjectsList( stars: repo.stargazers_count, order: -repo.stargazers_count, links: [{ href: repo.html_url, icon: 'logo-github', title: 'GitHub' }], + raw: repo, }) const overridesFile = path.join(overridesDir, `${project.name}.md`) @@ -75,7 +79,7 @@ export async function getProjectsList( const readmeResponse = await fetch( `https://raw.githubusercontent.com/${options.username}/${repo.name}/${repo.default_branch}/README.md`, - { headers }, + { headers: getAuthorization(options) }, ) const readme = readmeResponse.ok ? await readmeResponse.text() : undefined project.readme = readme @@ -85,14 +89,15 @@ export async function getProjectsList( projects.push(project) logger.log(`Added project ${repo.name}`) + logger.log() } return projects } function projectFilter( - project: Record, - { lastUpdated }: InternalLoaderOptions, + project: GitHubRepositoryAPIResponse, + { lastUpdated, filter }: InternalLoaderOptions, ): boolean { if (projectKeep.includes(project.name)) { return true @@ -100,10 +105,17 @@ function projectFilter( if (projectIgnore.includes(project.name)) { return false } - return [ - parseDate(project.updated_at) > lastUpdated, - !project.fork, - project.stargazers_count > 0, - // - ].every(Boolean) + const internalFilter = parseDate(project.updated_at!) > lastUpdated + if (!internalFilter) { + logger.log(`Skipping ${project.name} (internal filter)`) + return false + } + if (filter) { + const externalFilter = filter(project as any) + if (!externalFilter) { + logger.log(`Skipping ${project.name} (external filter)`) + return false + } + } + return true } diff --git a/src/types.ts b/src/types.ts index d278524..eb9b890 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,4 @@ +import { Endpoints } from '@octokit/types' import { z } from 'zod' /** @@ -37,9 +38,15 @@ export const GitHubProjectSchema = z.object({ links: z.array(LinkSchema), /** Whether the GitHub project is featured. */ featured: z.boolean().optional().default(false), + /** Raw response from GitHub API, containing all fields. */ + raw: z.any(), }) export type GitHubProjectSchema = z.infer +export type GitHubRepositoryAPIResponse = NonNullable< + Required +> + /** * Schema for loader options. */ @@ -50,6 +57,16 @@ export const LoaderOptions = z.object({ debug: z.boolean().optional(), /** GitHub organizations to fetch projects from. */ orgs: z.array(z.string()).optional(), + /** A function to filter out projects. Filtered projects will not be parsed or saved in the collection. */ + filter: z + .function() + .args(z.any() as z.ZodType) + .returns(z.boolean()) + .optional(), + /** The GitHub API token to use for fetching the repositories. */ + apiToken: z.string(), + /** Whether to force a reload of the projects, regardless of cache status. */ + force: z.boolean().optional(), }) export type LoaderOptions = z.infer diff --git a/tsconfig.json b/tsconfig.json index ce64e49..bbb1f4c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -58,8 +58,8 @@ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./build", // /* Specify an output folder for all emitted files. */ - // "removeComments": true, /* Disable emitting comments. */ + "outDir": "./build", // /* Specify an output folder for all emitted files. */ + "removeComments": false, // /* Disable emitting comments. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */