casraf/github-repos-astro-loader
1
0
Fork 0
mirror of https://github.com/chenasraf/github-repos-astro-loader.git synced 2026-05-17 17:38:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: extract and use config options

Browse Source
This commit is contained in:
Chen Asraf
2024-11-29 01:47:48 +02:00
parent 1759cd35e0
commit 9ef674fe5c
8 changed files with 89 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@@ -11,9 +11,21 @@ In your `src/content/config.ts` file, add a new collection and use the loader:
import githubReposLoader from 'github-repos-astro-loader'
const project = defineCollection({
loader: githubReposLoader({
username: 'myusername', // The GitHub username you want to fetch the repositories for (required)
// Required
apiToken: GITHUB_TOKEN, // GitHub API token to use for requests
username: 'myusername', // The GitHub username you want to fetch the repositories for
// Optional
orgs: ['myorg'], // A list of GitHub orgs to fetch repositories from
debug: true, // Output debug logs during processing
force: false, // Ignore cache and force a full re-fetch
filter: (repo) => // Filter repositories to include in the collection
[
!repo.fork,
repo.stargazers_count! > 0,
//
].every(Boolean),
}),
}),
})
```

3
package.json
View File

@@ -22,15 +22,16 @@
"devDependencies": {
"@astropub/md": "^1.0.0",
"@eslint/js": "^9.15.0",
"@octokit/types": "^13.6.2",
"@types/node": "^22.10.0",
"astro": "^4.16.16",
"prettier": "^3.4.1",
"typescript": "^5.7.2",
"typescript-eslint": "^8.16.0"
},
"dependencies": {},
"peerDependencies": {
"@astropub/md": "*",
"@octokit/types": "*",
"astro": "*",
"date-fns": "*",
"yaml": "*",

15
pnpm-lock.yaml generated
View File

@@ -24,6 +24,9 @@ importers:
'@eslint/js':
specifier: ^9.15.0
version: 9.15.0
'@octokit/types':
specifier: ^13.6.2
version: 13.6.2
'@types/node':
specifier: ^22.10.0
version: 22.10.0
@@ -481,6 +484,12 @@ packages:
resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
engines: {node: '>= 8'}
'@octokit/openapi-types@22.2.0':
resolution: {integrity: sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==}
'@octokit/types@13.6.2':
resolution: {integrity: sha512-WpbZfZUcZU77DrSW4wbsSgTPfKcp286q3ItaIgvSbBpZJlu6mnYXAkjZz6LVZPXkEvLIM8McanyZejKTYUHipA==}
'@oslojs/encoding@1.1.0':
resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==}
@@ -2371,6 +2380,12 @@ snapshots:
'@nodelib/fs.scandir': 2.1.5
fastq: 1.17.1
'@octokit/openapi-types@22.2.0': {}
'@octokit/types@13.6.2':
dependencies:
'@octokit/openapi-types': 22.2.0
'@oslojs/encoding@1.1.0': {}
'@rollup/pluginutils@5.1.3(rollup@4.27.4)':

20
src/github.ts
View File

@@ -1,10 +1,7 @@
import fs from 'node:fs/promises'
import path from 'node:path'
import { logger } from './logger.js'
const GITHUB_TOKEN = process.env.GH_TOKEN || process.env.GITHUB_TOKEN!
export const headers = new Headers()
headers.set('Authorization', `Bearer ${GITHUB_TOKEN}`)
import { InternalLoaderOptions } from './types.js'
export const overridesDir = path.join(process.cwd(), 'src', 'content', 'project-overrides')
export const projectIgnoreFile = path.join(overridesDir, '.projectignore')
@@ -13,21 +10,22 @@ export const projectKeepFile = path.join(overridesDir, '.projectkeep')
export let projectIgnore: string[] = []
export let projectKeep: string[] = []
export async function fetchRepos(endpoint: string) {
export async function fetchRepos(endpoint: string, options: InternalLoaderOptions) {
const repos = []
let response = await fetchReposPage(endpoint)
let response = await fetchReposPage(endpoint, options)
repos.push(...response.repos)
while (response.url) {
response = await fetchReposPage(response.url)
response = await fetchReposPage(response.url, options)
repos.push(...response.repos)
}
return repos
}
async function fetchReposPage(endpoint: string) {
async function fetchReposPage(endpoint: string, options: InternalLoaderOptions) {
const headers = getAuthorization(options)
logger.log('Fetching endpoint', endpoint)
const response = await fetch(endpoint, { headers })
if (!response.ok) {
@@ -61,3 +59,9 @@ export async function reloadOverrides() {
projectIgnore = await loadFileList(projectIgnoreFile)
projectKeep = await loadFileList(projectKeepFile)
}
export function getAuthorization(options: InternalLoaderOptions): Headers {
const headers = new Headers()
headers.set('Authorization', `Bearer ${options.apiToken}`)
return headers
}

4
src/loader.ts
View File

@@ -11,7 +11,9 @@ async function reloadProjects(
opts: LoaderOptions,
) {
logger.enabled = opts.debug ?? false
const lastUpdated = parseDate(meta.get('lastUpdated') ?? '1970-01-01T00:00:00Z')
const lastUpdated = parseDate(
(opts.force ? undefined : meta.get('lastUpdated')) ?? '1970-01-01T00:00:00Z',
)
await reloadOverrides()
const options = InternalLoaderOptions.parse({

38
src/parser.ts
View File

@@ -3,10 +3,10 @@ import path from 'node:path'
import yaml from 'yaml'
import { parseJSON as parseDate } from 'date-fns/parseJSON'
import { formatISO as formatDate } from 'date-fns/formatISO'
import { GitHubProjectSchema, InternalLoaderOptions } from './types.js'
import { GitHubProjectSchema, GitHubRepositoryAPIResponse, InternalLoaderOptions } from './types.js'
import { fileExists, parseMarkdown } from './utils.js'
import { logger } from './logger.js'
import { fetchRepos, headers, overridesDir, projectIgnore, projectKeep } from './github.js'
import { fetchRepos, getAuthorization, overridesDir, projectIgnore, projectKeep } from './github.js'
export async function getProjectsList(
options: InternalLoaderOptions,
@@ -15,9 +15,12 @@ export async function getProjectsList(
const repos = await fetchRepos(
`https://api.github.com/users/${options.username}/repos?per_page=100`,
options,
)
for (const org of options.orgs) {
repos.push(...(await fetchRepos(`https://api.github.com/orgs/${org}/repos?per_page=100`)))
repos.push(
...(await fetchRepos(`https://api.github.com/orgs/${org}/repos?per_page=100`, options)),
)
}
logger.log(`Fetched ${repos.length} projects from GitHub`)
@@ -26,7 +29,7 @@ export async function getProjectsList(
for (const repo of repos) {
logger.log(`Processing ${repo.name}`)
if (!projectFilter(repo, options)) {
logger.log(`Skipping ${repo.name}`)
logger.log()
continue
}
@@ -38,6 +41,7 @@ export async function getProjectsList(
stars: repo.stargazers_count,
order: -repo.stargazers_count,
links: [{ href: repo.html_url, icon: 'logo-github', title: 'GitHub' }],
raw: repo,
})
const overridesFile = path.join(overridesDir, `${project.name}.md`)
@@ -75,7 +79,7 @@ export async function getProjectsList(
const readmeResponse = await fetch(
`https://raw.githubusercontent.com/${options.username}/${repo.name}/${repo.default_branch}/README.md`,
{ headers },
{ headers: getAuthorization(options) },
)
const readme = readmeResponse.ok ? await readmeResponse.text() : undefined
project.readme = readme
@@ -85,14 +89,15 @@ export async function getProjectsList(
projects.push(project)
logger.log(`Added project ${repo.name}`)
logger.log()
}
return projects
}
function projectFilter(
project: Record<string, any>,
{ lastUpdated }: InternalLoaderOptions,
project: GitHubRepositoryAPIResponse,
{ lastUpdated, filter }: InternalLoaderOptions,
): boolean {
if (projectKeep.includes(project.name)) {
return true
@@ -100,10 +105,17 @@ function projectFilter(
if (projectIgnore.includes(project.name)) {
return false
}
return [
parseDate(project.updated_at) > lastUpdated,
!project.fork,
project.stargazers_count > 0,
//
].every(Boolean)
const internalFilter = parseDate(project.updated_at!) > lastUpdated
if (!internalFilter) {
logger.log(`Skipping ${project.name} (internal filter)`)
return false
}
if (filter) {
const externalFilter = filter(project as any)
if (!externalFilter) {
logger.log(`Skipping ${project.name} (external filter)`)
return false
}
}
return true
}

17
src/types.ts
View File

@@ -1,3 +1,4 @@
import { Endpoints } from '@octokit/types'
import { z } from 'zod'
/**
@@ -37,9 +38,15 @@ export const GitHubProjectSchema = z.object({
links: z.array(LinkSchema),
/** Whether the GitHub project is featured. */
featured: z.boolean().optional().default(false),
/** Raw response from GitHub API, containing all fields. */
raw: z.any(),
})
export type GitHubProjectSchema = z.infer<typeof GitHubProjectSchema>
export type GitHubRepositoryAPIResponse = NonNullable<
Required<Endpoints['GET /users/{username}/repos']['response']['data'][number]>
>
/**
* Schema for loader options.
*/
@@ -50,6 +57,16 @@ export const LoaderOptions = z.object({
debug: z.boolean().optional(),
/** GitHub organizations to fetch projects from. */
orgs: z.array(z.string()).optional(),
/** A function to filter out projects. Filtered projects will not be parsed or saved in the collection. */
filter: z
.function()
.args(z.any() as z.ZodType<GitHubRepositoryAPIResponse>)
.returns(z.boolean())
.optional(),
/** The GitHub API token to use for fetching the repositories. */
apiToken: z.string(),
/** Whether to force a reload of the projects, regardless of cache status. */
force: z.boolean().optional(),
})
export type LoaderOptions = z.infer<typeof LoaderOptions>

4
tsconfig.json
View File

@@ -58,8 +58,8 @@
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
"outDir": "./build", // /* Specify an output folder for all emitted files. */
// "removeComments": true, /* Disable emitting comments. */
"outDir": "./build", // /* Specify an output folder for all emitted files. */
"removeComments": false, // /* Disable emitting comments. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */