diff --git a/appconfig/appconfig.go b/appconfig/appconfig.go index 126e5b6..a71a696 100644 --- a/appconfig/appconfig.go +++ b/appconfig/appconfig.go @@ -12,6 +12,7 @@ import ( "github.com/chenasraf/sofmani/platform" "github.com/chenasraf/sofmani/utils" "github.com/eschao/config" + "gopkg.in/yaml.v3" ) // AppConfig represents the main application configuration. @@ -87,6 +88,16 @@ func ParseConfigFrom(file string) (*AppConfig, error) { return &appConfig, nil } +// ParseConfigFromContent parses the configuration from YAML content. +func ParseConfigFromContent(content []byte) (*AppConfig, error) { + appConfig := NewAppConfig() + err := yaml.Unmarshal(content, &appConfig) + if err != nil { + return nil, err + } + return &appConfig, nil +} + // FindConfigFile searches for the configuration file in standard locations. // It searches in the current working directory, then in ~/.config, and finally in the home directory. // It returns the path to the first file found, or an empty string if no file is found. diff --git a/docs/installer-configuration.md b/docs/installer-configuration.md index cfd823b..1c81bee 100644 --- a/docs/installer-configuration.md +++ b/docs/installer-configuration.md @@ -194,12 +194,16 @@ These fields are shared by all installer types. Some fields may vary in behavior installers. - `debug` and `check_updates` will be inherited by the loaded config. - `env` and `defaults` will be merged into the loaded config, overriding any existing values. + - Remote manifests are fetched directly via HTTP (no git clone required). - **Options**: - - `opts.source`: The local file, or remote git URL (https or SSH) containing the manifest. - - `opts.path`: The path to the manifest file within the repository. If `opts.source` is a local - file, `opts.path` will be appended to it. - - `opts.ref`: The branch, tag, or commit to checkout after cloning if `opts.source` is a git - URL. For local manifests, this value will be ignored. + - `opts.source`: The source of the manifest file. Supports: + - Local file paths (e.g., `~/.dotfiles/manifest.yml`) + - Git repository URLs (SSH or HTTPS) - GitHub, GitLab, Bitbucket, and self-hosted instances + - Raw HTTP URLs (e.g., `https://raw.githubusercontent.com/user/repo/master/manifest.yml`) + - `opts.path`: The path to the manifest file within the repository. Required for git URLs, + optional for local files (will be appended to source). Ignored for raw HTTP URLs. + - `opts.ref`: The branch, tag, or commit to use if `opts.source` is a git URL. Defaults to + `master`. Ignored for local files and raw HTTP URLs. - **`rsync`** diff --git a/installer/manifest_installer.go b/installer/manifest_installer.go index 00273ef..edcc938 100644 --- a/installer/manifest_installer.go +++ b/installer/manifest_installer.go @@ -2,9 +2,11 @@ package installer import ( "fmt" + "io" "maps" - "os" + "net/http" "path/filepath" + "strings" "github.com/chenasraf/sofmani/appconfig" "github.com/chenasraf/sofmani/logger" @@ -124,75 +126,121 @@ func (i *ManifestInstaller) GetOpts() *ManifestOpts { } // FetchManifest fetches and parses the manifest file. -// It handles both local and Git sources. +// It handles local files, Git repository URLs, and raw HTTP URLs. func (i *ManifestInstaller) FetchManifest() error { opts := i.GetOpts() source := *opts.Source - isGit := utils.IsGitURL(source) env := i.GetData().Environ() - var path string - if opts.Path == nil { - path = "" - } else { - path = *opts.Path - } - path = utils.GetRealPath(env, path) - if isGit { - src, err := i.getGitManifestConfig(source) + var config *appconfig.AppConfig + var err error + + switch { + case utils.IsGitURL(source): + // Git repository URL - convert to raw URL and fetch + content, fetchErr := i.getGitManifestConfig(source) + if fetchErr != nil { + return fetchErr + } + config, err = appconfig.ParseConfigFromContent([]byte(content)) + if err != nil { + return fmt.Errorf("failed to parse manifest content: %w", err) + } + case strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://"): + // Direct HTTP URL - fetch directly + content, fetchErr := i.fetchRawURL(source) + if fetchErr != nil { + return fetchErr + } + config, err = appconfig.ParseConfigFromContent([]byte(content)) + if err != nil { + return fmt.Errorf("failed to parse manifest content: %w", err) + } + default: + // Local file path + source = utils.GetRealPath(env, source) + var path string + if opts.Path == nil { + path = "" + } else { + path = *opts.Path + } + path = utils.GetRealPath(env, path) + fullPath := filepath.Join(source, path) + logger.Debug("Parsing manifest from %s", fullPath) + config, err = i.getLocalManifestConfig(fullPath) if err != nil { return err } - source = src - } else { - source = utils.GetRealPath(env, source) } - logger.Debug("Parsing manifest from %s", filepath.Join(source, path)) - config, err := i.getLocalManifestConfig(filepath.Join(source, path)) - if err != nil { - return err - } logger.Debug("Installers: %d", len(config.Install)) + config = i.inheritManifest(config) i.ManifestConfig = config return nil } -func (i *ManifestInstaller) getGitManifestConfig(source string) (string, error) { - opts := i.GetOpts() - tmpDir, err := os.MkdirTemp("", "sofmani") +// fetchRawURL fetches content directly from a raw HTTP URL. +func (i *ManifestInstaller) fetchRawURL(url string) (string, error) { + logger.Debug("Fetching manifest from raw URL: %s", url) + resp, err := http.Get(url) if err != nil { - return "", err + return "", fmt.Errorf("failed to fetch manifest: %w", err) } - defer func() { - if rmErr := os.RemoveAll(tmpDir); rmErr != nil { - logger.Warn("Failed to clean up tmp dir %s: %v", tmpDir, rmErr) + if cerr := resp.Body.Close(); cerr != nil { + logger.Warn("failed to close response body: %v", cerr) } }() - logger.Debug("Cloning %s to %s", source, tmpDir) - success, err := i.RunCmdGetSuccess("git", "clone", "--depth=1", source, tmpDir) - if err != nil { - return "", err + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to fetch manifest: HTTP %d", resp.StatusCode) } - if opts.Ref != nil { - logger.Debug("Checking out ref %s", *opts.Ref) - err = i.RunCmdPassThrough("git", "-C", tmpDir, "checkout", *opts.Ref) - if err != nil { - return "", err + content, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read manifest content: %w", err) + } + + return string(content), nil +} + +func (i *ManifestInstaller) getGitManifestConfig(source string) (string, error) { + opts := i.GetOpts() + + ref := "main" + if opts.Ref != nil && *opts.Ref != "" { + ref = *opts.Ref + } + + path := "" + if opts.Path != nil { + path = *opts.Path + } + + rawURL, err := utils.GetRawFileURL(source, ref, path) + if err != nil { + return "", fmt.Errorf("failed to construct raw file URL: %w", err) + } + + logger.Debug("Fetching manifest from %s", rawURL) + resp, err := http.Get(rawURL) + if err != nil { + return "", fmt.Errorf("failed to fetch manifest: %w", err) + } + defer func() { + if cerr := resp.Body.Close(); cerr != nil { + logger.Warn("failed to close response body: %v", cerr) } + }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to fetch manifest: HTTP %d", resp.StatusCode) } - if !success { - return "", fmt.Errorf("failed to clone %s", source) - } - - contentPath := filepath.Join(tmpDir, "manifest.yaml") - content, err := os.ReadFile(contentPath) + content, err := io.ReadAll(resp.Body) if err != nil { - return "", fmt.Errorf("failed to read manifest file: %w", err) + return "", fmt.Errorf("failed to read manifest content: %w", err) } return string(content), nil diff --git a/utils/git.go b/utils/git.go index a9d0c80..00d2e25 100644 --- a/utils/git.go +++ b/utils/git.go @@ -1,9 +1,146 @@ package utils -import "strings" +import ( + "fmt" + "regexp" + "strings" +) -// IsGitURL checks if a string is likely a Git URL. -// It checks for "https://" or "git@" prefixes. +// IsGitURL checks if a string is likely a Git repository URL (not a raw file URL). +// It checks for "git@" prefix or URLs ending with ".git". func IsGitURL(url string) bool { - return strings.HasPrefix(url, "https://") || strings.HasPrefix(url, "git@") + if strings.HasPrefix(url, "git@") { + return true + } + if strings.HasSuffix(url, ".git") { + return true + } + // Check for common git hosting patterns (but not raw file URLs) + gitPatterns := []string{ + "github.com/", + "gitlab.com/", + "bitbucket.org/", + } + for _, pattern := range gitPatterns { + if strings.Contains(url, pattern) && !IsRawFileURL(url) { + return true + } + } + return false +} + +// IsRawFileURL checks if a URL is a direct raw file URL. +func IsRawFileURL(url string) bool { + rawPatterns := []string{ + "raw.githubusercontent.com", + "/-/raw/", // GitLab raw URL pattern + "/raw/", // Bitbucket raw URL pattern + "raw.github.com", + } + for _, pattern := range rawPatterns { + if strings.Contains(url, pattern) { + return true + } + } + return false +} + +// GitURLInfo holds parsed information from a Git URL. +type GitURLInfo struct { + Host string // e.g., "github.com", "gitlab.com" + Owner string // e.g., "chenasraf" + Repo string // e.g., "sofmani" +} + +// ParseGitURL parses a Git URL (SSH or HTTPS) and extracts host, owner, and repo. +// Supports formats: +// - git@github.com:owner/repo.git +// - https://github.com/owner/repo.git +// - https://github.com/owner/repo +func ParseGitURL(url string) (*GitURLInfo, error) { + // SSH format: git@host:owner/repo.git + sshRegex := regexp.MustCompile(`^git@([^:]+):([^/]+)/(.+?)(?:\.git)?$`) + if matches := sshRegex.FindStringSubmatch(url); matches != nil { + return &GitURLInfo{ + Host: matches[1], + Owner: matches[2], + Repo: matches[3], + }, nil + } + + // HTTPS format: https://host/owner/repo.git or https://host/owner/repo + httpsRegex := regexp.MustCompile(`^https://([^/]+)/([^/]+)/(.+?)(?:\.git)?$`) + if matches := httpsRegex.FindStringSubmatch(url); matches != nil { + return &GitURLInfo{ + Host: matches[1], + Owner: matches[2], + Repo: matches[3], + }, nil + } + + return nil, fmt.Errorf("unable to parse Git URL: %s", url) +} + +// GitHostType represents the type of Git hosting service. +type GitHostType string + +const ( + GitHostGitHub GitHostType = "github" + GitHostGitLab GitHostType = "gitlab" + GitHostBitbucket GitHostType = "bitbucket" + GitHostUnknown GitHostType = "unknown" +) + +// DetectGitHostType detects the Git hosting service from a host string. +// It checks for known patterns in the hostname. +func DetectGitHostType(host string) GitHostType { + host = strings.ToLower(host) + switch { + case strings.Contains(host, "github"): + return GitHostGitHub + case strings.Contains(host, "gitlab"): + return GitHostGitLab + case strings.Contains(host, "bitbucket"): + return GitHostBitbucket + default: + return GitHostUnknown + } +} + +// GetRawFileURL converts a Git repository URL to a raw file URL for direct access. +// Supports GitHub, GitLab (including self-hosted), and Bitbucket (including self-hosted). +// For unknown hosts, it attempts to use GitLab-style raw URLs as a fallback. +// Parameters: +// - gitURL: The Git repository URL (SSH or HTTPS) +// - ref: The branch, tag, or commit (defaults to "master" if empty) +// - path: The file path within the repository +func GetRawFileURL(gitURL, ref, path string) (string, error) { + info, err := ParseGitURL(gitURL) + if err != nil { + return "", err + } + + if ref == "" { + ref = "master" + } + + // Remove leading slash from path if present + path = strings.TrimPrefix(path, "/") + + hostType := DetectGitHostType(info.Host) + + switch hostType { + case GitHostGitHub: + // GitHub: https://raw.githubusercontent.com/owner/repo/ref/path + return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", info.Owner, info.Repo, ref, path), nil + case GitHostGitLab: + // GitLab (including self-hosted): https://host/owner/repo/-/raw/ref/path + return fmt.Sprintf("https://%s/%s/%s/-/raw/%s/%s", info.Host, info.Owner, info.Repo, ref, path), nil + case GitHostBitbucket: + // Bitbucket (including self-hosted): https://host/owner/repo/raw/ref/path + return fmt.Sprintf("https://%s/%s/%s/raw/%s/%s", info.Host, info.Owner, info.Repo, ref, path), nil + default: + // For unknown hosts, try GitLab-style as it's common for self-hosted instances + return fmt.Sprintf("https://%s/%s/%s/-/raw/%s/%s", info.Host, info.Owner, info.Repo, ref, path), nil + } } diff --git a/utils/git_test.go b/utils/git_test.go index 0212e70..9a0e21d 100644 --- a/utils/git_test.go +++ b/utils/git_test.go @@ -14,9 +14,14 @@ func TestIsGitURL(t *testing.T) { }{ {"Valid HTTPS URL", "https://github.com/user/repo.git", true}, {"Valid SSH URL", "git@github.com:user/repo.git", true}, - {"Invalid URL", "ftp://github.com/user/repo.git", false}, + {"URL ending with .git", "ftp://example.com/user/repo.git", true}, {"Empty URL", "", false}, {"Random string", "not_a_url", false}, + {"GitHub URL without .git", "https://github.com/user/repo", true}, + {"GitLab URL without .git", "https://gitlab.com/user/repo", true}, + {"Raw GitHub URL", "https://raw.githubusercontent.com/user/repo/main/file.txt", false}, + {"Raw GitLab URL", "https://gitlab.com/user/repo/-/raw/main/file.txt", false}, + {"Generic HTTPS URL", "https://example.com/file.yaml", false}, } for _, tt := range tests { @@ -26,3 +31,191 @@ func TestIsGitURL(t *testing.T) { }) } } + +func TestParseGitURL(t *testing.T) { + tests := []struct { + name string + url string + expectError bool + expected *GitURLInfo + }{ + { + name: "GitHub SSH", + url: "git@github.com:chenasraf/sofmani.git", + expected: &GitURLInfo{ + Host: "github.com", + Owner: "chenasraf", + Repo: "sofmani", + }, + }, + { + name: "GitHub HTTPS with .git", + url: "https://github.com/chenasraf/sofmani.git", + expected: &GitURLInfo{ + Host: "github.com", + Owner: "chenasraf", + Repo: "sofmani", + }, + }, + { + name: "GitHub HTTPS without .git", + url: "https://github.com/chenasraf/sofmani", + expected: &GitURLInfo{ + Host: "github.com", + Owner: "chenasraf", + Repo: "sofmani", + }, + }, + { + name: "GitLab SSH", + url: "git@gitlab.com:user/project.git", + expected: &GitURLInfo{ + Host: "gitlab.com", + Owner: "user", + Repo: "project", + }, + }, + { + name: "Invalid URL", + url: "not-a-valid-url", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := ParseGitURL(tt.url) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + } + }) + } +} + +func TestGetRawFileURL(t *testing.T) { + tests := []struct { + name string + gitURL string + ref string + path string + expectError bool + expected string + }{ + { + name: "GitHub SSH with ref", + gitURL: "git@github.com:chenasraf/sofmani.git", + ref: "master", + path: "docs/recipes/lazygit.yml", + expected: "https://raw.githubusercontent.com/chenasraf/sofmani/master/docs/recipes/lazygit.yml", + }, + { + name: "GitHub HTTPS default ref", + gitURL: "https://github.com/chenasraf/sofmani.git", + ref: "", + path: "README.md", + expected: "https://raw.githubusercontent.com/chenasraf/sofmani/master/README.md", + }, + { + name: "GitLab SSH", + gitURL: "git@gitlab.com:user/project.git", + ref: "develop", + path: "config.yml", + expected: "https://gitlab.com/user/project/-/raw/develop/config.yml", + }, + { + name: "Bitbucket HTTPS", + gitURL: "https://bitbucket.org/user/repo.git", + ref: "main", + path: "file.txt", + expected: "https://bitbucket.org/user/repo/raw/main/file.txt", + }, + { + name: "Path with leading slash", + gitURL: "git@github.com:user/repo.git", + ref: "main", + path: "/path/to/file.yml", + expected: "https://raw.githubusercontent.com/user/repo/main/path/to/file.yml", + }, + { + name: "Self-hosted GitLab", + gitURL: "git@gitlab.company.com:team/project.git", + ref: "main", + path: "manifest.yml", + expected: "https://gitlab.company.com/team/project/-/raw/main/manifest.yml", + }, + { + name: "Self-hosted Bitbucket", + gitURL: "https://bitbucket.mycompany.org/user/repo.git", + ref: "master", + path: "config.yml", + expected: "https://bitbucket.mycompany.org/user/repo/raw/master/config.yml", + }, + { + name: "Unknown host falls back to GitLab style", + gitURL: "git@custom.host.com:user/repo.git", + ref: "main", + path: "file.txt", + expected: "https://custom.host.com/user/repo/-/raw/main/file.txt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := GetRawFileURL(tt.gitURL, tt.ref, tt.path) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + } + }) + } +} + +func TestIsRawFileURL(t *testing.T) { + tests := []struct { + name string + url string + expected bool + }{ + {"GitHub raw URL", "https://raw.githubusercontent.com/user/repo/main/file.txt", true}, + {"GitLab raw URL", "https://gitlab.com/user/repo/-/raw/main/file.txt", true}, + {"Bitbucket raw URL", "https://bitbucket.org/user/repo/raw/main/file.txt", true}, + {"Regular GitHub URL", "https://github.com/user/repo", false}, + {"Regular GitLab URL", "https://gitlab.com/user/repo", false}, + {"Local path", "/path/to/file.txt", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := IsRawFileURL(tt.url) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestDetectGitHostType(t *testing.T) { + tests := []struct { + name string + host string + expected GitHostType + }{ + {"GitHub", "github.com", GitHostGitHub}, + {"GitHub Enterprise", "github.mycompany.com", GitHostGitHub}, + {"GitLab", "gitlab.com", GitHostGitLab}, + {"Self-hosted GitLab", "gitlab.company.org", GitHostGitLab}, + {"Bitbucket", "bitbucket.org", GitHostBitbucket}, + {"Self-hosted Bitbucket", "bitbucket.mycompany.com", GitHostBitbucket}, + {"Unknown host", "git.custom.com", GitHostUnknown}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := DetectGitHostType(tt.host) + assert.Equal(t, tt.expected, result) + }) + } +}