feat: manifest git download without clone

2026-05-17 17:28:04 +00:00 · 2025-12-05 22:07:22 +02:00
parent a8fca5373c
commit a7053cfb2c
5 changed files with 446 additions and 53 deletions
--- a/appconfig/appconfig.go
+++ b/appconfig/appconfig.go
@@ -12,6 +12,7 @@ import (
 	"github.com/chenasraf/sofmani/platform"
 	"github.com/chenasraf/sofmani/utils"
 	"github.com/eschao/config"
+	"gopkg.in/yaml.v3"
 )

 // AppConfig represents the main application configuration.
@@ -87,6 +88,16 @@ func ParseConfigFrom(file string) (*AppConfig, error) {
 	return &appConfig, nil
 }

+// ParseConfigFromContent parses the configuration from YAML content.
+func ParseConfigFromContent(content []byte) (*AppConfig, error) {
+	appConfig := NewAppConfig()
+	err := yaml.Unmarshal(content, &appConfig)
+	if err != nil {
+		return nil, err
+	}
+	return &appConfig, nil
+}
+
 // FindConfigFile searches for the configuration file in standard locations.
 // It searches in the current working directory, then in ~/.config, and finally in the home directory.
 // It returns the path to the first file found, or an empty string if no file is found.
--- a/docs/installer-configuration.md
+++ b/docs/installer-configuration.md
@@ -194,12 +194,16 @@ These fields are shared by all installer types. Some fields may vary in behavior
      installers.
    - `debug` and `check_updates` will be inherited by the loaded config.
    - `env` and `defaults` will be merged into the loaded config, overriding any existing values.
+    - Remote manifests are fetched directly via HTTP (no git clone required).
  - **Options**:
-    - `opts.source`: The local file, or remote git URL (https or SSH) containing the manifest.
-    - `opts.path`: The path to the manifest file within the repository. If `opts.source` is a local
-      file, `opts.path` will be appended to it.
-    - `opts.ref`: The branch, tag, or commit to checkout after cloning if `opts.source` is a git
-      URL. For local manifests, this value will be ignored.
+    - `opts.source`: The source of the manifest file. Supports:
+      - Local file paths (e.g., `~/.dotfiles/manifest.yml`)
+      - Git repository URLs (SSH or HTTPS) - GitHub, GitLab, Bitbucket, and self-hosted instances
+      - Raw HTTP URLs (e.g., `https://raw.githubusercontent.com/user/repo/master/manifest.yml`)
+    - `opts.path`: The path to the manifest file within the repository. Required for git URLs,
+      optional for local files (will be appended to source). Ignored for raw HTTP URLs.
+    - `opts.ref`: The branch, tag, or commit to use if `opts.source` is a git URL. Defaults to
+      `master`. Ignored for local files and raw HTTP URLs.

 - **`rsync`**

--- a/installer/manifest_installer.go
+++ b/installer/manifest_installer.go
@@ -2,9 +2,11 @@ package installer

 import (
 	"fmt"
+	"io"
 	"maps"
-	"os"
+	"net/http"
 	"path/filepath"
+	"strings"

 	"github.com/chenasraf/sofmani/appconfig"
 	"github.com/chenasraf/sofmani/logger"
@@ -124,75 +126,121 @@ func (i *ManifestInstaller) GetOpts() *ManifestOpts {
 }

 // FetchManifest fetches and parses the manifest file.
-// It handles both local and Git sources.
+// It handles local files, Git repository URLs, and raw HTTP URLs.
 func (i *ManifestInstaller) FetchManifest() error {
 	opts := i.GetOpts()
 	source := *opts.Source
-	isGit := utils.IsGitURL(source)
 	env := i.GetData().Environ()
-	var path string
-	if opts.Path == nil {
-		path = ""
-	} else {
-		path = *opts.Path
-	}
-	path = utils.GetRealPath(env, path)

-	if isGit {
-		src, err := i.getGitManifestConfig(source)
+	var config *appconfig.AppConfig
+	var err error
+
+	switch {
+	case utils.IsGitURL(source):
+		// Git repository URL - convert to raw URL and fetch
+		content, fetchErr := i.getGitManifestConfig(source)
+		if fetchErr != nil {
+			return fetchErr
+		}
+		config, err = appconfig.ParseConfigFromContent([]byte(content))
+		if err != nil {
+			return fmt.Errorf("failed to parse manifest content: %w", err)
+		}
+	case strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://"):
+		// Direct HTTP URL - fetch directly
+		content, fetchErr := i.fetchRawURL(source)
+		if fetchErr != nil {
+			return fetchErr
+		}
+		config, err = appconfig.ParseConfigFromContent([]byte(content))
+		if err != nil {
+			return fmt.Errorf("failed to parse manifest content: %w", err)
+		}
+	default:
+		// Local file path
+		source = utils.GetRealPath(env, source)
+		var path string
+		if opts.Path == nil {
+			path = ""
+		} else {
+			path = *opts.Path
+		}
+		path = utils.GetRealPath(env, path)
+		fullPath := filepath.Join(source, path)
+		logger.Debug("Parsing manifest from %s", fullPath)
+		config, err = i.getLocalManifestConfig(fullPath)
 		if err != nil {
 			return err
 		}
-		source = src
-	} else {
-		source = utils.GetRealPath(env, source)
 	}

-	logger.Debug("Parsing manifest from %s", filepath.Join(source, path))
-	config, err := i.getLocalManifestConfig(filepath.Join(source, path))
-	if err != nil {
-		return err
-	}
 	logger.Debug("Installers: %d", len(config.Install))
+	config = i.inheritManifest(config)
 	i.ManifestConfig = config
 	return nil
 }

-func (i *ManifestInstaller) getGitManifestConfig(source string) (string, error) {
-	opts := i.GetOpts()
-	tmpDir, err := os.MkdirTemp("", "sofmani")
+// fetchRawURL fetches content directly from a raw HTTP URL.
+func (i *ManifestInstaller) fetchRawURL(url string) (string, error) {
+	logger.Debug("Fetching manifest from raw URL: %s", url)
+	resp, err := http.Get(url)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to fetch manifest: %w", err)
 	}
-
 	defer func() {
-		if rmErr := os.RemoveAll(tmpDir); rmErr != nil {
-			logger.Warn("Failed to clean up tmp dir %s: %v", tmpDir, rmErr)
+		if cerr := resp.Body.Close(); cerr != nil {
+			logger.Warn("failed to close response body: %v", cerr)
 		}
 	}()

-	logger.Debug("Cloning %s to %s", source, tmpDir)
-	success, err := i.RunCmdGetSuccess("git", "clone", "--depth=1", source, tmpDir)
-	if err != nil {
-		return "", err
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to fetch manifest: HTTP %d", resp.StatusCode)
 	}

-	if opts.Ref != nil {
-		logger.Debug("Checking out ref %s", *opts.Ref)
-		err = i.RunCmdPassThrough("git", "-C", tmpDir, "checkout", *opts.Ref)
-		if err != nil {
-			return "", err
+	content, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read manifest content: %w", err)
+	}
+
+	return string(content), nil
+}
+
+func (i *ManifestInstaller) getGitManifestConfig(source string) (string, error) {
+	opts := i.GetOpts()
+
+	ref := "main"
+	if opts.Ref != nil && *opts.Ref != "" {
+		ref = *opts.Ref
+	}
+
+	path := ""
+	if opts.Path != nil {
+		path = *opts.Path
+	}
+
+	rawURL, err := utils.GetRawFileURL(source, ref, path)
+	if err != nil {
+		return "", fmt.Errorf("failed to construct raw file URL: %w", err)
+	}
+
+	logger.Debug("Fetching manifest from %s", rawURL)
+	resp, err := http.Get(rawURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to fetch manifest: %w", err)
+	}
+	defer func() {
+		if cerr := resp.Body.Close(); cerr != nil {
+			logger.Warn("failed to close response body: %v", cerr)
 		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to fetch manifest: HTTP %d", resp.StatusCode)
 	}

-	if !success {
-		return "", fmt.Errorf("failed to clone %s", source)
-	}
-
-	contentPath := filepath.Join(tmpDir, "manifest.yaml")
-	content, err := os.ReadFile(contentPath)
+	content, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return "", fmt.Errorf("failed to read manifest file: %w", err)
+		return "", fmt.Errorf("failed to read manifest content: %w", err)
 	}

 	return string(content), nil
--- a/utils/git.go
+++ b/utils/git.go
@@ -1,9 +1,146 @@
 package utils

-import "strings"
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)

-// IsGitURL checks if a string is likely a Git URL.
-// It checks for "https://" or "git@" prefixes.
+// IsGitURL checks if a string is likely a Git repository URL (not a raw file URL).
+// It checks for "git@" prefix or URLs ending with ".git".
 func IsGitURL(url string) bool {
-	return strings.HasPrefix(url, "https://") || strings.HasPrefix(url, "git@")
+	if strings.HasPrefix(url, "git@") {
+		return true
+	}
+	if strings.HasSuffix(url, ".git") {
+		return true
+	}
+	// Check for common git hosting patterns (but not raw file URLs)
+	gitPatterns := []string{
+		"github.com/",
+		"gitlab.com/",
+		"bitbucket.org/",
+	}
+	for _, pattern := range gitPatterns {
+		if strings.Contains(url, pattern) && !IsRawFileURL(url) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsRawFileURL checks if a URL is a direct raw file URL.
+func IsRawFileURL(url string) bool {
+	rawPatterns := []string{
+		"raw.githubusercontent.com",
+		"/-/raw/", // GitLab raw URL pattern
+		"/raw/",   // Bitbucket raw URL pattern
+		"raw.github.com",
+	}
+	for _, pattern := range rawPatterns {
+		if strings.Contains(url, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+// GitURLInfo holds parsed information from a Git URL.
+type GitURLInfo struct {
+	Host  string // e.g., "github.com", "gitlab.com"
+	Owner string // e.g., "chenasraf"
+	Repo  string // e.g., "sofmani"
+}
+
+// ParseGitURL parses a Git URL (SSH or HTTPS) and extracts host, owner, and repo.
+// Supports formats:
+//   - git@github.com:owner/repo.git
+//   - https://github.com/owner/repo.git
+//   - https://github.com/owner/repo
+func ParseGitURL(url string) (*GitURLInfo, error) {
+	// SSH format: git@host:owner/repo.git
+	sshRegex := regexp.MustCompile(`^git@([^:]+):([^/]+)/(.+?)(?:\.git)?$`)
+	if matches := sshRegex.FindStringSubmatch(url); matches != nil {
+		return &GitURLInfo{
+			Host:  matches[1],
+			Owner: matches[2],
+			Repo:  matches[3],
+		}, nil
+	}
+
+	// HTTPS format: https://host/owner/repo.git or https://host/owner/repo
+	httpsRegex := regexp.MustCompile(`^https://([^/]+)/([^/]+)/(.+?)(?:\.git)?$`)
+	if matches := httpsRegex.FindStringSubmatch(url); matches != nil {
+		return &GitURLInfo{
+			Host:  matches[1],
+			Owner: matches[2],
+			Repo:  matches[3],
+		}, nil
+	}
+
+	return nil, fmt.Errorf("unable to parse Git URL: %s", url)
+}
+
+// GitHostType represents the type of Git hosting service.
+type GitHostType string
+
+const (
+	GitHostGitHub    GitHostType = "github"
+	GitHostGitLab    GitHostType = "gitlab"
+	GitHostBitbucket GitHostType = "bitbucket"
+	GitHostUnknown   GitHostType = "unknown"
+)
+
+// DetectGitHostType detects the Git hosting service from a host string.
+// It checks for known patterns in the hostname.
+func DetectGitHostType(host string) GitHostType {
+	host = strings.ToLower(host)
+	switch {
+	case strings.Contains(host, "github"):
+		return GitHostGitHub
+	case strings.Contains(host, "gitlab"):
+		return GitHostGitLab
+	case strings.Contains(host, "bitbucket"):
+		return GitHostBitbucket
+	default:
+		return GitHostUnknown
+	}
+}
+
+// GetRawFileURL converts a Git repository URL to a raw file URL for direct access.
+// Supports GitHub, GitLab (including self-hosted), and Bitbucket (including self-hosted).
+// For unknown hosts, it attempts to use GitLab-style raw URLs as a fallback.
+// Parameters:
+//   - gitURL: The Git repository URL (SSH or HTTPS)
+//   - ref: The branch, tag, or commit (defaults to "master" if empty)
+//   - path: The file path within the repository
+func GetRawFileURL(gitURL, ref, path string) (string, error) {
+	info, err := ParseGitURL(gitURL)
+	if err != nil {
+		return "", err
+	}
+
+	if ref == "" {
+		ref = "master"
+	}
+
+	// Remove leading slash from path if present
+	path = strings.TrimPrefix(path, "/")
+
+	hostType := DetectGitHostType(info.Host)
+
+	switch hostType {
+	case GitHostGitHub:
+		// GitHub: https://raw.githubusercontent.com/owner/repo/ref/path
+		return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", info.Owner, info.Repo, ref, path), nil
+	case GitHostGitLab:
+		// GitLab (including self-hosted): https://host/owner/repo/-/raw/ref/path
+		return fmt.Sprintf("https://%s/%s/%s/-/raw/%s/%s", info.Host, info.Owner, info.Repo, ref, path), nil
+	case GitHostBitbucket:
+		// Bitbucket (including self-hosted): https://host/owner/repo/raw/ref/path
+		return fmt.Sprintf("https://%s/%s/%s/raw/%s/%s", info.Host, info.Owner, info.Repo, ref, path), nil
+	default:
+		// For unknown hosts, try GitLab-style as it's common for self-hosted instances
+		return fmt.Sprintf("https://%s/%s/%s/-/raw/%s/%s", info.Host, info.Owner, info.Repo, ref, path), nil
+	}
 }
--- a/utils/git_test.go
+++ b/utils/git_test.go
@@ -14,9 +14,14 @@ func TestIsGitURL(t *testing.T) {
 	}{
 		{"Valid HTTPS URL", "https://github.com/user/repo.git", true},
 		{"Valid SSH URL", "git@github.com:user/repo.git", true},
-		{"Invalid URL", "ftp://github.com/user/repo.git", false},
+		{"URL ending with .git", "ftp://example.com/user/repo.git", true},
 		{"Empty URL", "", false},
 		{"Random string", "not_a_url", false},
+		{"GitHub URL without .git", "https://github.com/user/repo", true},
+		{"GitLab URL without .git", "https://gitlab.com/user/repo", true},
+		{"Raw GitHub URL", "https://raw.githubusercontent.com/user/repo/main/file.txt", false},
+		{"Raw GitLab URL", "https://gitlab.com/user/repo/-/raw/main/file.txt", false},
+		{"Generic HTTPS URL", "https://example.com/file.yaml", false},
 	}

 	for _, tt := range tests {
@@ -26,3 +31,191 @@ func TestIsGitURL(t *testing.T) {
 		})
 	}
 }
+
+func TestParseGitURL(t *testing.T) {
+	tests := []struct {
+		name        string
+		url         string
+		expectError bool
+		expected    *GitURLInfo
+	}{
+		{
+			name: "GitHub SSH",
+			url:  "git@github.com:chenasraf/sofmani.git",
+			expected: &GitURLInfo{
+				Host:  "github.com",
+				Owner: "chenasraf",
+				Repo:  "sofmani",
+			},
+		},
+		{
+			name: "GitHub HTTPS with .git",
+			url:  "https://github.com/chenasraf/sofmani.git",
+			expected: &GitURLInfo{
+				Host:  "github.com",
+				Owner: "chenasraf",
+				Repo:  "sofmani",
+			},
+		},
+		{
+			name: "GitHub HTTPS without .git",
+			url:  "https://github.com/chenasraf/sofmani",
+			expected: &GitURLInfo{
+				Host:  "github.com",
+				Owner: "chenasraf",
+				Repo:  "sofmani",
+			},
+		},
+		{
+			name: "GitLab SSH",
+			url:  "git@gitlab.com:user/project.git",
+			expected: &GitURLInfo{
+				Host:  "gitlab.com",
+				Owner: "user",
+				Repo:  "project",
+			},
+		},
+		{
+			name:        "Invalid URL",
+			url:         "not-a-valid-url",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := ParseGitURL(tt.url)
+			if tt.expectError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expected, result)
+			}
+		})
+	}
+}
+
+func TestGetRawFileURL(t *testing.T) {
+	tests := []struct {
+		name        string
+		gitURL      string
+		ref         string
+		path        string
+		expectError bool
+		expected    string
+	}{
+		{
+			name:     "GitHub SSH with ref",
+			gitURL:   "git@github.com:chenasraf/sofmani.git",
+			ref:      "master",
+			path:     "docs/recipes/lazygit.yml",
+			expected: "https://raw.githubusercontent.com/chenasraf/sofmani/master/docs/recipes/lazygit.yml",
+		},
+		{
+			name:     "GitHub HTTPS default ref",
+			gitURL:   "https://github.com/chenasraf/sofmani.git",
+			ref:      "",
+			path:     "README.md",
+			expected: "https://raw.githubusercontent.com/chenasraf/sofmani/master/README.md",
+		},
+		{
+			name:     "GitLab SSH",
+			gitURL:   "git@gitlab.com:user/project.git",
+			ref:      "develop",
+			path:     "config.yml",
+			expected: "https://gitlab.com/user/project/-/raw/develop/config.yml",
+		},
+		{
+			name:     "Bitbucket HTTPS",
+			gitURL:   "https://bitbucket.org/user/repo.git",
+			ref:      "main",
+			path:     "file.txt",
+			expected: "https://bitbucket.org/user/repo/raw/main/file.txt",
+		},
+		{
+			name:     "Path with leading slash",
+			gitURL:   "git@github.com:user/repo.git",
+			ref:      "main",
+			path:     "/path/to/file.yml",
+			expected: "https://raw.githubusercontent.com/user/repo/main/path/to/file.yml",
+		},
+		{
+			name:     "Self-hosted GitLab",
+			gitURL:   "git@gitlab.company.com:team/project.git",
+			ref:      "main",
+			path:     "manifest.yml",
+			expected: "https://gitlab.company.com/team/project/-/raw/main/manifest.yml",
+		},
+		{
+			name:     "Self-hosted Bitbucket",
+			gitURL:   "https://bitbucket.mycompany.org/user/repo.git",
+			ref:      "master",
+			path:     "config.yml",
+			expected: "https://bitbucket.mycompany.org/user/repo/raw/master/config.yml",
+		},
+		{
+			name:     "Unknown host falls back to GitLab style",
+			gitURL:   "git@custom.host.com:user/repo.git",
+			ref:      "main",
+			path:     "file.txt",
+			expected: "https://custom.host.com/user/repo/-/raw/main/file.txt",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := GetRawFileURL(tt.gitURL, tt.ref, tt.path)
+			if tt.expectError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expected, result)
+			}
+		})
+	}
+}
+
+func TestIsRawFileURL(t *testing.T) {
+	tests := []struct {
+		name     string
+		url      string
+		expected bool
+	}{
+		{"GitHub raw URL", "https://raw.githubusercontent.com/user/repo/main/file.txt", true},
+		{"GitLab raw URL", "https://gitlab.com/user/repo/-/raw/main/file.txt", true},
+		{"Bitbucket raw URL", "https://bitbucket.org/user/repo/raw/main/file.txt", true},
+		{"Regular GitHub URL", "https://github.com/user/repo", false},
+		{"Regular GitLab URL", "https://gitlab.com/user/repo", false},
+		{"Local path", "/path/to/file.txt", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := IsRawFileURL(tt.url)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}
+
+func TestDetectGitHostType(t *testing.T) {
+	tests := []struct {
+		name     string
+		host     string
+		expected GitHostType
+	}{
+		{"GitHub", "github.com", GitHostGitHub},
+		{"GitHub Enterprise", "github.mycompany.com", GitHostGitHub},
+		{"GitLab", "gitlab.com", GitHostGitLab},
+		{"Self-hosted GitLab", "gitlab.company.org", GitHostGitLab},
+		{"Bitbucket", "bitbucket.org", GitHostBitbucket},
+		{"Self-hosted Bitbucket", "bitbucket.mycompany.com", GitHostBitbucket},
+		{"Unknown host", "git.custom.com", GitHostUnknown},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := DetectGitHostType(tt.host)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}