From 241fe8c53b30aea16917c3da34948348248341f4 Mon Sep 17 00:00:00 2001 From: Simon Cornet Date: Mon, 7 Jul 2025 10:50:31 +0200 Subject: [PATCH] feat: use yaml config instead of args or envs --- cmd/gogitlabber/git.go | 14 +-- cmd/gogitlabber/gitea.go | 10 +- cmd/gogitlabber/gitlab.go | 10 +- cmd/gogitlabber/input.go | 243 ++++++++++++++++++++------------------ cmd/gogitlabber/main.go | 35 +++--- go.mod | 1 + go.sum | 1 + readme.md | 97 +++++---------- 8 files changed, 193 insertions(+), 218 deletions(-) diff --git a/cmd/gogitlabber/git.go b/cmd/gogitlabber/git.go index 2b8ac41..6546f32 100644 --- a/cmd/gogitlabber/git.go +++ b/cmd/gogitlabber/git.go @@ -12,11 +12,11 @@ import ( // add a mutex to safely increment shared counters var mu sync.Mutex -func checkoutRepositories(repositories []Repository, concurrency int) { +func checkoutRepositories(repositories []Repository) { // create a waitgroup + semaphore channel var wg sync.WaitGroup - semaphore := make(chan struct{}, concurrency) + semaphore := make(chan struct{}, config.Concurrency) // manage all repositories found for _, repo := range repositories { @@ -36,13 +36,13 @@ func checkoutRepositories(repositories []Repository, concurrency int) { // get repository name + create repo destination repoName := string(repo.PathWithNamespace) - repoDestination := repoDestinationPre + repoName + repoDestination := config.Destination + repoName // log activity logger.Print("Starting on repository: "+repoName, nil) // make git url - url := fmt.Sprintf("https://%s-token:%s@%s/%s.git", gitBackend, gitToken, gitHost, repoName) + url := fmt.Sprintf("https://%s-token:%s@%s/%s.git", config.GitBackend, config.GitToken, config.GitHost, repoName) // check current status of repoDestination checkRepo := func(repoDestination string) string { @@ -78,7 +78,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) { // set a lock, increment counters, update progressbar and unlock mu.Lock() clonedCount++ - if !debug { + if !config.Debug { _ = bar.Add(1) } mu.Unlock() @@ -87,7 +87,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) { case strings.Contains(string(repoStatus), url): logger.Print("Decided to pull repository: "+repoName, nil) pullRepository(repoName, repoDestination) - if !debug { + if !config.Debug { _ = bar.Add(1) } @@ -98,7 +98,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) { // set a lock, increment counters and unlock mu.Lock() errorCount++ - if !debug { + if !config.Debug { _ = bar.Add(1) } mu.Unlock() diff --git a/cmd/gogitlabber/gitea.go b/cmd/gogitlabber/gitea.go index 51006ee..c0d5101 100644 --- a/cmd/gogitlabber/gitea.go +++ b/cmd/gogitlabber/gitea.go @@ -22,7 +22,7 @@ func fetchRepositoriesGitea() ([]Repository, error) { // configure archived options var archived string - switch includeArchived { + switch config.IncludeArchived { case "excluded": archived = "&archived=false" case "only": @@ -32,7 +32,7 @@ func fetchRepositoriesGitea() ([]Repository, error) { } url := fmt.Sprintf("https://%s/api/v1/user/repos?%s&%s&%s%s", - gitHost, visibility, sort, perpage, archived) + config.GitHost, visibility, sort, perpage, archived) logger.Print("HTTP: Creating API request", nil) req, err := http.NewRequest("GET", url, nil) @@ -41,7 +41,7 @@ func fetchRepositoriesGitea() ([]Repository, error) { } logger.Print("HTTP: Adding Authorization header to API request", nil) - req.Header.Set("Authorization", fmt.Sprintf("token %s", gitToken)) + req.Header.Set("Authorization", fmt.Sprintf("token %s", config.GitToken)) logger.Print("HTTP: Making request", nil) client := &http.Client{} @@ -82,7 +82,7 @@ func fetchRepositoriesGitea() ([]Repository, error) { repoCount := len(repositories) logger.Print("BAR: Resetting the progressbar", nil) - if !debug { + if !config.Debug { err = bar.Set(0) if err != nil { logger.Fatal("Could not reset the progressbar", err) @@ -90,7 +90,7 @@ func fetchRepositoriesGitea() ([]Repository, error) { } logger.Print("BAR: Increasing the max value of the progressbar", nil) - if !debug { + if !config.Debug { bar.ChangeMax(repoCount) } diff --git a/cmd/gogitlabber/gitlab.go b/cmd/gogitlabber/gitlab.go index 0a72ef4..66f97b6 100644 --- a/cmd/gogitlabber/gitlab.go +++ b/cmd/gogitlabber/gitlab.go @@ -17,7 +17,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) { // configure archived options var archived string - switch includeArchived { + switch config.IncludeArchived { case "excluded": archived = "&archived=false" case "only": @@ -27,7 +27,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) { } url := fmt.Sprintf("https://%s/api/v4/projects?%s&%s&%s%s", - gitHost, membership, order, perpage, archived) + config.GitHost, membership, order, perpage, archived) logger.Print("HTTP: Creating API request", nil) req, err := http.NewRequest("GET", url, nil) @@ -36,7 +36,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) { } logger.Print("HTTP: Adding PRIVATE-TOKEN header to API request", nil) - req.Header.Set("PRIVATE-TOKEN", gitToken) + req.Header.Set("PRIVATE-TOKEN", config.GitToken) logger.Print("HTTP: Making request", nil) client := &http.Client{} @@ -67,7 +67,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) { repoCount := len(repositories) logger.Print("BAR: Resetting the progressbar", nil) - if !debug { + if !config.Debug { err = bar.Set(0) if err != nil { logger.Fatal("Could not reset the progressbar", err) @@ -75,7 +75,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) { } logger.Print("BAR: Increasing the max value of the progressbar", nil) - if !debug { + if !config.Debug { bar.ChangeMax(repoCount) } diff --git a/cmd/gogitlabber/input.go b/cmd/gogitlabber/input.go index 072fd28..cd99b41 100644 --- a/cmd/gogitlabber/input.go +++ b/cmd/gogitlabber/input.go @@ -4,151 +4,168 @@ import ( "flag" "fmt" "os" - "strconv" + "path/filepath" "strings" "github.com/scornet256/go-logger" + "gopkg.in/yaml.v3" ) -// set default values and override values from environment variables -func setDefaultsFromEnv() { +// config struct for config +type Config struct { + Concurrency int `yaml:"concurrency"` + Debug bool `yaml:"debug"` + Destination string `yaml:"destination"` + GitBackend string `yaml:"git_backend"` + GitHost string `yaml:"git_host"` + GitToken string `yaml:"git_token"` + IncludeArchived string `yaml:"include_archived"` +} - // set default values - debug = false - concurrency = 15 - gitHost = "gitlab.com" - gitToken = "" - includeArchived = "excluded" - repoDestinationPre = "$HOME/Documents" +// setdefaults sets default values for the configuration +func (conf *Config) setDefaults() { + conf.Concurrency = 15 + conf.Debug = false + conf.Destination = "$HOME/Documents" + conf.GitBackend = "" + conf.GitHost = "gitlab.com" + conf.GitToken = "" + conf.IncludeArchived = "excluded" +} - // override with environment variables if present - if envDebug := os.Getenv("GOGITLABBER_DEBUG"); envDebug != "" { - if debugVal, err := strconv.ParseBool(envDebug); err == nil { - debug = debugVal - } else { - logger.Print("Warning: Invalid debug value in environment, using default", nil) +// expand variable paths +func expandPath(path string) string { + + // expand environment variables like $home + expanded := os.ExpandEnv(path) + + // expand ~ + if strings.HasPrefix(expanded, "~/") { + home, err := os.UserHomeDir() + if err != nil { + return expanded } - } - - if envBackend := os.Getenv("GOGITLABBER_BACKEND"); envBackend != "" { - gitBackend = envBackend - } - - if envToken := os.Getenv("GIT_API_TOKEN"); envToken != "" { - gitToken = envToken - } - - if envHost := os.Getenv("GIT_URL"); envHost != "" { - gitHost = envHost - } - - if envRepoDest := os.Getenv("GOGITLABBER_DESTINATION"); envRepoDest != "" { - repoDestinationPre = envRepoDest - } - - if envConcurrency := os.Getenv("GOGITLABBER_CONCURRENCY"); envConcurrency != "" { - if concurrencyVal, err := strconv.Atoi(envConcurrency); err == nil { - concurrency = concurrencyVal - } else { - logger.Print("Warning: Invalid concurrency value in environment, using default", nil) + expanded = filepath.Join(home, expanded[2:]) + } else if expanded == "~" { + home, err := os.UserHomeDir() + if err != nil { + return expanded } + expanded = home } - if envArchived := os.Getenv("GOGITLABBER_ARCHIVED"); envArchived != "" { - switch envArchived { - case "any", "exclusive", "excluded": - includeArchived = envArchived - default: - logger.Print("Warning: Invalid archived value in environment, using default", nil) - } + return filepath.Clean(expanded) +} + +// loadconfig from yaml file +func loadConfig(configPath string) (*Config, error) { + config := &Config{} + config.setDefaults() + + // check if config file exists + if _, err := os.Stat(configPath); os.IsNotExist(err) { + return nil, fmt.Errorf("config file not found: %s", configPath) + } + + // read config + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + // parse yaml + if err := yaml.Unmarshal(data, config); err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + + return config, nil +} + +// validateConfig validates the configuration values +func (conf *Config) validateConfig() error { + // validate required parameters + if conf.GitToken == "" { + return fmt.Errorf("git_token is required") + } + + // validate archived option + switch conf.IncludeArchived { + case "any", "exclusive", "excluded": + default: + return fmt.Errorf("invalid include_archived option: %s (must be any|excluded|exclusive)", conf.IncludeArchived) + } + + // validate concurrency + if conf.Concurrency < 1 { + return fmt.Errorf("concurrency must be greater than 0") + } + + return nil +} + +// process config after loading +func (conf *Config) processConfig() { + // expand path variables + conf.Destination = expandPath(conf.Destination) + + // add trailing slash if not provided + if !strings.HasSuffix(conf.Destination, "/") { + conf.Destination += "/" } } -func manageArguments() { +// log active config +func (conf *Config) logConfig(configPath string) { + logger.Print("Configuration: Using config file: "+configPath, nil) + logger.Print("Configuration: Using host: "+conf.GitHost, nil) + logger.Print("Configuration: Using destination: "+conf.Destination, nil) + logger.Print("Configuration: Using concurrency: "+fmt.Sprintf("%d", conf.Concurrency), nil) + logger.Print("Configuration: Using archived option: "+conf.IncludeArchived, nil) + if conf.Debug { + logger.Print("Configuration: Debug mode enabled", nil) + } +} - // set defaults from environment variables - setDefaultsFromEnv() +// manage arguments +func manageArguments() *Config { - // define flags (which will override environment variables) - var archivedFlag = flag.String( - "archived", - includeArchived, - "To include archived repositories (any|excluded|exclusive)\n example: -archived=any\nenv = GOGITLABBER_ARCHIVED\n") + defaultConfigPath := "./$HOME./gogitlabber.yaml" - var concurrencyFlag = flag.Int( - "concurrency", - concurrency, - "Specify repository concurrency\n example: -concurrency=15\nenv = GOGITLABBER_CONCURRENCY\n") - - var destinationFlag = flag.String( - "destination", - repoDestinationPre, - "Specify where to check the repositories out\n example: -destination=$HOME/repos\nenv = GOGITLABBER_DESTINATION\n") - - var backendFlag = flag.String( - "backend", - gitBackend, - "Specify git backend\n example -backend=gitlab\nenv = GOGITLABBER_BACKEND\n") - - var hostFlag = flag.String( - "git-url", - gitHost, - "Specify GitLab/Gitea host\n example: -git-url=gitlab.com\nenv = GIT_URL\n") - - var tokenFlag = flag.String( - "git-api-token", - gitToken, - "Specify GitLab/Gitea API token\n example: -git-api=glpat-xxxx\nenv = GIT_API_TOKEN\n") - - var debugFlag = flag.Bool( - "debug", - debug, - "Toggle debug mode\n example: -debug=true\nenv = GOGITLABBER_DEBUG\n") + // Define only the config file flag + configFileFlag := flag.String( + "config", + defaultConfigPath, + "Specify config file path (YAML)\n example: -config=./config/app.yaml") versionFlag := flag.Bool("version", false, "Print the version and exit") flag.Parse() - // print version if *versionFlag { fmt.Println(version) os.Exit(0) } - // override with flag values (higher precedence) - concurrency = *concurrencyFlag - debug = *debugFlag - gitHost = *hostFlag - gitToken = *tokenFlag - gitBackend = *backendFlag - includeArchived = *archivedFlag - repoDestinationPre = *destinationFlag + configPath := *configFileFlag - // add slash 🎩🎸 if not provided - if !strings.HasSuffix(repoDestinationPre, "/") { - repoDestinationPre += "/" - } - - // validate required parameters - if gitToken == "" { + // Load configuration from YAML file + config, err := loadConfig(configPath) + if err != nil { flag.Usage() - logger.Fatal("Configuration: API Token not found", nil) + logger.Fatal("Configuration error: "+err.Error(), nil) } - // validate archived option - switch includeArchived { - case "any", "exclusive", "excluded": - default: + // Process configuration + config.processConfig() + + // Validate configuration + if err := config.validateConfig(); err != nil { flag.Usage() - logger.Fatal("Configuration: Invalid archive option: "+includeArchived, nil) + logger.Fatal("Configuration validation error: "+err.Error(), nil) } - // log configuration - logger.Print("Configuration: Using host: "+gitHost, nil) - logger.Print("Configuration: Using destination: "+repoDestinationPre, nil) - logger.Print("Configuration: Using concurrency: "+strconv.Itoa(concurrency), nil) - logger.Print("Configuration: Using archived option: "+includeArchived, nil) - if debug { - logger.Print("Configuration: Debug mode enabled", nil) - } + // Log configuration + config.logConfig(configPath) + + return config } diff --git a/cmd/gogitlabber/main.go b/cmd/gogitlabber/main.go index 7bdce2e..51e6eb1 100644 --- a/cmd/gogitlabber/main.go +++ b/cmd/gogitlabber/main.go @@ -1,22 +1,14 @@ package main import ( + "fmt" + "github.com/scornet256/go-logger" ) // version var version string - -// userdata -var concurrency int -var debug bool -var includeArchived string -var repoDestinationPre string - -// git -var gitHost string -var gitToken string -var gitBackend string +var config *Config // keep count 🧛 var clonedCount int @@ -34,16 +26,16 @@ type Repository struct { func main() { // set app version - version = "1.1.5" + version = "2.0.0" // set appname for logger logger.SetAppName("gogitlabber") - // manage all argument magic - manageArguments() + // manage all argument magic and load configuration + config = manageArguments() // set debugging - logger.SetDebug(debug) + logger.SetDebug(config.Debug) // check for git err := verifyGitAvailable() @@ -52,14 +44,19 @@ func main() { } logger.Print("VALIDATION: git found in path", nil) + // validate git backend is set + if config.GitBackend == "" { + logger.Fatal("Configuration error: git_backend is required (gitlab|gitea)", nil) + } + // make initial progressbar - if !debug { + if !config.Debug { progressBar() } // fetch repository information var repositories []Repository - switch gitBackend { + switch config.GitBackend { case "gitea": repositories, err = fetchRepositoriesGitea() if err != nil { @@ -71,11 +68,11 @@ func main() { logger.Fatal("Fetching repositories failed", err) } default: - logger.Fatal("Fetching repositories failed", err) + logger.Fatal(fmt.Sprintf("Unsupported git backend: %s (supported: gitlab|gitea)", config.GitBackend), nil) } // manage found repositories - checkoutRepositories(repositories, concurrency) + checkoutRepositories(repositories) printSummary() printPullErrorUnstaged(pullErrorMsgUnstaged) printPullErrorUncommitted(pullErrorMsgUncommitted) diff --git a/go.mod b/go.mod index dadac7e..cd29e95 100644 --- a/go.mod +++ b/go.mod @@ -14,4 +14,5 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/term v0.30.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c18b996..648e786 100644 --- a/go.sum +++ b/go.sum @@ -27,5 +27,6 @@ golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/readme.md b/readme.md index 9fca5c6..8218392 100644 --- a/readme.md +++ b/readme.md @@ -1,16 +1,16 @@ # GoGitlabber This project is inspired from the python application called gitlabber (https://github.com/ezbz/gitlabber). -It is mainly to learn Golang. But also to make something that specifically -solves my problem. 😆 +It is mainly to learn Golang. But also to make something that specifically solves my problem. 😆 It is definitely not as feature-rich as the original project... 😬 -The program can clone and pull all repositories you have access to on a selfhosted or SaaS provided Gitlab or Gitea server. +The program can clone and pull all repositories you have access to on a selfhosted or SaaS provided Gitlab or Gitea +server. It only supports the HTTP access method. -It will pull the repositories in a tree like structure same as on Gitlab or -Gitea. +It will pull the repositories in a tree like structure same as on Gitlab or Gitea. + ``` root [http://gitlab.example.com] ├── group1 [/group1] @@ -23,78 +23,37 @@ root [http://gitlab.example.com] └── subgroup3 [/group2/subgroup3] ``` -# Example -Gitea: -``` -$ ./gogitlabber -backend=gitea -destination=$HOME/Documents -git-url=gitea.example.com --git-api-token=supersecrettoken - 100% [====================] (30/30) [4s] ... -Summary: - Cloned repositories: 0 - Pulled repositories: 30 - Errors: 0 -``` -Gitlab: -``` -$ ./gogitlabber -backend=gitlab -destination=$HOME/Documents -git-url=gitlab.example.com --git-api-token=supersecrettoken - 100% [====================] (30/30) [4s] ... -Summary: - Cloned repositories: 0 - Pulled repositories: 30 - Errors: 0 +## Config file + +GitLab: + +```yaml +# ~/.config/gogitlabber/gitlab.example.com.yaml +debug: false +concurrency: 15 +git_host: "gitlab.example.net" +git_token: "glpat-" +git_backend: "gitlab" +include_archived: "excluded" +destination: "$HOME/Documents" ``` -# Usage -``` -Usage of gogitlabber: - -archived string - To include archived repositories (any|excluded|exclusive) - example: -archived=any - env = GOGITLABBER_ARCHIVED - (default "excluded") - -backend string - Specify git backend - example: -backend=gitlab - env = GOGITLABBER_BACKEND +## Usage - -concurrency int - Specify repository concurrency - example: -concurrency=15 - env = GOGITLABBER_CONCURRENCY - (default 15) - - -debug - Toggle debug mode - example: -debug=true - env = GOGITLABBER_DEBUG - (default false) - - -destination string - Specify where to check the repositories out - example: -destination=$HOME/repos - env = GOGITLABBER_DESTINATION - (default "$HOME/Documents") - - -git-api-token string - Specify API token - example: -git-api=glpat-xxxx - env = GIT_API_TOKEN - (default "") - - -git-url string - Specify Git host - example: -git-url=gitlab.example.com - env = GIT_URL - (default "gitlab.com") +```bash +gogitlabber -config=~/.config/gogitlabber/gitlab.example.com.yaml ``` -# Access Token Permissions -## Gitea + +## Access Token Permissions + +### Gitea + Make sure the Gitea Access Token has at least the following permissions: - user - read - repository - read -## Gitlab +### Gitlab + Make sure the Gitlab Access Token has the `api` scope.