golang/gogitlabber
golang/gogitlabber
3
0
Fork 0
Code Issues 1 Pull requests Releases 1 Activity

feat: use yaml config instead of args or envs

Browse source
This commit is contained in:
Simon Cornet 2025-07-07 10:50:31 +02:00
commit 241fe8c53b
8 changed files with 186 additions and 211 deletions
Download patch file Download diff file Expand all files Collapse all files

14
cmd/gogitlabber/git.go
View file

@ -12,11 +12,11 @@ import (
// add a mutex to safely increment shared counters // add a mutex to safely increment shared counters
var mu sync.Mutex var mu sync.Mutex
func checkoutRepositories(repositories []Repository, concurrency int) { func checkoutRepositories(repositories []Repository) {
// create a waitgroup + semaphore channel // create a waitgroup + semaphore channel
var wg sync.WaitGroup var wg sync.WaitGroup
semaphore := make(chan struct{}, concurrency) semaphore := make(chan struct{}, config.Concurrency)
// manage all repositories found // manage all repositories found
for _, repo := range repositories { for _, repo := range repositories {
@ -36,13 +36,13 @@ func checkoutRepositories(repositories []Repository, concurrency int) {
// get repository name + create repo destination // get repository name + create repo destination
repoName := string(repo.PathWithNamespace) repoName := string(repo.PathWithNamespace)
repoDestination := repoDestinationPre + repoName repoDestination := config.Destination + repoName
// log activity // log activity
logger.Print("Starting on repository: "+repoName, nil) logger.Print("Starting on repository: "+repoName, nil)
// make git url // make git url
url := fmt.Sprintf("https://%s-token:%s@%s/%s.git", gitBackend, gitToken, gitHost, repoName) url := fmt.Sprintf("https://%s-token:%s@%s/%s.git", config.GitBackend, config.GitToken, config.GitHost, repoName)
// check current status of repoDestination // check current status of repoDestination
checkRepo := func(repoDestination string) string { checkRepo := func(repoDestination string) string {
@ -78,7 +78,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) {
// set a lock, increment counters, update progressbar and unlock // set a lock, increment counters, update progressbar and unlock
mu.Lock() mu.Lock()
clonedCount++ clonedCount++
if !debug { if !config.Debug {
_ = bar.Add(1) _ = bar.Add(1)
} }
mu.Unlock() mu.Unlock()
@ -87,7 +87,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) {
case strings.Contains(string(repoStatus), url): case strings.Contains(string(repoStatus), url):
logger.Print("Decided to pull repository: "+repoName, nil) logger.Print("Decided to pull repository: "+repoName, nil)
pullRepository(repoName, repoDestination) pullRepository(repoName, repoDestination)
if !debug { if !config.Debug {
_ = bar.Add(1) _ = bar.Add(1)
} }
@ -98,7 +98,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) {
// set a lock, increment counters and unlock // set a lock, increment counters and unlock
mu.Lock() mu.Lock()
errorCount++ errorCount++
if !debug { if !config.Debug {
_ = bar.Add(1) _ = bar.Add(1)
} }
mu.Unlock() mu.Unlock()

10
cmd/gogitlabber/gitea.go
View file

@ -22,7 +22,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
// configure archived options // configure archived options
var archived string var archived string
switch includeArchived { switch config.IncludeArchived {
case "excluded": case "excluded":
archived = "&archived=false" archived = "&archived=false"
case "only": case "only":
@ -32,7 +32,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
} }
url := fmt.Sprintf("https://%s/api/v1/user/repos?%s&%s&%s%s", url := fmt.Sprintf("https://%s/api/v1/user/repos?%s&%s&%s%s",
gitHost, visibility, sort, perpage, archived) config.GitHost, visibility, sort, perpage, archived)
logger.Print("HTTP: Creating API request", nil) logger.Print("HTTP: Creating API request", nil)
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
@ -41,7 +41,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
} }
logger.Print("HTTP: Adding Authorization header to API request", nil) logger.Print("HTTP: Adding Authorization header to API request", nil)
req.Header.Set("Authorization", fmt.Sprintf("token %s", gitToken)) req.Header.Set("Authorization", fmt.Sprintf("token %s", config.GitToken))
logger.Print("HTTP: Making request", nil) logger.Print("HTTP: Making request", nil)
client := &http.Client{} client := &http.Client{}
@ -82,7 +82,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
repoCount := len(repositories) repoCount := len(repositories)
logger.Print("BAR: Resetting the progressbar", nil) logger.Print("BAR: Resetting the progressbar", nil)
if !debug { if !config.Debug {
err = bar.Set(0) err = bar.Set(0)
if err != nil { if err != nil {
logger.Fatal("Could not reset the progressbar", err) logger.Fatal("Could not reset the progressbar", err)
@ -90,7 +90,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
} }
logger.Print("BAR: Increasing the max value of the progressbar", nil) logger.Print("BAR: Increasing the max value of the progressbar", nil)
if !debug { if !config.Debug {
bar.ChangeMax(repoCount) bar.ChangeMax(repoCount)
} }

10
cmd/gogitlabber/gitlab.go
View file

@ -17,7 +17,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
// configure archived options // configure archived options
var archived string var archived string
switch includeArchived { switch config.IncludeArchived {
case "excluded": case "excluded":
archived = "&archived=false" archived = "&archived=false"
case "only": case "only":
@ -27,7 +27,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
} }
url := fmt.Sprintf("https://%s/api/v4/projects?%s&%s&%s%s", url := fmt.Sprintf("https://%s/api/v4/projects?%s&%s&%s%s",
gitHost, membership, order, perpage, archived) config.GitHost, membership, order, perpage, archived)
logger.Print("HTTP: Creating API request", nil) logger.Print("HTTP: Creating API request", nil)
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
@ -36,7 +36,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
} }
logger.Print("HTTP: Adding PRIVATE-TOKEN header to API request", nil) logger.Print("HTTP: Adding PRIVATE-TOKEN header to API request", nil)
req.Header.Set("PRIVATE-TOKEN", gitToken) req.Header.Set("PRIVATE-TOKEN", config.GitToken)
logger.Print("HTTP: Making request", nil) logger.Print("HTTP: Making request", nil)
client := &http.Client{} client := &http.Client{}
@ -67,7 +67,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
repoCount := len(repositories) repoCount := len(repositories)
logger.Print("BAR: Resetting the progressbar", nil) logger.Print("BAR: Resetting the progressbar", nil)
if !debug { if !config.Debug {
err = bar.Set(0) err = bar.Set(0)
if err != nil { if err != nil {
logger.Fatal("Could not reset the progressbar", err) logger.Fatal("Could not reset the progressbar", err)
@ -75,7 +75,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
} }
logger.Print("BAR: Increasing the max value of the progressbar", nil) logger.Print("BAR: Increasing the max value of the progressbar", nil)
if !debug { if !config.Debug {
bar.ChangeMax(repoCount) bar.ChangeMax(repoCount)
} }

221
cmd/gogitlabber/input.go
View file

@ -4,151 +4,168 @@ import (
"flag" "flag"
"fmt" "fmt"
"os" "os"
"strconv" "path/filepath"
"strings" "strings"
"github.com/scornet256/go-logger" "github.com/scornet256/go-logger"
"gopkg.in/yaml.v3"
) )
// set default values and override values from environment variables // config struct for config
func setDefaultsFromEnv() { type Config struct {
Concurrency int `yaml:"concurrency"`
// set default values Debug bool `yaml:"debug"`
debug = false Destination string `yaml:"destination"`
concurrency = 15 GitBackend string `yaml:"git_backend"`
gitHost = "gitlab.com" GitHost string `yaml:"git_host"`
gitToken = "" GitToken string `yaml:"git_token"`
includeArchived = "excluded" IncludeArchived string `yaml:"include_archived"`
repoDestinationPre = "$HOME/Documents"
// override with environment variables if present
if envDebug := os.Getenv("GOGITLABBER_DEBUG"); envDebug != "" {
if debugVal, err := strconv.ParseBool(envDebug); err == nil {
debug = debugVal
} else {
logger.Print("Warning: Invalid debug value in environment, using default", nil)
}
} }
if envBackend := os.Getenv("GOGITLABBER_BACKEND"); envBackend != "" { // setdefaults sets default values for the configuration
gitBackend = envBackend func (conf *Config) setDefaults() {
conf.Concurrency = 15
conf.Debug = false
conf.Destination = "$HOME/Documents"
conf.GitBackend = ""
conf.GitHost = "gitlab.com"
conf.GitToken = ""
conf.IncludeArchived = "excluded"
} }
if envToken := os.Getenv("GIT_API_TOKEN"); envToken != "" { // expand variable paths
gitToken = envToken func expandPath(path string) string {
// expand environment variables like $home
expanded := os.ExpandEnv(path)
// expand ~
if strings.HasPrefix(expanded, "~/") {
home, err := os.UserHomeDir()
if err != nil {
return expanded
}
expanded = filepath.Join(home, expanded[2:])
} else if expanded == "~" {
home, err := os.UserHomeDir()
if err != nil {
return expanded
}
expanded = home
} }
if envHost := os.Getenv("GIT_URL"); envHost != "" { return filepath.Clean(expanded)
gitHost = envHost
} }
if envRepoDest := os.Getenv("GOGITLABBER_DESTINATION"); envRepoDest != "" { // loadconfig from yaml file
repoDestinationPre = envRepoDest func loadConfig(configPath string) (*Config, error) {
config := &Config{}
config.setDefaults()
// check if config file exists
if _, err := os.Stat(configPath); os.IsNotExist(err) {
return nil, fmt.Errorf("config file not found: %s", configPath)
} }
if envConcurrency := os.Getenv("GOGITLABBER_CONCURRENCY"); envConcurrency != "" { // read config
if concurrencyVal, err := strconv.Atoi(envConcurrency); err == nil { data, err := os.ReadFile(configPath)
concurrency = concurrencyVal if err != nil {
} else { return nil, fmt.Errorf("failed to read config file: %w", err)
logger.Print("Warning: Invalid concurrency value in environment, using default", nil)
}
} }
if envArchived := os.Getenv("GOGITLABBER_ARCHIVED"); envArchived != "" { // parse yaml
switch envArchived { if err := yaml.Unmarshal(data, config); err != nil {
return nil, fmt.Errorf("failed to parse config file: %w", err)
}
return config, nil
}
// validateConfig validates the configuration values
func (conf *Config) validateConfig() error {
// validate required parameters
if conf.GitToken == "" {
return fmt.Errorf("git_token is required")
}
// validate archived option
switch conf.IncludeArchived {
case "any", "exclusive", "excluded": case "any", "exclusive", "excluded":
includeArchived = envArchived
default: default:
logger.Print("Warning: Invalid archived value in environment, using default", nil) return fmt.Errorf("invalid include_archived option: %s (must be any|excluded|exclusive)", conf.IncludeArchived)
}
}
} }
func manageArguments() { // validate concurrency
if conf.Concurrency < 1 {
return fmt.Errorf("concurrency must be greater than 0")
}
// set defaults from environment variables return nil
setDefaultsFromEnv() }
// define flags (which will override environment variables) // process config after loading
var archivedFlag = flag.String( func (conf *Config) processConfig() {
"archived", // expand path variables
includeArchived, conf.Destination = expandPath(conf.Destination)
"To include archived repositories (any|excluded|exclusive)\n example: -archived=any\nenv = GOGITLABBER_ARCHIVED\n")
var concurrencyFlag = flag.Int( // add trailing slash if not provided
"concurrency", if !strings.HasSuffix(conf.Destination, "/") {
concurrency, conf.Destination += "/"
"Specify repository concurrency\n example: -concurrency=15\nenv = GOGITLABBER_CONCURRENCY\n") }
}
var destinationFlag = flag.String( // log active config
"destination", func (conf *Config) logConfig(configPath string) {
repoDestinationPre, logger.Print("Configuration: Using config file: "+configPath, nil)
"Specify where to check the repositories out\n example: -destination=$HOME/repos\nenv = GOGITLABBER_DESTINATION\n") logger.Print("Configuration: Using host: "+conf.GitHost, nil)
logger.Print("Configuration: Using destination: "+conf.Destination, nil)
logger.Print("Configuration: Using concurrency: "+fmt.Sprintf("%d", conf.Concurrency), nil)
logger.Print("Configuration: Using archived option: "+conf.IncludeArchived, nil)
if conf.Debug {
logger.Print("Configuration: Debug mode enabled", nil)
}
}
var backendFlag = flag.String( // manage arguments
"backend", func manageArguments() *Config {
gitBackend,
"Specify git backend\n example -backend=gitlab\nenv = GOGITLABBER_BACKEND\n")
var hostFlag = flag.String( defaultConfigPath := "./$HOME./gogitlabber.yaml"
"git-url",
gitHost,
"Specify GitLab/Gitea host\n example: -git-url=gitlab.com\nenv = GIT_URL\n")
var tokenFlag = flag.String( // Define only the config file flag
"git-api-token", configFileFlag := flag.String(
gitToken, "config",
"Specify GitLab/Gitea API token\n example: -git-api=glpat-xxxx\nenv = GIT_API_TOKEN\n") defaultConfigPath,
"Specify config file path (YAML)\n example: -config=./config/app.yaml")
var debugFlag = flag.Bool(
"debug",
debug,
"Toggle debug mode\n example: -debug=true\nenv = GOGITLABBER_DEBUG\n")
versionFlag := flag.Bool("version", false, "Print the version and exit") versionFlag := flag.Bool("version", false, "Print the version and exit")
flag.Parse() flag.Parse()
// print version
if *versionFlag { if *versionFlag {
fmt.Println(version) fmt.Println(version)
os.Exit(0) os.Exit(0)
} }
// override with flag values (higher precedence) configPath := *configFileFlag
concurrency = *concurrencyFlag
debug = *debugFlag
gitHost = *hostFlag
gitToken = *tokenFlag
gitBackend = *backendFlag
includeArchived = *archivedFlag
repoDestinationPre = *destinationFlag
// add slash 🎩🎸 if not provided // Load configuration from YAML file
if !strings.HasSuffix(repoDestinationPre, "/") { config, err := loadConfig(configPath)
repoDestinationPre += "/" if err != nil {
}
// validate required parameters
if gitToken == "" {
flag.Usage() flag.Usage()
logger.Fatal("Configuration: API Token not found", nil) logger.Fatal("Configuration error: "+err.Error(), nil)
} }
// validate archived option // Process configuration
switch includeArchived { config.processConfig()
case "any", "exclusive", "excluded":
default: // Validate configuration
if err := config.validateConfig(); err != nil {
flag.Usage() flag.Usage()
logger.Fatal("Configuration: Invalid archive option: "+includeArchived, nil) logger.Fatal("Configuration validation error: "+err.Error(), nil)
} }
// log configuration // Log configuration
logger.Print("Configuration: Using host: "+gitHost, nil) config.logConfig(configPath)
logger.Print("Configuration: Using destination: "+repoDestinationPre, nil)
logger.Print("Configuration: Using concurrency: "+strconv.Itoa(concurrency), nil) return config
logger.Print("Configuration: Using archived option: "+includeArchived, nil)
if debug {
logger.Print("Configuration: Debug mode enabled", nil)
}
} }

35
cmd/gogitlabber/main.go
View file

@ -1,22 +1,14 @@
package main package main
import ( import (
"fmt"
"github.com/scornet256/go-logger" "github.com/scornet256/go-logger"
) )
// version // version
var version string var version string
var config *Config
// userdata
var concurrency int
var debug bool
var includeArchived string
var repoDestinationPre string
// git
var gitHost string
var gitToken string
var gitBackend string
// keep count 🧛 // keep count 🧛
var clonedCount int var clonedCount int
@ -34,16 +26,16 @@ type Repository struct {
func main() { func main() {
// set app version // set app version
version = "1.1.5" version = "2.0.0"
// set appname for logger // set appname for logger
logger.SetAppName("gogitlabber") logger.SetAppName("gogitlabber")
// manage all argument magic // manage all argument magic and load configuration
manageArguments() config = manageArguments()
// set debugging // set debugging
logger.SetDebug(debug) logger.SetDebug(config.Debug)
// check for git // check for git
err := verifyGitAvailable() err := verifyGitAvailable()
@ -52,14 +44,19 @@ func main() {
} }
logger.Print("VALIDATION: git found in path", nil) logger.Print("VALIDATION: git found in path", nil)
// validate git backend is set
if config.GitBackend == "" {
logger.Fatal("Configuration error: git_backend is required (gitlab|gitea)", nil)
}
// make initial progressbar // make initial progressbar
if !debug { if !config.Debug {
progressBar() progressBar()
} }
// fetch repository information // fetch repository information
var repositories []Repository var repositories []Repository
switch gitBackend { switch config.GitBackend {
case "gitea": case "gitea":
repositories, err = fetchRepositoriesGitea() repositories, err = fetchRepositoriesGitea()
if err != nil { if err != nil {
@ -71,11 +68,11 @@ func main() {
logger.Fatal("Fetching repositories failed", err) logger.Fatal("Fetching repositories failed", err)
} }
default: default:
logger.Fatal("Fetching repositories failed", err) logger.Fatal(fmt.Sprintf("Unsupported git backend: %s (supported: gitlab|gitea)", config.GitBackend), nil)
} }
// manage found repositories // manage found repositories
checkoutRepositories(repositories, concurrency) checkoutRepositories(repositories)
printSummary() printSummary()
printPullErrorUnstaged(pullErrorMsgUnstaged) printPullErrorUnstaged(pullErrorMsgUnstaged)
printPullErrorUncommitted(pullErrorMsgUncommitted) printPullErrorUncommitted(pullErrorMsgUncommitted)

1
go.mod
View file

@ -14,4 +14,5 @@ require (
github.com/rivo/uniseg v0.4.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect
golang.org/x/sys v0.31.0 // indirect golang.org/x/sys v0.31.0 // indirect
golang.org/x/term v0.30.0 // indirect golang.org/x/term v0.30.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
) )

1
go.sum
View file

@ -27,5 +27,6 @@ golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

97
readme.md
View file

@ -1,16 +1,16 @@
# GoGitlabber # GoGitlabber
This project is inspired from the python application called gitlabber (https://github.com/ezbz/gitlabber). This project is inspired from the python application called gitlabber (https://github.com/ezbz/gitlabber).
It is mainly to learn Golang. But also to make something that specifically It is mainly to learn Golang. But also to make something that specifically solves my problem. 😆
solves my problem. 😆
It is definitely not as feature-rich as the original project... 😬 It is definitely not as feature-rich as the original project... 😬
The program can clone and pull all repositories you have access to on a selfhosted or SaaS provided Gitlab or Gitea server. The program can clone and pull all repositories you have access to on a selfhosted or SaaS provided Gitlab or Gitea
server.
It only supports the HTTP access method. It only supports the HTTP access method.
It will pull the repositories in a tree like structure same as on Gitlab or It will pull the repositories in a tree like structure same as on Gitlab or Gitea.
Gitea.
``` ```
root [http://gitlab.example.com] root [http://gitlab.example.com]
├── group1 [/group1] ├── group1 [/group1]
@ -23,78 +23,37 @@ root [http://gitlab.example.com]
└── subgroup3 [/group2/subgroup3] └── subgroup3 [/group2/subgroup3]
``` ```
# Example ## Config file
Gitea:
``` GitLab:
$ ./gogitlabber -backend=gitea -destination=$HOME/Documents -git-url=gitea.example.com
-git-api-token=supersecrettoken ```yaml
100% [====================] (30/30) [4s] ... # ~/.config/gogitlabber/gitlab.example.com.yaml
Summary: debug: false
Cloned repositories: 0 concurrency: 15
Pulled repositories: 30 git_host: "gitlab.example.net"
Errors: 0 git_token: "glpat-"
``` git_backend: "gitlab"
Gitlab: include_archived: "excluded"
``` destination: "$HOME/Documents"
$ ./gogitlabber -backend=gitlab -destination=$HOME/Documents -git-url=gitlab.example.com
-git-api-token=supersecrettoken
100% [====================] (30/30) [4s] ...
Summary:
Cloned repositories: 0
Pulled repositories: 30
Errors: 0
``` ```
# Usage
```
Usage of gogitlabber:
-archived string
To include archived repositories (any|excluded|exclusive)
example: -archived=any
env = GOGITLABBER_ARCHIVED
(default "excluded")
-backend string ## Usage
Specify git backend
example: -backend=gitlab
env = GOGITLABBER_BACKEND
-concurrency int ```bash
Specify repository concurrency gogitlabber -config=~/.config/gogitlabber/gitlab.example.com.yaml
example: -concurrency=15
env = GOGITLABBER_CONCURRENCY
(default 15)
-debug
Toggle debug mode
example: -debug=true
env = GOGITLABBER_DEBUG
(default false)
-destination string
Specify where to check the repositories out
example: -destination=$HOME/repos
env = GOGITLABBER_DESTINATION
(default "$HOME/Documents")
-git-api-token string
Specify API token
example: -git-api=glpat-xxxx
env = GIT_API_TOKEN
(default "")
-git-url string
Specify Git host
example: -git-url=gitlab.example.com
env = GIT_URL
(default "gitlab.com")
``` ```
# Access Token Permissions
## Gitea ## Access Token Permissions
### Gitea
Make sure the Gitea Access Token has at least the following permissions: Make sure the Gitea Access Token has at least the following permissions:
- user - read - user - read
- repository - read - repository - read
## Gitlab ### Gitlab
Make sure the Gitlab Access Token has the `api` scope. Make sure the Gitlab Access Token has the `api` scope.