golang/gogitlabber
golang/gogitlabber
3
0
Fork 0
Code Issues 1 Pull requests Releases 1 Activity

feat: use yaml config instead of args or envs

Browse source
This commit is contained in:
Simon Cornet 2025-07-07 10:50:31 +02:00
commit 241fe8c53b
8 changed files with 186 additions and 211 deletions
Download patch file Download diff file Expand all files Collapse all files

14
cmd/gogitlabber/git.go
View file

@ -12,11 +12,11 @@ import (
// add a mutex to safely increment shared counters
var mu sync.Mutex
func checkoutRepositories(repositories []Repository, concurrency int) {
func checkoutRepositories(repositories []Repository) {
// create a waitgroup + semaphore channel
var wg sync.WaitGroup
semaphore := make(chan struct{}, concurrency)
semaphore := make(chan struct{}, config.Concurrency)
// manage all repositories found
for _, repo := range repositories {
@ -36,13 +36,13 @@ func checkoutRepositories(repositories []Repository, concurrency int) {
// get repository name + create repo destination
repoName := string(repo.PathWithNamespace)
repoDestination := repoDestinationPre + repoName
repoDestination := config.Destination + repoName
// log activity
logger.Print("Starting on repository: "+repoName, nil)
// make git url
url := fmt.Sprintf("https://%s-token:%s@%s/%s.git", gitBackend, gitToken, gitHost, repoName)
url := fmt.Sprintf("https://%s-token:%s@%s/%s.git", config.GitBackend, config.GitToken, config.GitHost, repoName)
// check current status of repoDestination
checkRepo := func(repoDestination string) string {
@ -78,7 +78,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) {
// set a lock, increment counters, update progressbar and unlock
mu.Lock()
clonedCount++
if !debug {
if !config.Debug {
_ = bar.Add(1)
}
mu.Unlock()
@ -87,7 +87,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) {
case strings.Contains(string(repoStatus), url):
logger.Print("Decided to pull repository: "+repoName, nil)
pullRepository(repoName, repoDestination)
if !debug {
if !config.Debug {
_ = bar.Add(1)
}
@ -98,7 +98,7 @@ func checkoutRepositories(repositories []Repository, concurrency int) {
// set a lock, increment counters and unlock
mu.Lock()
errorCount++
if !debug {
if !config.Debug {
_ = bar.Add(1)
}
mu.Unlock()

10
cmd/gogitlabber/gitea.go
View file

@ -22,7 +22,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
// configure archived options
var archived string
switch includeArchived {
switch config.IncludeArchived {
case "excluded":
archived = "&archived=false"
case "only":
@ -32,7 +32,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
}
url := fmt.Sprintf("https://%s/api/v1/user/repos?%s&%s&%s%s",
gitHost, visibility, sort, perpage, archived)
config.GitHost, visibility, sort, perpage, archived)
logger.Print("HTTP: Creating API request", nil)
req, err := http.NewRequest("GET", url, nil)
@ -41,7 +41,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
}
logger.Print("HTTP: Adding Authorization header to API request", nil)
req.Header.Set("Authorization", fmt.Sprintf("token %s", gitToken))
req.Header.Set("Authorization", fmt.Sprintf("token %s", config.GitToken))
logger.Print("HTTP: Making request", nil)
client := &http.Client{}
@ -82,7 +82,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
repoCount := len(repositories)
logger.Print("BAR: Resetting the progressbar", nil)
if !debug {
if !config.Debug {
err = bar.Set(0)
if err != nil {
logger.Fatal("Could not reset the progressbar", err)
@ -90,7 +90,7 @@ func fetchRepositoriesGitea() ([]Repository, error) {
}
logger.Print("BAR: Increasing the max value of the progressbar", nil)
if !debug {
if !config.Debug {
bar.ChangeMax(repoCount)
}

10
cmd/gogitlabber/gitlab.go
View file

@ -17,7 +17,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
// configure archived options
var archived string
switch includeArchived {
switch config.IncludeArchived {
case "excluded":
archived = "&archived=false"
case "only":
@ -27,7 +27,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
}
url := fmt.Sprintf("https://%s/api/v4/projects?%s&%s&%s%s",
gitHost, membership, order, perpage, archived)
config.GitHost, membership, order, perpage, archived)
logger.Print("HTTP: Creating API request", nil)
req, err := http.NewRequest("GET", url, nil)
@ -36,7 +36,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
}
logger.Print("HTTP: Adding PRIVATE-TOKEN header to API request", nil)
req.Header.Set("PRIVATE-TOKEN", gitToken)
req.Header.Set("PRIVATE-TOKEN", config.GitToken)
logger.Print("HTTP: Making request", nil)
client := &http.Client{}
@ -67,7 +67,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
repoCount := len(repositories)
logger.Print("BAR: Resetting the progressbar", nil)
if !debug {
if !config.Debug {
err = bar.Set(0)
if err != nil {
logger.Fatal("Could not reset the progressbar", err)
@ -75,7 +75,7 @@ func fetchRepositoriesGitlab() ([]Repository, error) {
}
logger.Print("BAR: Increasing the max value of the progressbar", nil)
if !debug {
if !config.Debug {
bar.ChangeMax(repoCount)
}

223
cmd/gogitlabber/input.go
View file

@ -4,151 +4,168 @@ import (
"flag"
"fmt"
"os"
"strconv"
"path/filepath"
"strings"
"github.com/scornet256/go-logger"
"gopkg.in/yaml.v3"
)
// set default values and override values from environment variables
func setDefaultsFromEnv() {
// config struct for config
type Config struct {
Concurrency int `yaml:"concurrency"`
Debug bool `yaml:"debug"`
Destination string `yaml:"destination"`
GitBackend string `yaml:"git_backend"`
GitHost string `yaml:"git_host"`
GitToken string `yaml:"git_token"`
IncludeArchived string `yaml:"include_archived"`
}
// set default values
debug = false
concurrency = 15
gitHost = "gitlab.com"
gitToken = ""
includeArchived = "excluded"
repoDestinationPre = "$HOME/Documents"
// setdefaults sets default values for the configuration
func (conf *Config) setDefaults() {
conf.Concurrency = 15
conf.Debug = false
conf.Destination = "$HOME/Documents"
conf.GitBackend = ""
conf.GitHost = "gitlab.com"
conf.GitToken = ""
conf.IncludeArchived = "excluded"
}
// override with environment variables if present
if envDebug := os.Getenv("GOGITLABBER_DEBUG"); envDebug != "" {
if debugVal, err := strconv.ParseBool(envDebug); err == nil {
debug = debugVal
} else {
logger.Print("Warning: Invalid debug value in environment, using default", nil)
// expand variable paths
func expandPath(path string) string {
// expand environment variables like $home
expanded := os.ExpandEnv(path)
// expand ~
if strings.HasPrefix(expanded, "~/") {
home, err := os.UserHomeDir()
if err != nil {
return expanded
}
expanded = filepath.Join(home, expanded[2:])
} else if expanded == "~" {
home, err := os.UserHomeDir()
if err != nil {
return expanded
}
expanded = home
}
if envBackend := os.Getenv("GOGITLABBER_BACKEND"); envBackend != "" {
gitBackend = envBackend
return filepath.Clean(expanded)
}
// loadconfig from yaml file
func loadConfig(configPath string) (*Config, error) {
config := &Config{}
config.setDefaults()
// check if config file exists
if _, err := os.Stat(configPath); os.IsNotExist(err) {
return nil, fmt.Errorf("config file not found: %s", configPath)
}
if envToken := os.Getenv("GIT_API_TOKEN"); envToken != "" {
gitToken = envToken
// read config
data, err := os.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
if envHost := os.Getenv("GIT_URL"); envHost != "" {
gitHost = envHost
// parse yaml
if err := yaml.Unmarshal(data, config); err != nil {
return nil, fmt.Errorf("failed to parse config file: %w", err)
}
if envRepoDest := os.Getenv("GOGITLABBER_DESTINATION"); envRepoDest != "" {
repoDestinationPre = envRepoDest
return config, nil
}
// validateConfig validates the configuration values
func (conf *Config) validateConfig() error {
// validate required parameters
if conf.GitToken == "" {
return fmt.Errorf("git_token is required")
}
if envConcurrency := os.Getenv("GOGITLABBER_CONCURRENCY"); envConcurrency != "" {
if concurrencyVal, err := strconv.Atoi(envConcurrency); err == nil {
concurrency = concurrencyVal
} else {
logger.Print("Warning: Invalid concurrency value in environment, using default", nil)
}
}
if envArchived := os.Getenv("GOGITLABBER_ARCHIVED"); envArchived != "" {
switch envArchived {
// validate archived option
switch conf.IncludeArchived {
case "any", "exclusive", "excluded":
includeArchived = envArchived
default:
logger.Print("Warning: Invalid archived value in environment, using default", nil)
return fmt.Errorf("invalid include_archived option: %s (must be any|excluded|exclusive)", conf.IncludeArchived)
}
// validate concurrency
if conf.Concurrency < 1 {
return fmt.Errorf("concurrency must be greater than 0")
}
return nil
}
// process config after loading
func (conf *Config) processConfig() {
// expand path variables
conf.Destination = expandPath(conf.Destination)
// add trailing slash if not provided
if !strings.HasSuffix(conf.Destination, "/") {
conf.Destination += "/"
}
}
func manageArguments() {
// log active config
func (conf *Config) logConfig(configPath string) {
logger.Print("Configuration: Using config file: "+configPath, nil)
logger.Print("Configuration: Using host: "+conf.GitHost, nil)
logger.Print("Configuration: Using destination: "+conf.Destination, nil)
logger.Print("Configuration: Using concurrency: "+fmt.Sprintf("%d", conf.Concurrency), nil)
logger.Print("Configuration: Using archived option: "+conf.IncludeArchived, nil)
if conf.Debug {
logger.Print("Configuration: Debug mode enabled", nil)
}
}
// set defaults from environment variables
setDefaultsFromEnv()
// manage arguments
func manageArguments() *Config {
// define flags (which will override environment variables)
var archivedFlag = flag.String(
"archived",
includeArchived,
"To include archived repositories (any|excluded|exclusive)\n example: -archived=any\nenv = GOGITLABBER_ARCHIVED\n")
defaultConfigPath := "./$HOME./gogitlabber.yaml"
var concurrencyFlag = flag.Int(
"concurrency",
concurrency,
"Specify repository concurrency\n example: -concurrency=15\nenv = GOGITLABBER_CONCURRENCY\n")
var destinationFlag = flag.String(
"destination",
repoDestinationPre,
"Specify where to check the repositories out\n example: -destination=$HOME/repos\nenv = GOGITLABBER_DESTINATION\n")
var backendFlag = flag.String(
"backend",
gitBackend,
"Specify git backend\n example -backend=gitlab\nenv = GOGITLABBER_BACKEND\n")
var hostFlag = flag.String(
"git-url",
gitHost,
"Specify GitLab/Gitea host\n example: -git-url=gitlab.com\nenv = GIT_URL\n")
var tokenFlag = flag.String(
"git-api-token",
gitToken,
"Specify GitLab/Gitea API token\n example: -git-api=glpat-xxxx\nenv = GIT_API_TOKEN\n")
var debugFlag = flag.Bool(
"debug",
debug,
"Toggle debug mode\n example: -debug=true\nenv = GOGITLABBER_DEBUG\n")
// Define only the config file flag
configFileFlag := flag.String(
"config",
defaultConfigPath,
"Specify config file path (YAML)\n example: -config=./config/app.yaml")
versionFlag := flag.Bool("version", false, "Print the version and exit")
flag.Parse()
// print version
if *versionFlag {
fmt.Println(version)
os.Exit(0)
}
// override with flag values (higher precedence)
concurrency = *concurrencyFlag
debug = *debugFlag
gitHost = *hostFlag
gitToken = *tokenFlag
gitBackend = *backendFlag
includeArchived = *archivedFlag
repoDestinationPre = *destinationFlag
configPath := *configFileFlag
// add slash 🎩🎸 if not provided
if !strings.HasSuffix(repoDestinationPre, "/") {
repoDestinationPre += "/"
}
// validate required parameters
if gitToken == "" {
// Load configuration from YAML file
config, err := loadConfig(configPath)
if err != nil {
flag.Usage()
logger.Fatal("Configuration: API Token not found", nil)
logger.Fatal("Configuration error: "+err.Error(), nil)
}
// validate archived option
switch includeArchived {
case "any", "exclusive", "excluded":
default:
// Process configuration
config.processConfig()
// Validate configuration
if err := config.validateConfig(); err != nil {
flag.Usage()
logger.Fatal("Configuration: Invalid archive option: "+includeArchived, nil)
logger.Fatal("Configuration validation error: "+err.Error(), nil)
}
// log configuration
logger.Print("Configuration: Using host: "+gitHost, nil)
logger.Print("Configuration: Using destination: "+repoDestinationPre, nil)
logger.Print("Configuration: Using concurrency: "+strconv.Itoa(concurrency), nil)
logger.Print("Configuration: Using archived option: "+includeArchived, nil)
if debug {
logger.Print("Configuration: Debug mode enabled", nil)
}
// Log configuration
config.logConfig(configPath)
return config
}

35
cmd/gogitlabber/main.go
View file

@ -1,22 +1,14 @@
package main
import (
"fmt"
"github.com/scornet256/go-logger"
)
// version
var version string
// userdata
var concurrency int
var debug bool
var includeArchived string
var repoDestinationPre string
// git
var gitHost string
var gitToken string
var gitBackend string
var config *Config
// keep count 🧛
var clonedCount int
@ -34,16 +26,16 @@ type Repository struct {
func main() {
// set app version
version = "1.1.5"
version = "2.0.0"
// set appname for logger
logger.SetAppName("gogitlabber")
// manage all argument magic
manageArguments()
// manage all argument magic and load configuration
config = manageArguments()
// set debugging
logger.SetDebug(debug)
logger.SetDebug(config.Debug)
// check for git
err := verifyGitAvailable()
@ -52,14 +44,19 @@ func main() {
}
logger.Print("VALIDATION: git found in path", nil)
// validate git backend is set
if config.GitBackend == "" {
logger.Fatal("Configuration error: git_backend is required (gitlab|gitea)", nil)
}
// make initial progressbar
if !debug {
if !config.Debug {
progressBar()
}
// fetch repository information
var repositories []Repository
switch gitBackend {
switch config.GitBackend {
case "gitea":
repositories, err = fetchRepositoriesGitea()
if err != nil {
@ -71,11 +68,11 @@ func main() {
logger.Fatal("Fetching repositories failed", err)
}
default:
logger.Fatal("Fetching repositories failed", err)
logger.Fatal(fmt.Sprintf("Unsupported git backend: %s (supported: gitlab|gitea)", config.GitBackend), nil)
}
// manage found repositories
checkoutRepositories(repositories, concurrency)
checkoutRepositories(repositories)
printSummary()
printPullErrorUnstaged(pullErrorMsgUnstaged)
printPullErrorUncommitted(pullErrorMsgUncommitted)

1
go.mod
View file

@ -14,4 +14,5 @@ require (
github.com/rivo/uniseg v0.4.7 // indirect
golang.org/x/sys v0.31.0 // indirect
golang.org/x/term v0.30.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

1
go.sum
View file

@ -27,5 +27,6 @@ golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

97
readme.md
View file

@ -1,16 +1,16 @@
# GoGitlabber
This project is inspired from the python application called gitlabber (https://github.com/ezbz/gitlabber).
It is mainly to learn Golang. But also to make something that specifically
solves my problem. 😆
It is mainly to learn Golang. But also to make something that specifically solves my problem. 😆
It is definitely not as feature-rich as the original project... 😬
The program can clone and pull all repositories you have access to on a selfhosted or SaaS provided Gitlab or Gitea server.
The program can clone and pull all repositories you have access to on a selfhosted or SaaS provided Gitlab or Gitea
server.
It only supports the HTTP access method.
It will pull the repositories in a tree like structure same as on Gitlab or
Gitea.
It will pull the repositories in a tree like structure same as on Gitlab or Gitea.
```
root [http://gitlab.example.com]
├── group1 [/group1]
@ -23,78 +23,37 @@ root [http://gitlab.example.com]
└── subgroup3 [/group2/subgroup3]
```
# Example
Gitea:
```
$ ./gogitlabber -backend=gitea -destination=$HOME/Documents -git-url=gitea.example.com
-git-api-token=supersecrettoken
100% [====================] (30/30) [4s] ...
Summary:
Cloned repositories: 0
Pulled repositories: 30
Errors: 0
```
Gitlab:
```
$ ./gogitlabber -backend=gitlab -destination=$HOME/Documents -git-url=gitlab.example.com
-git-api-token=supersecrettoken
100% [====================] (30/30) [4s] ...
Summary:
Cloned repositories: 0
Pulled repositories: 30
Errors: 0
## Config file
GitLab:
```yaml
# ~/.config/gogitlabber/gitlab.example.com.yaml
debug: false
concurrency: 15
git_host: "gitlab.example.net"
git_token: "glpat-"
git_backend: "gitlab"
include_archived: "excluded"
destination: "$HOME/Documents"
```
# Usage
```
Usage of gogitlabber:
-archived string
To include archived repositories (any|excluded|exclusive)
example: -archived=any
env = GOGITLABBER_ARCHIVED
(default "excluded")
-backend string
Specify git backend
example: -backend=gitlab
env = GOGITLABBER_BACKEND
## Usage
-concurrency int
Specify repository concurrency
example: -concurrency=15
env = GOGITLABBER_CONCURRENCY
(default 15)
-debug
Toggle debug mode
example: -debug=true
env = GOGITLABBER_DEBUG
(default false)
-destination string
Specify where to check the repositories out
example: -destination=$HOME/repos
env = GOGITLABBER_DESTINATION
(default "$HOME/Documents")
-git-api-token string
Specify API token
example: -git-api=glpat-xxxx
env = GIT_API_TOKEN
(default "")
-git-url string
Specify Git host
example: -git-url=gitlab.example.com
env = GIT_URL
(default "gitlab.com")
```bash
gogitlabber -config=~/.config/gogitlabber/gitlab.example.com.yaml
```
# Access Token Permissions
## Gitea
## Access Token Permissions
### Gitea
Make sure the Gitea Access Token has at least the following permissions:
- user - read
- repository - read
## Gitlab
### Gitlab
Make sure the Gitlab Access Token has the `api` scope.