From 02d022da5429681c10bef9e8b495e2311524256b Mon Sep 17 00:00:00 2001 From: Simon Cornet Date: Mon, 7 Jul 2025 22:14:42 +0200 Subject: [PATCH] feat: rewrite git.go and gitea.go --- cmd/gogitlabber/git.go | 393 +++++++++++++++++++++++--------------- cmd/gogitlabber/gitea.go | 263 +++++++++++++++++++------ cmd/gogitlabber/main.go | 18 +- cmd/gogitlabber/output.go | 21 +- 4 files changed, 461 insertions(+), 234 deletions(-) diff --git a/cmd/gogitlabber/git.go b/cmd/gogitlabber/git.go index 45494c9..4043b1d 100644 --- a/cmd/gogitlabber/git.go +++ b/cmd/gogitlabber/git.go @@ -3,198 +3,291 @@ package main import ( "fmt" "os/exec" + "path/filepath" "strings" "sync" "github.com/scornet256/go-logger" ) -// add a mutex to safely increment shared counters -var mu sync.Mutex +// collect git operations results +type GitOperationResult struct { + RepoName string + Operation string + Error error + ErrorType string +} -func checkoutRepositories(repositories []Repository) { +// collect git stats +type GitStats struct { + mu sync.Mutex + clonedCount int + pulledCount int + errorCount int + pullErrorMsgUnstaged []string + pullErrorMsgUncommitted []string +} + +// increment counters +func (stats *GitStats) IncrementCounter(operation string, repoPath string) { + + stats.mu.Lock() + defer stats.mu.Unlock() + + switch operation { + case "cloned": + stats.clonedCount++ + case "pulled": + stats.pulledCount++ + case "error": + stats.errorCount++ + case "unstaged": + stats.errorCount++ + stats.pullErrorMsgUnstaged = append(stats.pullErrorMsgUnstaged, repoPath) + case "uncommitted": + stats.errorCount++ + stats.pullErrorMsgUncommitted = append(stats.pullErrorMsgUncommitted, repoPath) + } +} + +// concurrent git operations +func CheckoutRepositories(repositories []Repository, stats *GitStats) { - // create a waitgroup + semaphore channel var wg sync.WaitGroup semaphore := make(chan struct{}, config.Concurrency) - // manage all repositories found for _, repo := range repositories { - - // increment waitgroup counter + acquire semaphore slot wg.Add(1) semaphore <- struct{}{} - // start go routine per repo go func(repo Repository) { - - // ensure we release the semaphore and close the goroutine defer func() { <-semaphore wg.Done() }() - // get repository name + create repo destination - repoName := string(repo.PathWithNamespace) - repoDestination := config.Destination + repoName - - // log activity - logger.Print("Starting on repository: "+repoName, nil) - - // make git url - url := fmt.Sprintf("https://%s-token:%s@%s/%s.git", config.GitBackend, config.GitToken, config.GitHost, repoName) - - // check current status of repoDestination - checkRepo := func(repoDestination string) string { - checkCmd := exec.Command("git", "-C", repoDestination, "remote", "-v") - checkOutput, _ := checkCmd.CombinedOutput() - logger.Print("Checking status for repository: "+repoName, nil) - - return string(checkOutput) - } - repoStatus := checkRepo(repoDestination) - - // report error if not cloned or pulled repository - // clone repository if it does not exist - switch { - case strings.Contains(string(repoStatus), "No such file or directory"): - - // log activity - logger.Print("Decided to clone repository: "+repoName, nil) - - // clone the repo - cloneRepository := func(repoDestination string, url string) (string, error) { - cloneCmd := exec.Command("git", "clone", url, repoDestination) - cloneOutput, err := cloneCmd.CombinedOutput() - - // set username and email - setGitUserName(repoName, repoDestination) - setGitUserMail(repoName, repoDestination) - - logger.Print("Cloning repository: "+repoName+" to "+repoDestination, nil) - - return string(cloneOutput), err - } - _, err := cloneRepository(repoDestination, url) - if err != nil { - logger.Print("ERROR: %v\n", err) - } - - // set a lock, increment counters, update progressbar and unlock - mu.Lock() - clonedCount++ - if !config.Debug { - _ = bar.Add(1) - } - mu.Unlock() - - // pull the latest - case strings.Contains(string(repoStatus), url): - logger.Print("Decided to pull repository: "+repoName, nil) - pullRepository(repoName, repoDestination) - - // set username and email - setGitUserName(repoName, repoDestination) - setGitUserMail(repoName, repoDestination) - - if !config.Debug { - _ = bar.Add(1) - } - - default: - logger.Print("ERROR: decided not to clone or pull repository: "+repoName, nil) - logger.Print("ERROR: this is why: "+repoStatus, nil) - - // set a lock, increment counters and unlock - mu.Lock() - errorCount++ - if !config.Debug { - _ = bar.Add(1) - } - mu.Unlock() - } + result := processRepository(repo) + handleResult(result, stats) }(repo) } - // wait for goroutines wg.Wait() } -func pullRepository(repoName string, repoDestination string) { +// manage single repo +func processRepository(repo Repository) GitOperationResult { + repoName := string(repo.PathWithNamespace) + repoDestination := filepath.Join(config.Destination, repoName) - // log activity - logger.Print("Pulling repository: "+repoName+" at "+repoDestination, nil) - - // find remote - findRemote := func(repoDestination string) (string, error) { - remoteCmd := exec.Command("git", "-C", repoDestination, "remote", "show") - remoteOutput, err := remoteCmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("finding remote: %v", err) - } - - logger.Print("Finding remote for repository: "+repoName+" at "+repoDestination, nil) - remote := strings.Split(strings.TrimSpace(string(remoteOutput)), "\n")[0] - logger.Print("Found remote; "+remote+" for repository: "+repoName+" at "+repoDestination, nil) - return remote, nil - } - remote, _ := findRemote(repoDestination) - - // pull repository - pullCmd := exec.Command("git", "-C", repoDestination, "pull", remote) - pullOutput, err := pullCmd.CombinedOutput() - - // set a lock, increment counters and unlock - mu.Lock() - pulledCount++ - mu.Unlock() + logger.Print("Starting on repository: "+repoName, nil) + // check repo status + status, err := checkRepositoryStatus(repoDestination) if err != nil { - - // set a lock, increment counters and unlock - mu.Lock() - errorCount++ - pulledCount-- - mu.Unlock() - - switch { - case strings.Contains(string(pullOutput), "You have unstaged changes"): - pullErrorMsgUnstaged = append(pullErrorMsgUnstaged, repoDestination) - logger.Print("Found unstaged changes in repository: "+repoName+" at "+repoDestination, nil) - - case strings.Contains(string(pullOutput), "Your index contains uncommitted changes"): - pullErrorMsgUncommitted = append(pullErrorMsgUncommitted, repoDestination) - logger.Print("Found uncommitted changes in repository: "+repoName+" at "+repoDestination, nil) - - default: - logger.Print("ERROR: pulling "+repoName, nil) + return GitOperationResult{ + RepoName: repoName, + Operation: "error", + Error: fmt.Errorf("checking repository status: %w", err), } } - // log activity - logger.Print("Pulled repository: "+repoName+" at "+repoDestination, nil) + gitURL := buildGitURL(repoName) + + switch { + case strings.Contains(status, "No such file or directory"): + return cloneRepository(repoName, repoDestination, gitURL) + case strings.Contains(status, gitURL): + return pullRepository(repoName, repoDestination) + default: + return GitOperationResult{ + RepoName: repoName, + Operation: "error", + Error: fmt.Errorf("unexpected repository status: %s", status), + } + } } -// function to set the git user name -func setGitUserName(repoName string, repoDestination string) { +// check repo status +func checkRepositoryStatus(repoDestination string) (string, error) { + cmd := exec.Command("git", "-C", repoDestination, "remote", "-v") + output, err := cmd.CombinedOutput() - gitUserNameCmd := exec.Command("git", "-C", repoDestination, "config", "user.name", config.GitUserName) - _, err := gitUserNameCmd.CombinedOutput() - if err != nil { - logger.Print("ERROR: %v\n", err) + // If directory doesn't exist, that's expected for new clones + if err != nil && strings.Contains(string(output), "No such file or directory") { + return string(output), nil } - logger.Print("Setting git username for: "+repoName, nil) + return string(output), err } -// function to set the git user mail -func setGitUserMail(repoName string, repoDestination string) { +// craft git url with auth +func buildGitURL(repoName string) string { + return fmt.Sprintf("https://%s-token:%s@%s/%s.git", + config.GitBackend, config.GitToken, config.GitHost, repoName) +} + +// clone new repository +func cloneRepository(repoName, repoDestination, gitURL string) GitOperationResult { + logger.Print("Cloning repository: "+repoName, nil) + + cmd := exec.Command("git", "clone", gitURL, repoDestination) + output, err := cmd.CombinedOutput() - gitUserMailCmd := exec.Command("git", "-C", repoDestination, "config", "user.mail", config.GitUserMail) - _, err := gitUserMailCmd.CombinedOutput() if err != nil { - logger.Print("ERROR: %v\n", err) + return GitOperationResult{ + RepoName: repoName, + Operation: "error", + Error: fmt.Errorf("cloning repository: %w, output: %s", err, string(output)), + } } - logger.Print("Setting git email for: "+repoName, nil) + // set git user config + if err := setGitUserConfig(repoName, repoDestination); err != nil { + logger.Print("WARNING: failed to set git user config: "+err.Error(), nil) + } + + return GitOperationResult{ + RepoName: repoName, + Operation: "cloned", + } +} + +// pull repo +func pullRepository(repoName, repoDestination string) GitOperationResult { + logger.Print("Pulling repository: "+repoName, nil) + + // Find remote + remote, err := findRemote(repoDestination) + if err != nil { + return GitOperationResult{ + RepoName: repoName, + Operation: "error", + Error: fmt.Errorf("finding remote: %w", err), + } + } + + // pull changes + cmd := exec.Command("git", "-C", repoDestination, "pull", remote) + output, err := cmd.CombinedOutput() + + if err != nil { + errorType := classifyPullError(string(output)) + return GitOperationResult{ + RepoName: repoName, + Operation: "error", + Error: fmt.Errorf("pulling repository: %w, output: %s", err, string(output)), + ErrorType: errorType, + } + } + + // set git user configuration + if err := setGitUserConfig(repoName, repoDestination); err != nil { + logger.Print("WARNING: failed to set git user config: "+err.Error(), nil) + } + + return GitOperationResult{ + RepoName: repoName, + Operation: "pulled", + } +} + +// find remote for repo +func findRemote(repoDestination string) (string, error) { + cmd := exec.Command("git", "-C", repoDestination, "remote", "show") + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("getting remote: %w", err) + } + + remotes := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(remotes) == 0 { + return "", fmt.Errorf("no remotes found") + } + + return remotes[0], nil +} + +// manage pull error +func classifyPullError(output string) string { + switch { + case strings.Contains(output, "You have unstaged changes"): + return "unstaged" + case strings.Contains(output, "Your index contains uncommitted changes"): + return "uncommitted" + default: + return "other" + } +} + +// set git user config +func setGitUserConfig(repoName, repoDestination string) error { + + // git user name + if err := setGitUserName(repoName, repoDestination); err != nil { + return fmt.Errorf("setting username: %w", err) + } + + // git user mail + if err := setGitUserEmail(repoName, repoDestination); err != nil { + return fmt.Errorf("setting email: %w", err) + } + + return nil +} + +// set git user name +func setGitUserName(repoName, repoDestination string) error { + + cmd := exec.Command("git", "-C", repoDestination, "config", "user.name", config.GitUserName) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("setting git username: %w, output: %s", err, string(output)) + } + + logger.Print("Set git username for: "+repoName, nil) + return nil +} + +// set git user mail +func setGitUserEmail(repoName, repoDestination string) error { + + cmd := exec.Command("git", "-C", repoDestination, "config", "user.email", config.GitUserMail) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("setting git email: %w, output: %s", err, string(output)) + } + + logger.Print("Set git email for: "+repoName, nil) + return nil +} + +// manage results +func handleResult(result GitOperationResult, stats *GitStats) { + + switch result.Operation { + case "cloned": + stats.IncrementCounter("cloned", "") + logger.Print("Successfully cloned: "+result.RepoName, nil) + + case "pulled": + stats.IncrementCounter("pulled", "") + logger.Print("Successfully pulled: "+result.RepoName, nil) + + case "error": + if result.ErrorType == "unstaged" { + stats.IncrementCounter("unstaged", result.RepoName) + logger.Print("Found unstaged changes in: "+result.RepoName, nil) + } else if result.ErrorType == "uncommitted" { + stats.IncrementCounter("uncommitted", result.RepoName) + logger.Print("Found uncommitted changes in: "+result.RepoName, nil) + } else { + stats.IncrementCounter("error", "") + logger.Print("ERROR processing "+result.RepoName+": "+result.Error.Error(), nil) + } + } + + // update progress bar + if !config.Debug { + _ = bar.Add(1) + } } diff --git a/cmd/gogitlabber/gitea.go b/cmd/gogitlabber/gitea.go index c0d5101..1604089 100644 --- a/cmd/gogitlabber/gitea.go +++ b/cmd/gogitlabber/gitea.go @@ -1,99 +1,238 @@ package main import ( + "context" "encoding/json" "fmt" "net/http" + "net/url" + "strconv" + "time" "github.com/scornet256/go-logger" ) -func fetchRepositoriesGitea() ([]Repository, error) { +// giteaClient struct +type GiteaClient struct { + httpClient *http.Client + baseURL string + token string +} - type GiteaRepository struct { - Name string `json:"name"` - FullName string `json:"full_name"` +// gitea repo information +type GiteaRepository struct { + Name string `json:"name"` + FullName string `json:"full_name"` +} + +// gitea api options +type GiteaAPIOptions struct { + Visibility string + IncludeArchived string + Sort string + Limit int + Page int +} + +// gitea api clien +func NewGiteaClient(baseURL, token string) *GiteaClient { + return &GiteaClient{ + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + baseURL: baseURL, + token: token, + } +} + +// fetch gitea repos +func FetchRepositoriesGitea() ([]Repository, error) { + client := NewGiteaClient(config.GitHost, config.GitToken) + + options := GiteaAPIOptions{ + Visibility: "all", + IncludeArchived: config.IncludeArchived, + Sort: "alpha", + Limit: 100, + Page: 1, } - // default options - visibility := "visibility=all" - perpage := "limit=100" - sort := "sort=alpha" - - // configure archived options - var archived string - switch config.IncludeArchived { - case "excluded": - archived = "&archived=false" - case "only": - archived = "&archived=true" - default: - archived = "" - } - - url := fmt.Sprintf("https://%s/api/v1/user/repos?%s&%s&%s%s", - config.GitHost, visibility, sort, perpage, archived) - - logger.Print("HTTP: Creating API request", nil) - req, err := http.NewRequest("GET", url, nil) + repositories, err := client.fetchAllRepositories(context.Background(), options) if err != nil { - return nil, fmt.Errorf("ERROR: creating request: %v", err) + return nil, fmt.Errorf("fetching repositories: %w", err) } - logger.Print("HTTP: Adding Authorization header to API request", nil) - req.Header.Set("Authorization", fmt.Sprintf("token %s", config.GitToken)) + if len(repositories) == 0 { + return repositories, fmt.Errorf("no repositories found") + } - logger.Print("HTTP: Making request", nil) - client := &http.Client{} - resp, err := client.Do(req) + // update progress bar + if err := updateProgressBar(len(repositories)); err != nil { + logger.Print("WARNING: failed to update progress bar: "+err.Error(), nil) + } + + logger.Print(fmt.Sprintf("Successfully fetched %d repositories", len(repositories)), nil) + return repositories, nil +} + +// fetch all repos with pagination +func (c *GiteaClient) fetchAllRepositories(ctx context.Context, options GiteaAPIOptions) ([]Repository, error) { + var allRepositories []Repository + + for { + giteaRepos, hasMore, err := c.fetchRepositoryPage(ctx, options) + if err != nil { + return nil, fmt.Errorf("fetching page %d: %w", options.Page, err) + } + + // convert gitea repositories to repo type + repositories := convertGiteaRepositories(giteaRepos) + allRepositories = append(allRepositories, repositories...) + + if !hasMore { + break + } + + options.Page++ + } + + return allRepositories, nil +} + +// fetch single page of repo +func (c *GiteaClient) fetchRepositoryPage(ctx context.Context, options GiteaAPIOptions) ([]GiteaRepository, bool, error) { + apiURL, err := c.buildAPIURL(options) if err != nil { - return nil, fmt.Errorf("ERROR: making request: %v", err) + return nil, false, fmt.Errorf("building API URL: %w", err) } + req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil) + if err != nil { + return nil, false, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("token %s", c.token)) + req.Header.Set("Accept", "application/json") + + logger.Print("Making API request to: "+apiURL, nil) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, false, fmt.Errorf("making request: %w", err) + } defer func() { - if err := resp.Body.Close(); err != nil { - logger.Fatal("HTTP: Error closing response body", err) + if closeErr := resp.Body.Close(); closeErr != nil { + logger.Print("WARNING: failed to close response body: "+closeErr.Error(), nil) } }() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("ERROR: API request failed with status: %d", resp.StatusCode) + return nil, false, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, resp.Status) } - logger.Print("HTTP: Decoding JSON response", nil) - // first decode into gitearepository slice var giteaRepos []GiteaRepository if err := json.NewDecoder(resp.Body).Decode(&giteaRepos); err != nil { - return nil, fmt.Errorf("ERROR: decoding response: %v", err) + return nil, false, fmt.Errorf("decoding JSON response: %w", err) } - // convert to repository slice + // check for more pages + hasMore := len(giteaRepos) == options.Limit + + return giteaRepos, hasMore, nil +} + +// build final api url +func (c *GiteaClient) buildAPIURL(options GiteaAPIOptions) (string, error) { + baseURL := fmt.Sprintf("https://%s/api/v1/user/repos", c.baseURL) + + u, err := url.Parse(baseURL) + if err != nil { + return "", fmt.Errorf("parsing base URL: %w", err) + } + + query := u.Query() + query.Set("visibility", options.Visibility) + query.Set("sort", options.Sort) + query.Set("limit", strconv.Itoa(options.Limit)) + query.Set("page", strconv.Itoa(options.Page)) + + // handle archived + switch options.IncludeArchived { + case "excluded": + query.Set("archived", "false") + case "only": + query.Set("archived", "true") + } + + u.RawQuery = query.Encode() + return u.String(), nil +} + +// convert gitea repos to repo type +func convertGiteaRepositories(giteaRepos []GiteaRepository) []Repository { repositories := make([]Repository, len(giteaRepos)) - for repo, giteaRepo := range giteaRepos { - repositories[repo] = Repository{ + for i, giteaRepo := range giteaRepos { + repositories[i] = Repository{ Name: giteaRepo.Name, PathWithNamespace: giteaRepo.FullName, } } - - if len(repositories) < 1 { - return repositories, fmt.Errorf("ERROR: no repositories found") - } - repoCount := len(repositories) - - logger.Print("BAR: Resetting the progressbar", nil) - if !config.Debug { - err = bar.Set(0) - if err != nil { - logger.Fatal("Could not reset the progressbar", err) - } - } - - logger.Print("BAR: Increasing the max value of the progressbar", nil) - if !config.Debug { - bar.ChangeMax(repoCount) - } - - logger.Print("HTTP: Returning repositories found", nil) - return repositories, nil + return repositories +} + +// update progressbar +func updateProgressBar(repoCount int) error { + if config.Debug { + return nil // Skip progress bar in debug mode + } + + logger.Print("Resetting progress bar", nil) + if err := bar.Set(0); err != nil { + return fmt.Errorf("resetting progress bar: %w", err) + } + + logger.Print("Setting progress bar maximum", nil) + bar.ChangeMax(repoCount) + + return nil +} + +// connection validation +func (c *GiteaClient) ValidateConnection(ctx context.Context) error { + apiURL := fmt.Sprintf("https://%s/api/v1/user", c.baseURL) + + req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil) + if err != nil { + return fmt.Errorf("creating validation request: %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("token %s", c.token)) + req.Header.Set("Accept", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("making validation request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("invalid or expired token") + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("API validation failed with status %d", resp.StatusCode) + } + + return nil +} + +// simply count git repos only +func (c *GiteaClient) GetRepositoryCount(ctx context.Context, options GiteaAPIOptions) (int, error) { + + repos, err := c.fetchAllRepositories(ctx, options) + if err != nil { + return 0, fmt.Errorf("counting repositories: %w", err) + } + + return len(repos), nil } diff --git a/cmd/gogitlabber/main.go b/cmd/gogitlabber/main.go index 306f7ee..63cc9bb 100644 --- a/cmd/gogitlabber/main.go +++ b/cmd/gogitlabber/main.go @@ -10,13 +10,6 @@ import ( var version string var config *Config -// keep count 🧛 -var clonedCount int -var errorCount int -var pulledCount int -var pullErrorMsgUnstaged []string -var pullErrorMsgUncommitted []string - // repository data type Repository struct { Name string `json:"name"` @@ -58,7 +51,7 @@ func main() { var repositories []Repository switch config.GitBackend { case "gitea": - repositories, err = fetchRepositoriesGitea() + repositories, err = FetchRepositoriesGitea() if err != nil { logger.Fatal("Fetching repositories failed", err) } @@ -72,8 +65,9 @@ func main() { } // manage found repositories - checkoutRepositories(repositories) - printSummary() - printPullErrorUnstaged(pullErrorMsgUnstaged) - printPullErrorUncommitted(pullErrorMsgUncommitted) + stats := &GitStats{} + CheckoutRepositories(repositories, stats) + printSummary(stats) + printPullErrorUnstaged(stats) + printPullErrorUncommitted(stats) } diff --git a/cmd/gogitlabber/output.go b/cmd/gogitlabber/output.go index 156b3d8..a1a7268 100644 --- a/cmd/gogitlabber/output.go +++ b/cmd/gogitlabber/output.go @@ -35,31 +35,32 @@ func progressBar() { logger.Print("Initialize progressbar", nil) } -func printSummary() { +func printSummary(stats *GitStats) { + // print stats fmt.Println("") fmt.Printf( "Summary:\n"+ " Cloned repositories: %v\n"+ " Pulled repositories: %v\n"+ " Errors: %v\n", - clonedCount, - pulledCount, - errorCount, + stats.clonedCount, + stats.pulledCount, + stats.errorCount, ) } -func printPullErrorUnstaged(pullErrorMsgUnstaged []string) { - if len(pullErrorMsgUnstaged) > 0 { - for _, repo := range pullErrorMsgUnstaged { +func printPullErrorUnstaged(stats *GitStats) { + if len(stats.pullErrorMsgUnstaged) > 0 { + for _, repo := range stats.pullErrorMsgUnstaged { fmt.Printf("❕%s has unstaged changes.\n", repo) } } } -func printPullErrorUncommitted(pullErrorMsgUncommitted []string) { - if len(pullErrorMsgUncommitted) > 0 { - for _, repo := range pullErrorMsgUncommitted { +func printPullErrorUncommitted(stats *GitStats) { + if len(stats.pullErrorMsgUncommitted) > 0 { + for _, repo := range stats.pullErrorMsgUncommitted { fmt.Printf("❕%s has uncommitted changes.\n", repo) } }