commit 309b240cdc27521e99d27d7b69ecb053374f64f5 Author: sonix Date: Sun Mar 22 09:43:45 2026 +0000 Initial commit diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8b188b5 --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +# Azure DevOps Configuration +AZURE_ORG=your-organization +AZURE_PAT=your-personal-access-token + +# Forgejo Configuration +FORGEJO_URL=https://git.yourdomain.com +FORGEJO_TOKEN=your-forgejo-token +FORGEJO_OWNER=target-username-or-org + +# Migration Settings +MIRROR_INTERVAL=8h +CONCURRENT_MIGRATIONS=3 +DRY_RUN=false \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5b90e79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# ---> Go +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..77014dc --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2026 sonix + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..48af95e --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# azure_migrate + +Migrate from Azure to Forgejo + +# Usage Instructions +Create your .env file: + +bash + +`cp .env.example .env` + +# List repositories first (dry view): + +bash + +`go run main.go list` + +# Run migration (dry run): + +bash + +`go run main.go migrate --dry-run` + + +# Execute actual migration: + +bash + +`go run main.go migrate` + +## Key Features ## + +# LFS Support: +The LFS: true flag in the migration request tells Forgejo to mirror LFS objects automatically. + +# Concurrent Processing: +Uses worker pools to migrate multiple repositories simultaneously (configurable via CONCURRENT_MIGRATIONS). + +# Detailed Error Handling: +Errors include repository name, project context, and specific failure reasons +Non-fatal errors don't stop the entire process +Summary report at the end + +# Azure Authentication: +Properly handles Azure DevOps PAT authentication via Basic Auth header. + +# Dry Run Mode: +Test the migration without creating any repositories. + +# Mirror Configuration: +Sets up Forgejo to periodically sync from Azure DevOps using the Interval setting. + + +## Important Notes ## +# Network Access: +Ensure the machine running this has access to both Azure DevOps and your Forgejo instance. + +# Storage: +Forgejo must have sufficient storage for LFS objects. + +# Permissions: +Azure PAT needs "Code (read)" permissions +Forgejo token needs repository creation permissions + +# Existing Repositories: +The tool skips if a repository already exists (to prevent overwrites). + +# LFS in Mirrors: +Forgejo will handle LFS objects during the initial clone and subsequent syncs automatically when LFS: true is set. diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..eb15de4 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,69 @@ +package cmd + +import ( + "github.com/spf13/cobra" + "vix.ro/sonix/azure_migrate/internal/config" + "vix.ro/sonix/azure_migrate/internal/migrator" +) + +var ( + cfgFile string + rootCmd = &cobra.Command{ + Use: "forgejo-migrator", + Short: "Mirror repositories from Azure DevOps to Forgejo", + Long: `A CLI tool to migrate Git repositories including LFS from Azure DevOps to Forgejo.`, + } +) + +func init() { + cobra.OnInitialize(initConfig) + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", ".env", "config file (default is .env)") + + rootCmd.AddCommand(migrateCmd) + rootCmd.AddCommand(listCmd) +} + +func initConfig() { + if err := config.Load(cfgFile); err != nil { + panic(err) + } +} + +func Execute() error { + return rootCmd.Execute() +} + +var migrateCmd = &cobra.Command{ + Use: "migrate", + Short: "Start migration process", + RunE: func(cmd *cobra.Command, args []string) error { + dryRun, _ := cmd.Flags().GetBool("dry-run") + cfg := config.Get() + cfg.DryRun = dryRun + + m, err := migrator.New(cfg) + if err != nil { + return err + } + + return m.Run() + }, +} + +var listCmd = &cobra.Command{ + Use: "list", + Short: "List available Azure repositories", + RunE: func(cmd *cobra.Command, args []string) error { + cfg := config.Get() + m, err := migrator.New(cfg) + if err != nil { + return err + } + + return m.ListRepositories() + }, +} + +func init() { + migrateCmd.Flags().BoolP("dry-run", "d", false, "Show what would be migrated without making changes") +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c926460 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module vix.ro/sonix/azure_migrate + +go 1.24.4 + +require github.com/spf13/cobra v1.10.2 + +require golang.org/x/net v0.43.0 // indirect + +require ( + github.com/go-resty/resty/v2 v2.17.2 + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/joho/godotenv v1.5.1 + github.com/spf13/pflag v1.0.9 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..bb29225 --- /dev/null +++ b/go.sum @@ -0,0 +1,18 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk= +github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/azure/client.go b/internal/azure/client.go new file mode 100644 index 0000000..106c424 --- /dev/null +++ b/internal/azure/client.go @@ -0,0 +1,81 @@ +package azure + +import ( + "encoding/base64" + "fmt" + "net/http" + "time" + + "github.com/go-resty/resty/v2" +) + +type Client struct { + client *resty.Client + organization string +} + +type Repository struct { + ID string `json:"id"` + Name string `json:"name"` + Project struct { + Name string `json:"name"` + } `json:"project"` + RemoteURL string `json:"remoteUrl"` + SSHURL string `json:"sshUrl"` + WebURL string `json:"webUrl"` +} + +type RepositoriesResponse struct { + Count int `json:"count"` + Value []Repository `json:"value"` +} + +func NewClient(org, pat string) *Client { + c := resty.New() + c.SetTimeout(30 * time.Second) + + // Azure DevOps uses Basic auth with empty username and PAT as password + auth := base64.StdEncoding.EncodeToString([]byte(":" + pat)) + c.SetHeader("Authorization", "Basic "+auth) + c.SetHeader("Content-Type", "application/json") + + return &Client{ + client: c, + organization: org, + } +} + +func (c *Client) GetRepositories() ([]Repository, error) { + url := fmt.Sprintf("https://dev.azure.com/%s/_apis/git/repositories", c.organization) + + var result RepositoriesResponse + resp, err := c.client.R(). + SetQueryParam("api-version", "7.0"). + SetResult(&result). + Get(url) + + if err != nil { + return nil, fmt.Errorf("failed to fetch repositories: %w", err) + } + + if resp.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("Azure API returned status %d: %s", resp.StatusCode(), resp.String()) + } + + // Handle pagination if needed (continuationToken) + // For simplicity, this example handles single page + if result.Count != len(result.Value) { + fmt.Printf("Warning: Repository count mismatch, expected %d got %d\n", result.Count, len(result.Value)) + } + + return result.Value, nil +} + +func (c *Client) GetAuthenticatedURL(remoteURL, pat string) string { + // Convert https://dev.azure.com/org/project/_git/repo + // to https://pat@dev.azure.com/org/project/_git/repo + if len(remoteURL) > 8 && remoteURL[:8] == "https://" { + return "https://:" + pat + "@" + remoteURL[8:] + } + return remoteURL +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..9d67630 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,84 @@ +package config + +import ( + "fmt" + "os" + "strconv" + + "github.com/joho/godotenv" +) + +type Config struct { + // Azure + AzureOrg string + AzurePAT string + + // Forgejo + ForgejoURL string + ForgejoToken string + ForgejoOwner string + + // Migration + MirrorInterval string + Concurrent int + DryRun bool +} + +var globalConfig *Config + +func Load(path string) error { + if err := godotenv.Load(path); err != nil { + // Try to load from environment if file doesn't exist + if !os.IsNotExist(err) { + return fmt.Errorf("error loading env file: %w", err) + } + } + + concurrent := 3 + if c := os.Getenv("CONCURRENT_MIGRATIONS"); c != "" { + if v, err := strconv.Atoi(c); err == nil { + concurrent = v + } + } + + globalConfig = &Config{ + AzureOrg: getEnvOrError("AZURE_ORG"), + AzurePAT: getEnvOrError("AZURE_PAT"), + ForgejoURL: getEnvOrError("FORGEJO_URL"), + ForgejoToken: getEnvOrError("FORGEJO_TOKEN"), + ForgejoOwner: getEnvOrError("FORGEJO_OWNER"), + MirrorInterval: getEnv("MIRROR_INTERVAL", "8h"), + Concurrent: concurrent, + } + + return validate(globalConfig) +} + +func Get() *Config { + return globalConfig +} + +func getEnv(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +func getEnvOrError(key string) string { + value := os.Getenv(key) + if value == "" { + panic(fmt.Sprintf("Required environment variable %s is not set", key)) + } + return value +} + +func validate(c *Config) error { + if c.AzureOrg == "" || c.AzurePAT == "" { + return fmt.Errorf("Azure organization and PAT must be configured") + } + if c.ForgejoURL == "" || c.ForgejoToken == "" { + return fmt.Errorf("Forgejo URL and token must be configured") + } + return nil +} diff --git a/internal/forgejo/client.go b/internal/forgejo/client.go new file mode 100644 index 0000000..40212d7 --- /dev/null +++ b/internal/forgejo/client.go @@ -0,0 +1,135 @@ +package forgejo + +import ( + "fmt" + "net/http" + "time" + + "github.com/go-resty/resty/v2" +) + +type Client struct { + client *resty.Client + token string + baseURL string +} + +type MigrateRequest struct { + CloneAddr string `json:"clone_addr"` + AuthUsername string `json:"auth_username"` + AuthPassword string `json:"auth_password"` + UID int `json:"uid"` + RepoName string `json:"repo_name"` + RepoOwner string `json:"repo_owner"` + Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` + Interval string `json:"interval"` + Description string `json:"description,omitempty"` + Private bool `json:"private"` +} + +type Repository struct { + ID int64 `json:"id"` + Name string `json:"name"` + FullName string `json:"full_name"` + HTMLURL string `json:"html_url"` + CloneURL string `json:"clone_url"` + Mirror bool `json:"mirror"` +} + +type User struct { + ID int64 `json:"id"` + UserName string `json:"login"` +} + +type APIError struct { + Message string `json:"message"` + URL string `json:"url"` +} + +func NewClient(baseURL, token string) *Client { + c := resty.New() + c.SetTimeout(60 * time.Second) // Longer timeout for migration + c.SetHeader("Authorization", "token "+token) + c.SetHeader("Content-Type", "application/json") + c.SetBaseURL(baseURL + "/api/v1") + + return &Client{ + client: c, + token: token, + baseURL: baseURL, + } +} + +func (c *Client) GetUser(username string) (*User, error) { + var user User + resp, err := c.client.R(). + SetResult(&user). + Get("/users/" + username) + + if err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + + if resp.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("failed to get user: status %d - %s", resp.StatusCode(), resp.String()) + } + + return &user, nil +} + +func (c *Client) RepositoryExists(owner, name string) (bool, error) { + resp, err := c.client.R(). + Head(fmt.Sprintf("/repos/%s/%s", owner, name)) + + if err != nil { + return false, err + } + + if resp.StatusCode() == http.StatusOK { + return true, nil + } else if resp.StatusCode() == http.StatusNotFound { + return false, nil + } + + return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode()) +} + +func (c *Client) CreateMirror(req MigrateRequest) (*Repository, error) { + var repo Repository + var apiErr APIError + + resp, err := c.client.R(). + SetBody(req). + SetResult(&repo). + SetError(&apiErr). + Post("/repos/migrate") + + if err != nil { + return nil, fmt.Errorf("API request failed: %w", err) + } + + if resp.StatusCode() != http.StatusCreated && resp.StatusCode() != http.StatusOK { + if apiErr.Message != "" { + return nil, fmt.Errorf("Forgejo API error: %s (status %d)", apiErr.Message, resp.StatusCode()) + } + return nil, fmt.Errorf("Forgejo API returned status %d: %s", resp.StatusCode(), resp.String()) + } + + return &repo, nil +} + +func (c *Client) DeleteRepository(owner, name string) error { + resp, err := c.client.R(). + Delete(fmt.Sprintf("/repos/%s/%s", owner, name)) + + if err != nil { + return err + } + + if resp.StatusCode() != http.StatusNoContent && resp.StatusCode() != http.StatusOK { + return fmt.Errorf("failed to delete repository: status %d", resp.StatusCode()) + } + + return nil +} diff --git a/internal/migrator/migrator.go b/internal/migrator/migrator.go new file mode 100644 index 0000000..55d7259 --- /dev/null +++ b/internal/migrator/migrator.go @@ -0,0 +1,204 @@ +package migrator + +import ( + "fmt" + "strings" + "sync" + + "vix.ro/sonix/azure_migrate/internal/azure" + "vix.ro/sonix/azure_migrate/internal/config" + "vix.ro/sonix/azure_migrate/internal/forgejo" +) + +type Migrator struct { + config *config.Config + azureClient *azure.Client + forgejoClient *forgejo.Client + errors []error + mu sync.Mutex +} + +type MigrationResult struct { + RepoName string + Success bool + Error error + ForgejoURL string +} + +func New(cfg *config.Config) (*Migrator, error) { + azClient := azure.NewClient(cfg.AzureOrg, cfg.AzurePAT) + fjClient := forgejo.NewClient(cfg.ForgejoURL, cfg.ForgejoToken) + + return &Migrator{ + config: cfg, + azureClient: azClient, + forgejoClient: fjClient, + errors: make([]error, 0), + }, nil +} + +func (m *Migrator) ListRepositories() error { + fmt.Println("Fetching repositories from Azure DevOps...") + + repos, err := m.azureClient.GetRepositories() + if err != nil { + return fmt.Errorf("failed to list repositories: %w", err) + } + + fmt.Printf("\nFound %d repositories:\n", len(repos)) + fmt.Println(strings.Repeat("-", 80)) + fmt.Printf("%-40s %-20s %s\n", "NAME", "PROJECT", "URL") + fmt.Println(strings.Repeat("-", 80)) + + for _, repo := range repos { + fmt.Printf("%-40s %-20s %s\n", + truncate(repo.Name, 40), + truncate(repo.Project.Name, 20), + repo.WebURL) + } + + return nil +} + +func (m *Migrator) Run() error { + fmt.Println("Starting migration process...") + if m.config.DryRun { + fmt.Println("*** DRY RUN MODE - No changes will be made ***") + } + + // Get Azure repos + repos, err := m.azureClient.GetRepositories() + if err != nil { + return fmt.Errorf("failed to fetch Azure repositories: %w", err) + } + + fmt.Printf("Found %d repositories to process\n\n", len(repos)) + + // Get Forgejo user/org ID + user, err := m.forgejoClient.GetUser(m.config.ForgejoOwner) + if err != nil { + return fmt.Errorf("failed to get Forgejo user/org '%s': %w", m.config.ForgejoOwner, err) + } + + // Process with concurrency + semaphore := make(chan struct{}, m.config.Concurrent) + var wg sync.WaitGroup + results := make(chan MigrationResult, len(repos)) + + for _, repo := range repos { + wg.Add(1) + semaphore <- struct{}{} // Acquire + + go func(r azure.Repository) { + defer wg.Done() + defer func() { <-semaphore }() // Release + + result := m.migrateRepository(r, user.ID) + results <- result + + if result.Error != nil { + m.addError(result.Error) + } + }(repo) + } + + // Close results channel when done + go func() { + wg.Wait() + close(results) + }() + + // Print results + var successCount, failCount int + for result := range results { + if result.Success { + successCount++ + fmt.Printf("✓ Migrated: %s -> %s\n", result.RepoName, result.ForgejoURL) + } else { + failCount++ + fmt.Printf("✗ Failed: %s - Error: %v\n", result.RepoName, result.Error) + } + } + + // Summary + fmt.Println(strings.Repeat("=", 80)) + fmt.Printf("Migration Complete: %d succeeded, %d failed\n", successCount, failCount) + + if len(m.errors) > 0 { + fmt.Println("\nDetailed errors:") + for i, err := range m.errors { + fmt.Printf("%d. %v\n", i+1, err) + } + return fmt.Errorf("migration completed with %d errors", len(m.errors)) + } + + return nil +} + +func (m *Migrator) migrateRepository(repo azure.Repository, uid int64) MigrationResult { + result := MigrationResult{ + RepoName: repo.Name, + } + + // Check if already exists + exists, err := m.forgejoClient.RepositoryExists(m.config.ForgejoOwner, repo.Name) + if err != nil { + result.Error = fmt.Errorf("failed to check if repository exists: %w", err) + return result + } + + if exists { + result.Error = fmt.Errorf("repository already exists in Forgejo") + return result + } + + // Prepare authenticated URL for Azure + cloneURL := m.azureClient.GetAuthenticatedURL(repo.RemoteURL, m.config.AzurePAT) + + // Prepare migration request + req := forgejo.MigrateRequest{ + CloneAddr: cloneURL, + AuthUsername: "", // Azure uses PAT as password with empty username + AuthPassword: m.config.AzurePAT, + UID: int(uid), + RepoName: repo.Name, + RepoOwner: m.config.ForgejoOwner, + Mirror: true, + LFS: true, // Enable LFS support + Interval: m.config.MirrorInterval, + Description: fmt.Sprintf("Mirrored from Azure DevOps: %s", repo.WebURL), + Private: true, // Default to private, adjust as needed + } + + if m.config.DryRun { + result.Success = true + result.ForgejoURL = fmt.Sprintf("%s/%s/%s", m.config.ForgejoURL, m.config.ForgejoOwner, repo.Name) + return result + } + + // Create mirror + createdRepo, err := m.forgejoClient.CreateMirror(req) + if err != nil { + // Provide detailed error context + result.Error = fmt.Errorf("failed to create mirror for '%s' (project: %s): %w", + repo.Name, repo.Project.Name, err) + return result + } + + result.Success = true + result.ForgejoURL = createdRepo.HTMLURL + return result +} + +func (m *Migrator) addError(err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.errors = append(m.errors, err) +} + +func truncate(s string, length int) string { + if len(s) <= length { + return s + } + return s[:length-3] + "..." +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..00a0c9d --- /dev/null +++ b/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + "os" + + "vix.ro/sonix/azure_migrate/cmd" +) + +func main() { + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +}