Initial commit

This commit is contained in:
sonix 2026-03-22 09:43:45 +00:00 committed by sonix
commit 309b240cdc
12 changed files with 738 additions and 0 deletions

13
.env.example Normal file
View file

@ -0,0 +1,13 @@
# Azure DevOps Configuration
AZURE_ORG=your-organization
AZURE_PAT=your-personal-access-token
# Forgejo Configuration
FORGEJO_URL=https://git.yourdomain.com
FORGEJO_TOKEN=your-forgejo-token
FORGEJO_OWNER=target-username-or-org
# Migration Settings
MIRROR_INTERVAL=8h
CONCURRENT_MIGRATIONS=3
DRY_RUN=false

27
.gitignore vendored Normal file
View file

@ -0,0 +1,27 @@
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
# Go workspace file
go.work
go.work.sum
# env file
.env

9
LICENSE Normal file
View file

@ -0,0 +1,9 @@
MIT License
Copyright (c) 2026 sonix
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

69
README.md Normal file
View file

@ -0,0 +1,69 @@
# azure_migrate
Migrate from Azure to Forgejo
# Usage Instructions
Create your .env file:
bash
`cp .env.example .env`
# List repositories first (dry view):
bash
`go run main.go list`
# Run migration (dry run):
bash
`go run main.go migrate --dry-run`
# Execute actual migration:
bash
`go run main.go migrate`
## Key Features ##
# LFS Support:
The LFS: true flag in the migration request tells Forgejo to mirror LFS objects automatically.
# Concurrent Processing:
Uses worker pools to migrate multiple repositories simultaneously (configurable via CONCURRENT_MIGRATIONS).
# Detailed Error Handling:
Errors include repository name, project context, and specific failure reasons
Non-fatal errors don't stop the entire process
Summary report at the end
# Azure Authentication:
Properly handles Azure DevOps PAT authentication via Basic Auth header.
# Dry Run Mode:
Test the migration without creating any repositories.
# Mirror Configuration:
Sets up Forgejo to periodically sync from Azure DevOps using the Interval setting.
## Important Notes ##
# Network Access:
Ensure the machine running this has access to both Azure DevOps and your Forgejo instance.
# Storage:
Forgejo must have sufficient storage for LFS objects.
# Permissions:
Azure PAT needs "Code (read)" permissions
Forgejo token needs repository creation permissions
# Existing Repositories:
The tool skips if a repository already exists (to prevent overwrites).
# LFS in Mirrors:
Forgejo will handle LFS objects during the initial clone and subsequent syncs automatically when LFS: true is set.

69
cmd/root.go Normal file
View file

@ -0,0 +1,69 @@
package cmd
import (
"github.com/spf13/cobra"
"vix.ro/sonix/azure_migrate/internal/config"
"vix.ro/sonix/azure_migrate/internal/migrator"
)
var (
cfgFile string
rootCmd = &cobra.Command{
Use: "forgejo-migrator",
Short: "Mirror repositories from Azure DevOps to Forgejo",
Long: `A CLI tool to migrate Git repositories including LFS from Azure DevOps to Forgejo.`,
}
)
func init() {
cobra.OnInitialize(initConfig)
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", ".env", "config file (default is .env)")
rootCmd.AddCommand(migrateCmd)
rootCmd.AddCommand(listCmd)
}
func initConfig() {
if err := config.Load(cfgFile); err != nil {
panic(err)
}
}
func Execute() error {
return rootCmd.Execute()
}
var migrateCmd = &cobra.Command{
Use: "migrate",
Short: "Start migration process",
RunE: func(cmd *cobra.Command, args []string) error {
dryRun, _ := cmd.Flags().GetBool("dry-run")
cfg := config.Get()
cfg.DryRun = dryRun
m, err := migrator.New(cfg)
if err != nil {
return err
}
return m.Run()
},
}
var listCmd = &cobra.Command{
Use: "list",
Short: "List available Azure repositories",
RunE: func(cmd *cobra.Command, args []string) error {
cfg := config.Get()
m, err := migrator.New(cfg)
if err != nil {
return err
}
return m.ListRepositories()
},
}
func init() {
migrateCmd.Flags().BoolP("dry-run", "d", false, "Show what would be migrated without making changes")
}

14
go.mod Normal file
View file

@ -0,0 +1,14 @@
module vix.ro/sonix/azure_migrate
go 1.24.4
require github.com/spf13/cobra v1.10.2
require golang.org/x/net v0.43.0 // indirect
require (
github.com/go-resty/resty/v2 v2.17.2
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/joho/godotenv v1.5.1
github.com/spf13/pflag v1.0.9 // indirect
)

18
go.sum Normal file
View file

@ -0,0 +1,18 @@
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

81
internal/azure/client.go Normal file
View file

@ -0,0 +1,81 @@
package azure
import (
"encoding/base64"
"fmt"
"net/http"
"time"
"github.com/go-resty/resty/v2"
)
type Client struct {
client *resty.Client
organization string
}
type Repository struct {
ID string `json:"id"`
Name string `json:"name"`
Project struct {
Name string `json:"name"`
} `json:"project"`
RemoteURL string `json:"remoteUrl"`
SSHURL string `json:"sshUrl"`
WebURL string `json:"webUrl"`
}
type RepositoriesResponse struct {
Count int `json:"count"`
Value []Repository `json:"value"`
}
func NewClient(org, pat string) *Client {
c := resty.New()
c.SetTimeout(30 * time.Second)
// Azure DevOps uses Basic auth with empty username and PAT as password
auth := base64.StdEncoding.EncodeToString([]byte(":" + pat))
c.SetHeader("Authorization", "Basic "+auth)
c.SetHeader("Content-Type", "application/json")
return &Client{
client: c,
organization: org,
}
}
func (c *Client) GetRepositories() ([]Repository, error) {
url := fmt.Sprintf("https://dev.azure.com/%s/_apis/git/repositories", c.organization)
var result RepositoriesResponse
resp, err := c.client.R().
SetQueryParam("api-version", "7.0").
SetResult(&result).
Get(url)
if err != nil {
return nil, fmt.Errorf("failed to fetch repositories: %w", err)
}
if resp.StatusCode() != http.StatusOK {
return nil, fmt.Errorf("Azure API returned status %d: %s", resp.StatusCode(), resp.String())
}
// Handle pagination if needed (continuationToken)
// For simplicity, this example handles single page
if result.Count != len(result.Value) {
fmt.Printf("Warning: Repository count mismatch, expected %d got %d\n", result.Count, len(result.Value))
}
return result.Value, nil
}
func (c *Client) GetAuthenticatedURL(remoteURL, pat string) string {
// Convert https://dev.azure.com/org/project/_git/repo
// to https://pat@dev.azure.com/org/project/_git/repo
if len(remoteURL) > 8 && remoteURL[:8] == "https://" {
return "https://:" + pat + "@" + remoteURL[8:]
}
return remoteURL
}

84
internal/config/config.go Normal file
View file

@ -0,0 +1,84 @@
package config
import (
"fmt"
"os"
"strconv"
"github.com/joho/godotenv"
)
type Config struct {
// Azure
AzureOrg string
AzurePAT string
// Forgejo
ForgejoURL string
ForgejoToken string
ForgejoOwner string
// Migration
MirrorInterval string
Concurrent int
DryRun bool
}
var globalConfig *Config
func Load(path string) error {
if err := godotenv.Load(path); err != nil {
// Try to load from environment if file doesn't exist
if !os.IsNotExist(err) {
return fmt.Errorf("error loading env file: %w", err)
}
}
concurrent := 3
if c := os.Getenv("CONCURRENT_MIGRATIONS"); c != "" {
if v, err := strconv.Atoi(c); err == nil {
concurrent = v
}
}
globalConfig = &Config{
AzureOrg: getEnvOrError("AZURE_ORG"),
AzurePAT: getEnvOrError("AZURE_PAT"),
ForgejoURL: getEnvOrError("FORGEJO_URL"),
ForgejoToken: getEnvOrError("FORGEJO_TOKEN"),
ForgejoOwner: getEnvOrError("FORGEJO_OWNER"),
MirrorInterval: getEnv("MIRROR_INTERVAL", "8h"),
Concurrent: concurrent,
}
return validate(globalConfig)
}
func Get() *Config {
return globalConfig
}
func getEnv(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
func getEnvOrError(key string) string {
value := os.Getenv(key)
if value == "" {
panic(fmt.Sprintf("Required environment variable %s is not set", key))
}
return value
}
func validate(c *Config) error {
if c.AzureOrg == "" || c.AzurePAT == "" {
return fmt.Errorf("Azure organization and PAT must be configured")
}
if c.ForgejoURL == "" || c.ForgejoToken == "" {
return fmt.Errorf("Forgejo URL and token must be configured")
}
return nil
}

135
internal/forgejo/client.go Normal file
View file

@ -0,0 +1,135 @@
package forgejo
import (
"fmt"
"net/http"
"time"
"github.com/go-resty/resty/v2"
)
type Client struct {
client *resty.Client
token string
baseURL string
}
type MigrateRequest struct {
CloneAddr string `json:"clone_addr"`
AuthUsername string `json:"auth_username"`
AuthPassword string `json:"auth_password"`
UID int `json:"uid"`
RepoName string `json:"repo_name"`
RepoOwner string `json:"repo_owner"`
Mirror bool `json:"mirror"`
LFS bool `json:"lfs"`
Interval string `json:"interval"`
Description string `json:"description,omitempty"`
Private bool `json:"private"`
}
type Repository struct {
ID int64 `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
HTMLURL string `json:"html_url"`
CloneURL string `json:"clone_url"`
Mirror bool `json:"mirror"`
}
type User struct {
ID int64 `json:"id"`
UserName string `json:"login"`
}
type APIError struct {
Message string `json:"message"`
URL string `json:"url"`
}
func NewClient(baseURL, token string) *Client {
c := resty.New()
c.SetTimeout(60 * time.Second) // Longer timeout for migration
c.SetHeader("Authorization", "token "+token)
c.SetHeader("Content-Type", "application/json")
c.SetBaseURL(baseURL + "/api/v1")
return &Client{
client: c,
token: token,
baseURL: baseURL,
}
}
func (c *Client) GetUser(username string) (*User, error) {
var user User
resp, err := c.client.R().
SetResult(&user).
Get("/users/" + username)
if err != nil {
return nil, fmt.Errorf("failed to get user: %w", err)
}
if resp.StatusCode() != http.StatusOK {
return nil, fmt.Errorf("failed to get user: status %d - %s", resp.StatusCode(), resp.String())
}
return &user, nil
}
func (c *Client) RepositoryExists(owner, name string) (bool, error) {
resp, err := c.client.R().
Head(fmt.Sprintf("/repos/%s/%s", owner, name))
if err != nil {
return false, err
}
if resp.StatusCode() == http.StatusOK {
return true, nil
} else if resp.StatusCode() == http.StatusNotFound {
return false, nil
}
return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode())
}
func (c *Client) CreateMirror(req MigrateRequest) (*Repository, error) {
var repo Repository
var apiErr APIError
resp, err := c.client.R().
SetBody(req).
SetResult(&repo).
SetError(&apiErr).
Post("/repos/migrate")
if err != nil {
return nil, fmt.Errorf("API request failed: %w", err)
}
if resp.StatusCode() != http.StatusCreated && resp.StatusCode() != http.StatusOK {
if apiErr.Message != "" {
return nil, fmt.Errorf("Forgejo API error: %s (status %d)", apiErr.Message, resp.StatusCode())
}
return nil, fmt.Errorf("Forgejo API returned status %d: %s", resp.StatusCode(), resp.String())
}
return &repo, nil
}
func (c *Client) DeleteRepository(owner, name string) error {
resp, err := c.client.R().
Delete(fmt.Sprintf("/repos/%s/%s", owner, name))
if err != nil {
return err
}
if resp.StatusCode() != http.StatusNoContent && resp.StatusCode() != http.StatusOK {
return fmt.Errorf("failed to delete repository: status %d", resp.StatusCode())
}
return nil
}

View file

@ -0,0 +1,204 @@
package migrator
import (
"fmt"
"strings"
"sync"
"vix.ro/sonix/azure_migrate/internal/azure"
"vix.ro/sonix/azure_migrate/internal/config"
"vix.ro/sonix/azure_migrate/internal/forgejo"
)
type Migrator struct {
config *config.Config
azureClient *azure.Client
forgejoClient *forgejo.Client
errors []error
mu sync.Mutex
}
type MigrationResult struct {
RepoName string
Success bool
Error error
ForgejoURL string
}
func New(cfg *config.Config) (*Migrator, error) {
azClient := azure.NewClient(cfg.AzureOrg, cfg.AzurePAT)
fjClient := forgejo.NewClient(cfg.ForgejoURL, cfg.ForgejoToken)
return &Migrator{
config: cfg,
azureClient: azClient,
forgejoClient: fjClient,
errors: make([]error, 0),
}, nil
}
func (m *Migrator) ListRepositories() error {
fmt.Println("Fetching repositories from Azure DevOps...")
repos, err := m.azureClient.GetRepositories()
if err != nil {
return fmt.Errorf("failed to list repositories: %w", err)
}
fmt.Printf("\nFound %d repositories:\n", len(repos))
fmt.Println(strings.Repeat("-", 80))
fmt.Printf("%-40s %-20s %s\n", "NAME", "PROJECT", "URL")
fmt.Println(strings.Repeat("-", 80))
for _, repo := range repos {
fmt.Printf("%-40s %-20s %s\n",
truncate(repo.Name, 40),
truncate(repo.Project.Name, 20),
repo.WebURL)
}
return nil
}
func (m *Migrator) Run() error {
fmt.Println("Starting migration process...")
if m.config.DryRun {
fmt.Println("*** DRY RUN MODE - No changes will be made ***")
}
// Get Azure repos
repos, err := m.azureClient.GetRepositories()
if err != nil {
return fmt.Errorf("failed to fetch Azure repositories: %w", err)
}
fmt.Printf("Found %d repositories to process\n\n", len(repos))
// Get Forgejo user/org ID
user, err := m.forgejoClient.GetUser(m.config.ForgejoOwner)
if err != nil {
return fmt.Errorf("failed to get Forgejo user/org '%s': %w", m.config.ForgejoOwner, err)
}
// Process with concurrency
semaphore := make(chan struct{}, m.config.Concurrent)
var wg sync.WaitGroup
results := make(chan MigrationResult, len(repos))
for _, repo := range repos {
wg.Add(1)
semaphore <- struct{}{} // Acquire
go func(r azure.Repository) {
defer wg.Done()
defer func() { <-semaphore }() // Release
result := m.migrateRepository(r, user.ID)
results <- result
if result.Error != nil {
m.addError(result.Error)
}
}(repo)
}
// Close results channel when done
go func() {
wg.Wait()
close(results)
}()
// Print results
var successCount, failCount int
for result := range results {
if result.Success {
successCount++
fmt.Printf("✓ Migrated: %s -> %s\n", result.RepoName, result.ForgejoURL)
} else {
failCount++
fmt.Printf("✗ Failed: %s - Error: %v\n", result.RepoName, result.Error)
}
}
// Summary
fmt.Println(strings.Repeat("=", 80))
fmt.Printf("Migration Complete: %d succeeded, %d failed\n", successCount, failCount)
if len(m.errors) > 0 {
fmt.Println("\nDetailed errors:")
for i, err := range m.errors {
fmt.Printf("%d. %v\n", i+1, err)
}
return fmt.Errorf("migration completed with %d errors", len(m.errors))
}
return nil
}
func (m *Migrator) migrateRepository(repo azure.Repository, uid int64) MigrationResult {
result := MigrationResult{
RepoName: repo.Name,
}
// Check if already exists
exists, err := m.forgejoClient.RepositoryExists(m.config.ForgejoOwner, repo.Name)
if err != nil {
result.Error = fmt.Errorf("failed to check if repository exists: %w", err)
return result
}
if exists {
result.Error = fmt.Errorf("repository already exists in Forgejo")
return result
}
// Prepare authenticated URL for Azure
cloneURL := m.azureClient.GetAuthenticatedURL(repo.RemoteURL, m.config.AzurePAT)
// Prepare migration request
req := forgejo.MigrateRequest{
CloneAddr: cloneURL,
AuthUsername: "", // Azure uses PAT as password with empty username
AuthPassword: m.config.AzurePAT,
UID: int(uid),
RepoName: repo.Name,
RepoOwner: m.config.ForgejoOwner,
Mirror: true,
LFS: true, // Enable LFS support
Interval: m.config.MirrorInterval,
Description: fmt.Sprintf("Mirrored from Azure DevOps: %s", repo.WebURL),
Private: true, // Default to private, adjust as needed
}
if m.config.DryRun {
result.Success = true
result.ForgejoURL = fmt.Sprintf("%s/%s/%s", m.config.ForgejoURL, m.config.ForgejoOwner, repo.Name)
return result
}
// Create mirror
createdRepo, err := m.forgejoClient.CreateMirror(req)
if err != nil {
// Provide detailed error context
result.Error = fmt.Errorf("failed to create mirror for '%s' (project: %s): %w",
repo.Name, repo.Project.Name, err)
return result
}
result.Success = true
result.ForgejoURL = createdRepo.HTMLURL
return result
}
func (m *Migrator) addError(err error) {
m.mu.Lock()
defer m.mu.Unlock()
m.errors = append(m.errors, err)
}
func truncate(s string, length int) string {
if len(s) <= length {
return s
}
return s[:length-3] + "..."
}

15
main.go Normal file
View file

@ -0,0 +1,15 @@
package main
import (
"fmt"
"os"
"vix.ro/sonix/azure_migrate/cmd"
)
func main() {
if err := cmd.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}