diff --git a/go.mod b/go.mod index c926460..b1eb1b3 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module vix.ro/sonix/azure_migrate go 1.24.4 -require github.com/spf13/cobra v1.10.2 +require ( + github.com/spf13/cobra v1.10.2 + gopkg.in/yaml.v3 v3.0.1 +) require golang.org/x/net v0.43.0 // indirect diff --git a/go.sum b/go.sum index bb29225..634c72b 100644 --- a/go.sum +++ b/go.sum @@ -15,4 +15,7 @@ golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/azure/client.go b/internal/azure/client.go index 106c424..79a48b6 100644 --- a/internal/azure/client.go +++ b/internal/azure/client.go @@ -4,11 +4,36 @@ import ( "encoding/base64" "fmt" "net/http" + "strings" "time" "github.com/go-resty/resty/v2" ) +type Pipeline struct { + ID int `json:"id"` + Name string `json:"name"` + Folder string `json:"folder"` + URL string `json:"url"` +} + +type BuildDefinition struct { + ID int `json:"id"` + Name string `json:"name"` + Process struct { + Type int `json:"type"` + YamlFilename string `json:"yamlFilename"` // for YAML pipelines + } `json:"process"` + Variables map[string]struct { + Value string `json:"value"` + IsSecret bool `json:"isSecret"` + } `json:"variables"` + Repository struct { + URL string `json:"url"` + Name string `json:"name"` + } `json:"repository"` +} + type Client struct { client *resty.Client organization string @@ -79,3 +104,96 @@ func (c *Client) GetAuthenticatedURL(remoteURL, pat string) string { } return remoteURL } + +// GetBuildDefinitions gets pipeline definitions for a project +func (c *Client) GetBuildDefinitions(project string) ([]BuildDefinition, error) { + url := fmt.Sprintf("https://dev.azure.com/%s/%s/_apis/build/definitions", c.organization, project) + + var result struct { + Count int `json:"count"` + Value []BuildDefinition `json:"value"` + } + + resp, err := c.client.R(). + SetQueryParam("api-version", "7.0"). + SetQueryParam("includeAll", "true"). + SetResult(&result). + Get(url) + + if err != nil { + return nil, fmt.Errorf("failed to fetch build definitions: %w", err) + } + + if resp.StatusCode() != 200 { + return nil, fmt.Errorf("Azure API returned %d: %s", resp.StatusCode(), resp.String()) + } + + return result.Value, nil +} + +// GetPipelineContent fetches the azure-pipelines.yml content from repository +func (c *Client) GetPipelineContent(repoURL, branch, filePath string) (string, error) { + // Azure DevOps Repos API for file content + // Extract project and repo name from URL + // URL format: https://dev.azure.com/org/project/_git/repo + + // For this example, construct the items API URL + parts := strings.Split(repoURL, "/_git/") + if len(parts) != 2 { + return "", fmt.Errorf("cannot parse repo URL: %s", repoURL) + } + + base := parts[0] + repoName := parts[1] + + url := fmt.Sprintf("%s/_apis/sourceProviders/TfsGit/filecontents?repository=%s&commitOrBranch=%s&path=%s&api-version=7.0", + base, repoName, branch, filePath) + + resp, err := c.client.R(). + Get(url) + + if err != nil { + return "", err + } + + if resp.StatusCode() != 200 { + // Try alternative: get via Git API (Items endpoint) + return "", fmt.Errorf("file not found or access denied") + } + + return string(resp.Body()), nil +} + +// GetPipelineVariables gets variables for a specific build definition +func (c *Client) GetPipelineVariables(project string, definitionID int) (map[string]string, error) { + url := fmt.Sprintf("https://dev.azure.com/%s/%s/_apis/build/definitions/%d/variables", + c.organization, project, definitionID) + + var result struct { + Count int `json:"count"` + Value map[string]struct { + Value string `json:"value"` + IsSecret bool `json:"isSecret"` + } `json:"value"` + } + + resp, err := c.client.R(). + SetQueryParam("api-version", "7.0"). + SetResult(&result). + Get(url) + + if err != nil { + return nil, err + } + + if resp.StatusCode() != 200 { + return nil, fmt.Errorf("failed to get variables: %d", resp.StatusCode()) + } + + vars := make(map[string]string) + for k, v := range result.Value { + vars[k] = v.Value // Note: secrets won't have values via API unless explicitly allowed + } + + return vars, nil +} diff --git a/internal/converter/pipeline.go b/internal/converter/pipeline.go new file mode 100644 index 0000000..f952f07 --- /dev/null +++ b/internal/converter/pipeline.go @@ -0,0 +1,295 @@ +package converter + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +// AzurePipeline represents the structure of azure-pipelines.yml +type AzurePipeline struct { + Trigger interface{} `yaml:"trigger"` + PR interface{} `yaml:"pr"` + Pool interface{} `yaml:"pool"` + Variables map[string]interface{} `yaml:"variables"` + Stages []AzureStage `yaml:"stages"` + Jobs []AzureJob `yaml:"jobs"` + Steps []AzureStep `yaml:"steps"` +} + +type AzureStage struct { + Stage string `yaml:"stage"` + Jobs []AzureJob `yaml:"jobs"` +} + +type AzureJob struct { + Job string `yaml:"job"` + Pool interface{} `yaml:"pool"` + Steps []AzureStep `yaml:"steps"` +} + +type AzureStep struct { + Task string `yaml:"task,omitempty"` + Script string `yaml:"script,omitempty"` + Powershell string `yaml:"powershell,omitempty"` + Bash string `yaml:"bash,omitempty"` + DisplayName string `yaml:"displayName,omitempty"` + Inputs map[string]string `yaml:"inputs,omitempty"` + Env map[string]string `yaml:"env,omitempty"` +} + +// ForgejoWorkflow represents GitHub Actions compatible workflow +type ForgejoWorkflow struct { + Name string `yaml:"name"` + On interface{} `yaml:"on"` + Jobs map[string]ForgejoJob `yaml:"jobs"` + Env map[string]string `yaml:"env,omitempty"` +} + +type ForgejoJob struct { + RunsOn string `yaml:"runs-on"` + Steps []ForgejoStep `yaml:"steps"` + Env map[string]string `yaml:"env,omitempty"` + Needs []string `yaml:"needs,omitempty"` + Timeout int `yaml:"timeout-minutes,omitempty"` +} + +type ForgejoStep struct { + Uses string `yaml:"uses,omitempty"` + Name string `yaml:"name,omitempty"` + Run string `yaml:"run,omitempty"` + With map[string]string `yaml:"with,omitempty"` + Env map[string]string `yaml:"env,omitempty"` +} + +// Converter handles the transformation +type Converter struct { + Warnings []string +} + +func New() *Converter { + return &Converter{ + Warnings: make([]string, 0), + } +} + +func (c *Converter) ConvertAzureToForgejo(azureYAML string, repoName string) (string, error) { + var azure AzurePipeline + if err := yaml.Unmarshal([]byte(azureYAML), &azure); err != nil { + return "", fmt.Errorf("failed to parse azure-pipelines.yml: %w", err) + } + + forgejo := ForgejoWorkflow{ + Name: fmt.Sprintf("Migrated Azure Pipeline - %s", repoName), + Jobs: make(map[string]ForgejoJob), + Env: make(map[string]string), + } + + // Convert trigger + forgejo.On = c.convertTrigger(azure.Trigger) + + // Convert variables + for k, v := range azure.Variables { + if str, ok := v.(string); ok { + forgejo.Env[k] = str + } else if m, ok := v.(map[string]interface{}); ok { + if val, exists := m["value"]; exists { + forgejo.Env[k] = fmt.Sprintf("%v", val) + } + } + } + + // Handle different structures: steps at root, jobs at root, or stages + if len(azure.Steps) > 0 { + // Simple pipeline with just steps + job := c.convertStepsToJob(azure.Steps, azure.Pool) + forgejo.Jobs["build"] = job + } else if len(azure.Jobs) > 0 { + // Multiple jobs + for _, job := range azure.Jobs { + fjJob := c.convertStepsToJob(job.Steps, job.Pool) + fjJob.Env = job.Steps[0].Env // Simplified + forgejo.Jobs[job.Job] = fjJob + } + } else if len(azure.Stages) > 0 { + // Stages become job dependencies + prevJob := "" + for _, stage := range azure.Stages { + for _, job := range stage.Jobs { + fjJob := c.convertStepsToJob(job.Steps, job.Pool) + if prevJob != "" { + fjJob.Needs = []string{prevJob} + } + jobName := fmt.Sprintf("%s_%s", stage.Stage, job.Job) + forgejo.Jobs[jobName] = fjJob + prevJob = jobName + } + } + } + + // Serialize + out, err := yaml.Marshal(forgejo) + if err != nil { + return "", err + } + + // Add header comment + header := fmt.Sprintf("# Migrated from Azure DevOps Pipeline\n# Repository: %s\n# WARNING: Review this file carefully. Azure-specific tasks require manual conversion.\n\n", repoName) + + return header + string(out), nil +} + +func (c *Converter) convertTrigger(trigger interface{}) interface{} { + // Simple conversion: Azure trigger -> GitHub on.push + if trigger == nil { + return map[string]interface{}{ + "push": map[string]interface{}{ + "branches": []string{"main", "master"}, + }, + } + } + + // If trigger is a list (branches), convert to push.branches + if branches, ok := trigger.([]interface{}); ok { + return map[string]interface{}{ + "push": map[string]interface{}{ + "branches": branches, + }, + "pull_request": map[string]interface{}{ + "branches": branches, + }, + } + } + + return trigger +} + +func (c *Converter) convertStepsToJob(steps []AzureStep, pool interface{}) ForgejoJob { + job := ForgejoJob{ + RunsOn: c.convertPool(pool), + Steps: make([]ForgejoStep, 0), + } + + // Add checkout step + job.Steps = append(job.Steps, ForgejoStep{ + Uses: "actions/checkout@v3", + Name: "Checkout", + }) + + for _, step := range steps { + fjStep := ForgejoStep{} + + if step.Script != "" { + fjStep.Run = step.Script + fjStep.Name = step.DisplayName + } else if step.Bash != "" { + fjStep.Run = step.Bash + fjStep.Name = step.DisplayName + fjStep.Run = "bash -c '" + strings.ReplaceAll(step.Bash, "'", "'\\''") + "'" + } else if step.Powershell != "" { + fjStep.Run = step.Powershell + fjStep.Name = step.DisplayName + c.Warnings = append(c.Warnings, fmt.Sprintf("PowerShell script detected: %s", step.DisplayName)) + } else if step.Task != "" { + // Convert known Azure tasks to actions + fjStep = c.convertAzureTask(step) + if fjStep.Uses == "" && fjStep.Run == "" { + fjStep.Run = fmt.Sprintf("# TODO: Convert Azure Task '%s' manually", step.Task) + fjStep.Name = fmt.Sprintf("[TODO] %s", step.DisplayName) + c.Warnings = append(c.Warnings, fmt.Sprintf("Cannot automatically convert task: %s", step.Task)) + } + } + + if fjStep.Name == "" { + fjStep.Name = step.DisplayName + } + + job.Steps = append(job.Steps, fjStep) + } + + return job +} + +func (c *Converter) convertPool(pool interface{}) string { + if pool == nil { + return "ubuntu-latest" + } + + // Pool can be a string or a map with vmImage + if str, ok := pool.(string); ok { + return c.mapAgent(str) + } + + if m, ok := pool.(map[string]interface{}); ok { + if vm, exists := m["vmImage"]; exists { + return c.mapAgent(vm.(string)) + } + if name, exists := m["name"]; exists { + // Self-hosted pool + return fmt.Sprintf("self-hosted,%s", name) + } + } + + return "ubuntu-latest" +} + +func (c *Converter) mapAgent(azureImage string) string { + // Map Azure VM images to Forgejo/Gitea runner labels + mappings := map[string]string{ + "ubuntu-latest": "ubuntu-latest", + "ubuntu-20.04": "ubuntu-20.04", + "ubuntu-22.04": "ubuntu-22.04", + "windows-latest": "windows-latest", + "macOS-latest": "macos-latest", + "macOS-11": "macos-11", + } + + if mapped, ok := mappings[azureImage]; ok { + return mapped + } + + c.Warnings = append(c.Warnings, fmt.Sprintf("Unknown VM image '%s', defaulting to ubuntu-latest", azureImage)) + return "ubuntu-latest" +} + +func (c *Converter) convertAzureTask(step AzureStep) ForgejoStep { + // Map common Azure tasks to GitHub Actions + switch step.Task { + case "AzureCLI@2", "AzureCLI@1": + return ForgejoStep{ + Name: step.DisplayName, + Run: "# Azure CLI step - configure azure/login action manually\n# az login ...\n" + step.Inputs["script"], + } + case "Docker@2": + return ForgejoStep{ + Uses: "docker/build-push-action@v4", + Name: step.DisplayName, + With: map[string]string{ + "push": step.Inputs["push"], + "tags": step.Inputs["containerRegistry"] + "/" + step.Inputs["repository"], + }, + } + case "PublishTestResults@2": + return ForgejoStep{ + Uses: "actions/upload-artifact@v3", + Name: step.DisplayName, + With: map[string]string{ + "name": "test-results", + "path": step.Inputs["testResultsFiles"], + }, + } + case "CopyFiles@2": + return ForgejoStep{ + Run: fmt.Sprintf("cp -r %s %s", step.Inputs["SourceFolder"], step.Inputs["TargetFolder"]), + Name: step.DisplayName, + } + case "ArchiveFiles@2": + return ForgejoStep{ + Run: fmt.Sprintf("zip -r %s %s", step.Inputs["archiveFile"], step.Inputs["rootFolderOrFile"]), + Name: step.DisplayName, + } + default: + return ForgejoStep{} + } +} diff --git a/internal/forgejo/client.go b/internal/forgejo/client.go index 40212d7..7973fc1 100644 --- a/internal/forgejo/client.go +++ b/internal/forgejo/client.go @@ -1,6 +1,7 @@ package forgejo import ( + "encoding/base64" "fmt" "net/http" "time" @@ -14,6 +15,11 @@ type Client struct { baseURL string } +type Secret struct { + Name string `json:"name"` + Data string `json:"data"` +} + type MigrateRequest struct { CloneAddr string `json:"clone_addr"` AuthUsername string `json:"auth_username"` @@ -133,3 +139,58 @@ func (c *Client) DeleteRepository(owner, name string) error { return nil } + +// CreateSecret creates a repository secret for Actions +func (c *Client) CreateSecret(owner, repo, name, data string) error { + // Forgejo/Gitea API for secrets + url := fmt.Sprintf("/repos/%s/%s/actions/secrets/%s", owner, repo, name) + + body := map[string]string{ + "data": data, + } + + resp, err := c.client.R(). + SetBody(body). + Put(url) + + if err != nil { + return fmt.Errorf("failed to create secret: %w", err) + } + + if resp.StatusCode() != 201 && resp.StatusCode() != 204 { + return fmt.Errorf("failed to create secret, status: %d", resp.StatusCode()) + } + + return nil +} + +// CreateFile creates a file in the repository (for workflow files) +func (c *Client) CreateFile(owner, repo, path, content, message string) error { + // Content needs to be base64 encoded + encoded := base64.StdEncoding.EncodeToString([]byte(content)) + + body := map[string]interface{}{ + "content": encoded, + "message": message, + } + + url := fmt.Sprintf("/repos/%s/%s/contents/%s", owner, repo, path) + + resp, err := c.client.R(). + SetBody(body). + Post(url) + + if err != nil { + return err + } + + // 201 = created, 422 = already exists (should handle update in production) + if resp.StatusCode() != 201 { + if resp.StatusCode() == 422 { + return fmt.Errorf("file %s already exists", path) + } + return fmt.Errorf("failed to create file, status %d: %s", resp.StatusCode(), resp.String()) + } + + return nil +} diff --git a/internal/migrator/migrator.go b/internal/migrator/migrator.go index 151991f..cda11d3 100644 --- a/internal/migrator/migrator.go +++ b/internal/migrator/migrator.go @@ -7,18 +7,24 @@ import ( "vix.ro/sonix/azure_migrate/internal/azure" "vix.ro/sonix/azure_migrate/internal/config" + "vix.ro/sonix/azure_migrate/internal/converter" "vix.ro/sonix/azure_migrate/internal/forgejo" ) type AzureClient interface { GetRepositories() ([]azure.Repository, error) GetAuthenticatedURL(remoteURL, pat string) string + GetBuildDefinitions(project string) ([]azure.BuildDefinition, error) + GetPipelineVariables(project string, definitionID int) (map[string]string, error) + GetPipelineContent(repoURL, branch, filePath string) (string, error) } type ForgejoClient interface { GetUser(username string) (*forgejo.User, error) RepositoryExists(owner, name string) (bool, error) CreateMirror(req forgejo.MigrateRequest) (*forgejo.Repository, error) + CreateFile(owner, repo, path, content, message string) error + CreateSecret(owner, repo, name, data string) error } type Migrator struct { @@ -146,6 +152,12 @@ func (m *Migrator) Run() error { return nil } +func (m *Migrator) addError(err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.errors = append(m.errors, err) +} + func (m *Migrator) migrateRepository(repo azure.Repository, uid int64) MigrationResult { result := MigrationResult{ RepoName: repo.Name, @@ -184,6 +196,9 @@ func (m *Migrator) migrateRepository(repo azure.Repository, uid int64) Migration if m.config.DryRun { result.Success = true result.ForgejoURL = fmt.Sprintf("%s/%s/%s", m.config.ForgejoURL, m.config.ForgejoOwner, repo.Name) + + // Show what would be migrated for pipelines + fmt.Printf(" [DRY-RUN] Would migrate pipeline for %s\n", repo.Name) return result } @@ -196,15 +211,135 @@ func (m *Migrator) migrateRepository(repo azure.Repository, uid int64) Migration return result } + // Migrate pipelines after repo creation + if err := m.migratePipeline(repo, createdRepo); err != nil { + // Log warning but don't fail the whole migration + fmt.Printf(" Warning: Pipeline migration failed for %s: %v\n", repo.Name, err) + } + result.Success = true result.ForgejoURL = createdRepo.HTMLURL return result } -func (m *Migrator) addError(err error) { - m.mu.Lock() - defer m.mu.Unlock() - m.errors = append(m.errors, err) +func (m *Migrator) migratePipeline(repo azure.Repository, forgejoRepo *forgejo.Repository) error { + fmt.Printf(" Checking for pipelines in %s...\n", repo.Name) + + // Get build definitions for this repo's project + defs, err := m.azureClient.GetBuildDefinitions(repo.Project.Name) + if err != nil { + return fmt.Errorf("failed to get pipeline definitions: %w", err) + } + + // Find pipelines associated with this repository + for _, def := range defs { + // Check if this pipeline belongs to the current repo + if !strings.Contains(def.Repository.URL, repo.Name) { + continue + } + + fmt.Printf(" Found pipeline: %s (ID: %d)\n", def.Name, def.ID) + + // Get variables + vars, err := m.azureClient.GetPipelineVariables(repo.Project.Name, def.ID) + if err != nil { + fmt.Printf(" Warning: Could not get variables: %v\n", err) + } + + // Migrate variables as secrets + for key, value := range vars { + secretName := "AZURE_" + strings.ToUpper(key) + if err := m.forgejoClient.CreateSecret(m.config.ForgejoOwner, repo.Name, secretName, value); err != nil { + fmt.Printf(" Warning: Failed to create secret %s: %v\n", secretName, err) + } else { + fmt.Printf(" Migrated secret: %s\n", secretName) + } + } + + // Try to get YAML content + var yamlContent string + if def.Process.YamlFilename != "" { + // YAML pipeline + content, err := m.azureClient.GetPipelineContent(repo.RemoteURL, "master", def.Process.YamlFilename) + if err != nil { + content, err = m.azureClient.GetPipelineContent(repo.RemoteURL, "main", def.Process.YamlFilename) + } + if err == nil { + yamlContent = content + } + } + + if yamlContent != "" { + // Convert YAML + conv := converter.New() + forgejoYAML, err := conv.ConvertAzureToForgejo(yamlContent, repo.Name) + if err != nil { + return fmt.Errorf("failed to convert pipeline YAML: %w", err) + } + + // Create workflow file + workflowPath := fmt.Sprintf(".forgejo/workflows/%s.yml", strings.ReplaceAll(def.Name, " ", "-")) + // Also create .github/workflows for compatibility + githubPath := fmt.Sprintf(".github/workflows/%s.yml", strings.ReplaceAll(def.Name, " ", "-")) + + commitMsg := fmt.Sprintf("Migrate Azure Pipeline: %s", def.Name) + + // Try Forgejo path first + err = m.forgejoClient.CreateFile(m.config.ForgejoOwner, repo.Name, workflowPath, forgejoYAML, commitMsg) + if err != nil { + // If .forgejo doesn't work (older versions), try .github + err = m.forgejoClient.CreateFile(m.config.ForgejoOwner, repo.Name, githubPath, forgejoYAML, commitMsg) + } + + if err != nil { + return fmt.Errorf("failed to create workflow file: %w", err) + } + + fmt.Printf(" Created workflow: %s\n", workflowPath) + + // Print conversion warnings + for _, warning := range conv.Warnings { + fmt.Printf(" ⚠️ %s\n", warning) + } + } else { + // Classic pipeline (UI-based) - create placeholder + placeholder := m.generatePlaceholderWorkflow(def) + workflowPath := fmt.Sprintf(".forgejo/workflows/%s-placeholder.yml", def.ID) + + err = m.forgejoClient.CreateFile(m.config.ForgejoOwner, repo.Name, workflowPath, placeholder, + "Placeholder for classic Azure Pipeline") + if err != nil { + fmt.Printf(" Warning: Could not create placeholder: %v\n", err) + } + fmt.Printf(" Created placeholder for classic pipeline (manual conversion required)\n") + } + } + + return nil +} + +func (m *Migrator) generatePlaceholderWorkflow(def azure.BuildDefinition) string { + return fmt.Sprintf(`# Placeholder for Classic Azure Pipeline: %s +# Pipeline ID: %d +# +# This pipeline was defined in Azure DevOps UI (Classic) and cannot be automatically converted. +# Please manually recreate this workflow using Forgejo Actions syntax. +# +# Variables defined in this pipeline: +name: Azure Pipeline Placeholder - %s + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: TODO - Add steps from Azure Pipeline + run: | + echo "Original Azure Pipeline: %s" + echo "Review the Azure DevOps UI for this pipeline's tasks and recreate them here" +`, def.Name, def.ID, def.Name, def.Name) } func truncate(s string, length int) string {