diff --git a/README.md b/README.md index ea1c710..006f44f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # go-gitea-vendor -Want to vendor all your Go projects on your personal Gitea instance? -Don't want to bloat your git history? -Use `go-gitea-vendor` to maintain a separate repo that is only the latest vendoring! +Want to vendor all your public Go projects hosted on your personal Gitea instance? +Don't want to bloat your git history and exhaust your storage? +Use `go-gitea-vendor` to maintain separate, shallow-clone repos that hold only the latest commit and vendored dependencies for each of your Go projects! ## Usage -`echo 'EXAMPLE_API_TOKEN' | go-gitea-vendor https://gitea.example.com` +`echo 'EXAMPLE_API_TOKEN' | go-gitea-vendor https://gitea.example.com [output organization]` -I recommend setting up a chron job or some form of automation for this. +I recommend setting up a chron job or some form of automation for this. My deployment involves running this on my Gitea mirror (this server mirrors my main Gitea instance as well as my GitHub) weekly using a chron job, then my main Gitea instance mirrors the output repos from this tool. diff --git a/go.mod b/go.mod index 494baeb..6a7606d 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,24 @@ module go-gitea-vendor go 1.26.4 -require github.com/rwinkhart/go-boilerplate v0.3.1 +require ( + github.com/go-git/go-git/v6 v6.0.0-alpha.4 + github.com/rwinkhart/go-boilerplate v0.3.1 +) + +require ( + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.4.1 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg/v2 v2.0.2 // indirect + github.com/go-git/go-billy/v6 v6.0.0-alpha.1 // indirect + github.com/kevinburke/ssh_config v1.6.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + golang.org/x/crypto v0.51.0 // indirect + golang.org/x/net v0.54.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.44.0 // indirect +) diff --git a/go.sum b/go.sum index 969f986..8c9acf3 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,62 @@ +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo= +github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs= +github.com/go-git/go-billy/v6 v6.0.0-alpha.1 h1:xVjAR4oUvrKy7/Xuw/lLlV3gkxR3KO2H8W+MamuVVsQ= +github.com/go-git/go-billy/v6 v6.0.0-alpha.1/go.mod h1:eaCUpHbedW7//EwcYmUDfJe2N6sJC9O12AT0OTqJR1E= +github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1 h1:gmqi2jvsreu0s8JMLylYDFq4sbjHwwlhktMw0DUg3mA= +github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1/go.mod h1:ECf1MqJlBdYpKggBrOXjo/0EnvRZx6D++I86UYjPgAQ= +github.com/go-git/go-git/v6 v6.0.0-alpha.4 h1:aDTc2UGanmaE7FkGLSlBEB9nohMnQ+RKXcfq/D+esDQ= +github.com/go-git/go-git/v6 v6.0.0-alpha.4/go.mod h1:4ODa/G7hPWrh4Y+7lmt59Ij3zW38IEfvRoAZxLYYBhc= +github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= +github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rwinkhart/go-boilerplate v0.3.1 h1:vkVRuptO2s1yPzzwpvtiiB0c/hPB6yr0p1mzZ2Myb9E= github.com/rwinkhart/go-boilerplate v0.3.1/go.mod h1:ES13A2r9fnCVfyezwMBgY/RgA4pOIudOUXz3Jk/ikes= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 587143b..b574256 100644 --- a/main.go +++ b/main.go @@ -8,25 +8,39 @@ import ( "os" "os/exec" + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/config" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + githttp "github.com/go-git/go-git/v6/plumbing/transport/http" "github.com/rwinkhart/go-boilerplate/back" "github.com/rwinkhart/go-boilerplate/other" ) +const workingDir = "repos-new" +const finishedDir = "repos" + type giteaRespT struct { Data []struct { - Language string `json:"language"` - MasterURL string `json:"original_url"` - Name string `json:"name"` + MirrorInterval string `json:"mirror_interval"` + Language string `json:"language"` + MasterURL string `json:"original_url"` + Name string `json:"name"` } } func main() { - if len(os.Args) != 2 { - fmt.Println("Usage: " + os.Args[0] + " \n\nAPI token can be piped into stdin to run non-interactively!") + // args + if len(os.Args) < 3 { + fmt.Println("Usage: " + os.Args[0] + " [output organization]\n\nAPI token can be piped into stdin to run non-interactively!") os.Exit(0) } - - url := os.Args[1] + "/api/v1/repos/search?sort=updated&order=desc&limit=999" + baseURL := os.Args[1] + apiURL := baseURL + "/api/v1/repos/search?sort=updated&order=desc&limit=999" + username := os.Args[2] + var organization string + if len(os.Args) == 4 { + organization = os.Args[3] + } fmt.Print("Input token: ") token := back.ReadFromStdin() if len(token) == 0 { @@ -34,61 +48,102 @@ func main() { } fmt.Print("\r \r") - req, err := http.NewRequest("GET", url, nil) - if err != nil { - other.PrintError("Failed to create request: "+err.Error(), 2) + // start cleanup + if err := os.RemoveAll(workingDir); err != nil { + other.PrintError("Failed to remove "+workingDir+": "+err.Error(), 2) } + // api call + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + other.PrintError("Failed to create request: "+err.Error(), 3) + } req.Header.Set("Accept", "application/json") req.Header.Set("Authorization", "token "+string(token)) - client := &http.Client{} resp, err := client.Do(req) if err != nil { - other.PrintError("Failed to make request: "+err.Error(), 3) + other.PrintError("Failed to make request: "+err.Error(), 4) } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) if err != nil { - other.PrintError("Failed to read response body: "+err.Error(), 4) + other.PrintError("Failed to read response body: "+err.Error(), 5) } + // cloning and vendoring var giteaResp giteaRespT if err = json.Unmarshal(body, &giteaResp); err != nil { - other.PrintError("Failed to unmarshal response: "+err.Error(), 5) + other.PrintError("Failed to unmarshal response: "+err.Error(), 6) } - - const workingDir = "repos-new" - const finishedDir = "repos" os.Mkdir(workingDir, 0755) for _, repo := range giteaResp.Data { - if repo.Language == "Go" { - cloneCmd := exec.Command("git", "clone", "--depth=1", repo.MasterURL) - cloneCmd.Dir = workingDir - if err = cloneCmd.Run(); err != nil { - other.PrintError("Failed to clone "+repo.MasterURL+": "+err.Error(), 6) + if repo.Language == "Go" && repo.MirrorInterval != "" { + //// clone + currentRepoDir := workingDir + "/" + repo.Name + "-ggv" + _, err = git.PlainClone(currentRepoDir, &git.CloneOptions{URL: repo.MasterURL, Progress: nil, Depth: 1}) + if err != nil { + other.PrintError("Failed to clone "+repo.MasterURL+": "+err.Error(), 7) } + //// vendor vendorCmd := exec.Command("go", "mod", "vendor") - vendorCmd.Dir = workingDir + "/" + repo.Name + vendorCmd.Dir = currentRepoDir if err = vendorCmd.Run(); err != nil { - other.PrintError("Failed to vendor "+repo.MasterURL+": "+err.Error(), 7) + other.PrintError("Failed to vendor "+repo.MasterURL+": "+err.Error(), 8) } - if err = os.RemoveAll(workingDir + "/" + repo.Name + "/.git"); err != nil { - other.PrintError("Failed to remove .git directory for "+repo.MasterURL, 8) + //// re-init + if err = os.RemoveAll(currentRepoDir + "/.git"); err != nil { + other.PrintError("Failed to remove .git directory for "+repo.MasterURL, 9) + } + goRepo, err := git.PlainInit(currentRepoDir, false) + if err != nil { + other.PrintError("Failed to init in "+currentRepoDir, 10) + } + //// set origin + if _, err = goRepo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{baseURL + "/" + organization + "/" + repo.Name + "-ggv.git"}, + }); err != nil { + other.PrintError("Failed to set origin for "+currentRepoDir+": "+err.Error(), 15) + } + //// add + wt, err := goRepo.Worktree() + if err != nil { + other.PrintError("Failed to get worktree for "+currentRepoDir, 11) + } + if err = wt.AddGlob("."); err != nil { + other.PrintError("Failed to add files in "+currentRepoDir+": "+err.Error(), 12) + } + //// commit + if _, err = wt.Commit("go-gitea-vendor", &git.CommitOptions{}); err != nil { + other.PrintError("Failed to commit in "+currentRepoDir+": "+err.Error(), 13) + } + //// push + if err = goRepo.Push(&git.PushOptions{ + Force: true, + ClientOptions: []gitclient.Option{ + gitclient.WithHTTPAuth(&githttp.BasicAuth{ + Username: username, + Password: string(token), + }), + }, + }); err != nil { + other.PrintError("Failed to push "+currentRepoDir+": "+err.Error(), 17) } } } + + // end cleanup isAccessible, err := back.TargetIsFile(finishedDir, false) if isAccessible && err == nil { if err = os.RemoveAll(finishedDir + ".bak"); err != nil { - other.PrintError("Failed to remove old backup dir", 9) + other.PrintError("Failed to remove old backup dir", 14) } if err = os.Rename(finishedDir, finishedDir+".bak"); err != nil { - other.PrintError("Failed to backup old repos dir", 10) + other.PrintError("Failed to backup old repos dir", 15) } } if err = os.Rename(workingDir, finishedDir); err != nil { - other.PrintError("Failed to rename new repos dir", 11) + other.PrintError("Failed to rename new repos dir", 16) } }