Wire up repo pushes

README.md

@@ -1,9 +1,9 @@
 # go-gitea-vendor
-Want to vendor all your Go projects on your personal Gitea instance?
+Want to vendor all your public Go projects hosted on your personal Gitea instance?
-Don't want to bloat your git history?
+Don't want to bloat your git history and exhaust your storage?
-Use `go-gitea-vendor` to maintain a separate repo that is only the latest vendoring!
+Use `go-gitea-vendor` to maintain separate, shallow-clone repos that hold only the latest commit and vendored dependencies for each of your Go projects!
 
 ## Usage
-`echo 'EXAMPLE_API_TOKEN' | go-gitea-vendor https://gitea.example.com`
+`echo 'EXAMPLE_API_TOKEN' | go-gitea-vendor https://gitea.example.com <username> [output organization]`
 
-I recommend setting up a chron job or some form of automation for this.
+I recommend setting up a chron job or some form of automation for this. My deployment involves running this on my Gitea mirror (this server mirrors my main Gitea instance as well as my GitHub) weekly using a chron job, then my main Gitea instance mirrors the output repos from this tool.

go.mod

@@ -2,4 +2,24 @@ module go-gitea-vendor
 
 go 1.26.4
 
-require github.com/rwinkhart/go-boilerplate v0.3.1
+require (
+	github.com/go-git/go-git/v6 v6.0.0-alpha.4
+	github.com/rwinkhart/go-boilerplate v0.3.1
+)
+
+require (
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/go-crypto v1.4.1 // indirect
+	github.com/cloudflare/circl v1.6.3 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg/v2 v2.0.2 // indirect
+	github.com/go-git/go-billy/v6 v6.0.0-alpha.1 // indirect
+	github.com/kevinburke/ssh_config v1.6.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/pjbgf/sha1cd v0.6.0 // indirect
+	github.com/sergi/go-diff v1.4.0 // indirect
+	golang.org/x/crypto v0.51.0 // indirect
+	golang.org/x/net v0.54.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.44.0 // indirect
+)

go.sum

@@ -1,2 +1,62 @@
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
+github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
+github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
+github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
+github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo=
+github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs=
+github.com/go-git/go-billy/v6 v6.0.0-alpha.1 h1:xVjAR4oUvrKy7/Xuw/lLlV3gkxR3KO2H8W+MamuVVsQ=
+github.com/go-git/go-billy/v6 v6.0.0-alpha.1/go.mod h1:eaCUpHbedW7//EwcYmUDfJe2N6sJC9O12AT0OTqJR1E=
+github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1 h1:gmqi2jvsreu0s8JMLylYDFq4sbjHwwlhktMw0DUg3mA=
+github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1/go.mod h1:ECf1MqJlBdYpKggBrOXjo/0EnvRZx6D++I86UYjPgAQ=
+github.com/go-git/go-git/v6 v6.0.0-alpha.4 h1:aDTc2UGanmaE7FkGLSlBEB9nohMnQ+RKXcfq/D+esDQ=
+github.com/go-git/go-git/v6 v6.0.0-alpha.4/go.mod h1:4ODa/G7hPWrh4Y+7lmt59Ij3zW38IEfvRoAZxLYYBhc=
+github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY=
+github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU=
+github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rwinkhart/go-boilerplate v0.3.1 h1:vkVRuptO2s1yPzzwpvtiiB0c/hPB6yr0p1mzZ2Myb9E=
 github.com/rwinkhart/go-boilerplate v0.3.1/go.mod h1:ES13A2r9fnCVfyezwMBgY/RgA4pOIudOUXz3Jk/ikes=
+github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
+github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
+golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
+golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
+golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
+golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -8,25 +8,39 @@ import (
 	"os"
 	"os/exec"
 
+	"github.com/go-git/go-git/v6"
+	"github.com/go-git/go-git/v6/config"
+	gitclient "github.com/go-git/go-git/v6/plumbing/client"
+	githttp "github.com/go-git/go-git/v6/plumbing/transport/http"
 	"github.com/rwinkhart/go-boilerplate/back"
 	"github.com/rwinkhart/go-boilerplate/other"
 )
 
+const workingDir = "repos-new"
+const finishedDir = "repos"
+
 type giteaRespT struct {
 	Data []struct {
-		Language  string `json:"language"`
+		MirrorInterval string `json:"mirror_interval"`
-		MasterURL string `json:"original_url"`
+		Language       string `json:"language"`
-		Name      string `json:"name"`
+		MasterURL      string `json:"original_url"`
+		Name           string `json:"name"`
 	}
 }
 
 func main() {
-	if len(os.Args) != 2 {
+	// args
-		fmt.Println("Usage: " + os.Args[0] + " <base url>\n\nAPI token can be piped into stdin to run non-interactively!")
+	if len(os.Args) < 3 {
+		fmt.Println("Usage: " + os.Args[0] + " <base url> <username> [output organization]\n\nAPI token can be piped into stdin to run non-interactively!")
 		os.Exit(0)
 	}
-
+	baseURL := os.Args[1]
-	url := os.Args[1] + "/api/v1/repos/search?sort=updated&order=desc&limit=999"
+	apiURL := baseURL + "/api/v1/repos/search?sort=updated&order=desc&limit=999"
+	username := os.Args[2]
+	var organization string
+	if len(os.Args) == 4 {
+		organization = os.Args[3]
+	}
 	fmt.Print("Input token: ")
 	token := back.ReadFromStdin()
 	if len(token) == 0 {
@@ -34,61 +48,102 @@ func main() {
 	}
 	fmt.Print("\r            \r")
 
-	req, err := http.NewRequest("GET", url, nil)
+	// start cleanup
-	if err != nil {
+	if err := os.RemoveAll(workingDir); err != nil {
-		other.PrintError("Failed to create request: "+err.Error(), 2)
+		other.PrintError("Failed to remove "+workingDir+": "+err.Error(), 2)
 	}
 
+	// api call
+	req, err := http.NewRequest("GET", apiURL, nil)
+	if err != nil {
+		other.PrintError("Failed to create request: "+err.Error(), 3)
+	}
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("Authorization", "token "+string(token))
-
 	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
-		other.PrintError("Failed to make request: "+err.Error(), 3)
+		other.PrintError("Failed to make request: "+err.Error(), 4)
 	}
 	defer resp.Body.Close()
-
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		other.PrintError("Failed to read response body: "+err.Error(), 4)
+		other.PrintError("Failed to read response body: "+err.Error(), 5)
 	}
 
+	// cloning and vendoring
 	var giteaResp giteaRespT
 	if err = json.Unmarshal(body, &giteaResp); err != nil {
-		other.PrintError("Failed to unmarshal response: "+err.Error(), 5)
+		other.PrintError("Failed to unmarshal response: "+err.Error(), 6)
 	}
-
-	const workingDir = "repos-new"
-	const finishedDir = "repos"
 	os.Mkdir(workingDir, 0755)
 	for _, repo := range giteaResp.Data {
-		if repo.Language == "Go" {
+		if repo.Language == "Go" && repo.MirrorInterval != "" {
-			cloneCmd := exec.Command("git", "clone", "--depth=1", repo.MasterURL)
+			//// clone
-			cloneCmd.Dir = workingDir
+			currentRepoDir := workingDir + "/" + repo.Name + "-ggv"
-			if err = cloneCmd.Run(); err != nil {
+			_, err = git.PlainClone(currentRepoDir, &git.CloneOptions{URL: repo.MasterURL, Progress: nil, Depth: 1})
-				other.PrintError("Failed to clone "+repo.MasterURL+": "+err.Error(), 6)
+			if err != nil {
+				other.PrintError("Failed to clone "+repo.MasterURL+": "+err.Error(), 7)
 			}
+			//// vendor
 			vendorCmd := exec.Command("go", "mod", "vendor")
-			vendorCmd.Dir = workingDir + "/" + repo.Name
+			vendorCmd.Dir = currentRepoDir
 			if err = vendorCmd.Run(); err != nil {
-				other.PrintError("Failed to vendor "+repo.MasterURL+": "+err.Error(), 7)
+				other.PrintError("Failed to vendor "+repo.MasterURL+": "+err.Error(), 8)
 			}
-			if err = os.RemoveAll(workingDir + "/" + repo.Name + "/.git"); err != nil {
+			//// re-init
-				other.PrintError("Failed to remove .git directory for "+repo.MasterURL, 8)
+			if err = os.RemoveAll(currentRepoDir + "/.git"); err != nil {
+				other.PrintError("Failed to remove .git directory for "+repo.MasterURL, 9)
+			}
+			goRepo, err := git.PlainInit(currentRepoDir, false)
+			if err != nil {
+				other.PrintError("Failed to init in "+currentRepoDir, 10)
+			}
+			//// set origin
+			if _, err = goRepo.CreateRemote(&config.RemoteConfig{
+				Name: "origin",
+				URLs: []string{baseURL + "/" + organization + "/" + repo.Name + "-ggv.git"},
+			}); err != nil {
+				other.PrintError("Failed to set origin for "+currentRepoDir+": "+err.Error(), 15)
+			}
+			//// add
+			wt, err := goRepo.Worktree()
+			if err != nil {
+				other.PrintError("Failed to get worktree for "+currentRepoDir, 11)
+			}
+			if err = wt.AddGlob("."); err != nil {
+				other.PrintError("Failed to add files in "+currentRepoDir+": "+err.Error(), 12)
+			}
+			//// commit
+			if _, err = wt.Commit("go-gitea-vendor", &git.CommitOptions{}); err != nil {
+				other.PrintError("Failed to commit in "+currentRepoDir+": "+err.Error(), 13)
+			}
+			//// push
+			if err = goRepo.Push(&git.PushOptions{
+				Force: true,
+				ClientOptions: []gitclient.Option{
+					gitclient.WithHTTPAuth(&githttp.BasicAuth{
+						Username: username,
+						Password: string(token),
+					}),
+				},
+			}); err != nil {
+				other.PrintError("Failed to push "+currentRepoDir+": "+err.Error(), 17)
 			}
 		}
 	}
+
+	// end cleanup
 	isAccessible, err := back.TargetIsFile(finishedDir, false)
 	if isAccessible && err == nil {
 		if err = os.RemoveAll(finishedDir + ".bak"); err != nil {
-			other.PrintError("Failed to remove old backup dir", 9)
+			other.PrintError("Failed to remove old backup dir", 14)
 		}
 		if err = os.Rename(finishedDir, finishedDir+".bak"); err != nil {
-			other.PrintError("Failed to backup old repos dir", 10)
+			other.PrintError("Failed to backup old repos dir", 15)
 		}
 	}
 	if err = os.Rename(workingDir, finishedDir); err != nil {
-		other.PrintError("Failed to rename new repos dir", 11)
+		other.PrintError("Failed to rename new repos dir", 16)
 	}
 }