Merge pull request #4 from zikani03/zikani03/issue#3

feat: Make in-memory option, closes #3

Author: zikani03

README.md

@@ -1,16 +1,29 @@
 git-monorepo
 ============
 
-Make Git monorepos like a boss.
+> Make Git monorepos like a boss.
+
+`git-monorepo` is a tool that allows you to combine two or more Git repositories into one repository and preserves the commit history for each repo as well as the directory structure.
 
 ## Features
 
-- Preserve commit history for each repo
-- Preserve Directory structure of each repo
-- Ordering Commits chronologically
+- Preserves commit history for each repo
+- Preserves Directory structure of each repo
+- Orders Commits chronologically
+
+## Building
+
+git-monorepo requires atleast Go 1.18 to build. 
+
+```sh
+$ git clone https://github.com/zikani03/git-monorepo
+$ go build -o git-monorepo ./cmd/main.go
+```
 
 ## Usage
 
+> NOTE: It's currently very CPU intensive at the moment, I'm gonna look for ways to reduce CPU usage. May also be memory intensive if you use the `--in-memory` flag. I also haven't tried it on very large repos (i.e. repos with very many many commits)
+
 ### Create monorepo locally
 
 ```shell
@@ -25,9 +38,13 @@ $ git-monorepo init --sources https://github.com/nndi-oss/ussdproxy,gh:nndi-oss/
 
 ## IDEAS / TODO
 
-- Make Submodules instead of mangling history
-- Parallel clones
-- Create monorepo locally and push to github
+- Support "squashing-merging" - that is merge the repos without preserving fine-grained commit history
+
+- Make Git Submodules instead of mangling history
+
+- Clone repositories in parallel
+
+- Create monorepo locally and push to github (and other git hosting services)
 
 ```shell
 $ git monorepo init --preserve-history \ 

cmd/main.go

@@ -30,6 +30,7 @@ type InitCmd struct {
 	Daemonize       bool     `help:"Daemonize or run in foreground"`
 	Mangle          bool     `help:"Combine files from repos in one directory (not recommended!)"`
 	PreserveHistory bool     `help:"Preserve history from the repos"`
+	InMemory        bool     `help:"Clone repos in-memory instead of a temporary directory on file system"`
 	MakeSubmodules  bool     `help:"Add child repositories as submodules (not ideal!)"`
 	TargetDir       string   `name:"target" help:"The target directory to create repo in. Must not exist" type:"path"`
 	Sources         []string `help:"Source repositories with support for 'git-down' shortcuts"`
@@ -47,6 +48,9 @@ func CheckIfError(err error) {
 
 func (r *InitCmd) Run(globals *Globals) error {
 	repo := gitMonorepo.NewMonorepoFromSources(r.Sources)
+	if r.InMemory {
+		repo = gitMonorepo.NewMonorepoFromSourcesInMemory(r.Sources)
+	}
 	err := repo.Init(r.TargetDir)
 	CheckIfError(err)
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/zikani03/git-monorepo
 
-go 1.17
+go 1.18
 
 require (
 	github.com/alecthomas/kong v0.5.0

monorepo.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"os"
 	"path"
 	"sort"
 	"strings"
@@ -21,6 +22,7 @@ type Monorepo struct {
 	// done chan bool
 	Dir                  string
 	SourceRepositoryURLs []string
+	UseInMemoryStorage   bool
 }
 
 type RepositoryCommit struct {
@@ -31,6 +33,14 @@ type RepositoryCommit struct {
 func NewMonorepoFromSources(repos []string) *Monorepo {
 	return &Monorepo{
 		SourceRepositoryURLs: repos,
+		UseInMemoryStorage:   false,
+	}
+}
+
+func NewMonorepoFromSourcesInMemory(repos []string) *Monorepo {
+	return &Monorepo{
+		SourceRepositoryURLs: repos,
+		UseInMemoryStorage:   true,
 	}
 }
 
@@ -63,14 +73,34 @@ func (m *Monorepo) initWithCommitsBetween(targetDir string, fromTime, toTime tim
 
 	worktreeDir := billyfs.New(targetDir)
 
+	dirsToClean := make([]string, 0)
 	for _, urlSpec := range m.SourceRepositoryURLs {
 		sourceRepositoryUrl := makeFullUrl(urlSpec)
-		repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
-			URL: sourceRepositoryUrl,
-		})
-		if err != nil {
-			return fmt.Errorf("failed to git clone %s : %v", sourceRepositoryUrl, err)
+		var repo *git.Repository
+		var err error
+		if m.UseInMemoryStorage {
+			storage := memory.NewStorage()
+			repo, err = git.Clone(storage, nil, &git.CloneOptions{
+				URL: sourceRepositoryUrl,
+			})
+			if err != nil {
+				fmt.Println(fmt.Errorf("failed to git clone %s : %v", sourceRepositoryUrl, err))
+				return err
+			}
+		} else {
+			tmpDirName, err := os.MkdirTemp(os.TempDir(), "monorepo_")
+			if err != nil {
+				return err
+			}
+			storage := filesystem.NewStorage(billyfs.New(tmpDirName), cache.NewObjectLRUDefault())
+			repo, err = git.Clone(storage, nil, &git.CloneOptions{URL: sourceRepositoryUrl})
+			if err != nil {
+				fmt.Println(fmt.Errorf("failed to git clone %s to %s: %v", sourceRepositoryUrl, tmpDirName, err))
+				return err
+			}
+			dirsToClean = append(dirsToClean, tmpDirName)
 		}
+
 		// retrieves the branch pointed by HEAD
 		repoRef, err := repo.Head()
 		if err != nil {
@@ -166,5 +196,14 @@ func (m *Monorepo) initWithCommitsBetween(targetDir string, fromTime, toTime tim
 		}
 	}
 
+	// Clean up the tmp directories
+	if len(dirsToClean) > 0 {
+		for _, dirToPurge := range dirsToClean {
+			if err := os.RemoveAll(dirToPurge); err != nil {
+				fmt.Printf("failed to remove tmp directory at %s got %v\n", dirToPurge, err)
+			}
+		}
+	}
+
 	return nil
 }