Merge pull request #464 from kuba--/cache-440

One configurable cache per `repository pool`

Author: ajnavarro

Gopkg.lock

@@ -17,7 +17,7 @@
 [[constraint]]
   name = "gopkg.in/src-d/go-git.v4"
   source = "github.com/src-d/go-git"
-  revision = "d3cec13ac0b195bfb897ed038a08b5130ab9969e"
+  revision = "2fb32d2a8601213b6db109d3e9028c6b64af1874"
 
 [[constraint]]
   name = "gopkg.in/src-d/go-git-fixtures.v3"

Gopkg.toml

@@ -16,6 +16,7 @@ import (
 	gopilosa "github.com/pilosa/go-pilosa"
 	"github.com/sirupsen/logrus"
 	"github.com/uber/jaeger-client-go/config"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	sqle "gopkg.in/src-d/go-mysql-server.v0"
 	"gopkg.in/src-d/go-mysql-server.v0/server"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
@@ -40,25 +41,25 @@ type Server struct {
 	pool   *gitbase.RepositoryPool
 	name   string
 
-	Version       string   // Version of the application.
-	Directories   []string `short:"d" long:"directories" description:"Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs."`
-	Depth         int      `long:"depth" default:"1000" description:"load repositories looking at less than <depth> nested subdirectories."`
-	Host          string   `long:"host" default:"localhost" description:"Host where the server is going to listen"`
-	Port          int      `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"`
-	User          string   `short:"u" long:"user" default:"root" description:"User name used for connection"`
-	Password      string   `short:"P" long:"password" default:"" description:"Password used for connection"`
-	PilosaURL     string   `long:"pilosa" default:"http://localhost:10101" description:"URL to your pilosa server" env:"PILOSA_ENDPOINT"`
-	IndexDir      string   `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"`
-	DisableSquash bool     `long:"no-squash" description:"Disables the table squashing."`
-	TraceEnabled  bool     `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"`
-	ReadOnly      bool     `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well." env:"GITBASE_READONLY"`
-	Parallelism   uint     `long:"parallelism" description:"Maximum number of parallel threads per table. By default, it's the number of CPU cores. 0 means default, 1 means disabled."`
-
-	SkipGitErrors bool // SkipGitErrors disables failing when Git errors are found.
-	DisableGit    bool `long:"no-git" description:"disable the load of git standard repositories."`
-	DisableSiva   bool `long:"no-siva" description:"disable the load of siva files."`
-	Verbose       bool `short:"v" description:"Activates the verbose mode"`
-	OldUast       bool `long:"old-uast-serialization" description:"serialize uast in the old format" env:"GITBASE_UAST_SERIALIZATION"`
+	Version       string         // Version of the application.
+	Directories   []string       `short:"d" long:"directories" description:"Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs."`
+	Depth         int            `long:"depth" default:"1000" description:"load repositories looking at less than <depth> nested subdirectories."`
+	Host          string         `long:"host" default:"localhost" description:"Host where the server is going to listen"`
+	Port          int            `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"`
+	User          string         `short:"u" long:"user" default:"root" description:"User name used for connection"`
+	Password      string         `short:"P" long:"password" default:"" description:"Password used for connection"`
+	PilosaURL     string         `long:"pilosa" default:"http://localhost:10101" description:"URL to your pilosa server" env:"PILOSA_ENDPOINT"`
+	IndexDir      string         `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"`
+	CacheSize     cache.FileSize `long:"cache" default:"512" description:"Object cache size in megabytes" env:"GITBASE_CACHESIZE_MB"`
+	Parallelism   uint           `long:"parallelism" description:"Maximum number of parallel threads per table. By default, it's the number of CPU cores. 0 means default, 1 means disabled."`
+	DisableSquash bool           `long:"no-squash" description:"Disables the table squashing."`
+	TraceEnabled  bool           `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"`
+	ReadOnly      bool           `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well." env:"GITBASE_READONLY"`
+	SkipGitErrors bool           // SkipGitErrors disables failing when Git errors are found.
+	DisableGit    bool           `long:"no-git" description:"disable the load of git standard repositories."`
+	DisableSiva   bool           `long:"no-siva" description:"disable the load of siva files."`
+	Verbose       bool           `short:"v" description:"Activates the verbose mode"`
+	OldUast       bool           `long:"old-uast-serialization" description:"serialize uast in the old format" env:"GITBASE_UAST_SERIALIZATION"`
 }
 
 type jaegerLogrus struct {
@@ -184,7 +185,7 @@ func (c *Server) buildDatabase() error {
 		)
 	}
 
-	c.pool = gitbase.NewRepositoryPool()
+	c.pool = gitbase.NewRepositoryPool(c.CacheSize * cache.MiByte)
 
 	if err := c.addDirectories(); err != nil {
 		return err

cmd/gitbase/command/server.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	fixtures "gopkg.in/src-d/go-git-fixtures.v3"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 	"gopkg.in/src-d/go-mysql-server.v0/sql/plan"
 )
@@ -36,7 +37,7 @@ func buildSession(t *testing.T, repos fixtures.Fixtures,
 
 	require.NoError(fixtures.Init())
 
-	pool := NewRepositoryPool()
+	pool := NewRepositoryPool(cache.DefaultMaxSize)
 	for _, fixture := range repos {
 		path := fixture.Worktree().Root()
 		ok, err := IsGitRepo(path)

common_test.go

@@ -13,6 +13,7 @@ import (
 	billy "gopkg.in/src-d/go-billy.v4"
 	fixtures "gopkg.in/src-d/go-git-fixtures.v3"
 	git "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 )
@@ -56,7 +57,7 @@ func setupErrorRepos(t *testing.T) (*sql.Context, CleanupFunc) {
 	fixture := fixtures.ByTag("worktree").One()
 	baseFS := fixture.Worktree()
 
-	pool := NewRepositoryPool()
+	pool := NewRepositoryPool(cache.DefaultMaxSize)
 
 	fs, err := brokenFS(brokenPackfile, baseFS)
 	require.NoError(err)
@@ -118,23 +119,21 @@ func testTable(t *testing.T, tableName string, number int) {
 }
 
 type billyRepository struct {
-	id string
-	fs billy.Filesystem
+	id    string
+	fs    billy.Filesystem
+	cache cache.Object
 }
 
 func billyRepo(id string, fs billy.Filesystem) repository {
-	return &billyRepository{id, fs}
+	return &billyRepository{id, fs, cache.NewObjectLRUDefault()}
 }
 
 func (r *billyRepository) ID() string {
 	return r.id
 }
 
 func (r *billyRepository) Repo() (*Repository, error) {
-	storage, err := filesystem.NewStorage(r.fs)
-	if err != nil {
-		return nil, err
-	}
+	storage := filesystem.NewStorage(r.fs, r.cache)
 
 	repo, err := git.Open(storage, r.fs)
 	if err != nil {
@@ -152,6 +151,10 @@ func (r *billyRepository) Path() string {
 	return r.id
 }
 
+func (r *billyRepository) Cache() cache.Object {
+	return r.cache
+}
+
 type brokenType uint64
 
 const (

fs_error_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/src-d/gitbase/internal/function"
 	"github.com/stretchr/testify/require"
 	fixtures "gopkg.in/src-d/go-git-fixtures.v3"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	sqle "gopkg.in/src-d/go-mysql-server.v0"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 	"gopkg.in/src-d/go-mysql-server.v0/sql/analyzer"
@@ -33,7 +34,7 @@ func TestIntegration(t *testing.T) {
 
 	path := fixtures.ByTag("worktree").One().Worktree().Root()
 
-	pool := gitbase.NewRepositoryPool()
+	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
 	require.NoError(t, pool.AddGitWithID("worktree", path))
 
 	testCases := []struct {
@@ -443,7 +444,7 @@ func TestMissingHeadRefs(t *testing.T) {
 		"_testdata",
 	)
 
-	pool := gitbase.NewRepositoryPool()
+	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
 	require.NoError(
 		filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
@@ -830,7 +831,7 @@ func setup(t testing.TB) (*sqle.Engine, *gitbase.RepositoryPool, func()) {
 		require.NoError(t, fixtures.Clean())
 	}
 
-	pool := gitbase.NewRepositoryPool()
+	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
 	for _, f := range fixtures.ByTag("worktree") {
 		pool.AddGitWithID("worktree", f.Worktree().Root())
 	}

integration_test.go

@@ -11,6 +11,7 @@ import (
 	"gopkg.in/bblfsh/sdk.v1/protocol"
 	"gopkg.in/bblfsh/sdk.v1/uast"
 	fixtures "gopkg.in/src-d/go-git-fixtures.v3"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 	"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
 )
@@ -421,7 +422,7 @@ func setup(t *testing.T) (*sql.Context, func()) {
 	t.Helper()
 	require.NoError(t, fixtures.Init())
 
-	pool := gitbase.NewRepositoryPool()
+	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
 	for _, f := range fixtures.ByTag("worktree") {
 		pool.AddGit(f.Worktree().Root())
 	}

internal/function/uast_test.go

@@ -166,10 +166,7 @@ func getUnpackedObject(repo repository, hash plumbing.Hash) (o object.Object, er
 
 	defer ioutil.CheckClose(f, &err)
 
-	storage, err := filesystem.NewStorage(fs)
-	if err != nil {
-		return nil, err
-	}
+	storage := filesystem.NewStorage(fs, repo.Cache())
 
 	obj := storage.NewEncodedObject()
 	r, err := objfile.NewReader(f)
@@ -224,11 +221,7 @@ func newRepoObjectDecoder(
 		return nil, err
 	}
 
-	storage, err := filesystem.NewStorage(fs)
-	if err != nil {
-		_ = packf.Close()
-		return nil, err
-	}
+	storage := filesystem.NewStorage(fs, repo.Cache())
 
 	idx, err := openPackfileIndex(dot, hash)
 	if err != nil {

packfiles.go

@@ -6,14 +6,15 @@ import (
 
 	"github.com/stretchr/testify/require"
 	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 )
 
 var testSivaFilePath = filepath.Join("_testdata", "fff7062de8474d10a67d417ccea87ba6f58ca81d.siva")
 
 func TestRepositoryPackfiles(t *testing.T) {
 	require := require.New(t)
 
-	fs, packfiles, err := repositoryPackfiles(sivaRepo("siva", testSivaFilePath))
+	fs, packfiles, err := repositoryPackfiles(sivaRepo("siva", testSivaFilePath, cache.NewObjectLRUDefault()))
 
 	require.NoError(err)
 	require.Equal([]plumbing.Hash{
@@ -24,7 +25,7 @@ func TestRepositoryPackfiles(t *testing.T) {
 }
 
 func TestRepositoryIndex(t *testing.T) {
-	idx, err := newRepositoryIndex(sivaRepo("siva", testSivaFilePath))
+	idx, err := newRepositoryIndex(sivaRepo("siva", testSivaFilePath, cache.NewObjectLRUDefault()))
 	require.NoError(t, err)
 
 	testCases := []struct {

packfiles_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 	"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
 )
@@ -17,10 +18,10 @@ func TestRepositoriesTable(t *testing.T) {
 		"seven", "eight", "nine",
 	}
 
-	pool := NewRepositoryPool()
+	pool := NewRepositoryPool(cache.DefaultMaxSize)
 
 	for _, id := range repoIDs {
-		pool.Add(gitRepo(id, ""))
+		pool.Add(gitRepo(id, "", pool.cache))
 	}
 
 	session := NewSession(pool)