Skip to content

Commit a8b9c07

Browse files
erizocosmicoerizocosmico
committed
gitbase: custom encode and decode of index keys to save space
Signed-off-by: Miguel Molina <[email protected]>
1 parent e8d7b16 commit a8b9c07

23 files changed

+1056
-149
lines changed

‎blobs.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ func (i *blobsKeyValueIter) Next() ([]interface{}, []byte, error) {
393393
hash = blob.Hash.String()
394394
}
395395

396-
key, err := encodeIndexKey(packOffsetIndexKey{
396+
key, err := encodeIndexKey(&packOffsetIndexKey{
397397
Repository: i.repo.ID,
398398
Packfile: packfile.String(),
399399
Offset: offset,

‎blobs_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
147147

148148
var expected = []keyValue{
149149
{
150-
assertEncodeKey(t, packOffsetIndexKey{
150+
assertEncodeKey(t, &packOffsetIndexKey{
151151
Repository: path,
152152
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
153153
Offset: 1591,
@@ -158,7 +158,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
158158
},
159159
},
160160
{
161-
assertEncodeKey(t, packOffsetIndexKey{
161+
assertEncodeKey(t, &packOffsetIndexKey{
162162
Repository: path,
163163
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
164164
Offset: 79864,
@@ -169,7 +169,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
169169
},
170170
},
171171
{
172-
assertEncodeKey(t, packOffsetIndexKey{
172+
assertEncodeKey(t, &packOffsetIndexKey{
173173
Repository: path,
174174
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
175175
Offset: 2418,
@@ -180,7 +180,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
180180
},
181181
},
182182
{
183-
assertEncodeKey(t, packOffsetIndexKey{
183+
assertEncodeKey(t, &packOffsetIndexKey{
184184
Repository: path,
185185
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
186186
Offset: 78932,
@@ -191,7 +191,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
191191
},
192192
},
193193
{
194-
assertEncodeKey(t, packOffsetIndexKey{
194+
assertEncodeKey(t, &packOffsetIndexKey{
195195
Repository: path,
196196
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
197197
Offset: 82000,
@@ -202,7 +202,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
202202
},
203203
},
204204
{
205-
assertEncodeKey(t, packOffsetIndexKey{
205+
assertEncodeKey(t, &packOffsetIndexKey{
206206
Repository: path,
207207
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
208208
Offset: 85438,
@@ -213,7 +213,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
213213
},
214214
},
215215
{
216-
assertEncodeKey(t, packOffsetIndexKey{
216+
assertEncodeKey(t, &packOffsetIndexKey{
217217
Repository: path,
218218
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
219219
Offset: 1780,
@@ -224,7 +224,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
224224
},
225225
},
226226
{
227-
assertEncodeKey(t, packOffsetIndexKey{
227+
assertEncodeKey(t, &packOffsetIndexKey{
228228
Repository: path,
229229
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
230230
Offset: 81707,
@@ -235,7 +235,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
235235
},
236236
},
237237
{
238-
assertEncodeKey(t, packOffsetIndexKey{
238+
assertEncodeKey(t, &packOffsetIndexKey{
239239
Repository: path,
240240
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
241241
Offset: 1752,
@@ -246,7 +246,7 @@ func TestBlobsIndexKeyValueIter(t *testing.T) {
246246
},
247247
},
248248
{
249-
assertEncodeKey(t, packOffsetIndexKey{
249+
assertEncodeKey(t, &packOffsetIndexKey{
250250
Repository: path,
251251
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
252252
Offset: 2436,

‎commit_blobs.go

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package gitbase
22

33
import (
4+
"bytes"
45
"io"
56

67
"gopkg.in/src-d/go-git.v4/plumbing/object"
@@ -101,7 +102,12 @@ func (*commitBlobsTable) IndexKeyValueIter(
101102
return nil, err
102103
}
103104

104-
return &rowKeyValueIter{iter, colNames, CommitBlobsSchema}, nil
105+
return &rowKeyValueIter{
106+
new(commitBlobsRowKeyMapper),
107+
iter,
108+
colNames,
109+
CommitBlobsSchema,
110+
}, nil
105111
}
106112

107113
// WithProjectFiltersAndIndex implements sql.Indexable interface.
@@ -117,7 +123,7 @@ func (*commitBlobsTable) WithProjectFiltersAndIndex(
117123
return nil, ErrInvalidGitbaseSession.New(ctx.Session)
118124
}
119125

120-
var iter sql.RowIter = &rowIndexIter{index}
126+
var iter sql.RowIter = &rowIndexIter{new(commitBlobsRowKeyMapper), index}
121127

122128
if len(filters) > 0 {
123129
iter = plan.NewFilterIter(ctx, expression.JoinAnd(filters...), iter)
@@ -126,6 +132,63 @@ func (*commitBlobsTable) WithProjectFiltersAndIndex(
126132
return sql.NewSpanIter(span, iter), nil
127133
}
128134

135+
type commitBlobsRowKeyMapper struct{}
136+
137+
func (commitBlobsRowKeyMapper) fromRow(row sql.Row) ([]byte, error) {
138+
if len(row) != 3 {
139+
return nil, errRowKeyMapperRowLength.New(3, len(row))
140+
}
141+
142+
repo, ok := row[0].(string)
143+
if !ok {
144+
return nil, errRowKeyMapperColType.New(0, repo, row[0])
145+
}
146+
147+
commit, ok := row[1].(string)
148+
if !ok {
149+
return nil, errRowKeyMapperColType.New(1, commit, row[1])
150+
}
151+
152+
blob, ok := row[2].(string)
153+
if !ok {
154+
return nil, errRowKeyMapperColType.New(2, blob, row[2])
155+
}
156+
157+
var buf bytes.Buffer
158+
writeString(&buf, repo)
159+
160+
if err := writeHash(&buf, commit); err != nil {
161+
return nil, err
162+
}
163+
164+
if err := writeHash(&buf, blob); err != nil {
165+
return nil, err
166+
}
167+
168+
return buf.Bytes(), nil
169+
}
170+
171+
func (commitBlobsRowKeyMapper) toRow(data []byte) (sql.Row, error) {
172+
var buf = bytes.NewBuffer(data)
173+
174+
repo, err := readString(buf)
175+
if err != nil {
176+
return nil, err
177+
}
178+
179+
commit, err := readHash(buf)
180+
if err != nil {
181+
return nil, err
182+
}
183+
184+
blob, err := readHash(buf)
185+
if err != nil {
186+
return nil, err
187+
}
188+
189+
return sql.Row{repo, commit, blob}, nil
190+
}
191+
129192
func commitBlobsIterBuilder(ctx *sql.Context, selectors selectors, columns []sql.Expression) (RowRepoIter, error) {
130193
repos, err := selectors.textValues("repository_id")
131194
if err != nil {

‎commit_blobs_test.go

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"testing"
66

77
"github.com/stretchr/testify/require"
8+
"gopkg.in/src-d/go-git.v4/plumbing"
89
"gopkg.in/src-d/go-mysql-server.v0/sql"
910
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
1011
)
@@ -180,14 +181,21 @@ func TestCommitBlobsIndexKeyValueIter(t *testing.T) {
180181
var expected []keyValue
181182
for _, row := range rows {
182183
var kv keyValue
183-
kv.key = assertEncodeKey(t, row)
184+
kv.key = assertEncodeCommitBlobsRow(t, row)
184185
kv.values = append(kv.values, row[2], row[1])
185186
expected = append(expected, kv)
186187
}
187188

188189
assertIndexKeyValueIter(t, iter, expected)
189190
}
190191

192+
func assertEncodeCommitBlobsRow(t *testing.T, row sql.Row) []byte {
193+
t.Helper()
194+
k, err := new(commitBlobsRowKeyMapper).fromRow(row)
195+
require.NoError(t, err)
196+
return k
197+
}
198+
191199
func TestCommitBlobsIndex(t *testing.T) {
192200
testTableIndex(
193201
t,
@@ -198,3 +206,17 @@ func TestCommitBlobsIndex(t *testing.T) {
198206
)},
199207
)
200208
}
209+
210+
func TestCommitBlobsRowKeyMapper(t *testing.T) {
211+
require := require.New(t)
212+
row := sql.Row{"repo1", plumbing.ZeroHash.String(), plumbing.ZeroHash.String()}
213+
mapper := new(commitBlobsRowKeyMapper)
214+
215+
k, err := mapper.fromRow(row)
216+
require.NoError(err)
217+
218+
row2, err := mapper.toRow(k)
219+
require.NoError(err)
220+
221+
require.Equal(row, row2)
222+
}

‎commit_trees.go

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package gitbase
22

33
import (
4+
"bytes"
45
"io"
56

67
"gopkg.in/src-d/go-git.v4/plumbing"
@@ -104,7 +105,12 @@ func (*commitTreesTable) IndexKeyValueIter(
104105
return nil, err
105106
}
106107

107-
return &rowKeyValueIter{iter, colNames, CommitTreesSchema}, nil
108+
return &rowKeyValueIter{
109+
new(commitTreesRowKeyMapper),
110+
iter,
111+
colNames,
112+
CommitTreesSchema,
113+
}, nil
108114
}
109115

110116
// WithProjectFiltersAndIndex implements sql.Indexable interface.
@@ -120,7 +126,7 @@ func (*commitTreesTable) WithProjectFiltersAndIndex(
120126
return nil, ErrInvalidGitbaseSession.New(ctx.Session)
121127
}
122128

123-
var iter sql.RowIter = &rowIndexIter{index}
129+
var iter sql.RowIter = &rowIndexIter{new(commitTreesRowKeyMapper), index}
124130

125131
if len(filters) > 0 {
126132
iter = plan.NewFilterIter(ctx, expression.JoinAnd(filters...), iter)
@@ -129,6 +135,63 @@ func (*commitTreesTable) WithProjectFiltersAndIndex(
129135
return sql.NewSpanIter(span, iter), nil
130136
}
131137

138+
type commitTreesRowKeyMapper struct{}
139+
140+
func (commitTreesRowKeyMapper) fromRow(row sql.Row) ([]byte, error) {
141+
if len(row) != 3 {
142+
return nil, errRowKeyMapperRowLength.New(3, len(row))
143+
}
144+
145+
repo, ok := row[0].(string)
146+
if !ok {
147+
return nil, errRowKeyMapperColType.New(0, repo, row[0])
148+
}
149+
150+
commit, ok := row[1].(string)
151+
if !ok {
152+
return nil, errRowKeyMapperColType.New(1, commit, row[1])
153+
}
154+
155+
tree, ok := row[2].(string)
156+
if !ok {
157+
return nil, errRowKeyMapperColType.New(2, tree, row[2])
158+
}
159+
160+
var buf bytes.Buffer
161+
writeString(&buf, repo)
162+
163+
if err := writeHash(&buf, commit); err != nil {
164+
return nil, err
165+
}
166+
167+
if err := writeHash(&buf, tree); err != nil {
168+
return nil, err
169+
}
170+
171+
return buf.Bytes(), nil
172+
}
173+
174+
func (commitTreesRowKeyMapper) toRow(data []byte) (sql.Row, error) {
175+
var buf = bytes.NewBuffer(data)
176+
177+
repo, err := readString(buf)
178+
if err != nil {
179+
return nil, err
180+
}
181+
182+
commit, err := readHash(buf)
183+
if err != nil {
184+
return nil, err
185+
}
186+
187+
tree, err := readHash(buf)
188+
if err != nil {
189+
return nil, err
190+
}
191+
192+
return sql.Row{repo, commit, tree}, nil
193+
}
194+
132195
func commitTreesIterBuilder(ctx *sql.Context, selectors selectors, columns []sql.Expression) (RowRepoIter, error) {
133196
repos, err := selectors.textValues("repository_id")
134197
if err != nil {

‎commit_trees_test.go

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"testing"
55

66
"github.com/stretchr/testify/require"
7+
"gopkg.in/src-d/go-git.v4/plumbing"
78
"gopkg.in/src-d/go-mysql-server.v0/sql"
89
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
910
)
@@ -135,14 +136,21 @@ func TestCommitTreesIndexKeyValueIter(t *testing.T) {
135136
var expected []keyValue
136137
for _, row := range rows {
137138
var kv keyValue
138-
kv.key = assertEncodeKey(t, row)
139+
kv.key = assertEncodeCommitTreesRow(t, row)
139140
kv.values = append(kv.values, row[2], row[1])
140141
expected = append(expected, kv)
141142
}
142143

143144
assertIndexKeyValueIter(t, iter, expected)
144145
}
145146

147+
func assertEncodeCommitTreesRow(t *testing.T, row sql.Row) []byte {
148+
t.Helper()
149+
k, err := new(commitTreesRowKeyMapper).fromRow(row)
150+
require.NoError(t, err)
151+
return k
152+
}
153+
146154
func TestCommitTreesIndex(t *testing.T) {
147155
testTableIndex(
148156
t,
@@ -153,3 +161,17 @@ func TestCommitTreesIndex(t *testing.T) {
153161
)},
154162
)
155163
}
164+
165+
func TestCommitTreesRowKeyMapper(t *testing.T) {
166+
require := require.New(t)
167+
row := sql.Row{"repo1", plumbing.ZeroHash.String(), plumbing.ZeroHash.String()}
168+
mapper := new(commitTreesRowKeyMapper)
169+
170+
k, err := mapper.fromRow(row)
171+
require.NoError(err)
172+
173+
row2, err := mapper.toRow(k)
174+
require.NoError(err)
175+
176+
require.Equal(row, row2)
177+
}

‎commits.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,7 @@ func (i *commitsKeyValueIter) Next() ([]interface{}, []byte, error) {
360360
hash = commit.Hash.String()
361361
}
362362

363-
key, err := encodeIndexKey(packOffsetIndexKey{
363+
key, err := encodeIndexKey(&packOffsetIndexKey{
364364
Repository: i.repo.ID,
365365
Packfile: packfile.String(),
366366
Offset: offset,

0 commit comments

Comments
 (0)