Merge pull request #481 from customerio/bit_slice_bulk

Add SetMany and SetBigMany support

Author: lemire

BitSliceIndexing/bsi.go

@@ -113,6 +113,29 @@ func (b *BSI) SetValue(columnID uint64, value int64) {
 	b.eBM.Add(uint32(columnID))
 }
 
+// SetMany sets a value for foundSet
+func (b *BSI) SetMany(foundSet *roaring.Bitmap, value int64) {
+
+	// If max/min values are set to zero then automatically determine bit array size
+	if b.MaxValue == 0 && b.MinValue == 0 {
+		for i := bits.Len64(uint64(value)) - b.BitCount(); i > 0; i-- {
+			b.bA = append(b.bA, roaring.NewBitmap())
+			if b.runOptimized {
+				b.bA[i].RunOptimize()
+			}
+		}
+	}
+
+	for i := 0; i < b.BitCount(); i++ {
+		if uint64(value)&(1<<uint64(i)) > 0 {
+			b.bA[i].Or(foundSet)
+		} else {
+			b.bA[i].AndNot(foundSet)
+		}
+	}
+	b.eBM.Or(foundSet)
+}
+
 // GetValue gets the value at the column ID.  Second param will be false for non-existent values.
 func (b *BSI) GetValue(columnID uint64) (int64, bool) {
 	value := int64(0)

BitSliceIndexing/bsi_test.go

@@ -24,6 +24,17 @@ func TestSetAndGet(t *testing.T) {
 	assert.Equal(t, int64(8), gv)
 }
 
+func TestSetMany(t *testing.T) {
+	bsi := setup()
+	// update with mix of existing and new columns
+	upd := roaring.BitmapOf(30, 31, 32, 33, 34, 35, 101, 102, 103)
+	bsi.SetMany(upd, 35)
+
+	matches := bsi.CompareValue(0, EQ, 35, 0, nil)
+
+	assert.True(t, upd.Equals(matches))
+}
+
 func setup() *BSI {
 
 	bsi := NewBSI(100, 0)

roaring64/bsi64.go

@@ -115,11 +115,37 @@ func (b *BSI) SetBigValue(columnID uint64, value *big.Int) {
 	b.eBM.Add(columnID)
 }
 
+func (b *BSI) SetBigMany(foundSet *Bitmap, value *big.Int) {
+	// If max/min values are set to zero then automatically determine bit array size
+	if b.MaxValue == 0 && b.MinValue == 0 {
+		minBits := value.BitLen() + 1
+		if minBits == 1 {
+			minBits = 2
+		}
+		for len(b.bA) < minBits {
+			b.bA = append(b.bA, Bitmap{})
+		}
+	}
+	for i := b.BitCount(); i >= 0; i-- {
+		if value.Bit(i) == 0 {
+			b.bA[i].AndNot(foundSet)
+		} else {
+			b.bA[i].Or(foundSet)
+		}
+	}
+	b.eBM.Or(foundSet)
+}
+
 // SetValue sets a value for a given columnID.
 func (b *BSI) SetValue(columnID uint64, value int64) {
 	b.SetBigValue(columnID, big.NewInt(value))
 }
 
+// SetMany sets a value for all columns in foundSet
+func (b *BSI) SetMany(foundSet *Bitmap, value int64) {
+	b.SetBigMany(foundSet, big.NewInt(value))
+}
+
 // GetValue gets the value at the column ID. Second param will be false for non-existent values.
 func (b *BSI) GetValue(columnID uint64) (value int64, exists bool) {
 	bv, exists := b.GetBigValue(columnID)

roaring64/bsi64_test.go

@@ -38,6 +38,18 @@ func TestSetAndGetSimple(t *testing.T) {
 	assert.Equal(t, int64(8), gv)
 }
 
+func TestSetMany(t *testing.T) {
+	bsi := setup()
+
+	upd := BitmapOf(30, 31, 32, 33, 34, 35, 101, 102, 103)
+	// update many including existing columns
+	bsi.SetMany(upd, 35)
+
+	matches := bsi.CompareValue(0, EQ, 35, 0, nil)
+
+	assert.True(t, upd.Equals(matches))
+}
+
 func TestSetAndGetBigValue(t *testing.T) {
 
 	// Set a large UUID value---