Skip to content

Commit

Permalink
Fix compaction of non-presorted tuples (#21)
Browse files Browse the repository at this point in the history
It covers the case when line pointers (lp) point to unsorted offsets
(tuples):
```
SELECT lp, lp_off, t_ctid FROM heap_page_items(get_raw_page('sbtest1', 0));
 lp | lp_off | t_ctid
----+--------+--------
  1 |   7960 | (0,1)
  2 |   7264 | (0,2)
  3 |   7032 | (0,3)
  4 |   6800 | (0,4)
  5 |   7728 | (0,5)
  6 |      0 |
  7 |   7496 | (0,7)
(7 rows)
```

This condition can be achieved by deleting some tuples and running
VACUUM

Also, the encoded bytes printing is hidden behind higher ENCRYPTION_DEBUG.
It floods log a lot otherwise.
  • Loading branch information
dAdAbird authored Sep 18, 2023
1 parent c69f1cd commit dfb14c6
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 2 deletions.
48 changes: 47 additions & 1 deletion src/access/pg_tde_prune.c
Original file line number Diff line number Diff line change
Expand Up @@ -1417,7 +1417,6 @@ pgtde_compactify_tuples(Relation rel, Buffer buffer, itemIdCompact itemidbase, i
}
else
{
Assert(0); // TODO: to be implemented with encryption
PGAlignedBlock scratch;
char *scratchptr = scratch.data;

Expand Down Expand Up @@ -1491,6 +1490,30 @@ pgtde_compactify_tuples(Relation rel, Buffer buffer, itemIdCompact itemidbase, i
ItemId lp;

itemidptr = &itemidbase[i];
if(copy_head < copy_tail) { // TODO: recheck this condition
// We leave the original loop as-is, and recrypt tuples one by one
// This is definitely not the fastest, but simple
Oid tableOid = RelationGetRelid(rel);
BlockNumber bn = BufferGetBlockNumber(buffer);
unsigned long headerSize = sizeof(HeapTupleHeaderData);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wvla"
char tmpData[itemidptr->alignedlen];
#pragma GCC diagnostic pop

#ifdef ENCRYPTION_DEBUG
fprintf(stderr, " >>>> [non-sorted] RELOCATING FROM %u to %u tail %u\n", copy_head, upper, copy_tail);
/* Decrypt with old offset */
fprintf(stderr, " >>>>> DECRYPTING\n");
#endif
PGTdeCryptTupInternal(tableOid, bn, copy_head, scratchptr + copy_head, tmpData, headerSize, itemidptr->len);
/* Reencrypt with new offset */
#ifdef ENCRYPTION_DEBUG
fprintf(stderr, " >>>>> ENCRYPTING\n");
#endif
PGTdeCryptTupInternal(tableOid, bn, upper, tmpData, scratchptr + copy_head, headerSize, itemidptr->len);
}

lp = PageGetItemId(page, itemidptr->offsetindex + 1);

/* copy pending tuples when we detect a gap */
Expand All @@ -1516,6 +1539,29 @@ pgtde_compactify_tuples(Relation rel, Buffer buffer, itemIdCompact itemidbase, i
lp->lp_off = upper;
}

/* Recrypt the remaining chunk */
if(copy_head < copy_tail) { // TODO: recheck this condition
Oid tableOid = RelationGetRelid(rel);
BlockNumber bn = BufferGetBlockNumber(buffer);
unsigned long headerSize = sizeof(HeapTupleHeaderData);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wvla"
char tmpData[itemidptr->alignedlen];
#pragma GCC diagnostic pop

#ifdef ENCRYPTION_DEBUG
fprintf(stderr, " >>>> [non-sorted] RELOCATING FROM %u to %u tail %u\n", copy_head, upper, copy_tail);
/* Decrypt with old offset */
fprintf(stderr, " >>>>> DECRYPTING\n");
#endif
PGTdeCryptTupInternal(tableOid, bn, copy_head, scratchptr + copy_head, tmpData, headerSize, itemidptr->len);
/* Reencrypt with new offset */
#ifdef ENCRYPTION_DEBUG
fprintf(stderr, " >>>>> ENCRYPTING\n");
#endif
PGTdeCryptTupInternal(tableOid, bn, upper, tmpData, scratchptr + copy_head, headerSize, itemidptr->len);
}

/* Copy the remaining chunk */
memcpy((char *) page + upper,
scratchptr + copy_head,
Expand Down
2 changes: 1 addition & 1 deletion src/encryption/enc_tuple.c
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ void PGTdeCryptTupInternal(Oid tableOid, BlockNumber bn, unsigned long offsetInP
for(unsigned i = 0; i < to - from; ++i) {
const char v = ((char*)(t_data))[i + from];
char realKey = encKey[aesBlockOffset + i];
#if ENCRYPTION_DEBUG
#if ENCRYPTION_DEBUG > 1
fprintf(stderr, " >> 0x%02hhX 0x%02hhX\n", v & 0xFF, (v ^ realKey) & 0xFF);
#endif
out[i + from] = v ^ realKey;
Expand Down
48 changes: 48 additions & 0 deletions src/test/non_sorted_off_compact.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
-- A test case for https://github.com/Percona-Lab/postgres-tde-ext/pull/21
--
DROP TABLE IF EXISTS sbtest1;
CREATE TABLE sbtest1(
id SERIAL,
k INTEGER DEFAULT '0' NOT NULL,
PRIMARY KEY (id)
) USING pg_tde;

INSERT INTO sbtest1(k) VALUES
(1),
(2),
(3),
(4),
(5),
(6),
(7),
(8),
(9),
(10);
DELETE FROM sbtest1 WHERE id IN (4,5,6);

VACUUM sbtest1;

INSERT INTO sbtest1(k) VALUES
(11),
(12),
(13);

-- Line pointers (lp) point to non-sorted offsets (lp_off):
-- CREATE EXTENSION pageinspect;
-- SELECT lp, lp_off, t_ctid FROM heap_page_items(get_raw_page('sbtest1', 0));
-- lp | lp_off | t_ctid
-- ----+--------+--------
-- 1 | 8160 | (0,1)
-- 2 | 8128 | (0,2)
-- 3 | 8096 | (0,3)
-- 4 | 7936 | (0,4)
-- 5 | 7904 | (0,5)
-- 6 | 7872 | (0,6)
-- 7 | 8064 | (0,7)
-- 8 | 8032 | (0,8)
-- 9 | 8000 | (0,9)
-- 10 | 7968 | (0,10)

---- Trigger comapction
delete from sbtest1 where id in (2);
VACUUM sbtest1;

0 comments on commit dfb14c6

Please sign in to comment.