Skip to content

Commit 8fd0c9a

Browse files
authored
Merge pull request #19 from MatrixAI/feature-control
Introduce Snapshot Isolation OCC to DBTransaction
2 parents a279855 + 879c5e2 commit 8fd0c9a

File tree

112 files changed

+13206
-2431
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

112 files changed

+13206
-2431
lines changed

.clang-format

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
BasedOnStyle: Google
2+
SortIncludes: false

.eslintrc

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@
5050
"ignoreConsecutiveComments": true
5151
}
5252
],
53+
"curly": [
54+
"error",
55+
"multi-line",
56+
"consistent"
57+
],
5358
"import/order": [
5459
"error",
5560
{
@@ -137,6 +142,10 @@
137142
"format": ["PascalCase"],
138143
"trailingUnderscore": "allowSingleOrDouble"
139144
},
145+
{
146+
"selector": "enumMember",
147+
"format": ["PascalCase", "UPPER_CASE"]
148+
},
140149
{
141150
"selector": "objectLiteralProperty",
142151
"format": null

.gitlab-ci.yml

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,13 +65,17 @@ check:test:
6565
- >
6666
nix-shell --run '
6767
npm run build --verbose;
68-
npm test -- --ci;
68+
npm test -- --ci --coverage;
6969
'
7070
artifacts:
7171
when: always
7272
reports:
7373
junit:
74-
- ./tmp/junit.xml
74+
- ./tmp/junit/junit.xml
75+
coverage_report:
76+
coverage_format: cobertura
77+
path: ./tmp/coverage/cobertura-coverage.xml
78+
coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
7579
rules:
7680
# Runs on staging commits and ignores version commits
7781
- if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -114,17 +118,21 @@ build:linux:
114118
- >
115119
nix-shell --run '
116120
npm run build --verbose;
117-
npm test -- --ci;
121+
npm test -- --ci --coverage;
118122
'
119123
artifacts:
120124
when: always
121125
reports:
122126
junit:
123-
- ./tmp/junit.xml
127+
- ./tmp/junit/junit.xml
128+
coverage_report:
129+
coverage_format: cobertura
130+
path: ./tmp/coverage/cobertura-coverage.xml
124131
paths:
125132
- ./prebuilds/
126133
# Only the build:linux preserves the dist
127134
- ./dist
135+
coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
128136
rules:
129137
# Runs on staging commits and ignores version commits
130138
- if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -150,7 +158,7 @@ build:windows:
150158
when: always
151159
reports:
152160
junit:
153-
- ./tmp/junit.xml
161+
- ./tmp/junit/junit.xml
154162
paths:
155163
- ./prebuilds/
156164
rules:
@@ -169,6 +177,7 @@ build:macos:
169177
HOMEBREW_NO_INSTALL_UPGRADE: "true"
170178
HOMEBREW_NO_INSTALL_CLEANUP: "true"
171179
before_script:
180+
- eval "$(brew shellenv)"
172181
- brew install node@16
173182
- brew link --overwrite node@16
174183
- brew install [email protected]
@@ -184,7 +193,7 @@ build:macos:
184193
when: always
185194
reports:
186195
junit:
187-
- ./tmp/junit.xml
196+
- ./tmp/junit/junit.xml
188197
paths:
189198
- ./prebuilds/
190199
rules:
@@ -209,6 +218,26 @@ build:prerelease:
209218
nix-shell --run '
210219
npm publish --tag prerelease --access public;
211220
'
221+
- >
222+
for d in prebuilds/*; do
223+
tar \
224+
--create \
225+
--verbose \
226+
--file="prebuilds/$(basename $d).tar" \
227+
--directory=prebuilds \
228+
"$(basename $d)";
229+
done
230+
- >
231+
nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
232+
gh release \
233+
create "$CI_COMMIT_TAG" \
234+
prebuilds/*.tar \
235+
--title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
236+
--notes "" \
237+
--prerelease \
238+
--target staging \
239+
--repo "$GH_PROJECT_PATH";
240+
'
212241
after_script:
213242
- rm -f ./.npmrc
214243
rules:
@@ -272,6 +301,25 @@ release:distribution:
272301
nix-shell --run '
273302
npm publish --access public;
274303
'
304+
- >
305+
for d in prebuilds/*; do
306+
tar \
307+
--create \
308+
--verbose \
309+
--file="prebuilds/$(basename $d).tar" \
310+
--directory=prebuilds \
311+
"$(basename $d)";
312+
done
313+
- >
314+
nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
315+
gh release \
316+
create "$CI_COMMIT_TAG" \
317+
prebuilds/*.tar \
318+
--title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
319+
--notes "" \
320+
--target master \
321+
--repo "$GH_PROJECT_PATH";
322+
'
275323
after_script:
276324
- rm -f ./.npmrc
277325
rules:

.gitmodules

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[submodule "deps/snappy/snappy"]
2+
path = deps/snappy/snappy
3+
url = https://github.com/google/snappy.git
4+
[submodule "deps/rocksdb/rocksdb"]
5+
path = deps/rocksdb/rocksdb
6+
url = https://github.com/facebook/rocksdb.git

README.md

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,76 @@ master: [![pipeline status](https://gitlab.com/MatrixAI/open-source/js-db/badges
55

66
DB is library managing key value state for MatrixAI's JavaScript/TypeScript applications.
77

8+
This forks classic-level's C++ binding code around LevelDB 1.20. Differences from classic-level:
9+
10+
* Uses TypeScript from ground-up
11+
* Supports Snapshot-Isolation based transactions via `DBTransaction`
12+
* API supports "key paths" which can be used to manipulate "levels" of nested keys
13+
* Value encryption (key-encryption is not supported yet) - requires additional work with block-encryption
14+
815
## Installation
916

1017
```sh
1118
npm install --save @matrixai/db
1219
```
1320

21+
## Usage
22+
23+
24+
```ts
25+
import { DB } from '@matrixai/db';
26+
27+
async function main () {
28+
29+
const key = Buffer.from([
30+
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
31+
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
32+
]);
33+
34+
const encrypt = async (
35+
key: ArrayBuffer,
36+
plainText: ArrayBuffer
37+
): Promise<ArrayBuffer> {
38+
return plainText;
39+
};
40+
41+
const decrypt = async (
42+
key: ArrayBuffer,
43+
cipherText: ArrayBuffer
44+
): Promise<ArrayBuffer | undefined> {
45+
return cipherText;
46+
}
47+
48+
const db = await DB.createDB({
49+
dbPath: './tmp/db',
50+
crypto: {
51+
key,
52+
ops: { encrypt, decrypt },
53+
},
54+
fresh: true,
55+
});
56+
57+
await db.put(['level', Buffer.from([0x30, 0x30]), 'a'], 'value');
58+
await db.put(['level', Buffer.from([0x30, 0x31]), 'b'], 'value');
59+
await db.put(['level', Buffer.from([0x30, 0x32]), 'c'], 'value');
60+
await db.put(['level', Buffer.from([0x30, 0x33]), 'c'], 'value');
61+
62+
console.log(await db.get(['level', Buffer.from([0x30, 0x32]), 'c']));
63+
64+
await db.del(['level', Buffer.from([0x30, 0x32]), 'c']);
65+
66+
for await (const [kP, v] of db.iterator({
67+
lt: [Buffer.from([0x30, 0x32]), ''],
68+
}, ['level'])) {
69+
console.log(kP, v);
70+
}
71+
72+
await db.stop();
73+
}
74+
75+
main();
76+
```
77+
1478
## Development
1579

1680
Run `nix-shell`, and once you're inside, you can use:

benches/DB1KiB.ts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,7 @@ async function main() {
5858
}
5959

6060
if (require.main === module) {
61-
(async () => {
62-
await main();
63-
})();
61+
void main();
6462
}
6563

6664
export default main;

benches/DB1MiB.ts

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ async function main() {
2020
const summary = await b.suite(
2121
'DB1MiB',
2222
b.add('get 1 MiB of data', async () => {
23-
await db.put('1kib', data1MiB, true);
23+
await db.put('1mib', data1MiB, true);
2424
return async () => {
25-
await db.get('1kib', true);
25+
await db.get('1mib', true);
2626
};
2727
}),
2828
b.add('put 1 MiB of data', async () => {
29-
await db.put('1kib', data1MiB, true);
29+
await db.put('1mib', data1MiB, true);
3030
}),
3131
b.add('put zero data', async () => {
3232
await db.put('0', data0, true);
@@ -58,9 +58,7 @@ async function main() {
5858
}
5959

6060
if (require.main === module) {
61-
(async () => {
62-
await main();
63-
})();
61+
void main();
6462
}
6563

6664
export default main;

benches/index.ts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ async function main(): Promise<void> {
2020
}
2121

2222
if (require.main === module) {
23-
(async () => {
24-
await main();
25-
})();
23+
void main();
2624
}
2725

2826
export default main;

benches/results/DB1KiB.chart.html

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
</head>
1717
<body>
1818
<div style="max-width: 800px">
19-
<canvas id="chart1654000546839" width="16" height="9"></canvas>
19+
<canvas id="chart1656585967881" width="16" height="9"></canvas>
2020
</div>
2121
<script>
2222
const format = (num) => {
@@ -34,10 +34,10 @@
3434

3535
return chunked.map((chunk) => chunk.join("")).join(" ");
3636
};
37-
const ctx1654000546839 = document
38-
.getElementById("chart1654000546839")
37+
const ctx1656585967881 = document
38+
.getElementById("chart1656585967881")
3939
.getContext("2d");
40-
const chart1654000546839 = new Chart(ctx1654000546839, {
40+
const chart1656585967881 = new Chart(ctx1656585967881, {
4141
type: "bar",
4242
data: {
4343
labels: [
@@ -48,7 +48,7 @@
4848
],
4949
datasets: [
5050
{
51-
data: [55227, 28769, 33029, 16935],
51+
data: [43375, 24504, 27503, 13785],
5252
backgroundColor: [
5353
"rgba(63, 142, 252, 0.8)",
5454
"rgba(116, 165, 127, 0.8)",

0 commit comments

Comments
 (0)