Skip to content

Commit e97ab40

Browse files
sebsasto-bkgsebsasto-bkg
authored
Merge pull request #22 from bookingcom/multivariant
Add multi-variant calculations for even splits
2 parents c14ef1c + 82af3fe commit e97ab40

26 files changed

+17090
-6866
lines changed

‎dist/powercalculator.css

+184-127
Large diffs are not rendered by default.

‎dist/powercalculator.js

+6,644-6,530
Large diffs are not rendered by default.

‎package-lock.json

+9,592
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+3
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,8 @@
3939
"babel-core": "^6.26.0",
4040
"babel-jest": "^22.4.1",
4141
"regenerator-runtime": "^0.11.1"
42+
},
43+
"jest": {
44+
"testURL": "http://localhost/"
4245
}
4346
}

‎rollup.config.js

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export default {
3636
})
3737
],
3838
output: {
39+
banner: banner,
3940
name: 'powercalculator',
4041
file: 'dist/powercalculator.js',
4142
format: 'umd',

‎src/components/impact-comp.vue

+1
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@
9393
<span class="pc-input-details">
9494
{{ testType == 'gTest' ? ' Incremental trials per day': ' Incremental change in the metric per day' }}
9595
</span>
96+
</label>
9697
</li>
9798
</ul>
9899
</div>

‎src/components/non-inferiority-comp.vue

+21-26
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,13 @@
1010
<ul class="pc-inputs">
1111
<li class="pc-input-item pc-input-left">
1212
<label>
13-
<span class="pc-input-title">Acceptable Cost
14-
<small class="pc-input-sub-title">
15-
{{isRelative ?
16-
'relative difference of' :
17-
'absolute impact per day of'
18-
}}
19-
</small>
20-
</span>
13+
<span class="pc-input-title">Relative <small class="pc-input-sub-title">change</small></span>
2114

2215
<pc-block-field
23-
fieldProp="threshold"
24-
:suffix="isRelative ? '%' : ''"
16+
fieldProp="thresholdRelative"
17+
suffix="%"
2518

26-
v-bind:fieldValue="threshold"
19+
v-bind:fieldValue="thresholdRelative"
2720
v-bind:fieldFromBlock="fieldFromBlock"
2821
v-bind:isBlockFocused="isBlockFocused"
2922
v-bind:isReadOnly="isReadOnly"
@@ -35,22 +28,18 @@
3528

3629
<li class="pc-input-item pc-input-right">
3730
<label>
38-
<span class="pc-input-title">
39-
Type {{ isRelative ?
40-
'' :
41-
'(per day)'
42-
}}
43-
<small class="pc-input-sub-title">
44-
</small>
45-
</span>
31+
<span class="pc-input-title">Absolute <small class="pc-input-sub-title">impact per day</small></span>
4632

47-
<div class="pc-non-inf-select-wrapper">
48-
<select v-model="selected" class="pc-non-inf-select">
49-
<option v-for="(option, index) in options" v-bind:key="index" v-bind:value="option.value">
50-
{{option.text}}
51-
</option>
52-
</select>
53-
</div>
33+
<pc-block-field
34+
fieldProp="thresholdAbsolute"
35+
suffix=""
36+
v-bind:fieldValue="thresholdAbsolute"
37+
v-bind:fieldFromBlock="fieldFromBlock"
38+
v-bind:isBlockFocused="isBlockFocused"
39+
v-bind:isReadOnly="isReadOnly"
40+
v-bind:enableEdit="true"
41+
42+
v-on:update:focus="updateFocus"></pc-block-field>
5443
</label>
5544
</li>
5645

@@ -114,6 +103,12 @@ export default {
114103
threshold () {
115104
return this.$store.state.nonInferiority.threshold
116105
},
106+
thresholdRelative () {
107+
return this.$store.state.nonInferiority.thresholdRelative
108+
},
109+
thresholdAbsolute () {
110+
return this.$store.state.nonInferiority.thresholdAbsolute
111+
},
117112
isRelative () {
118113
return this.$store.state.nonInferiority.selected == 'relative'
119114
},

‎src/components/pc-block-field.vue

+12-1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ let validateFunctions = {
6161
return value > 0
6262
},
6363
defaultVal: 0
64+
},
65+
variants: {
66+
fn (value) {
67+
return Number.isInteger(value) && value > 1
68+
},
69+
defaultVal: 1
6470
}
6571
},
6672
gTest: {
@@ -376,7 +382,8 @@ export default {
376382
377383
.pc-non-inf-treshold-input,
378384
.pc-power-input,
379-
.pc-false-positive-input {
385+
.pc-false-positive-input,
386+
.pc-variants-input {
380387
display: inline-block;
381388
vertical-align: middle;
382389
padding: 4px 8px;
@@ -387,6 +394,10 @@ export default {
387394
font-size: inherit;
388395
}
389396
397+
.pc-variants-input {
398+
width: 6.5em;
399+
}
400+
390401
.pc-top-fields-error {
391402
color: var(--red);
392403
}

‎src/components/sample-comp.vue

+13-7
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@
1313
</div>
1414

1515

16-
<ul class="pc-inputs">
16+
<ul class="pc-inputs" :class="{'pc-inputs-no-grid': onlyTotalVisitors}">
1717
<li class="pc-input-item pc-input-left">
1818
<label>
19-
<span class="pc-input-title">Total # <small class="pc-input-sub-title">of visitors</small></span>
19+
<span class="pc-input-title">Total # <small class="pc-input-sub-title">of new visitors</small></span>
2020

2121
<pc-block-field
2222
fieldProp="sample"
@@ -28,9 +28,9 @@
2828
v-on:update:focus="updateFocus"></pc-block-field>
2929
</label>
3030
</li>
31-
<li class="pc-input-item pc-input-right pc-value-field--lockable" :class="getLockedStateClass('visitorsPerDay')">
31+
<li class="pc-input-item pc-input-right pc-value-field--lockable" :class="[getLockedStateClass('visitorsPerDay'), {'pc-hidden': onlyTotalVisitors}]">
3232
<label>
33-
<span class="pc-input-title">Daily # <small class="pc-input-sub-title">of visitors</small></span>
33+
<span class="pc-input-title">Daily # <small class="pc-input-sub-title">of new visitors</small></span>
3434

3535
<pc-block-field
3636
fieldProp="visitorsPerDay"
@@ -74,7 +74,7 @@
7474
</button>
7575

7676
</li>
77-
<li class="pc-input-item pc-input-right-swap pc-value-field--lockable" :class="getLockedStateClass('days')">
77+
<li class="pc-input-item pc-input-right-swap pc-value-field--lockable" :class="[getLockedStateClass('days'), {'pc-hidden': onlyTotalVisitors}]">
7878
<label>
7979
<pc-block-field
8080
fieldProp="runtime"
@@ -104,7 +104,6 @@ export default {
104104
extends: pcBlock,
105105
data () {
106106
return {
107-
variants: 2,
108107
focusedBlock: ''
109108
}
110109
},
@@ -120,7 +119,10 @@ export default {
120119
},
121120
lockedField () {
122121
return this.$store.state.attributes.lockedField
123-
}
122+
},
123+
onlyTotalVisitors () {
124+
return this.$store.state.attributes.onlyTotalVisitors
125+
},
124126
},
125127
methods: {
126128
updateFocus ({fieldProp, value}) {
@@ -209,4 +211,8 @@ export default {
209211
background: linear-gradient(0deg, var(--light-gray) 0%, var(--white) 100%);
210212
}
211213
214+
.pc-inputs-no-grid {
215+
display: block;
216+
}
217+
212218
</style>

‎src/js/math.js

+46-85
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,52 @@
11
import jstat from 'jstat';
22

3-
// SOLVING FOR POWER
4-
function solveforpower_Gtest ({total_sample_size, base_rate, effect_size, alpha, alternative, mu}) {
5-
var sample_size = total_sample_size/2;
3+
function get_alpha_sidaks_correction(alpha, variants) {
4+
if (variants == 1) {
5+
return alpha;
6+
}
67

7-
var mean_base = base_rate;
8-
var mean_var = base_rate * (1+effect_size);
8+
return 1 - (Math.pow(1-alpha, 1/variants));
9+
}
910

10-
var mean_diff = mean_var - mean_base;
11-
var delta = mean_diff - mu
11+
// SOLVING FOR POWER
12+
function solveforpower_Gtest(data) {
13+
var { base_rate, effect_size } = data;
14+
var mean_var = base_rate*(1+effect_size);
15+
data.variance = base_rate*(1-base_rate) + mean_var*(1-mean_var);
1216

13-
var variance = mean_base * (1-mean_base) + mean_var * (1-mean_var);
14-
var z = jstat.normal.inv(1-alpha/2, 0, 1);
15-
var mean = delta*Math.sqrt(sample_size/variance);
17+
return solve_for_power(data);
18+
}
1619

17-
var power;
18-
if (alternative == 'lower') {
19-
power = jstat.normal.cdf(jstat.normal.inv(alpha, 0, 1), mean, 1)
20-
} else if (alternative == 'greater') {
21-
power = 1-jstat.normal.cdf(jstat.normal.inv(1-alpha, 0, 1), mean, 1)
22-
} else {
23-
power = 1 - (jstat.normal.cdf(z, mean, 1) -
24-
jstat.normal.cdf(-z, mean, 1))
25-
}
20+
function solveforpower_Ttest(data) {
21+
var { sd_rate } = data;
22+
data.variance = 2*sd_rate**2;
2623

27-
return power
24+
return solve_for_power(data);
2825
}
2926

30-
function solveforpower_Ttest({total_sample_size, base_rate, sd_rate, effect_size, alpha, alternative, mu}) {
31-
var sample_size = total_sample_size/2;
27+
function solve_for_power(data) {
28+
var {total_sample_size, base_rate, variance, effect_size, alpha, variants, alternative, mu} = data;
29+
var sample_size = total_sample_size / (1 + variants);
3230

3331
var mean_base = base_rate;
3432
var mean_var = base_rate * (1+effect_size);
3533

3634
var mean_diff = mean_var - mean_base;
37-
var delta = mean_diff - mu
35+
var delta = mean_diff - mu;
3836

39-
var variance = 2*sd_rate**2;
40-
var z = jstat.normal.inv(1-alpha/2, 0, 1)
37+
var z = jstat.normal.inv(1-alpha/2, 0, 1);
4138
var mean = delta*Math.sqrt(sample_size/variance);
4239

4340
var power;
4441
if (alternative == 'lower') {
45-
power = jstat.normal.cdf(jstat.normal.inv(alpha, 0, 1), mean, 1)
42+
power = jstat.normal.cdf(jstat.normal.inv(alpha, 0, 1), mean, 1);
4643
} else if (alternative == 'greater') {
47-
power = 1-jstat.normal.cdf(jstat.normal.inv(1-alpha, 0, 1), mean, 1)
44+
power = 1-jstat.normal.cdf(jstat.normal.inv(1-alpha, 0, 1), mean, 1);
4845
} else {
49-
power = 1 - (jstat.normal.cdf(z, mean, 1) -
50-
jstat.normal.cdf(-z, mean, 1))
46+
power = 1 - (jstat.normal.cdf(z, mean, 1) - jstat.normal.cdf(-z, mean, 1));
5147
}
5248

53-
return power
49+
return power;
5450
}
5551

5652

@@ -109,64 +105,28 @@ function solve_quadratic_for_sample({mean_diff, Z, days, threshold, variance}) {
109105
}
110106

111107
function solveforsample_Ttest(data){
112-
var { base_rate, sd_rate, effect_size, alpha, beta, alternative, mu, opts } = data;
113-
if (!is_valid_input(data)) {
114-
return NaN;
115-
}
116-
var mean_base = base_rate;
117-
var mean_var = base_rate * (1+effect_size);
118-
119-
var variance = 2*sd_rate**2;
120-
var mean_diff = mean_var - mean_base;
121-
122-
var multiplier;
123-
var sample_one_group;
124-
if (opts && opts.type == 'absolutePerDay') {
125-
if (opts.calculating == 'visitorsPerDay') {
126-
var Z;
127-
if (alternative == "greater") {
128-
Z = jstat.normal.inv(beta, 0, 1) - jstat.normal.inv(1-alpha, 0, 1);
129-
} else if (alternative == "lower") {
130-
Z = jstat.normal.inv(1-beta, 0, 1) - jstat.normal.inv(alpha, 0, 1);
131-
} else {
132-
Z = jstat.normal.inv(1-beta, 0, 1) + jstat.normal.inv(1-alpha/2, 0, 1);
133-
}
134-
var sqrt_visitors_per_day = solve_quadratic_for_sample({mean_diff: mean_diff, Z: Z,
135-
days: opts.days, threshold: opts.threshold, variance: variance});
136-
sample_one_group = opts.days*sqrt_visitors_per_day**2;
137-
} else {
138-
multiplier = variance/(mean_diff*Math.sqrt(opts.visitors_per_day/2) - opts.threshold/(Math.sqrt(2*opts.visitors_per_day)))**2;
139-
var days;
140-
if (alternative == "greater" || alternative == "lower") {
141-
days = multiplier * (jstat.normal.inv(beta, 0, 1) - jstat.normal.inv(1-alpha, 0, 1))**2
142-
} else {
143-
days = multiplier * (jstat.normal.inv(1-beta, 0, 1) + jstat.normal.inv(1-alpha/2, 0, 1))**2
144-
}
145-
sample_one_group = days*opts.visitors_per_day/2;
146-
}
147-
} else {
148-
multiplier = variance/(mu - mean_diff)**2
108+
var { sd_rate } = data;
109+
data.variance = 2*sd_rate**2;
110+
return sample_size_calculation(data);
111+
}
149112

150-
if (alternative == "greater" || alternative == "lower") {
151-
sample_one_group = multiplier * (jstat.normal.inv(beta, 0, 1) - jstat.normal.inv(1-alpha, 0, 1))**2
152-
} else {
153-
sample_one_group = multiplier * (jstat.normal.inv(1-beta, 0, 1) + jstat.normal.inv(1-alpha/2, 0, 1))**2
154-
}
155-
}
113+
function solveforsample_Gtest(data){
114+
var { base_rate, effect_size } = data;
115+
var mean_var = base_rate*(1+effect_size);
116+
data.variance = base_rate*(1-base_rate) + mean_var*(1-mean_var);
156117

157-
return 2*Math.ceil(sample_one_group);
118+
return sample_size_calculation(data);
158119
}
159120

160-
function solveforsample_Gtest(data){
161-
var { base_rate, effect_size, alpha, beta, alternative, mu, opts } = data;
121+
function sample_size_calculation(data) {
122+
var { base_rate, variance, effect_size, alpha, beta, variants, alternative, mu, opts } = data;
123+
162124
if (!is_valid_input(data)) {
163-
return NaN;
125+
return NaN;
164126
}
127+
165128
var mean_base = base_rate;
166129
var mean_var = base_rate*(1+effect_size);
167-
168-
var variance = mean_base*(1-mean_base) + mean_var*(1-mean_var);
169-
170130
var mean_diff = mean_var - mean_base;
171131

172132
var multiplier;
@@ -204,14 +164,14 @@ function solveforsample_Gtest(data){
204164
}
205165
}
206166

207-
return 2*Math.ceil(sample_one_group);
167+
return (1+variants)*Math.ceil(sample_one_group);
208168
}
209169

210170

211171

212172
// SOLVING FOR EFFECT SIZE
213-
function solveforeffectsize_Ttest({total_sample_size, base_rate, sd_rate, alpha, beta, alternative, mu}){
214-
var sample_size = total_sample_size/2;
173+
function solveforeffectsize_Ttest({total_sample_size, base_rate, sd_rate, alpha, beta, variants, alternative, mu}){
174+
var sample_size = total_sample_size / (1 + variants);
215175
var variance = 2*sd_rate**2;
216176

217177
var z = jstat.normal.inv(1-beta, 0, 1);
@@ -245,8 +205,8 @@ function solve_quadratic(Z, sample_size, control_rate, mu) {
245205
return [sol_h, sol_l];
246206
}
247207

248-
function solveforeffectsize_Gtest({total_sample_size, base_rate, alpha, beta, alternative, mu}){
249-
var sample_size = total_sample_size / 2;
208+
function solveforeffectsize_Gtest({total_sample_size, base_rate, alpha, beta, variants, alternative, mu}){
209+
var sample_size = total_sample_size / (1 + variants);
250210

251211
var rel_effect_size;
252212
var Z;
@@ -340,4 +300,5 @@ export default {
340300
getMuFromRelativeDifference: get_mu_from_relative_difference,
341301
getMuFromAbsolutePerDay: get_mu_from_absolute_per_day,
342302
getAlternative: get_alternative,
303+
getCorrectedAlpha: get_alpha_sidaks_correction,
343304
}

0 commit comments

Comments
 (0)