From ed36b7e2ae7b7b6c80a47ccc4b155d943d0c5078 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Fri, 24 Jul 2020 13:46:34 +0500 Subject: [PATCH 01/67] add distribution.h --- src/daemon/distribution.h | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/daemon/distribution.h diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h new file mode 100644 index 0000000000..c1c6dc75c5 --- /dev/null +++ b/src/daemon/distribution.h @@ -0,0 +1,8 @@ +// +// Created by Svetlana Shmidt on 24.07.2020. +// + +#ifndef COLLECTD_DISTRIBUTION_H +#define COLLECTD_DISTRIBUTION_H + +#endif // COLLECTD_DISTRIBUTION_H From 99ce818a1104993cb03bcd1a9c4bed560955359e Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Fri, 24 Jul 2020 15:04:18 +0500 Subject: [PATCH 02/67] add linear constructor --- src/daemon/distribution.c | 66 +++++++++++++++++++++++++++++++++++++++ src/daemon/distribution.h | 16 ++++++++++ 2 files changed, 82 insertions(+) create mode 100644 src/daemon/distribution.c diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c new file mode 100644 index 0000000000..fbd123abda --- /dev/null +++ b/src/daemon/distribution.c @@ -0,0 +1,66 @@ +// +// Created by Svetlana Shmidt on 24.07.2020. +// + +#include "distribution.h" + +typedef struct bucket_s { + uint64_t bucket_counter; + double minimum, maximum; +} bucket_t; + +struct distribution_s { + bucket_t *tree; + size_t num_buckets; +}; + +size_t left_child_index(size_t node_index, size_t left, size_t right) { + return node_index + 1; +} + +size_t right_child_index(size_t node_index, size_t left, size_t right) { + size_t mid = (left + right) / 2; + return node_index + 2 * (mid - left + 1); +} + +bucket_t merge_buckets(bucket_t left_child, bucket_t right_child) { + return (bucket_t) { + .bucket_counter = left_child.bucket_counter + right_child.bucket_counter, + .minimum = left_child.minimum, + .maximum = right_child.maximum, + }; +} + +void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t left, size_t right) { + if (left > right) + return; + if (left == right) { + d->tree[node_index] = buckets[left]; + return; + } + size_t mid = (left + right) / 2; + size_t left_child = left_child_index(node_index, left, right); + size_t right_child = right_child_index(node_index, left, right); + build_tree(d, buckets, left_child, left, mid); + build_tree(d, buckets, right_child, mid + 1, right); + d->tree[node_index] = merge_buckets(d->tree[left_child], d->tree[right_child]); +} + +distribution_t* distribution_new_linear(size_t num_buckets, double size) { + bucket_t *bucket_array = calloc(num_buckets, sizeof(bucket_t)); + for (size_t i = 0; i < num_buckets; i++) { + bucket_array[i].minimum = i * size; + bucket_array[i].maximum = (i + 1) * size; + } + + distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); + new_distribution->tree = calloc(2 * num_buckets - 1, sizeof(bucket_t)); + new_distribution->num_buckets = num_buckets; + build_tree(new_distribution, bucket_array, 0, 0, num_buckets - 1); + free(bucket_array); + return new_distribution; +} + +int main() { + distribution_t *p = distribution_new_linear(5, 2); +} \ No newline at end of file diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index c1c6dc75c5..fd1872e303 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -5,4 +5,20 @@ #ifndef COLLECTD_DISTRIBUTION_H #define COLLECTD_DISTRIBUTION_H +#include + +struct distribution_s; +typedef struct distribution_s distribution_t; + +//constructor functions: +distribution_t* distribution_new_linear(size_t num_buckets, double size); +distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor); +distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes); + +void distribution_update(distribution_t *dist, double gauge); +double distribution_percentile(distribution_t *dist, double percent); +double distribution_average(distribution_t *dist); +distribution_t distribution_clone(distribution_t *dist); +void distribution_destroy(distribution_t *d); + #endif // COLLECTD_DISTRIBUTION_H From 8e04c194c9dbbb1d5b934078988c81b11f800768 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Mon, 27 Jul 2020 15:11:00 +0500 Subject: [PATCH 03/67] add checks + add infinity bucket --- src/daemon/distribution.c | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index fbd123abda..b5dd53c0fe 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -3,6 +3,9 @@ // #include "distribution.h" +#include +#include +#include typedef struct bucket_s { uint64_t bucket_counter; @@ -47,20 +50,49 @@ void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t } distribution_t* distribution_new_linear(size_t num_buckets, double size) { + if (num_buckets == 0 || size <= 0) { + errno = EINVAL; + return NULL; + } + bucket_t *bucket_array = calloc(num_buckets, sizeof(bucket_t)); + if (bucket_array == NULL) + return NULL; + for (size_t i = 0; i < num_buckets; i++) { bucket_array[i].minimum = i * size; - bucket_array[i].maximum = (i + 1) * size; + if (i == num_buckets - 1) + bucket_array[i].maximum = INFINITY; + else + bucket_array[i].maximum = (i + 1) * size; } distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); + if (new_distribution == NULL) + return NULL; new_distribution->tree = calloc(2 * num_buckets - 1, sizeof(bucket_t)); + if (new_distribution->tree == NULL) { + free (new_distribution); + return NULL; + } + new_distribution->num_buckets = num_buckets; build_tree(new_distribution, bucket_array, 0, 0, num_buckets - 1); free(bucket_array); return new_distribution; } +void distribution_destroy(distribution_t *d) { + if (d == NULL) + return; + free(d->tree); + free(d); +} + int main() { - distribution_t *p = distribution_new_linear(5, 2); + distribution_t *p = distribution_new_linear(3, 2); + for (size_t i = 0; i < 5; i++) { + printf("%f %f\n", p->tree[i].minimum, p->tree[i].maximum); + } + distribution_destroy(p); } \ No newline at end of file From 0b57898aed5cb8bf9488daeb8bc10bc1e7d4ffae Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Mon, 27 Jul 2020 15:21:27 +0500 Subject: [PATCH 04/67] add build_distribution_from_bucket_array function --- src/daemon/distribution.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index b5dd53c0fe..c609383422 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -49,6 +49,22 @@ void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t d->tree[node_index] = merge_buckets(d->tree[left_child], d->tree[right_child]); } +distribution_t* build_distribution_from_bucket_array(size_t num_buckets, bucket_t *bucket_array) { + distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); + if (new_distribution == NULL) + return NULL; + new_distribution->tree = calloc(2 * num_buckets - 1, sizeof(bucket_t)); + if (new_distribution->tree == NULL) { + free (new_distribution); + return NULL; + } + + new_distribution->num_buckets = num_buckets; + build_tree(new_distribution, bucket_array, 0, 0, num_buckets - 1); + free(bucket_array); + return new_distribution; +} + distribution_t* distribution_new_linear(size_t num_buckets, double size) { if (num_buckets == 0 || size <= 0) { errno = EINVAL; @@ -66,20 +82,7 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { else bucket_array[i].maximum = (i + 1) * size; } - - distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); - if (new_distribution == NULL) - return NULL; - new_distribution->tree = calloc(2 * num_buckets - 1, sizeof(bucket_t)); - if (new_distribution->tree == NULL) { - free (new_distribution); - return NULL; - } - - new_distribution->num_buckets = num_buckets; - build_tree(new_distribution, bucket_array, 0, 0, num_buckets - 1); - free(bucket_array); - return new_distribution; + return build_distribution_from_bucket_array(num_buckets, bucket_array); } void distribution_destroy(distribution_t *d) { From 8170a9398f98c90261cd6255ac1949ece72850cc Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Mon, 27 Jul 2020 15:39:53 +0500 Subject: [PATCH 05/67] add exponential constructor --- src/daemon/distribution.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index c609383422..69faf1c09a 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -85,6 +85,29 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { return build_distribution_from_bucket_array(num_buckets, bucket_array); } +distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor) { + if (num_buckets == 0 || initial_size <= 0 || factor <= 0) { + errno = EINVAL; + return NULL; + } + + bucket_t *bucket_array = calloc(num_buckets, sizeof(bucket_t)); + if (bucket_array == NULL) + return NULL; + + bucket_array[0] = (bucket_t) { + .maximum = initial_size, + }; + for (size_t i = 1; i < num_buckets; i++) { + bucket_array[i].minimum = bucket_array[i - 1].maximum; + if (i == num_buckets - 1) + bucket_array[i].maximum = INFINITY; + else + bucket_array[i].maximum = bucket_array[i].minimum * factor; + } + return build_distribution_from_bucket_array(num_buckets, bucket_array); +} + void distribution_destroy(distribution_t *d) { if (d == NULL) return; @@ -93,8 +116,8 @@ void distribution_destroy(distribution_t *d) { } int main() { - distribution_t *p = distribution_new_linear(3, 2); - for (size_t i = 0; i < 5; i++) { + distribution_t *p = distribution_new_exponential(5, 0.5, 2); + for (size_t i = 0; i < 9; i++) { printf("%f %f\n", p->tree[i].minimum, p->tree[i].maximum); } distribution_destroy(p); From 88137d63569ac3886f72b679c3760e8437a15b63 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Mon, 27 Jul 2020 15:50:21 +0500 Subject: [PATCH 06/67] fix possible memory leaks in build_distribution_from_bucket_array --- src/daemon/distribution.c | 25 ++++++++++++++++++++----- src/daemon/distribution.h | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 69faf1c09a..2834297611 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -51,17 +51,17 @@ void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t distribution_t* build_distribution_from_bucket_array(size_t num_buckets, bucket_t *bucket_array) { distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); - if (new_distribution == NULL) + if (new_distribution == NULL) { return NULL; + } new_distribution->tree = calloc(2 * num_buckets - 1, sizeof(bucket_t)); if (new_distribution->tree == NULL) { - free (new_distribution); + free(new_distribution); return NULL; } new_distribution->num_buckets = num_buckets; build_tree(new_distribution, bucket_array, 0, 0, num_buckets - 1); - free(bucket_array); return new_distribution; } @@ -82,7 +82,10 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { else bucket_array[i].maximum = (i + 1) * size; } - return build_distribution_from_bucket_array(num_buckets, bucket_array); + + distribution_t *new_distribution = build_distribution_from_bucket_array(num_buckets, bucket_array); + free(bucket_array); + return new_distribution; } distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor) { @@ -105,9 +108,21 @@ distribution_t* distribution_new_exponential(size_t num_buckets, double initial_ else bucket_array[i].maximum = bucket_array[i].minimum * factor; } - return build_distribution_from_bucket_array(num_buckets, bucket_array); + + distribution_t *new_distribution = build_distribution_from_bucket_array(num_buckets, bucket_array); + free(bucket_array); + return new_distribution; } +/*distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes) { + if (num_buckets == 0) { + errno = EINVAL; + return NULL; + } + for (size_t i = 0; i < num_buckets; i++) + if (custom_buckets) +}*/ + void distribution_destroy(distribution_t *d) { if (d == NULL) return; diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index fd1872e303..0dac5cf4b1 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -13,7 +13,7 @@ typedef struct distribution_s distribution_t; //constructor functions: distribution_t* distribution_new_linear(size_t num_buckets, double size); distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor); -distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes); +distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes); void distribution_update(distribution_t *dist, double gauge); double distribution_percentile(distribution_t *dist, double percent); From b47f5a27d1a1d1eca144e03fc753b3c896d05705 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Mon, 27 Jul 2020 15:58:52 +0500 Subject: [PATCH 07/67] add custom constructor --- src/daemon/distribution.c | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 2834297611..a1ed1b13c1 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -114,14 +114,34 @@ distribution_t* distribution_new_exponential(size_t num_buckets, double initial_ return new_distribution; } -/*distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes) { +distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes) { if (num_buckets == 0) { errno = EINVAL; return NULL; } for (size_t i = 0; i < num_buckets; i++) - if (custom_buckets) -}*/ + if (custom_buckets_sizes[i] <= 0) { + errno = EINVAL; + return NULL; + } + + bucket_t *bucket_array = calloc(num_buckets + 1, sizeof(bucket_t)); + if (bucket_array == NULL) + return NULL; + + for (size_t i = 0; i < num_buckets + 1; i++) { + if (i != 0) + bucket_array[i].minimum = bucket_array[i - 1].maximum; + if (i == num_buckets) + bucket_array[i].maximum = INFINITY; + else + bucket_array[i].maximum = bucket_array[i].minimum + custom_buckets_sizes[i]; + } + + distribution_t *new_distribution = build_distribution_from_bucket_array(num_buckets + 1, bucket_array); + free(bucket_array); + return new_distribution; +} void distribution_destroy(distribution_t *d) { if (d == NULL) @@ -131,8 +151,9 @@ void distribution_destroy(distribution_t *d) { } int main() { - distribution_t *p = distribution_new_exponential(5, 0.5, 2); - for (size_t i = 0; i < 9; i++) { + double a[] = {3.0, 2.7, 1.0}; + distribution_t *p = distribution_new_custom(3, a); + for (size_t i = 0; i < 7; i++) { printf("%f %f\n", p->tree[i].minimum, p->tree[i].maximum); } distribution_destroy(p); From edbd83ca0c8d942d1fd682d9a5786a1cb3b74640 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Mon, 27 Jul 2020 16:45:59 +0500 Subject: [PATCH 08/67] add distribution_clone() --- src/daemon/distribution.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index a1ed1b13c1..55faf1cdf0 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -6,6 +6,7 @@ #include #include #include +#include typedef struct bucket_s { uint64_t bucket_counter; @@ -150,6 +151,15 @@ void distribution_destroy(distribution_t *d) { free(d); } +distribution_t* distribution_clone(distribution_t *dist) { + if (dist == NULL) + return NULL; + distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); + new_distribution->num_buckets = dist->num_buckets; + memcpy(new_distribution->tree, dist->tree, sizeof(bucket_t) * (2 * dist->num_buckets - 1)); + return new_distribution; +} + int main() { double a[] = {3.0, 2.7, 1.0}; distribution_t *p = distribution_new_custom(3, a); From 33ed94a7aecacfda5f17c04b4be1cec55e99b2c5 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Mon, 27 Jul 2020 19:32:30 +0500 Subject: [PATCH 09/67] add distribution_update() --- src/daemon/distribution.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 55faf1cdf0..6183f95a7d 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -3,9 +3,10 @@ // #include "distribution.h" -#include + #include #include +#include #include typedef struct bucket_s { @@ -160,11 +161,38 @@ distribution_t* distribution_clone(distribution_t *dist) { return new_distribution; } +void update_tree(distribution_t *dist, size_t node_index, size_t left, size_t right, double gauge) { + if (left > right) + return; + if (left == right) { + dist->tree[node_index].bucket_counter++; + return; + } + size_t mid = (left + right) / 2; + size_t left_child = left_child_index(node_index, left, right); + size_t right_child = right_child_index(node_index, left, right); + if (dist->tree[left_child].maximum > gauge) + update_tree(dist, left_child, left, mid, gauge); + else + update_tree(dist, right_child, mid + 1, right, gauge); + dist->tree[node_index] = merge_buckets(dist->tree[left_child], dist->tree[right_child]); +} + +void distribution_update(distribution_t *dist, double gauge) { + if (dist == NULL) + return; + update_tree(dist, 0, 0, dist->num_buckets - 1, gauge); +} + int main() { double a[] = {3.0, 2.7, 1.0}; distribution_t *p = distribution_new_custom(3, a); + distribution_update(p, 2); + distribution_update(p, 5); + distribution_update(p, 7.5); + distribution_update(p, 3.1); for (size_t i = 0; i < 7; i++) { - printf("%f %f\n", p->tree[i].minimum, p->tree[i].maximum); + printf("%f %f %llu\n", p->tree[i].minimum, p->tree[i].maximum, p->tree[i].bucket_counter); } distribution_destroy(p); } \ No newline at end of file From 175687a20c63b05cd67882954fd1ececc5603839 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Mon, 27 Jul 2020 20:29:42 +0500 Subject: [PATCH 10/67] add distribution_average() --- src/daemon/distribution.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 6183f95a7d..18746e655d 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -17,6 +17,7 @@ typedef struct bucket_s { struct distribution_s { bucket_t *tree; size_t num_buckets; + double total_sum; }; size_t left_child_index(size_t node_index, size_t left, size_t right) { @@ -175,13 +176,18 @@ void update_tree(distribution_t *dist, size_t node_index, size_t left, size_t ri update_tree(dist, left_child, left, mid, gauge); else update_tree(dist, right_child, mid + 1, right, gauge); - dist->tree[node_index] = merge_buckets(dist->tree[left_child], dist->tree[right_child]); + dist->tree[node_index].bucket_counter++; } void distribution_update(distribution_t *dist, double gauge) { if (dist == NULL) return; update_tree(dist, 0, 0, dist->num_buckets - 1, gauge); + dist->total_sum += gauge; +} + +double distribution_average(distribution_t *dist) { + return dist->total_sum / dist->tree[0].bucket_counter; } int main() { @@ -194,5 +200,6 @@ int main() { for (size_t i = 0; i < 7; i++) { printf("%f %f %llu\n", p->tree[i].minimum, p->tree[i].maximum, p->tree[i].bucket_counter); } + printf("%f\n", distribution_average(p)); distribution_destroy(p); } \ No newline at end of file From 674b70f2888fc08277a5d04a39347e90bc546c9d Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 14:23:20 +0500 Subject: [PATCH 11/67] add static keyword to private functions --- src/daemon/distribution.c | 12 ++++++------ src/daemon/distribution.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 18746e655d..9214e265aa 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -20,16 +20,16 @@ struct distribution_s { double total_sum; }; -size_t left_child_index(size_t node_index, size_t left, size_t right) { +static size_t left_child_index(size_t node_index, size_t left, size_t right) { return node_index + 1; } -size_t right_child_index(size_t node_index, size_t left, size_t right) { +static size_t right_child_index(size_t node_index, size_t left, size_t right) { size_t mid = (left + right) / 2; return node_index + 2 * (mid - left + 1); } -bucket_t merge_buckets(bucket_t left_child, bucket_t right_child) { +static bucket_t merge_buckets(bucket_t left_child, bucket_t right_child) { return (bucket_t) { .bucket_counter = left_child.bucket_counter + right_child.bucket_counter, .minimum = left_child.minimum, @@ -37,7 +37,7 @@ bucket_t merge_buckets(bucket_t left_child, bucket_t right_child) { }; } -void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t left, size_t right) { +static void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t left, size_t right) { if (left > right) return; if (left == right) { @@ -52,7 +52,7 @@ void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t d->tree[node_index] = merge_buckets(d->tree[left_child], d->tree[right_child]); } -distribution_t* build_distribution_from_bucket_array(size_t num_buckets, bucket_t *bucket_array) { +static distribution_t* build_distribution_from_bucket_array(size_t num_buckets, bucket_t *bucket_array) { distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); if (new_distribution == NULL) { return NULL; @@ -162,7 +162,7 @@ distribution_t* distribution_clone(distribution_t *dist) { return new_distribution; } -void update_tree(distribution_t *dist, size_t node_index, size_t left, size_t right, double gauge) { +static void update_tree(distribution_t *dist, size_t node_index, size_t left, size_t right, double gauge) { if (left > right) return; if (left == right) { diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 0dac5cf4b1..4c126b2735 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -18,7 +18,7 @@ distribution_t* distribution_new_custom(size_t num_buckets, double *custom_bucke void distribution_update(distribution_t *dist, double gauge); double distribution_percentile(distribution_t *dist, double percent); double distribution_average(distribution_t *dist); -distribution_t distribution_clone(distribution_t *dist); +distribution_t* distribution_clone(distribution_t *dist); void distribution_destroy(distribution_t *d); #endif // COLLECTD_DISTRIBUTION_H From c645475305b9f3eb07f01cae6a66c29dc434167a Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 14:25:18 +0500 Subject: [PATCH 12/67] add unused attribute --- src/daemon/distribution.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 9214e265aa..23bc3e5c03 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -24,7 +24,9 @@ static size_t left_child_index(size_t node_index, size_t left, size_t right) { return node_index + 1; } -static size_t right_child_index(size_t node_index, size_t left, size_t right) { +static size_t right_child_index(size_t node_index, + __attribute__((unused)) size_t left, + __attribute__((unused)) size_t right) { size_t mid = (left + right) / 2; return node_index + 2 * (mid - left + 1); } From ca2065e56de919684cc564add202e8c4b8afeb4d Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 14:27:04 +0500 Subject: [PATCH 13/67] fix attributes --- src/daemon/distribution.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 23bc3e5c03..e75f3daf75 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -20,13 +20,13 @@ struct distribution_s { double total_sum; }; -static size_t left_child_index(size_t node_index, size_t left, size_t right) { +static size_t left_child_index(size_t node_index, + __attribute__((unused)) size_t left, + __attribute__((unused)) size_t right) { return node_index + 1; } -static size_t right_child_index(size_t node_index, - __attribute__((unused)) size_t left, - __attribute__((unused)) size_t right) { +static size_t right_child_index(size_t node_index, size_t left, size_t right) { size_t mid = (left + right) / 2; return node_index + 2 * (mid - left + 1); } From bbec9d91ee4a6a42add6a975b4013d19c43bbdd0 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 14:30:44 +0500 Subject: [PATCH 14/67] fixed factor checker --- src/daemon/distribution.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index e75f3daf75..cedc7aa307 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -94,7 +94,7 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { } distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor) { - if (num_buckets == 0 || initial_size <= 0 || factor <= 0) { + if (num_buckets == 0 || initial_size <= 0 || factor <= 1) { errno = EINVAL; return NULL; } From c9e191fe75291923504d0b0119608c08a940035e Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 14:31:56 +0500 Subject: [PATCH 15/67] fix code duplication --- src/daemon/distribution.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index cedc7aa307..5ec2a5f43b 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -167,8 +167,8 @@ distribution_t* distribution_clone(distribution_t *dist) { static void update_tree(distribution_t *dist, size_t node_index, size_t left, size_t right, double gauge) { if (left > right) return; + dist->tree[node_index].bucket_counter++; if (left == right) { - dist->tree[node_index].bucket_counter++; return; } size_t mid = (left + right) / 2; @@ -178,7 +178,6 @@ static void update_tree(distribution_t *dist, size_t node_index, size_t left, si update_tree(dist, left_child, left, mid, gauge); else update_tree(dist, right_child, mid + 1, right, gauge); - dist->tree[node_index].bucket_counter++; } void distribution_update(distribution_t *dist, double gauge) { From 775fa032df3b5203ef5456e19d00ab540e0fdd55 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 16:44:11 +0500 Subject: [PATCH 16/67] change heap allocation to stack allocation --- src/daemon/distribution.c | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 5ec2a5f43b..dff2fff925 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -76,21 +76,16 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { return NULL; } - bucket_t *bucket_array = calloc(num_buckets, sizeof(bucket_t)); - if (bucket_array == NULL) - return NULL; - + bucket_t bucket_array[num_buckets]; for (size_t i = 0; i < num_buckets; i++) { + bucket_array[i].bucket_counter = 0; bucket_array[i].minimum = i * size; if (i == num_buckets - 1) bucket_array[i].maximum = INFINITY; else bucket_array[i].maximum = (i + 1) * size; } - - distribution_t *new_distribution = build_distribution_from_bucket_array(num_buckets, bucket_array); - free(bucket_array); - return new_distribution; + return build_distribution_from_bucket_array(num_buckets, bucket_array); } distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor) { @@ -99,24 +94,21 @@ distribution_t* distribution_new_exponential(size_t num_buckets, double initial_ return NULL; } - bucket_t *bucket_array = calloc(num_buckets, sizeof(bucket_t)); - if (bucket_array == NULL) - return NULL; - + bucket_t bucket_array[num_buckets]; bucket_array[0] = (bucket_t) { + .bucket_counter = 0, + .minimum = 0, .maximum = initial_size, }; for (size_t i = 1; i < num_buckets; i++) { + bucket_array[i].bucket_counter = 0; bucket_array[i].minimum = bucket_array[i - 1].maximum; if (i == num_buckets - 1) bucket_array[i].maximum = INFINITY; else bucket_array[i].maximum = bucket_array[i].minimum * factor; } - - distribution_t *new_distribution = build_distribution_from_bucket_array(num_buckets, bucket_array); - free(bucket_array); - return new_distribution; + return build_distribution_from_bucket_array(num_buckets, bucket_array); } distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes) { @@ -130,22 +122,19 @@ distribution_t* distribution_new_custom(size_t num_buckets, double *custom_bucke return NULL; } - bucket_t *bucket_array = calloc(num_buckets + 1, sizeof(bucket_t)); - if (bucket_array == NULL) - return NULL; - + bucket_t bucket_array[num_buckets + 1]; for (size_t i = 0; i < num_buckets + 1; i++) { + bucket_array[i].bucket_counter = 0; if (i != 0) bucket_array[i].minimum = bucket_array[i - 1].maximum; + else + bucket_array[i].minimum = 0; if (i == num_buckets) bucket_array[i].maximum = INFINITY; else bucket_array[i].maximum = bucket_array[i].minimum + custom_buckets_sizes[i]; } - - distribution_t *new_distribution = build_distribution_from_bucket_array(num_buckets + 1, bucket_array); - free(bucket_array); - return new_distribution; + return build_distribution_from_bucket_array(num_buckets + 1, bucket_array); } void distribution_destroy(distribution_t *d) { From cb589e29abc4bffc1a828e3c85d023ebb4354b8d Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 16:52:01 +0500 Subject: [PATCH 17/67] change custom sizes to custom boundaries --- src/daemon/distribution.c | 27 ++++++++++++++------------- src/daemon/distribution.h | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index dff2fff925..1b05ac183b 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -111,30 +111,31 @@ distribution_t* distribution_new_exponential(size_t num_buckets, double initial_ return build_distribution_from_bucket_array(num_buckets, bucket_array); } -distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes) { - if (num_buckets == 0) { - errno = EINVAL; - return NULL; - } - for (size_t i = 0; i < num_buckets; i++) - if (custom_buckets_sizes[i] <= 0) { +distribution_t* distribution_new_custom(size_t array_size, double *custom_buckets_boundaries) { + for (size_t i = 0; i < array_size; i++) { + double previous_boundary = 0; + if (i > 0) + previous_boundary = custom_buckets_boundaries[i - 1]; + if (custom_buckets_boundaries[i] <= previous_boundary) { errno = EINVAL; return NULL; } + } - bucket_t bucket_array[num_buckets + 1]; - for (size_t i = 0; i < num_buckets + 1; i++) { + size_t num_buckets = array_size + 1; + bucket_t bucket_array[num_buckets]; + for (size_t i = 0; i < num_buckets; i++) { bucket_array[i].bucket_counter = 0; if (i != 0) bucket_array[i].minimum = bucket_array[i - 1].maximum; else bucket_array[i].minimum = 0; - if (i == num_buckets) + if (i == num_buckets - 1) bucket_array[i].maximum = INFINITY; else - bucket_array[i].maximum = bucket_array[i].minimum + custom_buckets_sizes[i]; + bucket_array[i].maximum = custom_buckets_boundaries[i]; } - return build_distribution_from_bucket_array(num_buckets + 1, bucket_array); + return build_distribution_from_bucket_array(num_buckets, bucket_array); } void distribution_destroy(distribution_t *d) { @@ -181,7 +182,7 @@ double distribution_average(distribution_t *dist) { } int main() { - double a[] = {3.0, 2.7, 1.0}; + double a[] = {3.0, 5.7, 6.7}; distribution_t *p = distribution_new_custom(3, a); distribution_update(p, 2); distribution_update(p, 5); diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 4c126b2735..8e6706094e 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -13,7 +13,7 @@ typedef struct distribution_s distribution_t; //constructor functions: distribution_t* distribution_new_linear(size_t num_buckets, double size); distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor); -distribution_t* distribution_new_custom(size_t num_buckets, double *custom_buckets_sizes); +distribution_t* distribution_new_custom(size_t array_size, double *custom_buckets_boundaries); void distribution_update(distribution_t *dist, double gauge); double distribution_percentile(distribution_t *dist, double percent); From 2e75bd53a017ace89fd8a77a33157d096e2b8fb9 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 17:12:04 +0500 Subject: [PATCH 18/67] add percentile calculation --- src/daemon/distribution.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 1b05ac183b..601aaac047 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -177,7 +177,35 @@ void distribution_update(distribution_t *dist, double gauge) { dist->total_sum += gauge; } +static double tree_get_counter(distribution_t *d, size_t node_index, size_t left, + size_t right, size_t counter) { + if (left > right) + return NAN; + if (left == right) { + return d->tree[node_index].maximum; + } + size_t mid = (left + right) / 2; + size_t left_child = left_child_index(node_index, left, right); + size_t right_child = right_child_index(node_index, left, right); + if (d->tree[left_child].bucket_counter >= counter) + return tree_get_counter(d, left_child, left, mid, counter); + else + return tree_get_counter(d, right_child, mid + 1, right, counter - d->tree[left_child].bucket_counter); +} + +double distribution_percentile(distribution_t *dist, double percent) { + if (percent <= 0 || percent > 100) { + errno = EINVAL; + return NAN; + } + if (dist->tree[0].bucket_counter == 0) + return NAN; + size_t counter = ceil(dist->tree[0].bucket_counter * percent / 100.0); + return tree_get_counter(dist, 0, 0, dist->num_buckets - 1, counter); +} + double distribution_average(distribution_t *dist) { + //TODO: checker return dist->total_sum / dist->tree[0].bucket_counter; } @@ -192,5 +220,6 @@ int main() { printf("%f %f %llu\n", p->tree[i].minimum, p->tree[i].maximum, p->tree[i].bucket_counter); } printf("%f\n", distribution_average(p)); + printf("%f\n", distribution_percentile(p, 90)); distribution_destroy(p); } \ No newline at end of file From 7350276e6a48fcdb0a8302917f0d2458311c9c5e Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 17:25:29 +0500 Subject: [PATCH 19/67] add license --- src/daemon/distribution.c | 29 +++++++++++++++++++++++++---- src/daemon/distribution.h | 28 +++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 601aaac047..8d647f64a7 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -1,7 +1,28 @@ -// -// Created by Svetlana Shmidt on 24.07.2020. -// - +/** + * collectd - src/daemon/distribution.c + * Copyright (C) 2019-2020 Google LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Svetlana Shmidt + **/ #include "distribution.h" #include diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 8e6706094e..c0d365e5c8 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -1,6 +1,28 @@ -// -// Created by Svetlana Shmidt on 24.07.2020. -// +/** + * collectd - src/daemon/distribution.h + * Copyright (C) 2019-2020 Google LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Svetlana Shmidt + **/ #ifndef COLLECTD_DISTRIBUTION_H #define COLLECTD_DISTRIBUTION_H From 67075322e3c9bb05bda2a3efc69cb774a9058517 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 18:17:07 +0500 Subject: [PATCH 20/67] add comments about stroing tree as an array --- src/daemon/distribution.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 8d647f64a7..fca4d0f5e8 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -41,6 +41,14 @@ struct distribution_s { double total_sum; }; +/** + * This code uses an Euler path to avoid gaps in the tree-to-array mapping. + * This way the tree contained N buckets contains 2 * N - 1 nodes + * Thus, left subtree has 2 * (mid - left + 1) - 1 nodes, + * therefore the right subtree starts at node_index + 2 * (mid - left + 1). + * For a detailed explanation, see https://docs.google.com/document/d/1ccsg5ffUfqt9-mBDGTymRn8X-9Wk1CuGYeMlRxmxiok/edit?usp=sharing". + */ + static size_t left_child_index(size_t node_index, __attribute__((unused)) size_t left, __attribute__((unused)) size_t right) { @@ -52,6 +60,10 @@ static size_t right_child_index(size_t node_index, size_t left, size_t right) { return node_index + 2 * (mid - left + 1); } +static size_t tree_size(size_t num_buckets) { + return 2 * num_buckets - 1; +} + static bucket_t merge_buckets(bucket_t left_child, bucket_t right_child) { return (bucket_t) { .bucket_counter = left_child.bucket_counter + right_child.bucket_counter, @@ -80,7 +92,7 @@ static distribution_t* build_distribution_from_bucket_array(size_t num_buckets, if (new_distribution == NULL) { return NULL; } - new_distribution->tree = calloc(2 * num_buckets - 1, sizeof(bucket_t)); + new_distribution->tree = calloc(tree_size(num_buckets), sizeof(bucket_t)); if (new_distribution->tree == NULL) { free(new_distribution); return NULL; From 47bd4ab25427716e23e49063e73c46f8027bcd36 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 18:35:46 +0500 Subject: [PATCH 21/67] add comments to constructor functions --- src/daemon/distribution.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index c0d365e5c8..251df9f810 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -33,8 +33,33 @@ struct distribution_s; typedef struct distribution_s distribution_t; //constructor functions: +/** + * function creates new distribution with the linear buckets: + * [0; size) [size; 2 * size) ... [(num_buckets - 1) * size; infinity) + * @param num_buckets - number of buckets. Should be greater than 0 + * @param size - size of each bucket. Should be greater than 0 + * @return pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + */ distribution_t* distribution_new_linear(size_t num_buckets, double size); + +/** + * function creates new distribution with the exponential buckets: + * [0; initial_size) [initial_size; initial_size * factor) ... [initial_size * factor^{num_buckets - 2}; infinity) + * @param num_buckets - number of buckets. Should be greater than 0 + * @param initial_size - size of the first bucket. Should be greater than 0 + * @param factor - factor for buckets' upper bound. Should be greater than 1 + * @return pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + */ distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor); + +/** + * function creates new distribution with the custom buckets: + * [0; custom_bucket_boundaries[0]) [custom_bucket_boundaries[0]; custom_bucket_boundaries[1]) ... + * ... [custom_bucket_boundaries[array_size - 1], infinity) + * @param array_size - size of array of bucket boundaries. Number of buckets is array_size + 1 + * @param custom_buckets_boundaries - array with bucket boundaries. Should be increasing and positive + * @return pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + */ distribution_t* distribution_new_custom(size_t array_size, double *custom_buckets_boundaries); void distribution_update(distribution_t *dist, double gauge); From 535fba9bbc96ac6533ded8453bb92335c649d68a Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Tue, 28 Jul 2020 18:46:49 +0500 Subject: [PATCH 22/67] add comments to header file --- src/daemon/distribution.h | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 251df9f810..7c887b09ed 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -38,7 +38,7 @@ typedef struct distribution_s distribution_t; * [0; size) [size; 2 * size) ... [(num_buckets - 1) * size; infinity) * @param num_buckets - number of buckets. Should be greater than 0 * @param size - size of each bucket. Should be greater than 0 - * @return pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + * @return - pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails */ distribution_t* distribution_new_linear(size_t num_buckets, double size); @@ -48,7 +48,7 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size); * @param num_buckets - number of buckets. Should be greater than 0 * @param initial_size - size of the first bucket. Should be greater than 0 * @param factor - factor for buckets' upper bound. Should be greater than 1 - * @return pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + * @return - pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails */ distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor); @@ -58,14 +58,28 @@ distribution_t* distribution_new_exponential(size_t num_buckets, double initial_ * ... [custom_bucket_boundaries[array_size - 1], infinity) * @param array_size - size of array of bucket boundaries. Number of buckets is array_size + 1 * @param custom_buckets_boundaries - array with bucket boundaries. Should be increasing and positive - * @return pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + * @return - pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails */ distribution_t* distribution_new_custom(size_t array_size, double *custom_buckets_boundaries); +/** add new value to a distribution **/ void distribution_update(distribution_t *dist, double gauge); + +/** + * @param percent - should be in (0; 100] range + * @return - an approximation of percent percentile + * (upper bound of such bucket that all less or equal buckets contain more than percent percents of values) + * or NAN if parameters are wrong or distribution is empty + */ double distribution_percentile(distribution_t *dist, double percent); + +/** @return - average of all values in distribution or NAN if distribution is empty */ double distribution_average(distribution_t *dist); + +/** @return - pointer to the copy of distribution or null if memory allocation fails */ distribution_t* distribution_clone(distribution_t *dist); + +/** destroy the distribution and free memory **/ void distribution_destroy(distribution_t *d); #endif // COLLECTD_DISTRIBUTION_H From 15d383a20c8c486acef220124c3186c554a1a4ec Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Wed, 29 Jul 2020 14:23:23 +0500 Subject: [PATCH 23/67] change exponential constructor --- src/daemon/distribution.c | 26 ++++++++++---------------- src/daemon/distribution.h | 8 ++++---- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index fca4d0f5e8..faaf08de32 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -121,25 +121,19 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { return build_distribution_from_bucket_array(num_buckets, bucket_array); } -distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor) { - if (num_buckets == 0 || initial_size <= 0 || factor <= 1) { +distribution_t* distribution_new_exponential(size_t num_buckets, double base, double factor) { + if (num_buckets == 0 || base <= 1 || factor <= 0) { errno = EINVAL; return NULL; } bucket_t bucket_array[num_buckets]; - bucket_array[0] = (bucket_t) { - .bucket_counter = 0, - .minimum = 0, - .maximum = initial_size, - }; - for (size_t i = 1; i < num_buckets; i++) { - bucket_array[i].bucket_counter = 0; - bucket_array[i].minimum = bucket_array[i - 1].maximum; - if (i == num_buckets - 1) - bucket_array[i].maximum = INFINITY; - else - bucket_array[i].maximum = bucket_array[i].minimum * factor; + for (size_t i = 0; i < num_buckets; i++) { + bucket_array[i] = (bucket_t) { + .bucket_counter = 0, + .minimum = (i == 0) ? 0 : bucket_array[i - 1].maximum, + .maximum = (i == num_buckets - 1) ? INFINITY : factor * pow(base, i), //check if it's slow + }; } return build_distribution_from_bucket_array(num_buckets, bucket_array); } @@ -244,7 +238,7 @@ double distribution_average(distribution_t *dist) { int main() { double a[] = {3.0, 5.7, 6.7}; - distribution_t *p = distribution_new_custom(3, a); + distribution_t *p = distribution_new_exponential(4, 2, 3); distribution_update(p, 2); distribution_update(p, 5); distribution_update(p, 7.5); @@ -253,6 +247,6 @@ int main() { printf("%f %f %llu\n", p->tree[i].minimum, p->tree[i].maximum, p->tree[i].bucket_counter); } printf("%f\n", distribution_average(p)); - printf("%f\n", distribution_percentile(p, 90)); + printf("%f\n", distribution_percentile(p, 70)); distribution_destroy(p); } \ No newline at end of file diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 7c887b09ed..3c4e1eea69 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -44,13 +44,13 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size); /** * function creates new distribution with the exponential buckets: - * [0; initial_size) [initial_size; initial_size * factor) ... [initial_size * factor^{num_buckets - 2}; infinity) + * [0; factor) [factor; factor * base) ... [factor * base^{num_buckets - 2}; infinity) * @param num_buckets - number of buckets. Should be greater than 0 - * @param initial_size - size of the first bucket. Should be greater than 0 - * @param factor - factor for buckets' upper bound. Should be greater than 1 + * @param base - base of geometric progression. Should be greater than 1 + * @param factor - size of the first bucket. Should be greater than 0 * @return - pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails */ -distribution_t* distribution_new_exponential(size_t num_buckets, double initial_size, double factor); +distribution_t* distribution_new_exponential(size_t num_buckets, double base, double factor); /** * function creates new distribution with the custom buckets: From e3cbd05ff56a1caa3e2a33180bdd88b49284d362 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Wed, 29 Jul 2020 14:29:24 +0500 Subject: [PATCH 24/67] cosmetical change in linear constructor --- src/daemon/distribution.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index faaf08de32..76df086cfe 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -111,12 +111,11 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { bucket_t bucket_array[num_buckets]; for (size_t i = 0; i < num_buckets; i++) { - bucket_array[i].bucket_counter = 0; - bucket_array[i].minimum = i * size; - if (i == num_buckets - 1) - bucket_array[i].maximum = INFINITY; - else - bucket_array[i].maximum = (i + 1) * size; + bucket_array[i] = (bucket_t) { + .bucket_counter = 0, + .minimum = i * size, + .maximum = (i == num_buckets - 1) ? INFINITY : (i + 1) * size, + }; } return build_distribution_from_bucket_array(num_buckets, bucket_array); } @@ -238,7 +237,7 @@ double distribution_average(distribution_t *dist) { int main() { double a[] = {3.0, 5.7, 6.7}; - distribution_t *p = distribution_new_exponential(4, 2, 3); + distribution_t *p = distribution_new_linear(4, 3); distribution_update(p, 2); distribution_update(p, 5); distribution_update(p, 7.5); From 258513b83dd280fe90412f833238558f4e6dde37 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Wed, 29 Jul 2020 14:34:48 +0500 Subject: [PATCH 25/67] cosmetic changes in custom constructor --- src/daemon/distribution.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 76df086cfe..b7d90a139a 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -151,15 +151,11 @@ distribution_t* distribution_new_custom(size_t array_size, double *custom_bucket size_t num_buckets = array_size + 1; bucket_t bucket_array[num_buckets]; for (size_t i = 0; i < num_buckets; i++) { - bucket_array[i].bucket_counter = 0; - if (i != 0) - bucket_array[i].minimum = bucket_array[i - 1].maximum; - else - bucket_array[i].minimum = 0; - if (i == num_buckets - 1) - bucket_array[i].maximum = INFINITY; - else - bucket_array[i].maximum = custom_buckets_boundaries[i]; + bucket_array[i] = (bucket_t) { + .bucket_counter = 0, + .minimum = (i == 0) ? 0 : bucket_array[i - 1].maximum, + .maximum = (i == num_buckets - 1) ? INFINITY : custom_buckets_boundaries[i], + }; } return build_distribution_from_bucket_array(num_buckets, bucket_array); } @@ -237,7 +233,7 @@ double distribution_average(distribution_t *dist) { int main() { double a[] = {3.0, 5.7, 6.7}; - distribution_t *p = distribution_new_linear(4, 3); + distribution_t *p = distribution_new_custom(3, a); distribution_update(p, 2); distribution_update(p, 5); distribution_update(p, 7.5); From a8f8f29ceae8fe768e5740aa5d467b6c37fe33c4 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Wed, 29 Jul 2020 14:38:45 +0500 Subject: [PATCH 26/67] add checker in average function --- src/daemon/distribution.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index b7d90a139a..b7309faeb0 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -227,7 +227,9 @@ double distribution_percentile(distribution_t *dist, double percent) { } double distribution_average(distribution_t *dist) { - //TODO: checker + if (dist->tree[0].bucket_counter == 0) { + return NAN; + } return dist->total_sum / dist->tree[0].bucket_counter; } From 9774b56f2fca85bc067c06873692025508c7f2d7 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Wed, 29 Jul 2020 14:51:51 +0500 Subject: [PATCH 27/67] change memory allocation style --- src/daemon/distribution.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index b7309faeb0..a13e781959 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -88,15 +88,16 @@ static void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, } static distribution_t* build_distribution_from_bucket_array(size_t num_buckets, bucket_t *bucket_array) { - distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); - if (new_distribution == NULL) { - return NULL; - } - new_distribution->tree = calloc(tree_size(num_buckets), sizeof(bucket_t)); - if (new_distribution->tree == NULL) { + distribution_t *new_distribution = calloc(1, sizeof(*new_distribution)); + if (num_buckets == 0) + return new_distribution; + bucket_t *nodes = calloc(tree_size(num_buckets), sizeof(*nodes)); + if (new_distribution == NULL || nodes == NULL) { free(new_distribution); + free(nodes); return NULL; } + new_distribution->tree = nodes; new_distribution->num_buckets = num_buckets; build_tree(new_distribution, bucket_array, 0, 0, num_buckets - 1); @@ -170,9 +171,18 @@ void distribution_destroy(distribution_t *d) { distribution_t* distribution_clone(distribution_t *dist) { if (dist == NULL) return NULL; - distribution_t *new_distribution = calloc(1, sizeof(distribution_t)); + distribution_t *new_distribution = calloc(1, sizeof(*new_distribution)); + if (dist->num_buckets == 0) + return new_distribution; + + bucket_t *nodes = calloc(tree_size(dist->num_buckets), sizeof(*nodes)); + if (new_distribution == NULL || nodes == NULL) { + free(new_distribution); + free(nodes); + return NULL; + } new_distribution->num_buckets = dist->num_buckets; - memcpy(new_distribution->tree, dist->tree, sizeof(bucket_t) * (2 * dist->num_buckets - 1)); + new_distribution->tree = nodes; return new_distribution; } From 782ca671b77066b2be751743a4c3ed8745283f68 Mon Sep 17 00:00:00 2001 From: Svetlana Shmidt Date: Wed, 29 Jul 2020 14:54:11 +0500 Subject: [PATCH 28/67] change counter type from size_t to uint64_t --- src/daemon/distribution.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index a13e781959..7046442d8b 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -210,7 +210,7 @@ void distribution_update(distribution_t *dist, double gauge) { } static double tree_get_counter(distribution_t *d, size_t node_index, size_t left, - size_t right, size_t counter) { + size_t right, uint64_t counter) { if (left > right) return NAN; if (left == right) { @@ -232,7 +232,7 @@ double distribution_percentile(distribution_t *dist, double percent) { } if (dist->tree[0].bucket_counter == 0) return NAN; - size_t counter = ceil(dist->tree[0].bucket_counter * percent / 100.0); + uint64_t counter = ceil(dist->tree[0].bucket_counter * percent / 100.0); return tree_get_counter(dist, 0, 0, dist->num_buckets - 1, counter); } From 72952153f508eb13d67b43cf91acec26d5429b4f Mon Sep 17 00:00:00 2001 From: Svetlana Date: Wed, 29 Jul 2020 13:32:49 +0000 Subject: [PATCH 29/67] fix compilation errors --- src/daemon/distribution.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 7046442d8b..d1e380c325 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -251,9 +252,9 @@ int main() { distribution_update(p, 7.5); distribution_update(p, 3.1); for (size_t i = 0; i < 7; i++) { - printf("%f %f %llu\n", p->tree[i].minimum, p->tree[i].maximum, p->tree[i].bucket_counter); + printf("%f %f %lu\n", p->tree[i].minimum, p->tree[i].maximum, p->tree[i].bucket_counter); } printf("%f\n", distribution_average(p)); printf("%f\n", distribution_percentile(p, 70)); distribution_destroy(p); -} \ No newline at end of file +} From fa3386c3962152aa8a286abc06a2bc4fa60b65be Mon Sep 17 00:00:00 2001 From: Svetlana Date: Wed, 29 Jul 2020 13:33:30 +0000 Subject: [PATCH 30/67] add distribution_test.c file --- src/daemon/distribution_test.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 src/daemon/distribution_test.c diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c new file mode 100644 index 0000000000..52be1d29b2 --- /dev/null +++ b/src/daemon/distribution_test.c @@ -0,0 +1,34 @@ +/** + * collectd - src/daemon/distribution_test.c + * Copyright (C) 2020 Google LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Svetlana Shmidt + */ + +#include "collectd.h" + +#include "metric.h" +#include "testing.h" + +int main(void) { + +} From 461281643f472ec71ed8cd6a9c019e235d5eb93c Mon Sep 17 00:00:00 2001 From: Svetlana Date: Wed, 29 Jul 2020 13:34:08 +0000 Subject: [PATCH 31/67] add distribution files to build system --- Makefile.am | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile.am b/Makefile.am index fdc2908f53..3bac69bfe4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,6 +131,7 @@ noinst_LTLIBRARIES = \ libavltree.la \ libcmds.la \ libcommon.la \ + libdistribution.la \ libformat_graphite.la \ libformat_json.la \ libheap.la \ @@ -151,6 +152,7 @@ check_LTLIBRARIES = \ check_PROGRAMS = \ test_common \ + test_distribution \ test_format_graphite \ test_meta_data \ test_metric \ @@ -268,6 +270,7 @@ collectd_LDFLAGS = -export-dynamic collectd_LDADD = \ libavltree.la \ libcommon.la \ + libdistribution.la \ libheap.la \ libllist.la \ libmetric.la \ @@ -356,6 +359,11 @@ test_meta_data_SOURCES = \ src/testing.h test_meta_data_LDADD = libmetadata.la libplugin_mock.la +test_distribution_SOURCES = \ + src/daemon/distribution_test.c \ + src/testing.h +test_distribution_LDADD = libdistribution.la libplugin_mock.la + test_metric_SOURCES = \ src/daemon/metric_test.c \ src/testing.h @@ -425,7 +433,14 @@ libmetadata_la_SOURCES = \ src/utils/metadata/meta_data.c \ src/utils/metadata/meta_data.h +libdistribution_la_SOURCES = \ + src/daemon/distribution.c \ + src/daemon/distribution.h +libdistribution_la_LIBADD = libmetadata.la $(COMMON_LIBS) + libmetric_la_SOURCES = \ + src/daemon/distribution.c \ + src/daemon/distribution.h \ src/daemon/metric.c \ src/daemon/metric.h libmetric_la_LIBADD = libmetadata.la $(COMMON_LIBS) From d654246eef4d8d7000249f39d5e8e631bfd0b388 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Wed, 29 Jul 2020 13:44:44 +0000 Subject: [PATCH 32/67] test ssh commit --- src/daemon/distribution_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 52be1d29b2..b020d3da1f 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -30,5 +30,5 @@ #include "testing.h" int main(void) { - + printf("Hello World!"); } From 117b89f28d50d160a28971cbb1dfa9b958dbdc57 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Thu, 30 Jul 2020 10:32:48 +0000 Subject: [PATCH 33/67] add unit tests into a build system --- Makefile.am | 1 + src/daemon/distribution.c | 15 +-------------- src/daemon/distribution_test.c | 13 ++++++++++--- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/Makefile.am b/Makefile.am index 3bac69bfe4..f31b02b855 100644 --- a/Makefile.am +++ b/Makefile.am @@ -208,6 +208,7 @@ endif # Link to these libraries.. COMMON_LIBS = $(PTHREAD_LIBS) +COMMON_LIBS += -lm if BUILD_WIN32 COMMON_LIBS += -lws2_32 endif diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index d1e380c325..a4fc52f678 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -23,6 +23,7 @@ * Authors: * Svetlana Shmidt **/ + #include "distribution.h" #include @@ -244,17 +245,3 @@ double distribution_average(distribution_t *dist) { return dist->total_sum / dist->tree[0].bucket_counter; } -int main() { - double a[] = {3.0, 5.7, 6.7}; - distribution_t *p = distribution_new_custom(3, a); - distribution_update(p, 2); - distribution_update(p, 5); - distribution_update(p, 7.5); - distribution_update(p, 3.1); - for (size_t i = 0; i < 7; i++) { - printf("%f %f %lu\n", p->tree[i].minimum, p->tree[i].maximum, p->tree[i].bucket_counter); - } - printf("%f\n", distribution_average(p)); - printf("%f\n", distribution_percentile(p, 70)); - distribution_destroy(p); -} diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index b020d3da1f..45cf3ef009 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -26,9 +26,16 @@ #include "collectd.h" -#include "metric.h" +#include "distribution.h" #include "testing.h" -int main(void) { - printf("Hello World!"); +DEF_TEST(distribution_new_linear) { + distribution_t *d = distribution_new_linear(3, -1); + EXPECT_EQ_PTR(NULL, d); + return 0; +} + +int main() { + RUN_TEST(distribution_new_linear); + END_TEST; } From d91926ff66661af28230c85aec9e71d5052ee3e2 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Thu, 30 Jul 2020 10:37:32 +0000 Subject: [PATCH 34/67] move bucket_t declaration to the header file --- src/daemon/distribution.c | 6 ------ src/daemon/distribution.h | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index a4fc52f678..0e340bbde9 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -28,15 +28,9 @@ #include #include -#include #include #include -typedef struct bucket_s { - uint64_t bucket_counter; - double minimum, maximum; -} bucket_t; - struct distribution_s { bucket_t *tree; size_t num_buckets; diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 3c4e1eea69..183965eaa9 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -27,8 +27,14 @@ #ifndef COLLECTD_DISTRIBUTION_H #define COLLECTD_DISTRIBUTION_H +#include #include +typedef struct bucket_s { + uint64_t bucket_counter; + double minimum, maximum; +} bucket_t; + struct distribution_s; typedef struct distribution_s distribution_t; From 4a9d323dd7ecc21f7ac0357a2577ef609ab09076 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Thu, 30 Jul 2020 10:45:01 +0000 Subject: [PATCH 35/67] add distribution_num_buckets function --- src/daemon/distribution.c | 3 +++ src/daemon/distribution.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 0e340bbde9..89c74940f9 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -239,3 +239,6 @@ double distribution_average(distribution_t *dist) { return dist->total_sum / dist->tree[0].bucket_counter; } +size_t distribution_num_buckets(distribution_t *dist) { + return dist->num_buckets; +} diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 183965eaa9..d4bdf503a1 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -88,4 +88,7 @@ distribution_t* distribution_clone(distribution_t *dist); /** destroy the distribution and free memory **/ void distribution_destroy(distribution_t *d); +/** @return - number of buckets stored in the distribution **/ +size_t distribution_num_buckets(distribution_t *dist); + #endif // COLLECTD_DISTRIBUTION_H From 30e6b452fa94adfeb28af47606a07a809f5ccdb3 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Thu, 30 Jul 2020 12:58:53 +0000 Subject: [PATCH 36/67] add some unit tests for distribution_new_linear --- src/daemon/distribution_test.c | 44 ++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 45cf3ef009..cb9a203992 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -30,12 +30,46 @@ #include "testing.h" DEF_TEST(distribution_new_linear) { - distribution_t *d = distribution_new_linear(3, -1); - EXPECT_EQ_PTR(NULL, d); - return 0; + struct { + size_t num_buckets; + double size; + double *want_get; + } cases[] = { + { + .num_buckets = 0, + .size = 5, + .want_get = NULL, + }, + { + .num_buckets = 3, + .size = -5, + .want_get = NULL, + }, + { + .num_buckets = 5, + .size = 0, + .want_get = NULL, + }, + { + .num_buckets = 3, + .size = 2.5, + .want_get = (double[]){2.5, 5.0, INFINITY}, + }, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + printf("## Case %zu:\n", i); + distribution_t *d; + if (cases[i].want_get == NULL) { + EXPECT_EQ_PTR(NULL, d = distribution_new_linear(cases[i].num_buckets, cases[i].size)); + continue; + } + CHECK_NOT_NULL(d = distribution_new_linear(cases[i].num_buckets, cases[i].size)); + distribution_destroy(d); + } + return 0; } int main() { - RUN_TEST(distribution_new_linear); - END_TEST; + RUN_TEST(distribution_new_linear); + END_TEST; } From 8a57e8cee0e0be36f0c4234e392bc4e5e0131955 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 08:43:23 +0000 Subject: [PATCH 37/67] add getter for buckets --- src/daemon/distribution.c | 33 +++++++++++++++++++++++++++++---- src/daemon/distribution.h | 9 ++++++++- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 89c74940f9..e1a139e663 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -63,7 +63,6 @@ static size_t tree_size(size_t num_buckets) { static bucket_t merge_buckets(bucket_t left_child, bucket_t right_child) { return (bucket_t) { .bucket_counter = left_child.bucket_counter + right_child.bucket_counter, - .minimum = left_child.minimum, .maximum = right_child.maximum, }; } @@ -110,7 +109,6 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { for (size_t i = 0; i < num_buckets; i++) { bucket_array[i] = (bucket_t) { .bucket_counter = 0, - .minimum = i * size, .maximum = (i == num_buckets - 1) ? INFINITY : (i + 1) * size, }; } @@ -127,7 +125,6 @@ distribution_t* distribution_new_exponential(size_t num_buckets, double base, do for (size_t i = 0; i < num_buckets; i++) { bucket_array[i] = (bucket_t) { .bucket_counter = 0, - .minimum = (i == 0) ? 0 : bucket_array[i - 1].maximum, .maximum = (i == num_buckets - 1) ? INFINITY : factor * pow(base, i), //check if it's slow }; } @@ -150,7 +147,6 @@ distribution_t* distribution_new_custom(size_t array_size, double *custom_bucket for (size_t i = 0; i < num_buckets; i++) { bucket_array[i] = (bucket_t) { .bucket_counter = 0, - .minimum = (i == 0) ? 0 : bucket_array[i - 1].maximum, .maximum = (i == num_buckets - 1) ? INFINITY : custom_buckets_boundaries[i], }; } @@ -240,5 +236,34 @@ double distribution_average(distribution_t *dist) { } size_t distribution_num_buckets(distribution_t *dist) { + if (dist == NULL) + return 0; return dist->num_buckets; +} + +/* @return - pointer to the first byte after last written bucket **/ +static bucket_t *tree_write_leave_buckets(distribution_t *dist, bucket_t *write_ptr, size_t node_index, size_t left, size_t right) { + if (left > right) + return NULL; + if (left == right) { + *write_ptr = dist->tree[node_index]; + write_ptr++; + return write_ptr; + } + size_t mid = (left + right) / 2; + size_t left_child = left_child_index(node_index, left, right); + size_t right_child = right_child_index(node_index, left, right); + bucket_t *new_write_ptr = tree_write_leave_buckets(dist, write_ptr, left_child, left, mid); + return tree_write_leave_buckets(dist, new_write_ptr, right_child, mid + 1, right); } + +buckets_array_t get_buckets(distribution_t *dist) { + buckets_array_t bucket_array = { + .num_buckets = dist->num_buckets, + .buckets = calloc(dist->num_buckets, sizeof(*bucket_array.buckets)), + }; + bucket_t *write_ptr = bucket_array.buckets; + tree_write_leave_buckets(dist, write_ptr, 0, 0, dist->num_buckets - 1); + return bucket_array; +} + diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index d4bdf503a1..552a22914c 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -32,12 +32,17 @@ typedef struct bucket_s { uint64_t bucket_counter; - double minimum, maximum; + double maximum; } bucket_t; struct distribution_s; typedef struct distribution_s distribution_t; +typedef struct buckets_array_s { + size_t num_buckets; + bucket_t *buckets; +} buckets_array_t; + //constructor functions: /** * function creates new distribution with the linear buckets: @@ -91,4 +96,6 @@ void distribution_destroy(distribution_t *d); /** @return - number of buckets stored in the distribution **/ size_t distribution_num_buckets(distribution_t *dist); +/** @return - array of buckets in the distribution **/ +buckets_array_t get_buckets(distribution_t *dist); #endif // COLLECTD_DISTRIBUTION_H From 560569197830bbbbc799417aad13a4b5f721c7b6 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 08:58:41 +0000 Subject: [PATCH 38/67] add checker to bucket getter --- src/daemon/distribution.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index e1a139e663..21fa3f05c8 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -259,9 +259,11 @@ static bucket_t *tree_write_leave_buckets(distribution_t *dist, bucket_t *write_ buckets_array_t get_buckets(distribution_t *dist) { buckets_array_t bucket_array = { - .num_buckets = dist->num_buckets, - .buckets = calloc(dist->num_buckets, sizeof(*bucket_array.buckets)), + .num_buckets = dist == NULL ? 0 : dist->num_buckets, + .buckets = dist == NULL ? NULL : calloc(dist->num_buckets, sizeof(*bucket_array.buckets)), }; + if (dist == NULL) + return bucket_array; bucket_t *write_ptr = bucket_array.buckets; tree_write_leave_buckets(dist, write_ptr, 0, 0, dist->num_buckets - 1); return bucket_array; From 2a86e2061f284ffe28a69c8edb6c911c5dc9cc96 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 09:27:09 +0000 Subject: [PATCH 39/67] add some unit tests for distribution_new_linear --- src/daemon/distribution_test.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index cb9a203992..be6d2989cb 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -34,21 +34,25 @@ DEF_TEST(distribution_new_linear) { size_t num_buckets; double size; double *want_get; + int want_err; } cases[] = { { .num_buckets = 0, .size = 5, .want_get = NULL, + .want_err = EINVAL, }, { .num_buckets = 3, .size = -5, .want_get = NULL, + .want_err = EINVAL, }, { .num_buckets = 5, .size = 0, .want_get = NULL, + .want_err = EINVAL, }, { .num_buckets = 3, @@ -58,12 +62,17 @@ DEF_TEST(distribution_new_linear) { }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { printf("## Case %zu:\n", i); - distribution_t *d; - if (cases[i].want_get == NULL) { - EXPECT_EQ_PTR(NULL, d = distribution_new_linear(cases[i].num_buckets, cases[i].size)); + if (cases[i].want_err != 0) { + EXPECT_EQ_PTR(cases[i].want_get, distribution_new_linear(cases[i].num_buckets, cases[i].size)); + EXPECT_EQ_INT(cases[i].want_err, errno); continue; } + distribution_t *d; CHECK_NOT_NULL(d = distribution_new_linear(cases[i].num_buckets, cases[i].size)); + buckets_array_t buckets_array = get_buckets(d); + for (size_t j = 0; j < cases[i].num_buckets; j++) { + EXPECT_EQ_DOUBLE(cases[i].want_get[j], buckets_array.buckets[j].maximum); + } distribution_destroy(d); } return 0; From d58ca9ad7be7c73ba42356690bfa965eb18f9c3e Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 09:44:20 +0000 Subject: [PATCH 40/67] add some unit tests for distribution_new_linear --- src/daemon/distribution_test.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index be6d2989cb..01fb0f8143 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -29,6 +29,14 @@ #include "distribution.h" #include "testing.h" +static double *linear_upper_bounds(size_t num, double size) { + double *linear_upper_bounds = calloc(num, sizeof(*linear_upper_bounds)); + for (size_t i = 0; i + 1 < num; i++) + linear_upper_bounds[i] = (i + 1) * size; + linear_upper_bounds[num - 1] = INFINITY; + return linear_upper_bounds; +} + DEF_TEST(distribution_new_linear) { struct { size_t num_buckets; @@ -57,8 +65,23 @@ DEF_TEST(distribution_new_linear) { { .num_buckets = 3, .size = 2.5, - .want_get = (double[]){2.5, 5.0, INFINITY}, - }, + .want_get = linear_upper_bounds(3, 2.5), + }, + { + .num_buckets = 5, + .size = 5.75, + .want_get = linear_upper_bounds(5, 5.75), + }, + { + .num_buckets = 151, + .size = 0.7, + .want_get = linear_upper_bounds(151, 0.7), + }, + { + .num_buckets = 111, + .size = 1074, + .want_get = linear_upper_bounds(111, 1074), + } }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { printf("## Case %zu:\n", i); @@ -75,6 +98,9 @@ DEF_TEST(distribution_new_linear) { } distribution_destroy(d); } + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + free(cases[i].want_get); + } return 0; } From 71b1f17e66c8eb27ffa7150b761034c6c0795924 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 09:53:49 +0000 Subject: [PATCH 41/67] add some test for distribution_new_linear --- src/daemon/distribution_test.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 01fb0f8143..6e549cd9ce 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -81,6 +81,11 @@ DEF_TEST(distribution_new_linear) { .num_buckets = 111, .size = 1074, .want_get = linear_upper_bounds(111, 1074), + }, + { + .num_buckets = 77, + .size = 1.0 / 3.0, + .want_get = linear_upper_bounds(77, 1.0 / 3.0), } }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { From 8978487ac20a3dce68bca05e617bc8d019a320cd Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 11:34:32 +0000 Subject: [PATCH 42/67] add some unit tests for distribution_new_exponential --- src/daemon/distribution_test.c | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 6e549cd9ce..5bf47859d9 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -109,7 +109,80 @@ DEF_TEST(distribution_new_linear) { return 0; } +DEF_TEST(distribution_new_exponential) { + struct { + size_t num_buckets; + double factor; + double base; + double *want_get; + int want_err; + } cases[] = { + { + .num_buckets = 0, + .factor = 5, + .base = 8, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 5, + .factor = 0.2, + .base = -1, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 8, + .factor = 100, + .base = 0.5, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 100, + .factor = 5.87, + .base = 1, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 6, + .factor = 0, + .base = 9.005, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 16, + .factor = -153, + .base = 1.41, + .want_get = NULL, + .want_err = EINVAL, + }, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + printf("## Case %zu:\n", i); + if (cases[i].want_err != 0) { + EXPECT_EQ_PTR(cases[i].want_get, distribution_new_exponential(cases[i].num_buckets, cases[i].base, cases[i].factor)); + EXPECT_EQ_INT(cases[i].want_err, errno); + continue; + } + /*distribution_t *d; + CHECK_NOT_NULL(d = distribution_new_linear(cases[i].num_buckets, cases[i].size)); + buckets_array_t buckets_array = get_buckets(d); + for (size_t j = 0; j < cases[i].num_buckets; j++) { + EXPECT_EQ_DOUBLE(cases[i].want_get[j], buckets_array.buckets[j].maximum); + } + distribution_destroy(d);*/ + } + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + free(cases[i].want_get); + } + return 0; +} + int main() { RUN_TEST(distribution_new_linear); + RUN_TEST(distribution_new_exponential); END_TEST; } From 64adecabdd3d1c14a6f8260efc19954c06c1e95d Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 11:54:59 +0000 Subject: [PATCH 43/67] add destroy function for bucket_array_t data structure --- src/daemon/distribution.c | 3 +++ src/daemon/distribution.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 21fa3f05c8..c440fa469f 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -269,3 +269,6 @@ buckets_array_t get_buckets(distribution_t *dist) { return bucket_array; } +void destroy_buckets_array(buckets_array_t buckets_array) { + free(buckets_array.buckets); +} diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 552a22914c..82e1c8e1f3 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -98,4 +98,7 @@ size_t distribution_num_buckets(distribution_t *dist); /** @return - array of buckets in the distribution **/ buckets_array_t get_buckets(distribution_t *dist); + +void destroy_buckets_array(buckets_array_t buckets_array); + #endif // COLLECTD_DISTRIBUTION_H From 9b6695fc46ff45f8630ecf0bddd1d4a1bae380d1 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 12:11:20 +0000 Subject: [PATCH 44/67] add some tests for exponential constructor (test failed because of doubles equality check) --- src/daemon/distribution_test.c | 46 +++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 5bf47859d9..77cefe97a3 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -37,6 +37,15 @@ static double *linear_upper_bounds(size_t num, double size) { return linear_upper_bounds; } +static double *exponential_upper_bounds(size_t num, double base, double factor) { + double *exponential_upper_bounds = calloc(num, sizeof(*exponential_upper_bounds)); + exponential_upper_bounds[0] = factor; + for (size_t i = 1; i + 1 < num; i++) + exponential_upper_bounds[i] = exponential_upper_bounds[i - 1] * base; + exponential_upper_bounds[num - 1] = INFINITY; + return exponential_upper_bounds; +} + DEF_TEST(distribution_new_linear) { struct { size_t num_buckets; @@ -86,7 +95,12 @@ DEF_TEST(distribution_new_linear) { .num_buckets = 77, .size = 1.0 / 3.0, .want_get = linear_upper_bounds(77, 1.0 / 3.0), - } + }, + { + .num_buckets = 1, + .size = 100, + .want_get = linear_upper_bounds(1, 100), + }, }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { printf("## Case %zu:\n", i); @@ -100,7 +114,8 @@ DEF_TEST(distribution_new_linear) { buckets_array_t buckets_array = get_buckets(d); for (size_t j = 0; j < cases[i].num_buckets; j++) { EXPECT_EQ_DOUBLE(cases[i].want_get[j], buckets_array.buckets[j].maximum); - } + } + destroy_buckets_array(buckets_array); distribution_destroy(d); } for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { @@ -159,6 +174,24 @@ DEF_TEST(distribution_new_exponential) { .want_get = NULL, .want_err = EINVAL, }, + { + .num_buckets = 1, + .factor = 10, + .base = 1.05, + .want_get = exponential_upper_bounds(1, 10, 1.05), + }, + { + .num_buckets = 63, + .factor = 1, + .base = 2, + .want_get = exponential_upper_bounds(63, 2, 1), + }, + { + .num_buckets = 600, + .factor = 0.55, + .base = 1.055, + .want_get = exponential_upper_bounds(600, 1.055, 0.55), + }, }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { printf("## Case %zu:\n", i); @@ -167,13 +200,14 @@ DEF_TEST(distribution_new_exponential) { EXPECT_EQ_INT(cases[i].want_err, errno); continue; } - /*distribution_t *d; - CHECK_NOT_NULL(d = distribution_new_linear(cases[i].num_buckets, cases[i].size)); + distribution_t *d; + CHECK_NOT_NULL(d = distribution_new_exponential(cases[i].num_buckets, cases[i].base, cases[i].factor)); buckets_array_t buckets_array = get_buckets(d); for (size_t j = 0; j < cases[i].num_buckets; j++) { EXPECT_EQ_DOUBLE(cases[i].want_get[j], buckets_array.buckets[j].maximum); - } - distribution_destroy(d);*/ + } + destroy_buckets_array(buckets_array); + distribution_destroy(d); } for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { free(cases[i].want_get); From 434680951efad3c9f25e02f20485e4c7c72f0d93 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 31 Jul 2020 12:24:26 +0000 Subject: [PATCH 45/67] fix exponential constructor unit tests --- src/daemon/distribution_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 77cefe97a3..6b3660445f 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -41,7 +41,7 @@ static double *exponential_upper_bounds(size_t num, double base, double factor) double *exponential_upper_bounds = calloc(num, sizeof(*exponential_upper_bounds)); exponential_upper_bounds[0] = factor; for (size_t i = 1; i + 1 < num; i++) - exponential_upper_bounds[i] = exponential_upper_bounds[i - 1] * base; + exponential_upper_bounds[i] = factor * pow(base, i); //exponential_upper_bounds[i - 1] * base; exponential_upper_bounds[num - 1] = INFINITY; return exponential_upper_bounds; } From 92219cddfc1020a8709ad2b6898d89b0f52ac396 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 3 Aug 2020 08:27:20 +0000 Subject: [PATCH 46/67] remove libdistribution.la and remove -lm from COMMON_LIBS --- Makefile.am | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Makefile.am b/Makefile.am index f31b02b855..92277d830e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,7 +131,6 @@ noinst_LTLIBRARIES = \ libavltree.la \ libcmds.la \ libcommon.la \ - libdistribution.la \ libformat_graphite.la \ libformat_json.la \ libheap.la \ @@ -208,7 +207,6 @@ endif # Link to these libraries.. COMMON_LIBS = $(PTHREAD_LIBS) -COMMON_LIBS += -lm if BUILD_WIN32 COMMON_LIBS += -lws2_32 endif @@ -271,7 +269,6 @@ collectd_LDFLAGS = -export-dynamic collectd_LDADD = \ libavltree.la \ libcommon.la \ - libdistribution.la \ libheap.la \ libllist.la \ libmetric.la \ @@ -363,7 +360,7 @@ test_meta_data_LDADD = libmetadata.la libplugin_mock.la test_distribution_SOURCES = \ src/daemon/distribution_test.c \ src/testing.h -test_distribution_LDADD = libdistribution.la libplugin_mock.la +test_distribution_LDADD = libmetric.la libplugin_mock.la test_metric_SOURCES = \ src/daemon/metric_test.c \ @@ -434,17 +431,12 @@ libmetadata_la_SOURCES = \ src/utils/metadata/meta_data.c \ src/utils/metadata/meta_data.h -libdistribution_la_SOURCES = \ - src/daemon/distribution.c \ - src/daemon/distribution.h -libdistribution_la_LIBADD = libmetadata.la $(COMMON_LIBS) - libmetric_la_SOURCES = \ src/daemon/distribution.c \ src/daemon/distribution.h \ src/daemon/metric.c \ src/daemon/metric.h -libmetric_la_LIBADD = libmetadata.la $(COMMON_LIBS) +libmetric_la_LIBADD = libmetadata.la $(COMMON_LIBS) -lm libplugin_mock_la_SOURCES = \ src/daemon/plugin_mock.c \ From 544654470aea21f2e201dd93fced69023d5b1075 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 3 Aug 2020 08:36:15 +0000 Subject: [PATCH 47/67] change return value from tree getter function --- src/daemon/distribution.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index c440fa469f..df1579b32c 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -241,20 +241,18 @@ size_t distribution_num_buckets(distribution_t *dist) { return dist->num_buckets; } -/* @return - pointer to the first byte after last written bucket **/ -static bucket_t *tree_write_leave_buckets(distribution_t *dist, bucket_t *write_ptr, size_t node_index, size_t left, size_t right) { +static void tree_write_leave_buckets(distribution_t *dist, bucket_t *write_ptr, size_t node_index, size_t left, size_t right) { if (left > right) - return NULL; + return; if (left == right) { - *write_ptr = dist->tree[node_index]; - write_ptr++; - return write_ptr; + write_ptr[left] = dist->tree[node_index]; + return; } size_t mid = (left + right) / 2; size_t left_child = left_child_index(node_index, left, right); size_t right_child = right_child_index(node_index, left, right); - bucket_t *new_write_ptr = tree_write_leave_buckets(dist, write_ptr, left_child, left, mid); - return tree_write_leave_buckets(dist, new_write_ptr, right_child, mid + 1, right); + tree_write_leave_buckets(dist, write_ptr, left_child, left, mid); + tree_write_leave_buckets(dist, write_ptr, right_child, mid + 1, right); } buckets_array_t get_buckets(distribution_t *dist) { From 348bdeb1d0b49379b1f13ce66809a2d6eaa2ab4c Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 3 Aug 2020 09:25:22 +0000 Subject: [PATCH 48/67] add unit tests for custom constructor + add inf checker to custom constructor --- src/daemon/distribution.c | 4 ++ src/daemon/distribution_test.c | 74 +++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index df1579b32c..0269eb8461 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -141,6 +141,10 @@ distribution_t* distribution_new_custom(size_t array_size, double *custom_bucket return NULL; } } + if (array_size > 0 && custom_buckets_boundaries[array_size - 1] == INFINITY) { + errno = EINVAL; + return NULL; + } size_t num_buckets = array_size + 1; bucket_t bucket_array[num_buckets]; diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 6b3660445f..f175fbcc5a 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -215,8 +215,80 @@ DEF_TEST(distribution_new_exponential) { return 0; } +DEF_TEST(distribution_new_custom) { + struct { + size_t array_size; + double *custom_boundaries; + double *want_get; + int want_err; + } cases[] = { + { + .array_size = 0, + .want_get = (double[]){INFINITY}, + }, + { + .array_size = 5, + .custom_boundaries = (double[]){0, 1, 2, 3, 4}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 3, + .custom_boundaries = (double[]){-5, 7, 3}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 4, + .custom_boundaries = (double[]){5.7, 6.0, 6.0, 7.0}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 1, + .custom_boundaries = (double[]){105.055}, + .want_get = (double[]){105.055, INFINITY}, + }, + { + .array_size = 5, + .custom_boundaries = (double[]){8, 100, 1000, 1008, INFINITY}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 7, + .custom_boundaries = (double[]){2, 4, 8, 6, 2, 16, 77.5}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 10, + .custom_boundaries = (double[]){77.5, 100.203, 122.01, 137.23, 200, 205, 210, 220, 230, 256}, + .want_get = (double[]){77.5, 100.203, 122.01, 137.23, 200, 205, 210, 220, 230, 256, INFINITY}, + }, + }; + + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + if (cases[i].want_err != 0) { + EXPECT_EQ_PTR(cases[i].want_get, distribution_new_custom(cases[i].array_size, cases[i].custom_boundaries)); + EXPECT_EQ_INT(cases[i].want_err, errno); + continue; + } + distribution_t *d; + CHECK_NOT_NULL(d = distribution_new_custom(cases[i].array_size, cases[i].custom_boundaries)); + buckets_array_t buckets_array = get_buckets(d); + for (size_t j = 0; j < cases[i].array_size + 1; j++) { + EXPECT_EQ_DOUBLE(cases[i].want_get[j], buckets_array.buckets[j].maximum); + } + destroy_buckets_array(buckets_array); + distribution_destroy(d); + } + return 0; +} + int main() { RUN_TEST(distribution_new_linear); - RUN_TEST(distribution_new_exponential); + RUN_TEST(distribution_new_exponential); + RUN_TEST(distribution_new_custom); END_TEST; } From 6935af15e7ec4066ffddff16ba83079c18071b76 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 3 Aug 2020 13:16:17 +0000 Subject: [PATCH 49/67] add unit tests for update function --- src/daemon/distribution_test.c | 58 ++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index f175fbcc5a..aedb81ef8c 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -286,9 +286,67 @@ DEF_TEST(distribution_new_custom) { return 0; } +DEF_TEST(update) { + struct { + distribution_t *dist; + size_t num_gauges; + double *gauges; + uint64_t *want_counters; + int want_err; + } cases[] = { + { + .dist = distribution_new_linear(6, 5), + .num_gauges = 10, + .gauges = (double[]){25, 30, 5, 7, 11, 10.5, 8.03, 1112.4, 35, 12.7}, + .want_counters = (uint64_t[]){0, 3, 3, 0, 0, 4}, + }, + { + .dist = distribution_new_exponential(4, 1.41, 1), + .num_gauges = 0, + .gauges = NULL, + .want_counters = (uint64_t[]){0, 0, 0, 0}, + }, + { + .dist = distribution_new_exponential(5, 2, 3), + .num_gauges = 5, + .gauges = (double[]){1, 7, 3, 10, 77}, + .want_counters = (uint64_t[]){1, 1, 2, 0, 1}, + }, + { + .dist = distribution_new_linear(100, 22), + .num_gauges = 3, + .gauges = (double[]){1000, 2, -8}, + .want_err = EINVAL, + }, + { + .dist = distribution_new_custom(3, (double[]) {5, 20, 35}), + .num_gauges = 7, + .gauges = (double[]){7.05, 22.37, 40.83, 90.55, 12.34, 14.2, 6.0}, + .want_counters = (uint64_t[]){0, 4, 1, 2}, + }, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + for (size_t j = 0; j < cases[i].num_gauges; j++) { + distribution_update(cases[i].dist, cases[i].gauges[j]); + } + if (cases[i].want_err != 0) { + EXPECT_EQ_INT(cases[i].want_err, errno); + continue; + } + buckets_array_t buckets_array = get_buckets(cases[i].dist); + for (size_t j = 0; j < buckets_array.num_buckets; j++) { + EXPECT_EQ_INT(cases[i].want_counters[j], buckets_array.buckets[j].bucket_counter); + } + destroy_buckets_array(buckets_array); + distribution_destroy(cases[i].dist); + } + return 0; +} + int main() { RUN_TEST(distribution_new_linear); RUN_TEST(distribution_new_exponential); RUN_TEST(distribution_new_custom); + RUN_TEST(update); END_TEST; } From a9e98cd2d357c34384b534f482278124bec95097 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 3 Aug 2020 13:56:20 +0000 Subject: [PATCH 50/67] add unit tests for average function --- src/daemon/distribution_test.c | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index aedb81ef8c..93e5f76f32 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -331,6 +331,7 @@ DEF_TEST(update) { } if (cases[i].want_err != 0) { EXPECT_EQ_INT(cases[i].want_err, errno); + distribution_destroy(cases[i].dist); continue; } buckets_array_t buckets_array = get_buckets(cases[i].dist); @@ -343,10 +344,46 @@ DEF_TEST(update) { return 0; } +DEF_TEST(average) { + struct { + distribution_t *dist; + size_t num_gauges; + double *update_gauges; + double want_average; + } cases[] = { + { + .dist = distribution_new_linear(6, 2), + .num_gauges = 0, + .want_average = NAN, + }, + { + .dist = distribution_new_linear(7, 10), + .num_gauges = 5, + .update_gauges = (double[]){3, 2, 5.7, 22.3, 7.5}, + .want_average = 40.5 / 5.0, + }, + { + .dist = distribution_new_exponential(10, 2, 0.75), + .num_gauges = 8, + .update_gauges = (double[]){2, 4, 6, 8, 22, 11, 77, 1005}, + .want_average = 1135.0 / 8.0, + }, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + for (size_t j = 0; j < cases[i].num_gauges; j++) { + distribution_update(cases[i].dist, cases[i].update_gauges[j]); + } + EXPECT_EQ_DOUBLE(cases[i].want_average, distribution_average(cases[i].dist)); + distribution_destroy(cases[i].dist); + } + return 0; +} + int main() { RUN_TEST(distribution_new_linear); RUN_TEST(distribution_new_exponential); RUN_TEST(distribution_new_custom); RUN_TEST(update); + RUN_TEST(average); END_TEST; } From 648aa5e724014ce13b30d313e2c154566652f820 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 3 Aug 2020 15:05:40 +0000 Subject: [PATCH 51/67] add unit tests for percentile calculation --- src/daemon/distribution.c | 4 +++ src/daemon/distribution_test.c | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 0269eb8461..764effa13d 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -201,6 +201,10 @@ static void update_tree(distribution_t *dist, size_t node_index, size_t left, si void distribution_update(distribution_t *dist, double gauge) { if (dist == NULL) return; + if (gauge < 0) { + errno = EINVAL; + return; + } update_tree(dist, 0, 0, dist->num_buckets - 1, gauge); dist->total_sum += gauge; } diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 93e5f76f32..be6db19bb4 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -379,11 +379,70 @@ DEF_TEST(average) { return 0; } +DEF_TEST(percentile) { + struct { + distribution_t *dist; + size_t num_gauges; + double *update_gauges; + double percent; + double want_percentile; + int want_err; + } cases[] = { + { + .dist = distribution_new_linear(5, 7), + .num_gauges = 1, + .update_gauges = (double[]){4}, + .percent = 105, + .want_percentile = NAN, + .want_err = EINVAL, + }, + { + .dist = distribution_new_linear(8, 10), + .num_gauges = 0, + .percent = 20, + .want_percentile = NAN, + }, + { + .dist = distribution_new_exponential(5, 2, 0.2), + .num_gauges = 2, + .update_gauges = (double[]){4, 30.08}, + .percent = -5, + .want_percentile = NAN, + .want_err = EINVAL, + }, + { + .dist = distribution_new_exponential(10, 2, 0.75), + .num_gauges = 8, + .update_gauges = (double[]){2, 4, 6, 8, 22, 11, 77, 1005}, + .percent = 50, + .want_percentile = 12, + }, + { + .dist = distribution_new_custom(3, (double[]){5, 20, 35}), + .num_gauges = 7, + .update_gauges = (double[]){5.5, 10.5, 11.3, 6.7, 24.7, 40.05, 35}, + .percent = 4.0 / 7.0 * 100, + .want_percentile = 20, + }, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + for (size_t j = 0; j < cases[i].num_gauges; j++) { + distribution_update(cases[i].dist, cases[i].update_gauges[j]); + } + EXPECT_EQ_DOUBLE(cases[i].want_percentile, distribution_percentile(cases[i].dist, cases[i].percent)); + if (cases[i].want_err != 0) + EXPECT_EQ_INT(cases[i].want_err, errno); + distribution_destroy(cases[i].dist); + } + return 0; +} + int main() { RUN_TEST(distribution_new_linear); RUN_TEST(distribution_new_exponential); RUN_TEST(distribution_new_custom); RUN_TEST(update); RUN_TEST(average); + RUN_TEST(percentile); END_TEST; } From 92854a681c1deae2e5260c19d8388aa491fe4c4b Mon Sep 17 00:00:00 2001 From: Svetlana Date: Thu, 6 Aug 2020 14:27:33 +0000 Subject: [PATCH 52/67] add benchmark --- Makefile.am | 10 ++- src/daemon/distribution_benchmark.c | 115 ++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 src/daemon/distribution_benchmark.c diff --git a/Makefile.am b/Makefile.am index 92277d830e..3823c64967 100644 --- a/Makefile.am +++ b/Makefile.am @@ -123,7 +123,8 @@ sbin_PROGRAMS = \ bin_PROGRAMS = \ collectd-nagios \ collectd-tg \ - collectdctl + collectdctl \ + distribution_benchmark endif # BUILD_WIN32 @@ -362,6 +363,13 @@ test_distribution_SOURCES = \ src/testing.h test_distribution_LDADD = libmetric.la libplugin_mock.la +distribution_benchmark_SOURCES = src/daemon/distribution_benchmark.c +#distribution_benchmark_CPPFLAGS = $(AM_CPPFLAGS) \ +# -I$(srcdir)/src/libcollectdclient \ +# -I$(top_builddir)/src/libcollectdclient +distribution_benchmark_LDADD = \ + libmetric.la + test_metric_SOURCES = \ src/daemon/metric_test.c \ src/testing.h diff --git a/src/daemon/distribution_benchmark.c b/src/daemon/distribution_benchmark.c new file mode 100644 index 0000000000..94d7c6d65c --- /dev/null +++ b/src/daemon/distribution_benchmark.c @@ -0,0 +1,115 @@ +#include +#include +#include +#include + +#include "distribution.h" + +const size_t NUM_UPDATES = 1000000; +const size_t NUM_PERCENTILES = 1000000; +const size_t MIXED = 1000000; + +distribution_t *build(size_t num_buckets) { + srand(5); + double custom[num_buckets - 1]; + custom[0] = rand() % 100; + for (size_t i = 1; i < num_buckets - 1; i++) { + custom[i] = custom[i - 1] + rand() % 100 + 1; + } + distribution_t *dist = distribution_new_custom(num_buckets - 1, custom); + return dist; +} + +double calculate_update_time(distribution_t *dist) { + double updates[NUM_UPDATES]; + for (size_t i = 0; i < NUM_UPDATES; i++) { + updates[i] = (rand() * RAND_MAX + rand()) % (distribution_num_buckets(dist) * 100); + } + struct timespec start, finish; + clock_gettime(CLOCK_MONOTONIC, &start); + for (size_t i = 0; i < NUM_UPDATES; i++) { + distribution_update(dist, updates[i]); + } + clock_gettime(CLOCK_MONOTONIC, &finish); + double update_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); + double average = update_dur / NUM_UPDATES * 1000000.0; + return average; +} + +double calculate_percentile_time(distribution_t *dist) { + double percentiles[NUM_PERCENTILES]; + for (size_t i = 0; i < NUM_PERCENTILES; i++) { + percentiles[i] = 100.0 * rand() / RAND_MAX; + } + struct timespec start, finish; + clock_gettime(CLOCK_MONOTONIC, &start); + for (size_t i = 0; i < NUM_PERCENTILES; i++) { + distribution_percentile(dist, percentiles[i]); + } + clock_gettime(CLOCK_MONOTONIC, &finish); + double percentile_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); + double average = percentile_dur / NUM_PERCENTILES * 1000000.0; + return average; +} + +double mixed(size_t num_buckets) { + distribution_t *dist = build(num_buckets); + double updates[MIXED / 10 * 9]; + double percentiles[MIXED / 10]; + for (size_t i = 0; i < MIXED / 10 * 9; i++) + updates[i] = (rand() * RAND_MAX + rand()) % (num_buckets * 100); + for (size_t i = 0; i < MIXED / 10; i++) + percentiles[i] = 100.0 * rand() / RAND_MAX; + + size_t uid = 0; + size_t pid = 0; + struct timespec start, finish; + clock_gettime(CLOCK_MONOTONIC, &start); + for (size_t i = 0; i < MIXED; i++) { + if (i % 10 == 9) + distribution_percentile(dist, percentiles[pid++]); + else + distribution_update(dist, updates[uid++]); + } + clock_gettime(CLOCK_MONOTONIC, &finish); + double dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); + distribution_destroy(dist); + return dur; +} + +int main() { + size_t *bucket_nums = (size_t[]){ 5, 10, 30, 50, 100, 300, 500, 1000}; + for (size_t i = 0; i < 8; i++) { + distribution_t *dist = build(bucket_nums[i]); + //printf("%lu %f %f %f\n", bucket_nums[i], calculate_update_time(dist), calculate_percentile_time(dist), mixed(bucket_nums[i])); + printf("Using %lu buckets one update takes %f ns in average\n", bucket_nums[i], calculate_update_time(dist)); + printf("Using %lu buckets one percentile calculation takes %f ns in average\n", bucket_nums[i], calculate_percentile_time(dist)); + distribution_destroy(dist); + printf("Using %lu buckets mixed function work in %f ms\n", bucket_nums[i], mixed(bucket_nums[i])); + printf("\n"); + } + /*printf("\n"); + for (size_t i = 0; i < 8; i++) { + printf("Using %lu buckets one percentile calculation takes %f ns in average\n", bucket_nums[i], calculate_percentile_time(bucket_nums[i])); + } + distribution_t *dist = distribution_new_linear(1000, 1); + struct timespec start, finish; + clock_gettime(CLOCK_MONOTONIC, &start); + size_t num_updates = 50000000; + for (size_t i = 0; i < num_updates; i++) { + distribution_update(dist, rand()); + } + clock_gettime(CLOCK_MONOTONIC, &finish); + double update_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); + printf("%lu updates takes %f ms\n", num_updates, update_dur); + size_t num_requests = 50000000; + start = clock(); + for (size_t i = 0; i < num_requests; i++) { + distribution_percentile(dist, 100.0 * rand() / RAND_MAX); + } + finish = clock(); + double request_duration = 1000.0 * (finish - start) / CLOCKS_PER_SEC; + printf("%lu percentile requests takes %f ms\n", num_requests, request_duration); + distribution_destroy(dist);*/ + return 0; +} From fb739ea5b6e8f86436fdee67d92b2814fa0ea544 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 7 Aug 2020 10:39:56 +0000 Subject: [PATCH 53/67] benchmark done --- benchmark.csv | 201 ++++++++++++++++++++++++++++ src/daemon/distribution_benchmark.c | 69 ++++------ 2 files changed, 230 insertions(+), 40 deletions(-) create mode 100644 benchmark.csv diff --git a/benchmark.csv b/benchmark.csv new file mode 100644 index 0000000000..49845c2fc0 --- /dev/null +++ b/benchmark.csv @@ -0,0 +1,201 @@ +Number of buckets,Average for update,Average for percentile,Total for 20000000 mixed iterations +20,22.399243,32.234964,495.857988 +40,25.528343,39.397647,586.628428 +60,26.706422,39.726522,613.791265 +80,29.402889,40.906750,662.411160 +100,30.655842,42.539808,708.406044 +120,34.291907,43.694596,705.757070 +140,33.157924,45.571501,743.901288 +160,33.644566,46.156432,759.979938 +180,34.511660,47.186986,783.354591 +200,35.149506,47.878792,796.423202 +220,37.831280,49.476695,826.503587 +240,37.810377,49.449305,834.809788 +260,38.861861,49.604123,904.369460 +280,38.918660,49.902678,872.406038 +300,38.925808,50.110397,853.687773 +320,39.479305,50.639268,858.783989 +340,40.030510,51.000832,872.098210 +360,40.726870,51.404635,883.838316 +380,40.538804,52.283202,884.844450 +400,40.926581,52.701877,996.370751 +420,41.231143,52.846620,913.055910 +440,41.718648,53.063876,920.263605 +460,43.048154,53.113507,924.775671 +480,43.041926,53.553811,934.157700 +500,43.594351,55.233892,960.866283 +520,44.275040,55.598164,976.063588 +540,45.630621,55.787917,1005.145613 +560,44.490061,56.098219,969.730471 +580,45.058746,56.669739,994.013328 +600,52.034014,112.826753,1975.138961 +620,88.859372,113.832529,1849.151438 +640,89.456311,114.281936,1989.765445 +660,89.228945,113.969544,1982.857811 +680,88.766927,114.673313,1865.125717 +700,88.835999,124.337479,1857.993412 +720,95.613325,114.395997,1987.750687 +740,89.408266,114.856012,2008.157167 +760,89.467911,120.347940,2188.573381 +780,89.366811,120.754142,1993.740474 +800,95.797516,114.512331,1998.330264 +820,89.829861,121.037966,2246.061960 +840,90.721193,114.551185,1996.292408 +860,90.967448,114.471501,2141.292135 +880,90.322395,115.502224,2005.003311 +900,90.439771,121.248371,2010.633220 +920,90.336257,121.453370,2154.557553 +940,97.830837,114.744609,2035.455975 +960,97.560298,124.358657,2147.040151 +980,91.284913,114.965916,2029.633665 +1000,97.645668,115.297393,2036.853279 +1020,97.900111,121.972760,2056.089064 +1040,98.664974,122.582933,2057.384955 +1060,98.395977,122.772813,2174.978407 +1080,98.666000,122.994144,2264.207825 +1100,98.711500,117.559049,2173.536969 +1120,92.185249,116.915751,2057.390656 +1140,92.377632,117.521651,2050.937698 +1160,98.680102,123.755452,2054.072184 +1180,99.158158,124.549303,2272.022697 +1200,99.646351,124.117236,2184.281674 +1220,101.982639,117.557074,2058.727049 +1240,98.888999,124.306981,2062.533151 +1260,99.822836,124.610359,2193.067156 +1280,102.808674,124.552191,2199.530214 +1300,99.306467,124.492050,2194.443489 +1320,99.310711,124.483635,2065.641481 +1340,99.287784,125.843748,2074.245138 +1360,93.382068,124.615543,2202.156389 +1380,99.644128,125.216063,2203.002227 +1400,100.553860,129.372198,2249.826176 +1420,109.489562,126.523210,2203.201478 +1440,100.466458,132.329895,2207.193528 +1460,100.196757,129.455762,2212.528889 +1480,100.230294,125.621451,2229.034263 +1500,100.605787,126.587007,2219.757186 +1520,101.217948,137.205395,2102.151091 +1540,107.391109,126.238699,2230.320982 +1560,100.878290,126.206870,2237.882363 +1580,101.183033,135.072038,2102.196121 +1600,107.182170,125.917355,2106.545995 +1620,107.585180,126.291577,2234.253909 +1640,102.031631,135.888359,2245.166293 +1660,101.206551,132.539892,2236.452972 +1680,101.436838,126.436447,2238.680233 +1700,101.594393,126.444108,2233.731741 +1720,101.555702,126.671375,2271.848751 +1740,113.119558,135.696346,2261.747305 +1760,102.795511,137.782247,2245.001044 +1780,108.483841,126.631449,2246.727265 +1800,102.508803,126.921568,2249.141683 +1820,102.687729,136.692892,2258.234661 +1840,108.833782,126.906014,2257.492740 +1860,102.688856,127.141704,2254.117380 +1880,102.527625,135.988985,2271.611313 +1900,109.361359,134.864022,2266.529140 +1920,103.025290,133.682820,2275.012767 +1940,109.580171,137.707916,2265.962114 +1960,103.094684,127.560911,2266.769230 +1980,103.311189,127.791572,2282.952863 +2000,110.083324,138.040303,2275.421372 +2020,103.601840,127.758244,2279.271022 +2040,103.895368,127.945153,2295.604751 +2060,110.884598,149.718550,2317.779055 +2080,122.670660,145.312687,2428.870867 +2100,111.247926,141.633110,2460.202474 +2120,110.788395,159.577851,2448.006245 +2140,111.250531,139.412448,2489.443471 +2160,113.838216,147.780204,2526.135227 +2180,137.275589,148.194148,2315.952410 +2200,112.445140,140.419958,2328.341468 +2220,111.510765,140.289054,2315.095706 +2240,105.090494,145.624897,2447.984542 +2260,111.198342,139.443637,2307.857417 +2280,104.572990,145.399085,2322.874036 +2300,110.460552,139.324778,2438.091755 +2320,104.863633,145.679558,2311.670992 +2340,111.360183,140.231518,2510.918524 +2360,112.314716,159.751326,2506.200920 +2380,105.173334,147.367201,2437.342323 +2400,111.619067,146.268009,2517.296059 +2420,104.997344,146.346595,2322.563188 +2440,111.203465,140.616630,2340.052027 +2460,111.406283,140.515532,2482.412814 +2480,105.144235,146.750382,2460.042421 +2500,111.572781,147.084099,2450.266062 +2520,111.684148,148.968831,2329.320582 +2540,111.213965,140.727502,2330.622722 +2560,111.859550,140.865379,2330.593810 +2580,111.900021,150.308166,2453.505446 +2600,120.310252,141.742834,2334.284653 +2620,111.487308,140.208695,2459.989314 +2640,111.840898,152.067253,2445.501799 +2660,111.560318,146.568121,2326.674731 +2680,111.638034,147.995733,2460.056224 +2700,112.486015,159.016382,2461.335657 +2720,111.902233,140.496837,2480.887975 +2740,112.330382,147.384770,2472.814407 +2760,123.394367,140.439287,2481.515891 +2780,119.380938,141.370731,2342.750913 +2800,113.392702,146.628905,2457.904515 +2820,122.945232,140.741310,2338.583099 +2840,112.013293,141.255852,2343.701157 +2860,112.358293,141.378250,2335.374902 +2880,112.596029,146.762496,2453.475048 +2900,112.590254,146.851270,2471.138050 +2920,112.535694,147.432182,2488.290312 +2940,112.613238,157.535682,2529.762852 +2960,113.063133,142.421984,2531.230776 +2980,113.996503,147.988168,2574.907647 +3000,112.395466,148.512368,2473.847280 +3020,112.662706,147.717789,2484.870015 +3040,112.751169,141.797787,2480.262393 +3060,112.943086,141.480118,2473.682775 +3080,113.045151,147.873888,2489.555987 +3100,113.193668,151.193176,2479.767150 +3120,120.292310,141.820743,2360.347057 +3140,113.406801,149.215052,2481.318891 +3160,113.284436,151.615125,2475.488664 +3180,113.322834,141.661712,2508.288343 +3200,119.253639,148.187514,2483.856596 +3220,112.872198,142.185911,2483.238299 +3240,113.175942,147.818862,2508.661840 +3260,113.945839,147.689601,2479.562502 +3280,123.385029,148.853810,2498.865767 +3300,113.194655,141.766002,2485.897106 +3320,113.304257,148.473288,2485.970065 +3340,113.798410,159.486118,2353.265265 +3360,114.359538,149.572928,2489.941027 +3380,113.276930,148.416204,2705.149188 +3400,113.622436,142.343168,2485.203570 +3420,113.788884,148.026343,2486.338179 +3440,113.457103,150.178732,2520.515980 +3460,121.441333,152.931507,2492.193183 +3480,113.474869,148.264927,2504.224769 +3500,113.470946,151.794485,2507.076630 +3520,113.968273,148.403653,2492.054662 +3540,114.480909,148.155640,2522.386916 +3560,121.346178,158.830787,2508.947161 +3580,114.899105,149.509165,2495.815424 +3600,114.242812,150.010535,2507.197538 +3620,118.202930,148.679725,2493.807606 +3640,114.514038,149.427490,2645.846938 +3660,120.542348,149.428224,2505.968862 +3680,114.222355,148.632125,2505.880338 +3700,114.185415,149.197959,2508.405181 +3720,114.377059,149.393723,2695.284406 +3740,121.262389,142.440306,2523.311926 +3760,114.664916,148.543349,2504.437980 +3780,115.535792,149.572713,2506.087070 +3800,120.848231,148.636450,2508.384652 +3820,115.213962,148.980192,2510.306562 +3840,114.638801,159.020896,2531.194653 +3860,115.745036,149.627541,2513.522442 +3880,121.288082,149.275561,2514.996486 +3900,116.176657,150.790998,2523.083664 +3920,114.678989,149.406514,2648.662511 +3940,123.565242,150.138635,2519.979079 +3960,125.292389,150.058899,2650.704602 +3980,121.412504,142.562508,2514.636836 +4000,121.880036,149.022176,2540.520500 diff --git a/src/daemon/distribution_benchmark.c b/src/daemon/distribution_benchmark.c index 94d7c6d65c..6111d3c778 100644 --- a/src/daemon/distribution_benchmark.c +++ b/src/daemon/distribution_benchmark.c @@ -5,9 +5,9 @@ #include "distribution.h" -const size_t NUM_UPDATES = 1000000; -const size_t NUM_PERCENTILES = 1000000; -const size_t MIXED = 1000000; +const size_t NUM_UPDATES = 20000000; +const size_t NUM_PERCENTILES = 20000000; +const size_t MIXED = 20000000; distribution_t *build(size_t num_buckets) { srand(5); @@ -21,7 +21,7 @@ distribution_t *build(size_t num_buckets) { } double calculate_update_time(distribution_t *dist) { - double updates[NUM_UPDATES]; + double *updates = calloc(NUM_UPDATES, sizeof(*updates)); for (size_t i = 0; i < NUM_UPDATES; i++) { updates[i] = (rand() * RAND_MAX + rand()) % (distribution_num_buckets(dist) * 100); } @@ -33,11 +33,12 @@ double calculate_update_time(distribution_t *dist) { clock_gettime(CLOCK_MONOTONIC, &finish); double update_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); double average = update_dur / NUM_UPDATES * 1000000.0; + free(updates); return average; } double calculate_percentile_time(distribution_t *dist) { - double percentiles[NUM_PERCENTILES]; + double *percentiles = calloc(NUM_PERCENTILES, sizeof(*percentiles)); for (size_t i = 0; i < NUM_PERCENTILES; i++) { percentiles[i] = 100.0 * rand() / RAND_MAX; } @@ -49,13 +50,14 @@ double calculate_percentile_time(distribution_t *dist) { clock_gettime(CLOCK_MONOTONIC, &finish); double percentile_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); double average = percentile_dur / NUM_PERCENTILES * 1000000.0; + free(percentiles); return average; } double mixed(size_t num_buckets) { distribution_t *dist = build(num_buckets); - double updates[MIXED / 10 * 9]; - double percentiles[MIXED / 10]; + double *updates = calloc(MIXED / 10 * 9, sizeof(*updates)); + double *percentiles = calloc(MIXED / 10, sizeof(*percentiles)); for (size_t i = 0; i < MIXED / 10 * 9; i++) updates[i] = (rand() * RAND_MAX + rand()) % (num_buckets * 100); for (size_t i = 0; i < MIXED / 10; i++) @@ -65,51 +67,38 @@ double mixed(size_t num_buckets) { size_t pid = 0; struct timespec start, finish; clock_gettime(CLOCK_MONOTONIC, &start); + double val = 0; for (size_t i = 0; i < MIXED; i++) { - if (i % 10 == 9) - distribution_percentile(dist, percentiles[pid++]); + if (i % 10 == 9) { + double d = distribution_percentile(dist, percentiles[pid++]); + if (d != INFINITY) + val += d; + } else distribution_update(dist, updates[uid++]); } clock_gettime(CLOCK_MONOTONIC, &finish); double dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); distribution_destroy(dist); + free(percentiles); + free(updates); + //printf("%f\n", val); return dur; } int main() { - size_t *bucket_nums = (size_t[]){ 5, 10, 30, 50, 100, 300, 500, 1000}; - for (size_t i = 0; i < 8; i++) { - distribution_t *dist = build(bucket_nums[i]); - //printf("%lu %f %f %f\n", bucket_nums[i], calculate_update_time(dist), calculate_percentile_time(dist), mixed(bucket_nums[i])); - printf("Using %lu buckets one update takes %f ns in average\n", bucket_nums[i], calculate_update_time(dist)); - printf("Using %lu buckets one percentile calculation takes %f ns in average\n", bucket_nums[i], calculate_percentile_time(dist)); + FILE *fout = fopen("benchmark.csv", "w"); + fprintf(fout, "Number of buckets,Average for update,Average for percentile,Total for %lu mixed iterations\n", MIXED); + for (size_t num_buckets = 20; num_buckets <= 4000; num_buckets += 20) { + distribution_t *dist = build(num_buckets); + fprintf(fout, "%lu,", num_buckets); + fprintf(fout, "%f,", calculate_update_time(dist)); + fprintf(fout, "%f,", calculate_percentile_time(dist)); + fprintf(fout, "%f\n", mixed(num_buckets)); + //fprintf(fout, "%lu,%f,%f,%f\n", num_buckets, calculate_update_time(dist), calculate_percentile_time(dist), mixed(num_buckets)); distribution_destroy(dist); - printf("Using %lu buckets mixed function work in %f ms\n", bucket_nums[i], mixed(bucket_nums[i])); - printf("\n"); + printf("OK %lu\n", num_buckets); } - /*printf("\n"); - for (size_t i = 0; i < 8; i++) { - printf("Using %lu buckets one percentile calculation takes %f ns in average\n", bucket_nums[i], calculate_percentile_time(bucket_nums[i])); - } - distribution_t *dist = distribution_new_linear(1000, 1); - struct timespec start, finish; - clock_gettime(CLOCK_MONOTONIC, &start); - size_t num_updates = 50000000; - for (size_t i = 0; i < num_updates; i++) { - distribution_update(dist, rand()); - } - clock_gettime(CLOCK_MONOTONIC, &finish); - double update_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); - printf("%lu updates takes %f ms\n", num_updates, update_dur); - size_t num_requests = 50000000; - start = clock(); - for (size_t i = 0; i < num_requests; i++) { - distribution_percentile(dist, 100.0 * rand() / RAND_MAX); - } - finish = clock(); - double request_duration = 1000.0 * (finish - start) / CLOCKS_PER_SEC; - printf("%lu percentile requests takes %f ms\n", num_requests, request_duration); - distribution_destroy(dist);*/ + fclose(fout); return 0; } From 1bcd249bfaf57570ed3bd1ff28980a8be714bdc6 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 7 Aug 2020 12:15:24 +0000 Subject: [PATCH 54/67] add benchmark for million updates --- benchmark.csv | 400 ++++++++++++++++++++++---------------------- benchmark_small.csv | 201 ++++++++++++++++++++++ 2 files changed, 401 insertions(+), 200 deletions(-) create mode 100644 benchmark_small.csv diff --git a/benchmark.csv b/benchmark.csv index 49845c2fc0..aaf9c3deba 100644 --- a/benchmark.csv +++ b/benchmark.csv @@ -1,201 +1,201 @@ Number of buckets,Average for update,Average for percentile,Total for 20000000 mixed iterations -20,22.399243,32.234964,495.857988 -40,25.528343,39.397647,586.628428 -60,26.706422,39.726522,613.791265 -80,29.402889,40.906750,662.411160 -100,30.655842,42.539808,708.406044 -120,34.291907,43.694596,705.757070 -140,33.157924,45.571501,743.901288 -160,33.644566,46.156432,759.979938 -180,34.511660,47.186986,783.354591 -200,35.149506,47.878792,796.423202 -220,37.831280,49.476695,826.503587 -240,37.810377,49.449305,834.809788 -260,38.861861,49.604123,904.369460 -280,38.918660,49.902678,872.406038 -300,38.925808,50.110397,853.687773 -320,39.479305,50.639268,858.783989 -340,40.030510,51.000832,872.098210 -360,40.726870,51.404635,883.838316 -380,40.538804,52.283202,884.844450 -400,40.926581,52.701877,996.370751 -420,41.231143,52.846620,913.055910 -440,41.718648,53.063876,920.263605 -460,43.048154,53.113507,924.775671 -480,43.041926,53.553811,934.157700 -500,43.594351,55.233892,960.866283 -520,44.275040,55.598164,976.063588 -540,45.630621,55.787917,1005.145613 -560,44.490061,56.098219,969.730471 -580,45.058746,56.669739,994.013328 -600,52.034014,112.826753,1975.138961 -620,88.859372,113.832529,1849.151438 -640,89.456311,114.281936,1989.765445 -660,89.228945,113.969544,1982.857811 -680,88.766927,114.673313,1865.125717 -700,88.835999,124.337479,1857.993412 -720,95.613325,114.395997,1987.750687 -740,89.408266,114.856012,2008.157167 -760,89.467911,120.347940,2188.573381 -780,89.366811,120.754142,1993.740474 -800,95.797516,114.512331,1998.330264 -820,89.829861,121.037966,2246.061960 -840,90.721193,114.551185,1996.292408 -860,90.967448,114.471501,2141.292135 -880,90.322395,115.502224,2005.003311 -900,90.439771,121.248371,2010.633220 -920,90.336257,121.453370,2154.557553 -940,97.830837,114.744609,2035.455975 -960,97.560298,124.358657,2147.040151 -980,91.284913,114.965916,2029.633665 -1000,97.645668,115.297393,2036.853279 -1020,97.900111,121.972760,2056.089064 -1040,98.664974,122.582933,2057.384955 -1060,98.395977,122.772813,2174.978407 -1080,98.666000,122.994144,2264.207825 -1100,98.711500,117.559049,2173.536969 -1120,92.185249,116.915751,2057.390656 -1140,92.377632,117.521651,2050.937698 -1160,98.680102,123.755452,2054.072184 -1180,99.158158,124.549303,2272.022697 -1200,99.646351,124.117236,2184.281674 -1220,101.982639,117.557074,2058.727049 -1240,98.888999,124.306981,2062.533151 -1260,99.822836,124.610359,2193.067156 -1280,102.808674,124.552191,2199.530214 -1300,99.306467,124.492050,2194.443489 -1320,99.310711,124.483635,2065.641481 -1340,99.287784,125.843748,2074.245138 -1360,93.382068,124.615543,2202.156389 -1380,99.644128,125.216063,2203.002227 -1400,100.553860,129.372198,2249.826176 -1420,109.489562,126.523210,2203.201478 -1440,100.466458,132.329895,2207.193528 -1460,100.196757,129.455762,2212.528889 -1480,100.230294,125.621451,2229.034263 -1500,100.605787,126.587007,2219.757186 -1520,101.217948,137.205395,2102.151091 -1540,107.391109,126.238699,2230.320982 -1560,100.878290,126.206870,2237.882363 -1580,101.183033,135.072038,2102.196121 -1600,107.182170,125.917355,2106.545995 -1620,107.585180,126.291577,2234.253909 -1640,102.031631,135.888359,2245.166293 -1660,101.206551,132.539892,2236.452972 -1680,101.436838,126.436447,2238.680233 -1700,101.594393,126.444108,2233.731741 -1720,101.555702,126.671375,2271.848751 -1740,113.119558,135.696346,2261.747305 -1760,102.795511,137.782247,2245.001044 -1780,108.483841,126.631449,2246.727265 -1800,102.508803,126.921568,2249.141683 -1820,102.687729,136.692892,2258.234661 -1840,108.833782,126.906014,2257.492740 -1860,102.688856,127.141704,2254.117380 -1880,102.527625,135.988985,2271.611313 -1900,109.361359,134.864022,2266.529140 -1920,103.025290,133.682820,2275.012767 -1940,109.580171,137.707916,2265.962114 -1960,103.094684,127.560911,2266.769230 -1980,103.311189,127.791572,2282.952863 -2000,110.083324,138.040303,2275.421372 -2020,103.601840,127.758244,2279.271022 -2040,103.895368,127.945153,2295.604751 -2060,110.884598,149.718550,2317.779055 -2080,122.670660,145.312687,2428.870867 -2100,111.247926,141.633110,2460.202474 -2120,110.788395,159.577851,2448.006245 -2140,111.250531,139.412448,2489.443471 -2160,113.838216,147.780204,2526.135227 -2180,137.275589,148.194148,2315.952410 -2200,112.445140,140.419958,2328.341468 -2220,111.510765,140.289054,2315.095706 -2240,105.090494,145.624897,2447.984542 -2260,111.198342,139.443637,2307.857417 -2280,104.572990,145.399085,2322.874036 -2300,110.460552,139.324778,2438.091755 -2320,104.863633,145.679558,2311.670992 -2340,111.360183,140.231518,2510.918524 -2360,112.314716,159.751326,2506.200920 -2380,105.173334,147.367201,2437.342323 -2400,111.619067,146.268009,2517.296059 -2420,104.997344,146.346595,2322.563188 -2440,111.203465,140.616630,2340.052027 -2460,111.406283,140.515532,2482.412814 -2480,105.144235,146.750382,2460.042421 -2500,111.572781,147.084099,2450.266062 -2520,111.684148,148.968831,2329.320582 -2540,111.213965,140.727502,2330.622722 -2560,111.859550,140.865379,2330.593810 -2580,111.900021,150.308166,2453.505446 -2600,120.310252,141.742834,2334.284653 -2620,111.487308,140.208695,2459.989314 -2640,111.840898,152.067253,2445.501799 -2660,111.560318,146.568121,2326.674731 -2680,111.638034,147.995733,2460.056224 -2700,112.486015,159.016382,2461.335657 -2720,111.902233,140.496837,2480.887975 -2740,112.330382,147.384770,2472.814407 -2760,123.394367,140.439287,2481.515891 -2780,119.380938,141.370731,2342.750913 -2800,113.392702,146.628905,2457.904515 -2820,122.945232,140.741310,2338.583099 -2840,112.013293,141.255852,2343.701157 -2860,112.358293,141.378250,2335.374902 -2880,112.596029,146.762496,2453.475048 -2900,112.590254,146.851270,2471.138050 -2920,112.535694,147.432182,2488.290312 -2940,112.613238,157.535682,2529.762852 -2960,113.063133,142.421984,2531.230776 -2980,113.996503,147.988168,2574.907647 -3000,112.395466,148.512368,2473.847280 -3020,112.662706,147.717789,2484.870015 -3040,112.751169,141.797787,2480.262393 -3060,112.943086,141.480118,2473.682775 -3080,113.045151,147.873888,2489.555987 -3100,113.193668,151.193176,2479.767150 -3120,120.292310,141.820743,2360.347057 -3140,113.406801,149.215052,2481.318891 -3160,113.284436,151.615125,2475.488664 -3180,113.322834,141.661712,2508.288343 -3200,119.253639,148.187514,2483.856596 -3220,112.872198,142.185911,2483.238299 -3240,113.175942,147.818862,2508.661840 -3260,113.945839,147.689601,2479.562502 -3280,123.385029,148.853810,2498.865767 -3300,113.194655,141.766002,2485.897106 -3320,113.304257,148.473288,2485.970065 -3340,113.798410,159.486118,2353.265265 -3360,114.359538,149.572928,2489.941027 -3380,113.276930,148.416204,2705.149188 -3400,113.622436,142.343168,2485.203570 -3420,113.788884,148.026343,2486.338179 -3440,113.457103,150.178732,2520.515980 -3460,121.441333,152.931507,2492.193183 -3480,113.474869,148.264927,2504.224769 -3500,113.470946,151.794485,2507.076630 -3520,113.968273,148.403653,2492.054662 -3540,114.480909,148.155640,2522.386916 -3560,121.346178,158.830787,2508.947161 -3580,114.899105,149.509165,2495.815424 -3600,114.242812,150.010535,2507.197538 -3620,118.202930,148.679725,2493.807606 -3640,114.514038,149.427490,2645.846938 -3660,120.542348,149.428224,2505.968862 -3680,114.222355,148.632125,2505.880338 -3700,114.185415,149.197959,2508.405181 -3720,114.377059,149.393723,2695.284406 -3740,121.262389,142.440306,2523.311926 -3760,114.664916,148.543349,2504.437980 -3780,115.535792,149.572713,2506.087070 -3800,120.848231,148.636450,2508.384652 -3820,115.213962,148.980192,2510.306562 -3840,114.638801,159.020896,2531.194653 -3860,115.745036,149.627541,2513.522442 -3880,121.288082,149.275561,2514.996486 -3900,116.176657,150.790998,2523.083664 -3920,114.678989,149.406514,2648.662511 -3940,123.565242,150.138635,2519.979079 -3960,125.292389,150.058899,2650.704602 -3980,121.412504,142.562508,2514.636836 -4000,121.880036,149.022176,2540.520500 +20,20.833077,33.140944,511.968124 +40,24.871834,36.059857,572.377243 +60,26.073686,37.543635,593.776276 +80,28.916161,40.072926,652.529222 +100,30.699940,41.640226,689.026993 +120,30.874444,43.060979,706.172506 +140,32.513817,44.898618,730.686766 +160,33.147428,45.237810,741.695803 +180,34.185429,46.078374,760.467895 +200,34.645466,47.461673,765.846071 +220,36.452024,47.081149,799.555336 +240,36.574275,47.175475,804.132166 +260,37.990334,48.397980,835.876786 +280,38.147707,48.813378,837.765745 +300,38.229090,49.249399,837.098446 +320,38.666436,49.609447,849.488966 +340,38.884782,50.036148,859.496535 +360,39.732383,50.695363,870.593929 +380,39.818529,51.305621,875.971119 +400,40.178807,51.326839,880.893379 +420,41.070184,51.743950,898.445908 +440,41.383053,52.057908,902.647839 +460,41.634752,52.080829,909.758109 +480,42.467967,52.259897,918.638426 +500,43.841562,52.723497,933.817474 +520,43.602939,54.488960,953.311065 +540,43.703239,54.542689,951.637020 +560,43.761980,54.937751,956.513530 +580,43.635702,55.228712,949.896027 +600,43.730006,55.379645,954.048957 +620,44.265220,73.653216,2049.773595 +640,87.804661,112.974539,1839.897381 +660,87.919513,112.181002,1840.538112 +680,88.073011,112.938754,1968.444859 +700,88.702958,112.665691,1842.752254 +720,88.375925,112.585872,1974.837199 +740,88.840096,113.038652,1974.233849 +760,90.908834,113.212586,1978.170734 +780,95.286165,113.068265,1981.879820 +800,88.795886,113.185687,1978.529653 +820,88.925599,113.166689,1982.555826 +840,89.364005,113.247402,1984.327452 +860,89.368505,113.336711,1990.374602 +880,95.664670,113.535643,1990.111667 +900,95.710216,125.671841,1991.283647 +920,89.609183,113.464495,1997.597951 +940,96.399109,114.115178,1996.475791 +960,89.852888,113.841985,2030.167979 +980,90.287138,113.866146,2008.447629 +1000,96.756945,114.020647,2009.202512 +1020,90.521636,113.999672,2014.223830 +1040,109.586508,121.804300,2031.565246 +1060,91.391154,121.650012,2030.098948 +1080,97.382280,115.310143,2029.597605 +1100,97.670673,116.513335,2030.564083 +1120,91.430033,121.643045,2032.548356 +1140,97.691063,116.071757,2156.563029 +1160,97.840235,122.371703,2065.363011 +1180,97.853581,116.037525,2039.342027 +1200,98.244354,122.702684,2045.133338 +1220,98.214568,116.407017,2045.108250 +1240,98.379457,116.688352,2046.490916 +1260,98.418106,116.628397,2172.218517 +1280,98.356146,123.278195,2179.401885 +1300,100.234367,123.205637,2054.172170 +1320,98.509031,123.493302,2055.784867 +1340,98.835003,123.745883,2062.280398 +1360,99.198711,125.008648,2185.379563 +1380,92.625308,123.388537,2183.551084 +1400,92.834658,123.812956,2185.762610 +1420,93.061516,117.403229,2084.653107 +1440,99.535905,124.130576,2193.218954 +1460,99.568697,123.754583,2199.334700 +1480,99.739003,117.861438,2090.727712 +1500,99.625641,124.262356,2200.258397 +1520,125.020176,237.067336,4114.030042 +1540,114.768170,225.467440,3261.872936 +1560,141.056171,184.780434,2945.742771 +1580,127.651321,214.795761,2745.877877 +1600,105.848378,124.598698,2083.859083 +1620,100.399835,124.828749,2215.968172 +1640,101.735051,125.568009,2100.750527 +1660,100.573839,124.950148,2217.729341 +1680,100.746419,124.952319,2217.206621 +1700,100.877626,126.725119,2230.364136 +1720,100.824358,125.212768,2223.283675 +1740,100.989016,125.215075,2227.256856 +1760,101.110792,126.430718,2226.582502 +1780,107.485283,125.412331,2231.998505 +1800,107.573218,125.420763,2232.855943 +1820,108.015956,126.026536,2270.014637 +1840,101.403769,125.725790,2239.077757 +1860,107.998218,125.765768,2240.477846 +1880,101.656547,125.951881,2482.842574 +1900,101.871661,126.104945,2243.934176 +1920,108.504175,126.155524,2248.066204 +1940,102.196268,126.236956,2252.192447 +1960,108.511859,127.044316,2258.801320 +1980,109.259507,126.513031,2253.070304 +2000,102.689637,132.684199,2258.167335 +2020,109.340887,126.450371,2258.617640 +2040,109.530100,126.827447,2269.020560 +2060,110.121377,137.492058,2296.144207 +2080,110.437918,137.612732,2288.097336 +2100,110.001937,137.432756,2291.184698 +2120,104.015173,137.713119,2290.005208 +2140,110.573933,138.030538,2291.041676 +2160,103.582606,148.803305,2415.789096 +2180,110.002547,137.537891,2424.537886 +2200,104.339575,137.897688,2294.260777 +2220,103.810388,137.518623,2415.582786 +2240,110.110051,137.786946,2417.294705 +2260,110.332857,137.764054,2417.864443 +2280,110.185616,137.879868,2292.689286 +2300,110.213115,137.828498,2296.664906 +2320,110.512877,138.079455,2299.789117 +2340,110.672938,138.461842,2429.465366 +2360,110.268692,138.231321,2423.405380 +2380,110.431845,138.104614,2300.020491 +2400,110.260652,138.290315,2426.660833 +2420,110.543089,138.440825,2428.891426 +2440,104.216735,138.465425,2301.749309 +2460,104.534811,138.583958,2306.114981 +2480,104.281298,144.665867,2304.187363 +2500,104.636953,145.571055,2304.536292 +2520,111.029655,138.965993,2432.819168 +2540,110.634383,138.562527,2433.879082 +2560,105.132859,145.164625,2309.720445 +2580,110.780880,139.103996,2436.214106 +2600,111.360110,145.572759,2341.336910 +2620,111.030853,139.234106,2307.375960 +2640,104.698380,145.200897,2310.936674 +2660,110.874977,139.220138,2465.121658 +2680,104.940546,145.362425,2316.032035 +2700,111.194037,139.282455,2434.735995 +2720,111.050057,138.982100,2550.520384 +2740,111.378580,139.508965,2439.719376 +2760,111.046686,139.291950,2434.424358 +2780,104.849723,145.628202,2467.623322 +2800,111.166878,139.818303,2330.182971 +2820,111.162130,139.235084,2445.749065 +2840,111.287927,139.246122,2547.285897 +2860,105.075993,145.598091,2318.814249 +2880,111.241276,139.430584,2442.596791 +2900,104.985098,145.575055,2454.444098 +2920,111.707930,139.339785,2320.744852 +2940,111.492077,139.581928,2448.412518 +2960,111.689481,146.128188,2450.082252 +2980,112.283823,146.503239,2336.981280 +3000,111.956611,139.752156,2331.032225 +3020,111.771459,140.816526,2454.019025 +3040,112.037193,139.790790,2453.675997 +3060,112.007624,146.527122,2465.502764 +3080,112.899524,158.011578,2464.691865 +3100,112.112632,146.319260,2337.380178 +3120,112.455636,140.228836,2337.890865 +3140,112.339785,147.763334,2472.226790 +3160,112.318814,146.384890,2339.854964 +3180,112.274367,140.037612,2337.972203 +3200,112.745264,148.608836,2473.053663 +3220,112.558855,146.682280,2538.914378 +3240,112.350903,146.720200,2460.985845 +3260,113.668348,147.022313,2469.381490 +3280,113.254116,140.919530,2483.555515 +3300,112.571139,140.507502,2469.895952 +3320,114.155043,146.837161,2467.429174 +3340,112.608822,146.811137,2466.785663 +3360,112.668541,146.728846,2470.770591 +3380,113.850032,146.861805,2475.941773 +3400,113.064849,140.728432,2356.416699 +3420,112.960434,140.411812,2348.290113 +3440,112.898934,146.561037,2469.482121 +3460,112.824672,146.838621,2474.293737 +3480,112.833938,140.839812,2499.793787 +3500,112.931204,140.424361,2348.342307 +3520,113.112293,146.763790,2475.950533 +3540,113.100448,146.941451,2705.810569 +3560,113.171508,146.894100,2475.882187 +3580,113.117937,146.852640,2487.424122 +3600,113.335796,140.744408,2495.167452 +3620,113.343334,147.442549,2475.834720 +3640,113.644828,147.295195,2488.476125 +3660,113.340550,150.215762,2483.566793 +3680,113.694441,140.644473,2481.981054 +3700,113.719818,147.323338,2485.999012 +3720,113.322047,148.061643,2489.702341 +3740,113.745410,141.009083,2483.818606 +3760,113.817257,147.169938,2486.387063 +3780,120.061682,152.833642,2487.592561 +3800,114.377361,147.891419,2501.453644 +3820,114.378227,147.957765,2496.925381 +3840,121.595965,141.166425,2494.710316 +3860,120.270576,147.282998,2497.846362 +3880,113.788127,147.516815,2495.682980 +3900,125.957699,147.536896,2493.837051 +3920,114.114346,147.472841,2496.612114 +3940,120.309828,141.236431,2501.958535 +3960,114.174887,147.504054,2500.873344 +3980,114.318758,147.380126,2510.417850 +4000,114.880214,148.085930,2539.786191 diff --git a/benchmark_small.csv b/benchmark_small.csv new file mode 100644 index 0000000000..7e52464b82 --- /dev/null +++ b/benchmark_small.csv @@ -0,0 +1,201 @@ +Number of buckets,Average for update,Average for percentile,Total for 1000000 mixed iterations +20,30.279575,31.594688,24.608609 +40,25.582838,36.222115,28.614193 +60,25.900862,37.503846,29.833795 +80,28.824710,44.583059,34.125163 +100,31.397022,43.217300,34.030339 +120,30.930586,43.033250,34.720907 +140,32.201488,44.678667,36.524133 +160,33.270566,45.343786,37.057255 +180,33.882487,46.062419,37.678757 +200,34.498888,46.060612,38.469573 +220,36.505075,46.776692,40.164427 +240,36.319747,47.257512,40.504973 +260,38.245813,48.557370,42.324541 +280,38.237893,48.890312,41.976441 +300,38.007175,49.376568,41.978078 +320,38.557740,49.588676,42.309814 +340,39.014167,50.209515,43.082755 +360,39.569006,50.597916,43.400479 +380,39.920028,51.100854,43.536472 +400,40.495439,51.530447,44.088921 +420,41.168983,51.864774,44.627674 +440,41.023137,51.829657,45.128834 +460,41.813872,52.331192,45.451797 +480,42.303883,52.617606,45.956554 +500,42.680624,52.972633,46.501718 +520,43.275708,54.647199,47.607325 +540,43.709591,54.540344,47.779972 +560,44.208340,55.048302,50.228766 +580,43.988163,55.385142,47.752993 +600,44.755797,55.771408,47.598055 +620,43.880436,55.797855,48.109657 +640,43.477199,55.998780,47.825107 +660,43.720069,55.800412,48.112947 +680,43.967458,56.343278,48.324987 +700,44.138272,56.260159,49.403632 +720,44.736586,56.540755,48.222454 +740,44.665820,56.585378,48.781087 +760,44.791490,56.598541,48.842271 +780,44.818675,56.548026,48.696944 +800,44.569383,56.666266,48.522515 +820,45.403978,56.957057,49.220015 +840,45.504865,57.213265,48.981029 +860,45.456769,58.700875,53.015213 +880,47.868071,56.907230,49.734639 +900,45.477546,57.178660,49.376460 +920,45.539323,57.284126,49.892168 +940,45.773609,57.229608,50.307060 +960,46.052729,57.476211,50.219397 +980,46.304167,57.591356,50.430984 +1000,46.663843,57.855602,50.207105 +1020,46.549401,57.894820,50.560559 +1040,47.729281,58.948228,51.424784 +1060,47.378295,58.974178,51.562945 +1080,47.732550,59.061161,51.299944 +1100,47.422751,59.378432,51.637271 +1120,47.535279,59.636300,51.875102 +1140,47.416219,59.638969,51.466828 +1160,47.541496,62.372862,74.780864 +1180,47.906223,59.923371,51.598257 +1200,47.601045,60.024814,51.763881 +1220,47.891179,60.354214,51.882209 +1240,48.220709,60.296880,52.716259 +1260,48.033805,60.363671,52.174321 +1280,48.070075,60.587039,52.578795 +1300,48.078878,60.707833,52.541964 +1320,48.346834,60.796052,52.847289 +1340,48.318736,60.667365,52.490966 +1360,48.635272,60.693801,52.755617 +1380,48.441798,60.904811,52.719066 +1400,48.958179,60.829070,53.777083 +1420,49.176889,61.105055,52.712338 +1440,48.993922,61.261518,52.996312 +1460,49.085647,61.581158,53.430562 +1480,49.894456,61.792587,53.565978 +1500,49.434430,61.511730,53.422607 +1520,50.430850,61.927736,54.680125 +1540,50.042466,61.782948,54.188356 +1560,50.626234,64.630917,54.073191 +1580,50.309620,62.148451,54.046681 +1600,49.901222,62.090722,54.137603 +1620,50.680344,62.304322,54.434820 +1640,50.573025,63.131581,55.134403 +1660,50.219907,62.483734,54.429006 +1680,50.350649,62.609521,54.523934 +1700,50.678749,62.234025,54.367081 +1720,50.820963,62.359883,54.830536 +1740,50.525967,62.812270,54.786726 +1760,51.686882,62.690574,54.782847 +1780,50.961380,62.736886,55.055650 +1800,51.068817,63.122665,55.235915 +1820,51.913093,63.497917,56.002548 +1840,51.529774,63.302531,55.521424 +1860,51.403516,63.392513,56.115667 +1880,51.671650,63.508614,55.470308 +1900,51.873017,63.334177,56.133747 +1920,51.868056,63.675988,56.083354 +1940,52.322543,63.687997,55.982485 +1960,52.116989,63.613090,55.865830 +1980,52.204028,63.766823,56.670002 +2000,52.254905,64.452683,56.876680 +2020,53.174767,64.272465,57.751285 +2040,53.043060,64.672269,56.915322 +2060,55.507562,69.321633,58.568302 +2080,56.004583,69.513837,58.229247 +2100,55.014491,69.174953,58.248046 +2120,53.742333,69.074059,58.629298 +2140,54.171234,69.234462,59.027992 +2160,53.592842,69.366887,58.327551 +2180,53.530757,69.494130,58.592062 +2200,53.822003,72.140826,60.888050 +2220,55.144748,69.123258,58.079006 +2240,53.652790,68.990011,58.305601 +2260,53.816040,69.158665,58.560090 +2280,54.079412,69.447263,58.530278 +2300,53.635252,69.421756,58.419882 +2320,53.830386,69.419201,58.471403 +2340,54.230589,69.673856,59.232988 +2360,53.969463,69.547158,58.587357 +2380,72.362578,71.715374,59.185883 +2400,54.227213,69.759882,58.637069 +2420,54.081270,69.901676,58.877024 +2440,54.055792,69.751263,58.408177 +2460,54.622058,77.442312,59.086894 +2480,54.046031,70.215774,58.994780 +2500,54.450404,70.134064,58.841864 +2520,54.457982,70.260155,59.197099 +2540,54.221169,70.023464,59.198274 +2560,54.944813,70.301828,58.922012 +2580,54.754265,70.357278,58.936917 +2600,55.024159,70.198531,58.903710 +2620,54.392080,70.193272,59.135476 +2640,54.745841,70.043982,59.009490 +2660,55.502013,70.164227,59.511555 +2680,54.725092,70.249758,59.114002 +2700,54.678079,70.214245,59.660478 +2720,54.784445,70.491713,60.235601 +2740,55.622628,70.603363,59.324161 +2760,55.180589,145.421244,59.341203 +2780,54.743429,147.806926,59.185746 +2800,54.760565,145.517308,59.900930 +2820,55.074581,146.295855,59.774567 +2840,180.290891,145.835769,59.573428 +2860,180.266554,133.459063,60.386303 +2880,180.327998,133.231126,59.969272 +2900,180.077131,133.311422,60.091939 +2920,179.759498,134.347039,60.589298 +2940,181.058266,134.212296,62.299925 +2960,180.227624,147.481052,61.013138 +2980,56.531547,146.368397,60.678622 +3000,55.459108,146.145743,60.090091 +3020,56.254363,146.427030,60.213138 +3040,55.377735,145.989103,60.222552 +3060,180.646183,146.165692,61.401884 +3080,180.556840,148.744368,60.621099 +3100,55.649209,146.090926,60.579003 +3120,55.761004,146.516187,60.678824 +3140,55.810243,146.310782,60.496865 +3160,56.008478,146.558079,60.706033 +3180,55.824185,146.214299,60.948261 +3200,55.994874,146.706851,212.093123 +3220,210.480636,146.650935,60.792510 +3240,180.725920,146.515604,60.619125 +3260,180.741523,146.677484,60.808386 +3280,57.415287,147.186995,61.582389 +3300,56.172186,146.707906,60.814673 +3320,56.063071,146.760821,60.791379 +3340,56.143963,146.914444,61.206165 +3360,56.330267,146.859216,185.841481 +3380,57.790392,149.485072,185.456421 +3400,56.428401,146.907141,186.172229 +3420,182.340170,146.884048,185.807056 +3440,182.550517,134.510603,185.736134 +3460,182.516556,134.539685,61.375244 +3480,182.458924,134.261983,61.104031 +3500,182.012471,134.205243,61.304505 +3520,182.606687,134.307559,61.176607 +3540,182.571038,147.016236,61.508035 +3560,182.423023,147.284216,61.805698 +3580,57.253225,146.867475,61.401099 +3600,56.833081,146.813521,61.413788 +3620,56.868331,147.138662,187.114517 +3640,57.440923,147.524592,187.448921 +3660,56.891377,147.107504,186.616561 +3680,57.148944,148.761962,62.569509 +3700,182.235366,134.566847,61.675157 +3720,182.286954,147.222844,61.627292 +3740,182.169898,147.138237,61.815352 +3760,57.631382,147.573349,61.931089 +3780,57.576468,147.490552,187.423907 +3800,57.583399,148.076982,186.966925 +3820,57.927596,147.251650,186.923013 +3840,182.614051,147.516222,187.818504 +3860,183.301656,135.222421,62.612584 +3880,183.038547,147.786395,62.222333 +3900,183.303946,147.456878,62.268387 +3920,58.120228,147.746213,188.515040 +3940,58.711964,147.543244,188.105778 +3960,58.026069,148.767576,192.508633 +3980,189.943186,148.242416,62.449116 +4000,183.389327,148.411055,62.740622 From bf6ac1c1f3acbefc8faca88d96d751cddc66eeaf Mon Sep 17 00:00:00 2001 From: Svetlana Date: Tue, 11 Aug 2020 13:20:18 +0000 Subject: [PATCH 55/67] removed unnecessary if condition from the building function --- src/daemon/distribution.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 764effa13d..53a0b49d42 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -84,8 +84,6 @@ static void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, static distribution_t* build_distribution_from_bucket_array(size_t num_buckets, bucket_t *bucket_array) { distribution_t *new_distribution = calloc(1, sizeof(*new_distribution)); - if (num_buckets == 0) - return new_distribution; bucket_t *nodes = calloc(tree_size(num_buckets), sizeof(*nodes)); if (new_distribution == NULL || nodes == NULL) { free(new_distribution); From 609775cbe5094fe7a005dcd49774a97cd7f5e66f Mon Sep 17 00:00:00 2001 From: Svetlana Date: Tue, 11 Aug 2020 13:21:45 +0000 Subject: [PATCH 56/67] fix codestyle --- src/daemon/distribution.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 53a0b49d42..bc469bde97 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -132,8 +132,9 @@ distribution_t* distribution_new_exponential(size_t num_buckets, double base, do distribution_t* distribution_new_custom(size_t array_size, double *custom_buckets_boundaries) { for (size_t i = 0; i < array_size; i++) { double previous_boundary = 0; - if (i > 0) + if (i > 0) { previous_boundary = custom_buckets_boundaries[i - 1]; + } if (custom_buckets_boundaries[i] <= previous_boundary) { errno = EINVAL; return NULL; From 834ef7ac17a7f563d57ba947c6b7f851a426024d Mon Sep 17 00:00:00 2001 From: Svetlana Date: Wed, 12 Aug 2020 11:57:34 +0000 Subject: [PATCH 57/67] fix distribution clone function --- src/daemon/distribution.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index bc469bde97..6f807faa3b 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -24,13 +24,9 @@ * Svetlana Shmidt **/ +#include "collectd.h" #include "distribution.h" -#include -#include -#include -#include - struct distribution_s { bucket_t *tree; size_t num_buckets; @@ -167,16 +163,15 @@ distribution_t* distribution_clone(distribution_t *dist) { if (dist == NULL) return NULL; distribution_t *new_distribution = calloc(1, sizeof(*new_distribution)); - if (dist->num_buckets == 0) - return new_distribution; - bucket_t *nodes = calloc(tree_size(dist->num_buckets), sizeof(*nodes)); if (new_distribution == NULL || nodes == NULL) { free(new_distribution); free(nodes); return NULL; } + memcpy(nodes, dist->tree, tree_size(dist->num_buckets) * sizeof(bucket_t)); new_distribution->num_buckets = dist->num_buckets; + new_distribution->total_sum = dist->total_sum; new_distribution->tree = nodes; return new_distribution; } From 0ae46c4589cc1c21d0c3f16f6f6dc59fc20e834e Mon Sep 17 00:00:00 2001 From: Svetlana Date: Wed, 12 Aug 2020 12:29:20 +0000 Subject: [PATCH 58/67] add mutexes --- src/daemon/distribution.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index 6f807faa3b..f8f165e6c5 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -27,10 +27,13 @@ #include "collectd.h" #include "distribution.h" +#include + struct distribution_s { bucket_t *tree; size_t num_buckets; double total_sum; + pthread_mutex_t mutex; }; /** @@ -90,6 +93,7 @@ static distribution_t* build_distribution_from_bucket_array(size_t num_buckets, new_distribution->num_buckets = num_buckets; build_tree(new_distribution, bucket_array, 0, 0, num_buckets - 1); + pthread_mutex_init(&new_distribution->mutex, NULL); return new_distribution; } @@ -155,6 +159,7 @@ distribution_t* distribution_new_custom(size_t array_size, double *custom_bucket void distribution_destroy(distribution_t *d) { if (d == NULL) return; + pthread_mutex_destroy(&d->mutex); free(d->tree); free(d); } @@ -169,10 +174,13 @@ distribution_t* distribution_clone(distribution_t *dist) { free(nodes); return NULL; } + pthread_mutex_lock(&dist->mutex); memcpy(nodes, dist->tree, tree_size(dist->num_buckets) * sizeof(bucket_t)); new_distribution->num_buckets = dist->num_buckets; new_distribution->total_sum = dist->total_sum; + pthread_mutex_unlock(&dist->mutex); new_distribution->tree = nodes; + pthread_mutex_init(&new_distribution->mutex, NULL); return new_distribution; } @@ -199,8 +207,10 @@ void distribution_update(distribution_t *dist, double gauge) { errno = EINVAL; return; } + pthread_mutex_lock(&dist->mutex); update_tree(dist, 0, 0, dist->num_buckets - 1, gauge); dist->total_sum += gauge; + pthread_mutex_unlock(&dist->mutex); } static double tree_get_counter(distribution_t *d, size_t node_index, size_t left, @@ -224,17 +234,23 @@ double distribution_percentile(distribution_t *dist, double percent) { errno = EINVAL; return NAN; } + pthread_mutex_lock(&dist->mutex); if (dist->tree[0].bucket_counter == 0) return NAN; uint64_t counter = ceil(dist->tree[0].bucket_counter * percent / 100.0); - return tree_get_counter(dist, 0, 0, dist->num_buckets - 1, counter); + double percentile = tree_get_counter(dist, 0, 0, dist->num_buckets - 1, counter); + pthread_mutex_unlock(&dist->mutex); + return percentile; } double distribution_average(distribution_t *dist) { - if (dist->tree[0].bucket_counter == 0) { + pthread_mutex_lock(&dist->mutex); + if (dist == NULL || dist->tree[0].bucket_counter == 0) { return NAN; } - return dist->total_sum / dist->tree[0].bucket_counter; + double average = dist->total_sum / dist->tree[0].bucket_counter; + pthread_mutex_unlock(&dist->mutex); + return average; } size_t distribution_num_buckets(distribution_t *dist) { @@ -265,7 +281,9 @@ buckets_array_t get_buckets(distribution_t *dist) { if (dist == NULL) return bucket_array; bucket_t *write_ptr = bucket_array.buckets; + pthread_mutex_lock(&dist->mutex); tree_write_leave_buckets(dist, write_ptr, 0, 0, dist->num_buckets - 1); + pthread_mutex_unlock(&dist->mutex); return bucket_array; } From cd7c0c154bf9071bdd0d0a575a53f825820b0dcc Mon Sep 17 00:00:00 2001 From: Svetlana Date: Wed, 12 Aug 2020 12:53:17 +0000 Subject: [PATCH 59/67] add unit test for clone function --- src/daemon/distribution_test.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index be6db19bb4..bfc8e96ccf 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -437,6 +437,21 @@ DEF_TEST(percentile) { return 0; } +DEF_TEST(clone) { + distribution_t *dist = distribution_new_linear(5, 7); + distribution_update(dist, 5); + distribution_update(dist, 7); + distribution_t *clone = distribution_clone(dist); + EXPECT_EQ_INT(distribution_num_buckets(clone), 5); + EXPECT_EQ_DOUBLE(distribution_percentile(clone, 50), 7); + distribution_update(dist, 10); + EXPECT_EQ_DOUBLE(distribution_percentile(clone, 50), 7); + EXPECT_EQ_DOUBLE(distribution_percentile(dist, 50), 14); + distribution_update(clone, 2); + EXPECT_EQ_DOUBLE(distribution_percentile(dist, 50), 14); + return 0; +} + int main() { RUN_TEST(distribution_new_linear); RUN_TEST(distribution_new_exponential); @@ -444,5 +459,6 @@ int main() { RUN_TEST(update); RUN_TEST(average); RUN_TEST(percentile); + RUN_TEST(clone); END_TEST; } From 80ee320414f16d9bf9b7dfad025150901d2780fd Mon Sep 17 00:00:00 2001 From: Svetlana Date: Wed, 12 Aug 2020 13:01:25 +0000 Subject: [PATCH 60/67] change benchmark --- src/daemon/distribution_benchmark.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/daemon/distribution_benchmark.c b/src/daemon/distribution_benchmark.c index 6111d3c778..c81e1d529d 100644 --- a/src/daemon/distribution_benchmark.c +++ b/src/daemon/distribution_benchmark.c @@ -5,9 +5,9 @@ #include "distribution.h" -const size_t NUM_UPDATES = 20000000; -const size_t NUM_PERCENTILES = 20000000; -const size_t MIXED = 20000000; +const size_t NUM_UPDATES = 1e6; +const size_t NUM_PERCENTILES = 1e6; +const size_t MIXED = 1e6; distribution_t *build(size_t num_buckets) { srand(5); @@ -87,9 +87,9 @@ double mixed(size_t num_buckets) { } int main() { - FILE *fout = fopen("benchmark.csv", "w"); + FILE *fout = fopen("benchmark_small.csv", "w"); fprintf(fout, "Number of buckets,Average for update,Average for percentile,Total for %lu mixed iterations\n", MIXED); - for (size_t num_buckets = 20; num_buckets <= 4000; num_buckets += 20) { + for (size_t num_buckets = 50; num_buckets <= 5000; num_buckets += 50) { distribution_t *dist = build(num_buckets); fprintf(fout, "%lu,", num_buckets); fprintf(fout, "%f,", calculate_update_time(dist)); From 9c04dfee209131dcb407139137b8bd78ba42fca2 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Thu, 13 Aug 2020 09:36:34 +0000 Subject: [PATCH 61/67] change includes to collectd.h --- src/daemon/distribution.c | 1 - src/daemon/distribution.h | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index f8f165e6c5..f573e1a434 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -24,7 +24,6 @@ * Svetlana Shmidt **/ -#include "collectd.h" #include "distribution.h" #include diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 82e1c8e1f3..8f5f4ba629 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -27,8 +27,7 @@ #ifndef COLLECTD_DISTRIBUTION_H #define COLLECTD_DISTRIBUTION_H -#include -#include +#include"collectd.h" typedef struct bucket_s { uint64_t bucket_counter; From 347a84b84c760b4460a51580cd5e171fa557fb8e Mon Sep 17 00:00:00 2001 From: Svetlana Date: Fri, 14 Aug 2020 11:12:02 +0000 Subject: [PATCH 62/67] fix codestyle in distribution data structure --- src/daemon/distribution.c | 69 ++-- src/daemon/distribution.h | 46 ++- src/daemon/distribution_benchmark.c | 37 +- src/daemon/distribution_test.c | 531 ++++++++++++++-------------- 4 files changed, 367 insertions(+), 316 deletions(-) diff --git a/src/daemon/distribution.c b/src/daemon/distribution.c index f573e1a434..2297914461 100644 --- a/src/daemon/distribution.c +++ b/src/daemon/distribution.c @@ -40,7 +40,8 @@ struct distribution_s { * This way the tree contained N buckets contains 2 * N - 1 nodes * Thus, left subtree has 2 * (mid - left + 1) - 1 nodes, * therefore the right subtree starts at node_index + 2 * (mid - left + 1). - * For a detailed explanation, see https://docs.google.com/document/d/1ccsg5ffUfqt9-mBDGTymRn8X-9Wk1CuGYeMlRxmxiok/edit?usp=sharing". + * For a detailed explanation, see + * https://docs.google.com/document/d/1ccsg5ffUfqt9-mBDGTymRn8X-9Wk1CuGYeMlRxmxiok/edit?usp=sharing". */ static size_t left_child_index(size_t node_index, @@ -54,18 +55,17 @@ static size_t right_child_index(size_t node_index, size_t left, size_t right) { return node_index + 2 * (mid - left + 1); } -static size_t tree_size(size_t num_buckets) { - return 2 * num_buckets - 1; -} +static size_t tree_size(size_t num_buckets) { return 2 * num_buckets - 1; } static bucket_t merge_buckets(bucket_t left_child, bucket_t right_child) { - return (bucket_t) { + return (bucket_t){ .bucket_counter = left_child.bucket_counter + right_child.bucket_counter, .maximum = right_child.maximum, }; } -static void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t left, size_t right) { +static void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, + size_t left, size_t right) { if (left > right) return; if (left == right) { @@ -77,10 +77,13 @@ static void build_tree(distribution_t *d, bucket_t *buckets, size_t node_index, size_t right_child = right_child_index(node_index, left, right); build_tree(d, buckets, left_child, left, mid); build_tree(d, buckets, right_child, mid + 1, right); - d->tree[node_index] = merge_buckets(d->tree[left_child], d->tree[right_child]); + d->tree[node_index] = + merge_buckets(d->tree[left_child], d->tree[right_child]); } -static distribution_t* build_distribution_from_bucket_array(size_t num_buckets, bucket_t *bucket_array) { +static distribution_t * +build_distribution_from_bucket_array(size_t num_buckets, + bucket_t *bucket_array) { distribution_t *new_distribution = calloc(1, sizeof(*new_distribution)); bucket_t *nodes = calloc(tree_size(num_buckets), sizeof(*nodes)); if (new_distribution == NULL || nodes == NULL) { @@ -96,7 +99,7 @@ static distribution_t* build_distribution_from_bucket_array(size_t num_buckets, return new_distribution; } -distribution_t* distribution_new_linear(size_t num_buckets, double size) { +distribution_t *distribution_new_linear(size_t num_buckets, double size) { if (num_buckets == 0 || size <= 0) { errno = EINVAL; return NULL; @@ -104,7 +107,7 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { bucket_t bucket_array[num_buckets]; for (size_t i = 0; i < num_buckets; i++) { - bucket_array[i] = (bucket_t) { + bucket_array[i] = (bucket_t){ .bucket_counter = 0, .maximum = (i == num_buckets - 1) ? INFINITY : (i + 1) * size, }; @@ -112,7 +115,8 @@ distribution_t* distribution_new_linear(size_t num_buckets, double size) { return build_distribution_from_bucket_array(num_buckets, bucket_array); } -distribution_t* distribution_new_exponential(size_t num_buckets, double base, double factor) { +distribution_t *distribution_new_exponential(size_t num_buckets, double base, + double factor) { if (num_buckets == 0 || base <= 1 || factor <= 0) { errno = EINVAL; return NULL; @@ -120,15 +124,18 @@ distribution_t* distribution_new_exponential(size_t num_buckets, double base, do bucket_t bucket_array[num_buckets]; for (size_t i = 0; i < num_buckets; i++) { - bucket_array[i] = (bucket_t) { + bucket_array[i] = (bucket_t){ .bucket_counter = 0, - .maximum = (i == num_buckets - 1) ? INFINITY : factor * pow(base, i), //check if it's slow + .maximum = (i == num_buckets - 1) + ? INFINITY + : factor * pow(base, i), // check if it's slow }; } return build_distribution_from_bucket_array(num_buckets, bucket_array); } -distribution_t* distribution_new_custom(size_t array_size, double *custom_buckets_boundaries) { +distribution_t *distribution_new_custom(size_t array_size, + double *custom_buckets_boundaries) { for (size_t i = 0; i < array_size; i++) { double previous_boundary = 0; if (i > 0) { @@ -147,9 +154,10 @@ distribution_t* distribution_new_custom(size_t array_size, double *custom_bucket size_t num_buckets = array_size + 1; bucket_t bucket_array[num_buckets]; for (size_t i = 0; i < num_buckets; i++) { - bucket_array[i] = (bucket_t) { + bucket_array[i] = (bucket_t){ .bucket_counter = 0, - .maximum = (i == num_buckets - 1) ? INFINITY : custom_buckets_boundaries[i], + .maximum = + (i == num_buckets - 1) ? INFINITY : custom_buckets_boundaries[i], }; } return build_distribution_from_bucket_array(num_buckets, bucket_array); @@ -163,7 +171,7 @@ void distribution_destroy(distribution_t *d) { free(d); } -distribution_t* distribution_clone(distribution_t *dist) { +distribution_t *distribution_clone(distribution_t *dist) { if (dist == NULL) return NULL; distribution_t *new_distribution = calloc(1, sizeof(*new_distribution)); @@ -183,7 +191,8 @@ distribution_t* distribution_clone(distribution_t *dist) { return new_distribution; } -static void update_tree(distribution_t *dist, size_t node_index, size_t left, size_t right, double gauge) { +static void update_tree(distribution_t *dist, size_t node_index, size_t left, + size_t right, double gauge) { if (left > right) return; dist->tree[node_index].bucket_counter++; @@ -212,8 +221,8 @@ void distribution_update(distribution_t *dist, double gauge) { pthread_mutex_unlock(&dist->mutex); } -static double tree_get_counter(distribution_t *d, size_t node_index, size_t left, - size_t right, uint64_t counter) { +static double tree_get_counter(distribution_t *d, size_t node_index, + size_t left, size_t right, uint64_t counter) { if (left > right) return NAN; if (left == right) { @@ -225,7 +234,8 @@ static double tree_get_counter(distribution_t *d, size_t node_index, size_t left if (d->tree[left_child].bucket_counter >= counter) return tree_get_counter(d, left_child, left, mid, counter); else - return tree_get_counter(d, right_child, mid + 1, right, counter - d->tree[left_child].bucket_counter); + return tree_get_counter(d, right_child, mid + 1, right, + counter - d->tree[left_child].bucket_counter); } double distribution_percentile(distribution_t *dist, double percent) { @@ -237,7 +247,8 @@ double distribution_percentile(distribution_t *dist, double percent) { if (dist->tree[0].bucket_counter == 0) return NAN; uint64_t counter = ceil(dist->tree[0].bucket_counter * percent / 100.0); - double percentile = tree_get_counter(dist, 0, 0, dist->num_buckets - 1, counter); + double percentile = + tree_get_counter(dist, 0, 0, dist->num_buckets - 1, counter); pthread_mutex_unlock(&dist->mutex); return percentile; } @@ -258,7 +269,9 @@ size_t distribution_num_buckets(distribution_t *dist) { return dist->num_buckets; } -static void tree_write_leave_buckets(distribution_t *dist, bucket_t *write_ptr, size_t node_index, size_t left, size_t right) { +static void tree_write_leave_buckets(distribution_t *dist, bucket_t *write_ptr, + size_t node_index, size_t left, + size_t right) { if (left > right) return; if (left == right) { @@ -269,13 +282,15 @@ static void tree_write_leave_buckets(distribution_t *dist, bucket_t *write_ptr, size_t left_child = left_child_index(node_index, left, right); size_t right_child = right_child_index(node_index, left, right); tree_write_leave_buckets(dist, write_ptr, left_child, left, mid); - tree_write_leave_buckets(dist, write_ptr, right_child, mid + 1, right); -} + tree_write_leave_buckets(dist, write_ptr, right_child, mid + 1, right); +} buckets_array_t get_buckets(distribution_t *dist) { buckets_array_t bucket_array = { - .num_buckets = dist == NULL ? 0 : dist->num_buckets, - .buckets = dist == NULL ? NULL : calloc(dist->num_buckets, sizeof(*bucket_array.buckets)), + .num_buckets = dist == NULL ? 0 : dist->num_buckets, + .buckets = dist == NULL + ? NULL + : calloc(dist->num_buckets, sizeof(*bucket_array.buckets)), }; if (dist == NULL) return bucket_array; diff --git a/src/daemon/distribution.h b/src/daemon/distribution.h index 8f5f4ba629..fdb771293c 100644 --- a/src/daemon/distribution.h +++ b/src/daemon/distribution.h @@ -27,7 +27,7 @@ #ifndef COLLECTD_DISTRIBUTION_H #define COLLECTD_DISTRIBUTION_H -#include"collectd.h" +#include "collectd.h" typedef struct bucket_s { uint64_t bucket_counter; @@ -42,35 +42,44 @@ typedef struct buckets_array_s { bucket_t *buckets; } buckets_array_t; -//constructor functions: +// constructor functions: /** * function creates new distribution with the linear buckets: * [0; size) [size; 2 * size) ... [(num_buckets - 1) * size; infinity) * @param num_buckets - number of buckets. Should be greater than 0 * @param size - size of each bucket. Should be greater than 0 - * @return - pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + * @return - pointer to a new distribution or null pointer if parameters are + * wrong or memory allocation fails */ -distribution_t* distribution_new_linear(size_t num_buckets, double size); +distribution_t *distribution_new_linear(size_t num_buckets, double size); /** * function creates new distribution with the exponential buckets: - * [0; factor) [factor; factor * base) ... [factor * base^{num_buckets - 2}; infinity) + * [0; factor) [factor; factor * base) ... [factor * base^{num_buckets - 2}; + * infinity) * @param num_buckets - number of buckets. Should be greater than 0 * @param base - base of geometric progression. Should be greater than 1 * @param factor - size of the first bucket. Should be greater than 0 - * @return - pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + * @return - pointer to a new distribution or null pointer if parameters are + * wrong or memory allocation fails */ -distribution_t* distribution_new_exponential(size_t num_buckets, double base, double factor); +distribution_t *distribution_new_exponential(size_t num_buckets, double base, + double factor); /** * function creates new distribution with the custom buckets: - * [0; custom_bucket_boundaries[0]) [custom_bucket_boundaries[0]; custom_bucket_boundaries[1]) ... + * [0; custom_bucket_boundaries[0]) [custom_bucket_boundaries[0]; + * custom_bucket_boundaries[1]) ... * ... [custom_bucket_boundaries[array_size - 1], infinity) - * @param array_size - size of array of bucket boundaries. Number of buckets is array_size + 1 - * @param custom_buckets_boundaries - array with bucket boundaries. Should be increasing and positive - * @return - pointer to a new distribution or null pointer if parameters are wrong or memory allocation fails + * @param array_size - size of array of bucket boundaries. Number of buckets is + * array_size + 1 + * @param custom_buckets_boundaries - array with bucket boundaries. Should be + * increasing and positive + * @return - pointer to a new distribution or null pointer if parameters are + * wrong or memory allocation fails */ -distribution_t* distribution_new_custom(size_t array_size, double *custom_buckets_boundaries); +distribution_t *distribution_new_custom(size_t array_size, + double *custom_buckets_boundaries); /** add new value to a distribution **/ void distribution_update(distribution_t *dist, double gauge); @@ -78,16 +87,19 @@ void distribution_update(distribution_t *dist, double gauge); /** * @param percent - should be in (0; 100] range * @return - an approximation of percent percentile - * (upper bound of such bucket that all less or equal buckets contain more than percent percents of values) - * or NAN if parameters are wrong or distribution is empty + * (upper bound of such bucket that all less or equal buckets contain more than + * percent percents of values) or NAN if parameters are wrong or distribution is + * empty */ double distribution_percentile(distribution_t *dist, double percent); -/** @return - average of all values in distribution or NAN if distribution is empty */ +/** @return - average of all values in distribution or NAN if distribution is + * empty */ double distribution_average(distribution_t *dist); -/** @return - pointer to the copy of distribution or null if memory allocation fails */ -distribution_t* distribution_clone(distribution_t *dist); +/** @return - pointer to the copy of distribution or null if memory allocation + * fails */ +distribution_t *distribution_clone(distribution_t *dist); /** destroy the distribution and free memory **/ void distribution_destroy(distribution_t *d); diff --git a/src/daemon/distribution_benchmark.c b/src/daemon/distribution_benchmark.c index c81e1d529d..b48069d758 100644 --- a/src/daemon/distribution_benchmark.c +++ b/src/daemon/distribution_benchmark.c @@ -1,7 +1,7 @@ +#include #include #include #include -#include #include "distribution.h" @@ -14,7 +14,7 @@ distribution_t *build(size_t num_buckets) { double custom[num_buckets - 1]; custom[0] = rand() % 100; for (size_t i = 1; i < num_buckets - 1; i++) { - custom[i] = custom[i - 1] + rand() % 100 + 1; + custom[i] = custom[i - 1] + rand() % 100 + 1; } distribution_t *dist = distribution_new_custom(num_buckets - 1, custom); return dist; @@ -23,7 +23,8 @@ distribution_t *build(size_t num_buckets) { double calculate_update_time(distribution_t *dist) { double *updates = calloc(NUM_UPDATES, sizeof(*updates)); for (size_t i = 0; i < NUM_UPDATES; i++) { - updates[i] = (rand() * RAND_MAX + rand()) % (distribution_num_buckets(dist) * 100); + updates[i] = + (rand() * RAND_MAX + rand()) % (distribution_num_buckets(dist) * 100); } struct timespec start, finish; clock_gettime(CLOCK_MONOTONIC, &start); @@ -31,10 +32,11 @@ double calculate_update_time(distribution_t *dist) { distribution_update(dist, updates[i]); } clock_gettime(CLOCK_MONOTONIC, &finish); - double update_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); + double update_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + + 1e-6 * (finish.tv_nsec - start.tv_nsec); double average = update_dur / NUM_UPDATES * 1000000.0; free(updates); - return average; + return average; } double calculate_percentile_time(distribution_t *dist) { @@ -48,10 +50,11 @@ double calculate_percentile_time(distribution_t *dist) { distribution_percentile(dist, percentiles[i]); } clock_gettime(CLOCK_MONOTONIC, &finish); - double percentile_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); + double percentile_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + + 1e-6 * (finish.tv_nsec - start.tv_nsec); double average = percentile_dur / NUM_PERCENTILES * 1000000.0; free(percentiles); - return average; + return average; } double mixed(size_t num_buckets) { @@ -59,7 +62,7 @@ double mixed(size_t num_buckets) { double *updates = calloc(MIXED / 10 * 9, sizeof(*updates)); double *percentiles = calloc(MIXED / 10, sizeof(*percentiles)); for (size_t i = 0; i < MIXED / 10 * 9; i++) - updates[i] = (rand() * RAND_MAX + rand()) % (num_buckets * 100); + updates[i] = (rand() * RAND_MAX + rand()) % (num_buckets * 100); for (size_t i = 0; i < MIXED / 10; i++) percentiles[i] = 100.0 * rand() / RAND_MAX; @@ -69,33 +72,37 @@ double mixed(size_t num_buckets) { clock_gettime(CLOCK_MONOTONIC, &start); double val = 0; for (size_t i = 0; i < MIXED; i++) { - if (i % 10 == 9) { + if (i % 10 == 9) { double d = distribution_percentile(dist, percentiles[pid++]); if (d != INFINITY) val += d; - } - else + } else distribution_update(dist, updates[uid++]); } clock_gettime(CLOCK_MONOTONIC, &finish); - double dur = 1000.0 * (finish.tv_sec - start.tv_sec) + 1e-6 * (finish.tv_nsec - start.tv_nsec); + double dur = 1000.0 * (finish.tv_sec - start.tv_sec) + + 1e-6 * (finish.tv_nsec - start.tv_nsec); distribution_destroy(dist); free(percentiles); free(updates); - //printf("%f\n", val); + // printf("%f\n", val); return dur; } int main() { FILE *fout = fopen("benchmark_small.csv", "w"); - fprintf(fout, "Number of buckets,Average for update,Average for percentile,Total for %lu mixed iterations\n", MIXED); + fprintf(fout, + "Number of buckets,Average for update,Average for percentile,Total " + "for %lu mixed iterations\n", + MIXED); for (size_t num_buckets = 50; num_buckets <= 5000; num_buckets += 50) { distribution_t *dist = build(num_buckets); fprintf(fout, "%lu,", num_buckets); fprintf(fout, "%f,", calculate_update_time(dist)); fprintf(fout, "%f,", calculate_percentile_time(dist)); fprintf(fout, "%f\n", mixed(num_buckets)); - //fprintf(fout, "%lu,%f,%f,%f\n", num_buckets, calculate_update_time(dist), calculate_percentile_time(dist), mixed(num_buckets)); + // fprintf(fout, "%lu,%f,%f,%f\n", num_buckets, calculate_update_time(dist), + // calculate_percentile_time(dist), mixed(num_buckets)); distribution_destroy(dist); printf("OK %lu\n", num_buckets); } diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index bfc8e96ccf..1238d7f714 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -37,13 +37,16 @@ static double *linear_upper_bounds(size_t num, double size) { return linear_upper_bounds; } -static double *exponential_upper_bounds(size_t num, double base, double factor) { - double *exponential_upper_bounds = calloc(num, sizeof(*exponential_upper_bounds)); +static double *exponential_upper_bounds(size_t num, double base, + double factor) { + double *exponential_upper_bounds = + calloc(num, sizeof(*exponential_upper_bounds)); exponential_upper_bounds[0] = factor; for (size_t i = 1; i + 1 < num; i++) - exponential_upper_bounds[i] = factor * pow(base, i); //exponential_upper_bounds[i - 1] * base; + exponential_upper_bounds[i] = + factor * pow(base, i); // exponential_upper_bounds[i - 1] * base; exponential_upper_bounds[num - 1] = INFINITY; - return exponential_upper_bounds; + return exponential_upper_bounds; } DEF_TEST(distribution_new_linear) { @@ -53,71 +56,74 @@ DEF_TEST(distribution_new_linear) { double *want_get; int want_err; } cases[] = { - { - .num_buckets = 0, - .size = 5, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 3, - .size = -5, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 5, - .size = 0, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 3, - .size = 2.5, - .want_get = linear_upper_bounds(3, 2.5), - }, - { - .num_buckets = 5, - .size = 5.75, - .want_get = linear_upper_bounds(5, 5.75), - }, - { - .num_buckets = 151, - .size = 0.7, - .want_get = linear_upper_bounds(151, 0.7), - }, - { - .num_buckets = 111, - .size = 1074, - .want_get = linear_upper_bounds(111, 1074), - }, - { - .num_buckets = 77, - .size = 1.0 / 3.0, - .want_get = linear_upper_bounds(77, 1.0 / 3.0), - }, - { - .num_buckets = 1, - .size = 100, - .want_get = linear_upper_bounds(1, 100), - }, + { + .num_buckets = 0, + .size = 5, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 3, + .size = -5, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 5, + .size = 0, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 3, + .size = 2.5, + .want_get = linear_upper_bounds(3, 2.5), + }, + { + .num_buckets = 5, + .size = 5.75, + .want_get = linear_upper_bounds(5, 5.75), + }, + { + .num_buckets = 151, + .size = 0.7, + .want_get = linear_upper_bounds(151, 0.7), + }, + { + .num_buckets = 111, + .size = 1074, + .want_get = linear_upper_bounds(111, 1074), + }, + { + .num_buckets = 77, + .size = 1.0 / 3.0, + .want_get = linear_upper_bounds(77, 1.0 / 3.0), + }, + { + .num_buckets = 1, + .size = 100, + .want_get = linear_upper_bounds(1, 100), + }, }; - for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { printf("## Case %zu:\n", i); if (cases[i].want_err != 0) { - EXPECT_EQ_PTR(cases[i].want_get, distribution_new_linear(cases[i].num_buckets, cases[i].size)); + EXPECT_EQ_PTR( + cases[i].want_get, + distribution_new_linear(cases[i].num_buckets, cases[i].size)); EXPECT_EQ_INT(cases[i].want_err, errno); continue; } distribution_t *d; - CHECK_NOT_NULL(d = distribution_new_linear(cases[i].num_buckets, cases[i].size)); + CHECK_NOT_NULL( + d = distribution_new_linear(cases[i].num_buckets, cases[i].size)); buckets_array_t buckets_array = get_buckets(d); for (size_t j = 0; j < cases[i].num_buckets; j++) { EXPECT_EQ_DOUBLE(cases[i].want_get[j], buckets_array.buckets[j].maximum); - } - destroy_buckets_array(buckets_array); + } + destroy_buckets_array(buckets_array); distribution_destroy(d); - } + } for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { free(cases[i].want_get); } @@ -132,83 +138,86 @@ DEF_TEST(distribution_new_exponential) { double *want_get; int want_err; } cases[] = { - { - .num_buckets = 0, - .factor = 5, - .base = 8, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 5, - .factor = 0.2, - .base = -1, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 8, - .factor = 100, - .base = 0.5, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 100, - .factor = 5.87, - .base = 1, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 6, - .factor = 0, - .base = 9.005, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 16, - .factor = -153, - .base = 1.41, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .num_buckets = 1, - .factor = 10, - .base = 1.05, - .want_get = exponential_upper_bounds(1, 10, 1.05), - }, - { - .num_buckets = 63, - .factor = 1, - .base = 2, - .want_get = exponential_upper_bounds(63, 2, 1), - }, - { - .num_buckets = 600, - .factor = 0.55, - .base = 1.055, - .want_get = exponential_upper_bounds(600, 1.055, 0.55), - }, + { + .num_buckets = 0, + .factor = 5, + .base = 8, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 5, + .factor = 0.2, + .base = -1, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 8, + .factor = 100, + .base = 0.5, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 100, + .factor = 5.87, + .base = 1, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 6, + .factor = 0, + .base = 9.005, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 16, + .factor = -153, + .base = 1.41, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .num_buckets = 1, + .factor = 10, + .base = 1.05, + .want_get = exponential_upper_bounds(1, 10, 1.05), + }, + { + .num_buckets = 63, + .factor = 1, + .base = 2, + .want_get = exponential_upper_bounds(63, 2, 1), + }, + { + .num_buckets = 600, + .factor = 0.55, + .base = 1.055, + .want_get = exponential_upper_bounds(600, 1.055, 0.55), + }, }; - for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { printf("## Case %zu:\n", i); if (cases[i].want_err != 0) { - EXPECT_EQ_PTR(cases[i].want_get, distribution_new_exponential(cases[i].num_buckets, cases[i].base, cases[i].factor)); + EXPECT_EQ_PTR(cases[i].want_get, + distribution_new_exponential( + cases[i].num_buckets, cases[i].base, cases[i].factor)); EXPECT_EQ_INT(cases[i].want_err, errno); continue; } distribution_t *d; - CHECK_NOT_NULL(d = distribution_new_exponential(cases[i].num_buckets, cases[i].base, cases[i].factor)); + CHECK_NOT_NULL(d = distribution_new_exponential( + cases[i].num_buckets, cases[i].base, cases[i].factor)); buckets_array_t buckets_array = get_buckets(d); for (size_t j = 0; j < cases[i].num_buckets; j++) { EXPECT_EQ_DOUBLE(cases[i].want_get[j], buckets_array.buckets[j].maximum); } - destroy_buckets_array(buckets_array); + destroy_buckets_array(buckets_array); distribution_destroy(d); - } + } for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { free(cases[i].want_get); } @@ -222,60 +231,65 @@ DEF_TEST(distribution_new_custom) { double *want_get; int want_err; } cases[] = { - { - .array_size = 0, - .want_get = (double[]){INFINITY}, - }, - { - .array_size = 5, - .custom_boundaries = (double[]){0, 1, 2, 3, 4}, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .array_size = 3, - .custom_boundaries = (double[]){-5, 7, 3}, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .array_size = 4, - .custom_boundaries = (double[]){5.7, 6.0, 6.0, 7.0}, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .array_size = 1, - .custom_boundaries = (double[]){105.055}, - .want_get = (double[]){105.055, INFINITY}, - }, - { - .array_size = 5, - .custom_boundaries = (double[]){8, 100, 1000, 1008, INFINITY}, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .array_size = 7, - .custom_boundaries = (double[]){2, 4, 8, 6, 2, 16, 77.5}, - .want_get = NULL, - .want_err = EINVAL, - }, - { - .array_size = 10, - .custom_boundaries = (double[]){77.5, 100.203, 122.01, 137.23, 200, 205, 210, 220, 230, 256}, - .want_get = (double[]){77.5, 100.203, 122.01, 137.23, 200, 205, 210, 220, 230, 256, INFINITY}, - }, + { + .array_size = 0, + .want_get = (double[]){INFINITY}, + }, + { + .array_size = 5, + .custom_boundaries = (double[]){0, 1, 2, 3, 4}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 3, + .custom_boundaries = (double[]){-5, 7, 3}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 4, + .custom_boundaries = (double[]){5.7, 6.0, 6.0, 7.0}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 1, + .custom_boundaries = (double[]){105.055}, + .want_get = (double[]){105.055, INFINITY}, + }, + { + .array_size = 5, + .custom_boundaries = (double[]){8, 100, 1000, 1008, INFINITY}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 7, + .custom_boundaries = (double[]){2, 4, 8, 6, 2, 16, 77.5}, + .want_get = NULL, + .want_err = EINVAL, + }, + { + .array_size = 10, + .custom_boundaries = (double[]){77.5, 100.203, 122.01, 137.23, 200, + 205, 210, 220, 230, 256}, + .want_get = (double[]){77.5, 100.203, 122.01, 137.23, 200, 205, 210, + 220, 230, 256, INFINITY}, + }, }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { if (cases[i].want_err != 0) { - EXPECT_EQ_PTR(cases[i].want_get, distribution_new_custom(cases[i].array_size, cases[i].custom_boundaries)); + EXPECT_EQ_PTR(cases[i].want_get, + distribution_new_custom(cases[i].array_size, + cases[i].custom_boundaries)); EXPECT_EQ_INT(cases[i].want_err, errno); continue; } distribution_t *d; - CHECK_NOT_NULL(d = distribution_new_custom(cases[i].array_size, cases[i].custom_boundaries)); + CHECK_NOT_NULL(d = distribution_new_custom(cases[i].array_size, + cases[i].custom_boundaries)); buckets_array_t buckets_array = get_buckets(d); for (size_t j = 0; j < cases[i].array_size + 1; j++) { EXPECT_EQ_DOUBLE(cases[i].want_get[j], buckets_array.buckets[j].maximum); @@ -294,36 +308,36 @@ DEF_TEST(update) { uint64_t *want_counters; int want_err; } cases[] = { - { - .dist = distribution_new_linear(6, 5), - .num_gauges = 10, - .gauges = (double[]){25, 30, 5, 7, 11, 10.5, 8.03, 1112.4, 35, 12.7}, - .want_counters = (uint64_t[]){0, 3, 3, 0, 0, 4}, - }, - { - .dist = distribution_new_exponential(4, 1.41, 1), - .num_gauges = 0, - .gauges = NULL, - .want_counters = (uint64_t[]){0, 0, 0, 0}, - }, - { - .dist = distribution_new_exponential(5, 2, 3), - .num_gauges = 5, - .gauges = (double[]){1, 7, 3, 10, 77}, - .want_counters = (uint64_t[]){1, 1, 2, 0, 1}, - }, - { - .dist = distribution_new_linear(100, 22), - .num_gauges = 3, - .gauges = (double[]){1000, 2, -8}, - .want_err = EINVAL, - }, - { - .dist = distribution_new_custom(3, (double[]) {5, 20, 35}), - .num_gauges = 7, - .gauges = (double[]){7.05, 22.37, 40.83, 90.55, 12.34, 14.2, 6.0}, - .want_counters = (uint64_t[]){0, 4, 1, 2}, - }, + { + .dist = distribution_new_linear(6, 5), + .num_gauges = 10, + .gauges = (double[]){25, 30, 5, 7, 11, 10.5, 8.03, 1112.4, 35, 12.7}, + .want_counters = (uint64_t[]){0, 3, 3, 0, 0, 4}, + }, + { + .dist = distribution_new_exponential(4, 1.41, 1), + .num_gauges = 0, + .gauges = NULL, + .want_counters = (uint64_t[]){0, 0, 0, 0}, + }, + { + .dist = distribution_new_exponential(5, 2, 3), + .num_gauges = 5, + .gauges = (double[]){1, 7, 3, 10, 77}, + .want_counters = (uint64_t[]){1, 1, 2, 0, 1}, + }, + { + .dist = distribution_new_linear(100, 22), + .num_gauges = 3, + .gauges = (double[]){1000, 2, -8}, + .want_err = EINVAL, + }, + { + .dist = distribution_new_custom(3, (double[]){5, 20, 35}), + .num_gauges = 7, + .gauges = (double[]){7.05, 22.37, 40.83, 90.55, 12.34, 14.2, 6.0}, + .want_counters = (uint64_t[]){0, 4, 1, 2}, + }, }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { for (size_t j = 0; j < cases[i].num_gauges; j++) { @@ -336,7 +350,8 @@ DEF_TEST(update) { } buckets_array_t buckets_array = get_buckets(cases[i].dist); for (size_t j = 0; j < buckets_array.num_buckets; j++) { - EXPECT_EQ_INT(cases[i].want_counters[j], buckets_array.buckets[j].bucket_counter); + EXPECT_EQ_INT(cases[i].want_counters[j], + buckets_array.buckets[j].bucket_counter); } destroy_buckets_array(buckets_array); distribution_destroy(cases[i].dist); @@ -351,29 +366,30 @@ DEF_TEST(average) { double *update_gauges; double want_average; } cases[] = { - { - .dist = distribution_new_linear(6, 2), - .num_gauges = 0, - .want_average = NAN, - }, - { - .dist = distribution_new_linear(7, 10), - .num_gauges = 5, - .update_gauges = (double[]){3, 2, 5.7, 22.3, 7.5}, - .want_average = 40.5 / 5.0, - }, - { - .dist = distribution_new_exponential(10, 2, 0.75), - .num_gauges = 8, - .update_gauges = (double[]){2, 4, 6, 8, 22, 11, 77, 1005}, - .want_average = 1135.0 / 8.0, - }, - }; + { + .dist = distribution_new_linear(6, 2), + .num_gauges = 0, + .want_average = NAN, + }, + { + .dist = distribution_new_linear(7, 10), + .num_gauges = 5, + .update_gauges = (double[]){3, 2, 5.7, 22.3, 7.5}, + .want_average = 40.5 / 5.0, + }, + { + .dist = distribution_new_exponential(10, 2, 0.75), + .num_gauges = 8, + .update_gauges = (double[]){2, 4, 6, 8, 22, 11, 77, 1005}, + .want_average = 1135.0 / 8.0, + }, + }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { for (size_t j = 0; j < cases[i].num_gauges; j++) { distribution_update(cases[i].dist, cases[i].update_gauges[j]); } - EXPECT_EQ_DOUBLE(cases[i].want_average, distribution_average(cases[i].dist)); + EXPECT_EQ_DOUBLE(cases[i].want_average, + distribution_average(cases[i].dist)); distribution_destroy(cases[i].dist); } return 0; @@ -388,48 +404,49 @@ DEF_TEST(percentile) { double want_percentile; int want_err; } cases[] = { - { - .dist = distribution_new_linear(5, 7), - .num_gauges = 1, - .update_gauges = (double[]){4}, - .percent = 105, - .want_percentile = NAN, - .want_err = EINVAL, - }, - { - .dist = distribution_new_linear(8, 10), - .num_gauges = 0, - .percent = 20, - .want_percentile = NAN, - }, - { - .dist = distribution_new_exponential(5, 2, 0.2), - .num_gauges = 2, - .update_gauges = (double[]){4, 30.08}, - .percent = -5, - .want_percentile = NAN, - .want_err = EINVAL, - }, - { - .dist = distribution_new_exponential(10, 2, 0.75), - .num_gauges = 8, - .update_gauges = (double[]){2, 4, 6, 8, 22, 11, 77, 1005}, - .percent = 50, - .want_percentile = 12, - }, - { - .dist = distribution_new_custom(3, (double[]){5, 20, 35}), - .num_gauges = 7, - .update_gauges = (double[]){5.5, 10.5, 11.3, 6.7, 24.7, 40.05, 35}, - .percent = 4.0 / 7.0 * 100, - .want_percentile = 20, - }, + { + .dist = distribution_new_linear(5, 7), + .num_gauges = 1, + .update_gauges = (double[]){4}, + .percent = 105, + .want_percentile = NAN, + .want_err = EINVAL, + }, + { + .dist = distribution_new_linear(8, 10), + .num_gauges = 0, + .percent = 20, + .want_percentile = NAN, + }, + { + .dist = distribution_new_exponential(5, 2, 0.2), + .num_gauges = 2, + .update_gauges = (double[]){4, 30.08}, + .percent = -5, + .want_percentile = NAN, + .want_err = EINVAL, + }, + { + .dist = distribution_new_exponential(10, 2, 0.75), + .num_gauges = 8, + .update_gauges = (double[]){2, 4, 6, 8, 22, 11, 77, 1005}, + .percent = 50, + .want_percentile = 12, + }, + { + .dist = distribution_new_custom(3, (double[]){5, 20, 35}), + .num_gauges = 7, + .update_gauges = (double[]){5.5, 10.5, 11.3, 6.7, 24.7, 40.05, 35}, + .percent = 4.0 / 7.0 * 100, + .want_percentile = 20, + }, }; for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { for (size_t j = 0; j < cases[i].num_gauges; j++) { distribution_update(cases[i].dist, cases[i].update_gauges[j]); } - EXPECT_EQ_DOUBLE(cases[i].want_percentile, distribution_percentile(cases[i].dist, cases[i].percent)); + EXPECT_EQ_DOUBLE(cases[i].want_percentile, + distribution_percentile(cases[i].dist, cases[i].percent)); if (cases[i].want_err != 0) EXPECT_EQ_INT(cases[i].want_err, errno); distribution_destroy(cases[i].dist); @@ -455,7 +472,7 @@ DEF_TEST(clone) { int main() { RUN_TEST(distribution_new_linear); RUN_TEST(distribution_new_exponential); - RUN_TEST(distribution_new_custom); + RUN_TEST(distribution_new_custom); RUN_TEST(update); RUN_TEST(average); RUN_TEST(percentile); From 8d2095fadd5e67e09d0298da474a7c8bfcc7ed23 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 17 Aug 2020 11:31:54 +0000 Subject: [PATCH 63/67] remove distribution benchmark as full benchmark was created in google-interns-2020 branch --- src/daemon/distribution_benchmark.c | 111 ---------------------------- 1 file changed, 111 deletions(-) delete mode 100644 src/daemon/distribution_benchmark.c diff --git a/src/daemon/distribution_benchmark.c b/src/daemon/distribution_benchmark.c deleted file mode 100644 index b48069d758..0000000000 --- a/src/daemon/distribution_benchmark.c +++ /dev/null @@ -1,111 +0,0 @@ -#include -#include -#include -#include - -#include "distribution.h" - -const size_t NUM_UPDATES = 1e6; -const size_t NUM_PERCENTILES = 1e6; -const size_t MIXED = 1e6; - -distribution_t *build(size_t num_buckets) { - srand(5); - double custom[num_buckets - 1]; - custom[0] = rand() % 100; - for (size_t i = 1; i < num_buckets - 1; i++) { - custom[i] = custom[i - 1] + rand() % 100 + 1; - } - distribution_t *dist = distribution_new_custom(num_buckets - 1, custom); - return dist; -} - -double calculate_update_time(distribution_t *dist) { - double *updates = calloc(NUM_UPDATES, sizeof(*updates)); - for (size_t i = 0; i < NUM_UPDATES; i++) { - updates[i] = - (rand() * RAND_MAX + rand()) % (distribution_num_buckets(dist) * 100); - } - struct timespec start, finish; - clock_gettime(CLOCK_MONOTONIC, &start); - for (size_t i = 0; i < NUM_UPDATES; i++) { - distribution_update(dist, updates[i]); - } - clock_gettime(CLOCK_MONOTONIC, &finish); - double update_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + - 1e-6 * (finish.tv_nsec - start.tv_nsec); - double average = update_dur / NUM_UPDATES * 1000000.0; - free(updates); - return average; -} - -double calculate_percentile_time(distribution_t *dist) { - double *percentiles = calloc(NUM_PERCENTILES, sizeof(*percentiles)); - for (size_t i = 0; i < NUM_PERCENTILES; i++) { - percentiles[i] = 100.0 * rand() / RAND_MAX; - } - struct timespec start, finish; - clock_gettime(CLOCK_MONOTONIC, &start); - for (size_t i = 0; i < NUM_PERCENTILES; i++) { - distribution_percentile(dist, percentiles[i]); - } - clock_gettime(CLOCK_MONOTONIC, &finish); - double percentile_dur = 1000.0 * (finish.tv_sec - start.tv_sec) + - 1e-6 * (finish.tv_nsec - start.tv_nsec); - double average = percentile_dur / NUM_PERCENTILES * 1000000.0; - free(percentiles); - return average; -} - -double mixed(size_t num_buckets) { - distribution_t *dist = build(num_buckets); - double *updates = calloc(MIXED / 10 * 9, sizeof(*updates)); - double *percentiles = calloc(MIXED / 10, sizeof(*percentiles)); - for (size_t i = 0; i < MIXED / 10 * 9; i++) - updates[i] = (rand() * RAND_MAX + rand()) % (num_buckets * 100); - for (size_t i = 0; i < MIXED / 10; i++) - percentiles[i] = 100.0 * rand() / RAND_MAX; - - size_t uid = 0; - size_t pid = 0; - struct timespec start, finish; - clock_gettime(CLOCK_MONOTONIC, &start); - double val = 0; - for (size_t i = 0; i < MIXED; i++) { - if (i % 10 == 9) { - double d = distribution_percentile(dist, percentiles[pid++]); - if (d != INFINITY) - val += d; - } else - distribution_update(dist, updates[uid++]); - } - clock_gettime(CLOCK_MONOTONIC, &finish); - double dur = 1000.0 * (finish.tv_sec - start.tv_sec) + - 1e-6 * (finish.tv_nsec - start.tv_nsec); - distribution_destroy(dist); - free(percentiles); - free(updates); - // printf("%f\n", val); - return dur; -} - -int main() { - FILE *fout = fopen("benchmark_small.csv", "w"); - fprintf(fout, - "Number of buckets,Average for update,Average for percentile,Total " - "for %lu mixed iterations\n", - MIXED); - for (size_t num_buckets = 50; num_buckets <= 5000; num_buckets += 50) { - distribution_t *dist = build(num_buckets); - fprintf(fout, "%lu,", num_buckets); - fprintf(fout, "%f,", calculate_update_time(dist)); - fprintf(fout, "%f,", calculate_percentile_time(dist)); - fprintf(fout, "%f\n", mixed(num_buckets)); - // fprintf(fout, "%lu,%f,%f,%f\n", num_buckets, calculate_update_time(dist), - // calculate_percentile_time(dist), mixed(num_buckets)); - distribution_destroy(dist); - printf("OK %lu\n", num_buckets); - } - fclose(fout); - return 0; -} From 547fd70b98aa99667d6de3cc4fb22fec49fd59d7 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 17 Aug 2020 11:53:11 +0000 Subject: [PATCH 64/67] remove benchmark csv files --- benchmark.csv | 201 -------------------------------------------- benchmark_small.csv | 201 -------------------------------------------- 2 files changed, 402 deletions(-) delete mode 100644 benchmark.csv delete mode 100644 benchmark_small.csv diff --git a/benchmark.csv b/benchmark.csv deleted file mode 100644 index aaf9c3deba..0000000000 --- a/benchmark.csv +++ /dev/null @@ -1,201 +0,0 @@ -Number of buckets,Average for update,Average for percentile,Total for 20000000 mixed iterations -20,20.833077,33.140944,511.968124 -40,24.871834,36.059857,572.377243 -60,26.073686,37.543635,593.776276 -80,28.916161,40.072926,652.529222 -100,30.699940,41.640226,689.026993 -120,30.874444,43.060979,706.172506 -140,32.513817,44.898618,730.686766 -160,33.147428,45.237810,741.695803 -180,34.185429,46.078374,760.467895 -200,34.645466,47.461673,765.846071 -220,36.452024,47.081149,799.555336 -240,36.574275,47.175475,804.132166 -260,37.990334,48.397980,835.876786 -280,38.147707,48.813378,837.765745 -300,38.229090,49.249399,837.098446 -320,38.666436,49.609447,849.488966 -340,38.884782,50.036148,859.496535 -360,39.732383,50.695363,870.593929 -380,39.818529,51.305621,875.971119 -400,40.178807,51.326839,880.893379 -420,41.070184,51.743950,898.445908 -440,41.383053,52.057908,902.647839 -460,41.634752,52.080829,909.758109 -480,42.467967,52.259897,918.638426 -500,43.841562,52.723497,933.817474 -520,43.602939,54.488960,953.311065 -540,43.703239,54.542689,951.637020 -560,43.761980,54.937751,956.513530 -580,43.635702,55.228712,949.896027 -600,43.730006,55.379645,954.048957 -620,44.265220,73.653216,2049.773595 -640,87.804661,112.974539,1839.897381 -660,87.919513,112.181002,1840.538112 -680,88.073011,112.938754,1968.444859 -700,88.702958,112.665691,1842.752254 -720,88.375925,112.585872,1974.837199 -740,88.840096,113.038652,1974.233849 -760,90.908834,113.212586,1978.170734 -780,95.286165,113.068265,1981.879820 -800,88.795886,113.185687,1978.529653 -820,88.925599,113.166689,1982.555826 -840,89.364005,113.247402,1984.327452 -860,89.368505,113.336711,1990.374602 -880,95.664670,113.535643,1990.111667 -900,95.710216,125.671841,1991.283647 -920,89.609183,113.464495,1997.597951 -940,96.399109,114.115178,1996.475791 -960,89.852888,113.841985,2030.167979 -980,90.287138,113.866146,2008.447629 -1000,96.756945,114.020647,2009.202512 -1020,90.521636,113.999672,2014.223830 -1040,109.586508,121.804300,2031.565246 -1060,91.391154,121.650012,2030.098948 -1080,97.382280,115.310143,2029.597605 -1100,97.670673,116.513335,2030.564083 -1120,91.430033,121.643045,2032.548356 -1140,97.691063,116.071757,2156.563029 -1160,97.840235,122.371703,2065.363011 -1180,97.853581,116.037525,2039.342027 -1200,98.244354,122.702684,2045.133338 -1220,98.214568,116.407017,2045.108250 -1240,98.379457,116.688352,2046.490916 -1260,98.418106,116.628397,2172.218517 -1280,98.356146,123.278195,2179.401885 -1300,100.234367,123.205637,2054.172170 -1320,98.509031,123.493302,2055.784867 -1340,98.835003,123.745883,2062.280398 -1360,99.198711,125.008648,2185.379563 -1380,92.625308,123.388537,2183.551084 -1400,92.834658,123.812956,2185.762610 -1420,93.061516,117.403229,2084.653107 -1440,99.535905,124.130576,2193.218954 -1460,99.568697,123.754583,2199.334700 -1480,99.739003,117.861438,2090.727712 -1500,99.625641,124.262356,2200.258397 -1520,125.020176,237.067336,4114.030042 -1540,114.768170,225.467440,3261.872936 -1560,141.056171,184.780434,2945.742771 -1580,127.651321,214.795761,2745.877877 -1600,105.848378,124.598698,2083.859083 -1620,100.399835,124.828749,2215.968172 -1640,101.735051,125.568009,2100.750527 -1660,100.573839,124.950148,2217.729341 -1680,100.746419,124.952319,2217.206621 -1700,100.877626,126.725119,2230.364136 -1720,100.824358,125.212768,2223.283675 -1740,100.989016,125.215075,2227.256856 -1760,101.110792,126.430718,2226.582502 -1780,107.485283,125.412331,2231.998505 -1800,107.573218,125.420763,2232.855943 -1820,108.015956,126.026536,2270.014637 -1840,101.403769,125.725790,2239.077757 -1860,107.998218,125.765768,2240.477846 -1880,101.656547,125.951881,2482.842574 -1900,101.871661,126.104945,2243.934176 -1920,108.504175,126.155524,2248.066204 -1940,102.196268,126.236956,2252.192447 -1960,108.511859,127.044316,2258.801320 -1980,109.259507,126.513031,2253.070304 -2000,102.689637,132.684199,2258.167335 -2020,109.340887,126.450371,2258.617640 -2040,109.530100,126.827447,2269.020560 -2060,110.121377,137.492058,2296.144207 -2080,110.437918,137.612732,2288.097336 -2100,110.001937,137.432756,2291.184698 -2120,104.015173,137.713119,2290.005208 -2140,110.573933,138.030538,2291.041676 -2160,103.582606,148.803305,2415.789096 -2180,110.002547,137.537891,2424.537886 -2200,104.339575,137.897688,2294.260777 -2220,103.810388,137.518623,2415.582786 -2240,110.110051,137.786946,2417.294705 -2260,110.332857,137.764054,2417.864443 -2280,110.185616,137.879868,2292.689286 -2300,110.213115,137.828498,2296.664906 -2320,110.512877,138.079455,2299.789117 -2340,110.672938,138.461842,2429.465366 -2360,110.268692,138.231321,2423.405380 -2380,110.431845,138.104614,2300.020491 -2400,110.260652,138.290315,2426.660833 -2420,110.543089,138.440825,2428.891426 -2440,104.216735,138.465425,2301.749309 -2460,104.534811,138.583958,2306.114981 -2480,104.281298,144.665867,2304.187363 -2500,104.636953,145.571055,2304.536292 -2520,111.029655,138.965993,2432.819168 -2540,110.634383,138.562527,2433.879082 -2560,105.132859,145.164625,2309.720445 -2580,110.780880,139.103996,2436.214106 -2600,111.360110,145.572759,2341.336910 -2620,111.030853,139.234106,2307.375960 -2640,104.698380,145.200897,2310.936674 -2660,110.874977,139.220138,2465.121658 -2680,104.940546,145.362425,2316.032035 -2700,111.194037,139.282455,2434.735995 -2720,111.050057,138.982100,2550.520384 -2740,111.378580,139.508965,2439.719376 -2760,111.046686,139.291950,2434.424358 -2780,104.849723,145.628202,2467.623322 -2800,111.166878,139.818303,2330.182971 -2820,111.162130,139.235084,2445.749065 -2840,111.287927,139.246122,2547.285897 -2860,105.075993,145.598091,2318.814249 -2880,111.241276,139.430584,2442.596791 -2900,104.985098,145.575055,2454.444098 -2920,111.707930,139.339785,2320.744852 -2940,111.492077,139.581928,2448.412518 -2960,111.689481,146.128188,2450.082252 -2980,112.283823,146.503239,2336.981280 -3000,111.956611,139.752156,2331.032225 -3020,111.771459,140.816526,2454.019025 -3040,112.037193,139.790790,2453.675997 -3060,112.007624,146.527122,2465.502764 -3080,112.899524,158.011578,2464.691865 -3100,112.112632,146.319260,2337.380178 -3120,112.455636,140.228836,2337.890865 -3140,112.339785,147.763334,2472.226790 -3160,112.318814,146.384890,2339.854964 -3180,112.274367,140.037612,2337.972203 -3200,112.745264,148.608836,2473.053663 -3220,112.558855,146.682280,2538.914378 -3240,112.350903,146.720200,2460.985845 -3260,113.668348,147.022313,2469.381490 -3280,113.254116,140.919530,2483.555515 -3300,112.571139,140.507502,2469.895952 -3320,114.155043,146.837161,2467.429174 -3340,112.608822,146.811137,2466.785663 -3360,112.668541,146.728846,2470.770591 -3380,113.850032,146.861805,2475.941773 -3400,113.064849,140.728432,2356.416699 -3420,112.960434,140.411812,2348.290113 -3440,112.898934,146.561037,2469.482121 -3460,112.824672,146.838621,2474.293737 -3480,112.833938,140.839812,2499.793787 -3500,112.931204,140.424361,2348.342307 -3520,113.112293,146.763790,2475.950533 -3540,113.100448,146.941451,2705.810569 -3560,113.171508,146.894100,2475.882187 -3580,113.117937,146.852640,2487.424122 -3600,113.335796,140.744408,2495.167452 -3620,113.343334,147.442549,2475.834720 -3640,113.644828,147.295195,2488.476125 -3660,113.340550,150.215762,2483.566793 -3680,113.694441,140.644473,2481.981054 -3700,113.719818,147.323338,2485.999012 -3720,113.322047,148.061643,2489.702341 -3740,113.745410,141.009083,2483.818606 -3760,113.817257,147.169938,2486.387063 -3780,120.061682,152.833642,2487.592561 -3800,114.377361,147.891419,2501.453644 -3820,114.378227,147.957765,2496.925381 -3840,121.595965,141.166425,2494.710316 -3860,120.270576,147.282998,2497.846362 -3880,113.788127,147.516815,2495.682980 -3900,125.957699,147.536896,2493.837051 -3920,114.114346,147.472841,2496.612114 -3940,120.309828,141.236431,2501.958535 -3960,114.174887,147.504054,2500.873344 -3980,114.318758,147.380126,2510.417850 -4000,114.880214,148.085930,2539.786191 diff --git a/benchmark_small.csv b/benchmark_small.csv deleted file mode 100644 index 7e52464b82..0000000000 --- a/benchmark_small.csv +++ /dev/null @@ -1,201 +0,0 @@ -Number of buckets,Average for update,Average for percentile,Total for 1000000 mixed iterations -20,30.279575,31.594688,24.608609 -40,25.582838,36.222115,28.614193 -60,25.900862,37.503846,29.833795 -80,28.824710,44.583059,34.125163 -100,31.397022,43.217300,34.030339 -120,30.930586,43.033250,34.720907 -140,32.201488,44.678667,36.524133 -160,33.270566,45.343786,37.057255 -180,33.882487,46.062419,37.678757 -200,34.498888,46.060612,38.469573 -220,36.505075,46.776692,40.164427 -240,36.319747,47.257512,40.504973 -260,38.245813,48.557370,42.324541 -280,38.237893,48.890312,41.976441 -300,38.007175,49.376568,41.978078 -320,38.557740,49.588676,42.309814 -340,39.014167,50.209515,43.082755 -360,39.569006,50.597916,43.400479 -380,39.920028,51.100854,43.536472 -400,40.495439,51.530447,44.088921 -420,41.168983,51.864774,44.627674 -440,41.023137,51.829657,45.128834 -460,41.813872,52.331192,45.451797 -480,42.303883,52.617606,45.956554 -500,42.680624,52.972633,46.501718 -520,43.275708,54.647199,47.607325 -540,43.709591,54.540344,47.779972 -560,44.208340,55.048302,50.228766 -580,43.988163,55.385142,47.752993 -600,44.755797,55.771408,47.598055 -620,43.880436,55.797855,48.109657 -640,43.477199,55.998780,47.825107 -660,43.720069,55.800412,48.112947 -680,43.967458,56.343278,48.324987 -700,44.138272,56.260159,49.403632 -720,44.736586,56.540755,48.222454 -740,44.665820,56.585378,48.781087 -760,44.791490,56.598541,48.842271 -780,44.818675,56.548026,48.696944 -800,44.569383,56.666266,48.522515 -820,45.403978,56.957057,49.220015 -840,45.504865,57.213265,48.981029 -860,45.456769,58.700875,53.015213 -880,47.868071,56.907230,49.734639 -900,45.477546,57.178660,49.376460 -920,45.539323,57.284126,49.892168 -940,45.773609,57.229608,50.307060 -960,46.052729,57.476211,50.219397 -980,46.304167,57.591356,50.430984 -1000,46.663843,57.855602,50.207105 -1020,46.549401,57.894820,50.560559 -1040,47.729281,58.948228,51.424784 -1060,47.378295,58.974178,51.562945 -1080,47.732550,59.061161,51.299944 -1100,47.422751,59.378432,51.637271 -1120,47.535279,59.636300,51.875102 -1140,47.416219,59.638969,51.466828 -1160,47.541496,62.372862,74.780864 -1180,47.906223,59.923371,51.598257 -1200,47.601045,60.024814,51.763881 -1220,47.891179,60.354214,51.882209 -1240,48.220709,60.296880,52.716259 -1260,48.033805,60.363671,52.174321 -1280,48.070075,60.587039,52.578795 -1300,48.078878,60.707833,52.541964 -1320,48.346834,60.796052,52.847289 -1340,48.318736,60.667365,52.490966 -1360,48.635272,60.693801,52.755617 -1380,48.441798,60.904811,52.719066 -1400,48.958179,60.829070,53.777083 -1420,49.176889,61.105055,52.712338 -1440,48.993922,61.261518,52.996312 -1460,49.085647,61.581158,53.430562 -1480,49.894456,61.792587,53.565978 -1500,49.434430,61.511730,53.422607 -1520,50.430850,61.927736,54.680125 -1540,50.042466,61.782948,54.188356 -1560,50.626234,64.630917,54.073191 -1580,50.309620,62.148451,54.046681 -1600,49.901222,62.090722,54.137603 -1620,50.680344,62.304322,54.434820 -1640,50.573025,63.131581,55.134403 -1660,50.219907,62.483734,54.429006 -1680,50.350649,62.609521,54.523934 -1700,50.678749,62.234025,54.367081 -1720,50.820963,62.359883,54.830536 -1740,50.525967,62.812270,54.786726 -1760,51.686882,62.690574,54.782847 -1780,50.961380,62.736886,55.055650 -1800,51.068817,63.122665,55.235915 -1820,51.913093,63.497917,56.002548 -1840,51.529774,63.302531,55.521424 -1860,51.403516,63.392513,56.115667 -1880,51.671650,63.508614,55.470308 -1900,51.873017,63.334177,56.133747 -1920,51.868056,63.675988,56.083354 -1940,52.322543,63.687997,55.982485 -1960,52.116989,63.613090,55.865830 -1980,52.204028,63.766823,56.670002 -2000,52.254905,64.452683,56.876680 -2020,53.174767,64.272465,57.751285 -2040,53.043060,64.672269,56.915322 -2060,55.507562,69.321633,58.568302 -2080,56.004583,69.513837,58.229247 -2100,55.014491,69.174953,58.248046 -2120,53.742333,69.074059,58.629298 -2140,54.171234,69.234462,59.027992 -2160,53.592842,69.366887,58.327551 -2180,53.530757,69.494130,58.592062 -2200,53.822003,72.140826,60.888050 -2220,55.144748,69.123258,58.079006 -2240,53.652790,68.990011,58.305601 -2260,53.816040,69.158665,58.560090 -2280,54.079412,69.447263,58.530278 -2300,53.635252,69.421756,58.419882 -2320,53.830386,69.419201,58.471403 -2340,54.230589,69.673856,59.232988 -2360,53.969463,69.547158,58.587357 -2380,72.362578,71.715374,59.185883 -2400,54.227213,69.759882,58.637069 -2420,54.081270,69.901676,58.877024 -2440,54.055792,69.751263,58.408177 -2460,54.622058,77.442312,59.086894 -2480,54.046031,70.215774,58.994780 -2500,54.450404,70.134064,58.841864 -2520,54.457982,70.260155,59.197099 -2540,54.221169,70.023464,59.198274 -2560,54.944813,70.301828,58.922012 -2580,54.754265,70.357278,58.936917 -2600,55.024159,70.198531,58.903710 -2620,54.392080,70.193272,59.135476 -2640,54.745841,70.043982,59.009490 -2660,55.502013,70.164227,59.511555 -2680,54.725092,70.249758,59.114002 -2700,54.678079,70.214245,59.660478 -2720,54.784445,70.491713,60.235601 -2740,55.622628,70.603363,59.324161 -2760,55.180589,145.421244,59.341203 -2780,54.743429,147.806926,59.185746 -2800,54.760565,145.517308,59.900930 -2820,55.074581,146.295855,59.774567 -2840,180.290891,145.835769,59.573428 -2860,180.266554,133.459063,60.386303 -2880,180.327998,133.231126,59.969272 -2900,180.077131,133.311422,60.091939 -2920,179.759498,134.347039,60.589298 -2940,181.058266,134.212296,62.299925 -2960,180.227624,147.481052,61.013138 -2980,56.531547,146.368397,60.678622 -3000,55.459108,146.145743,60.090091 -3020,56.254363,146.427030,60.213138 -3040,55.377735,145.989103,60.222552 -3060,180.646183,146.165692,61.401884 -3080,180.556840,148.744368,60.621099 -3100,55.649209,146.090926,60.579003 -3120,55.761004,146.516187,60.678824 -3140,55.810243,146.310782,60.496865 -3160,56.008478,146.558079,60.706033 -3180,55.824185,146.214299,60.948261 -3200,55.994874,146.706851,212.093123 -3220,210.480636,146.650935,60.792510 -3240,180.725920,146.515604,60.619125 -3260,180.741523,146.677484,60.808386 -3280,57.415287,147.186995,61.582389 -3300,56.172186,146.707906,60.814673 -3320,56.063071,146.760821,60.791379 -3340,56.143963,146.914444,61.206165 -3360,56.330267,146.859216,185.841481 -3380,57.790392,149.485072,185.456421 -3400,56.428401,146.907141,186.172229 -3420,182.340170,146.884048,185.807056 -3440,182.550517,134.510603,185.736134 -3460,182.516556,134.539685,61.375244 -3480,182.458924,134.261983,61.104031 -3500,182.012471,134.205243,61.304505 -3520,182.606687,134.307559,61.176607 -3540,182.571038,147.016236,61.508035 -3560,182.423023,147.284216,61.805698 -3580,57.253225,146.867475,61.401099 -3600,56.833081,146.813521,61.413788 -3620,56.868331,147.138662,187.114517 -3640,57.440923,147.524592,187.448921 -3660,56.891377,147.107504,186.616561 -3680,57.148944,148.761962,62.569509 -3700,182.235366,134.566847,61.675157 -3720,182.286954,147.222844,61.627292 -3740,182.169898,147.138237,61.815352 -3760,57.631382,147.573349,61.931089 -3780,57.576468,147.490552,187.423907 -3800,57.583399,148.076982,186.966925 -3820,57.927596,147.251650,186.923013 -3840,182.614051,147.516222,187.818504 -3860,183.301656,135.222421,62.612584 -3880,183.038547,147.786395,62.222333 -3900,183.303946,147.456878,62.268387 -3920,58.120228,147.746213,188.515040 -3940,58.711964,147.543244,188.105778 -3960,58.026069,148.767576,192.508633 -3980,189.943186,148.242416,62.449116 -4000,183.389327,148.411055,62.740622 From 5573bbba5924d172ace591272b65e7bb77fac189 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 17 Aug 2020 11:56:09 +0000 Subject: [PATCH 65/67] remove benchmark from Makefile --- Makefile.am | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Makefile.am b/Makefile.am index 3823c64967..4ac4c24817 100644 --- a/Makefile.am +++ b/Makefile.am @@ -124,7 +124,6 @@ bin_PROGRAMS = \ collectd-nagios \ collectd-tg \ collectdctl \ - distribution_benchmark endif # BUILD_WIN32 @@ -363,13 +362,6 @@ test_distribution_SOURCES = \ src/testing.h test_distribution_LDADD = libmetric.la libplugin_mock.la -distribution_benchmark_SOURCES = src/daemon/distribution_benchmark.c -#distribution_benchmark_CPPFLAGS = $(AM_CPPFLAGS) \ -# -I$(srcdir)/src/libcollectdclient \ -# -I$(top_builddir)/src/libcollectdclient -distribution_benchmark_LDADD = \ - libmetric.la - test_metric_SOURCES = \ src/daemon/metric_test.c \ src/testing.h From b8611e255ac2191bded92600eba5235f0872e1ab Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 17 Aug 2020 12:41:39 +0000 Subject: [PATCH 66/67] fix Makefile.am --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 4ac4c24817..a6706f7848 100644 --- a/Makefile.am +++ b/Makefile.am @@ -123,7 +123,7 @@ sbin_PROGRAMS = \ bin_PROGRAMS = \ collectd-nagios \ collectd-tg \ - collectdctl \ + collectdctl endif # BUILD_WIN32 From ae97265744d9cd106e11410d0e2b541abb83ec19 Mon Sep 17 00:00:00 2001 From: Svetlana Date: Mon, 17 Aug 2020 13:06:35 +0000 Subject: [PATCH 67/67] fix memory leaks in clone unit test --- src/daemon/distribution_test.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/daemon/distribution_test.c b/src/daemon/distribution_test.c index 1238d7f714..4d29c000cd 100644 --- a/src/daemon/distribution_test.c +++ b/src/daemon/distribution_test.c @@ -466,6 +466,11 @@ DEF_TEST(clone) { EXPECT_EQ_DOUBLE(distribution_percentile(dist, 50), 14); distribution_update(clone, 2); EXPECT_EQ_DOUBLE(distribution_percentile(dist, 50), 14); + distribution_destroy(dist); + distribution_update(clone, 25); + distribution_update(clone, 200); + EXPECT_EQ_DOUBLE(distribution_percentile(clone, 80), 28); + distribution_destroy(clone); return 0; }