|
|
|
kevin brintnall
|
---
doc/librrd.pod | 18 ++++++++++++++++++ src/rrd.h | 4 ++++ src/rrd_utils.c | 53 ++++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/doc/librrd.pod b/doc/librrd.pod index 038746c..396e50c 100644 --- a/doc/librrd.pod +++ b/doc/librrd.pod @@ -81,6 +81,16 @@ end of the new C<dest>. Returns 1 on success, 0 on failure. if (!rrd_add_ptr(&arr, &arr_size, elem)) handle_failure(); +=item B<rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, size_t *alloc, size_t chunk)> + +Like C<rrd_add_ptr>, except the destination is allocated in chunks of +C<chunk>. C<alloc> points to the number of entries allocated, whereas +C<dest_size> points to the number of valid pointers. If more pointers are +needed, C<chunk> pointers are allocated and C<alloc> is increased +accordingly. C<alloc> must be E<gt>= C<dest_size>. + +This method improves performance on hosts with expensive C<realloc()>. + =item B<rrd_add_strdup(char ***dest, size_t *dest_size, char *src)> Like C<rrd_add_ptr>, except adds a C<strdup> of the source string. @@ -91,6 +101,14 @@ Like C<rrd_add_ptr>, except adds a C<strdup> of the source string. if (!rrd_add_strdup(&arr, &arr_size, str)) handle_failure(); +=item B<rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, size_t *alloc, size_t chunk)> + +Like C<rrd_add_strdup>, except the destination is allocated in chunks of +C<chunk>. C<alloc> points to the number of entries allocated, whereas +C<dest_size> points to the number of valid pointers. If more pointers are +needed, C<chunk> pointers are allocated and C<alloc> is increased +accordingly. C<alloc> must be E<gt>= C<dest_size>. + =item B<rrd_free_ptrs(void ***src, size_t *cnt)> Free an array of pointers allocated by C<rrd_add_ptr> or diff --git a/src/rrd.h b/src/rrd.h index ce7e749..6008794 100644 --- a/src/rrd.h +++ b/src/rrd.h @@ -333,8 +333,12 @@ int rrd_proc_start_end( long rrd_random(void); + int rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, + size_t *alloc, size_t chunk); int rrd_add_ptr(void ***dest, size_t *dest_size, void *src); int rrd_add_strdup(char ***dest, size_t *dest_size, char *src); + int rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, + size_t *alloc, size_t chunk); void rrd_free_ptrs(void ***src, size_t *cnt); int rrd_mkdir_p(const char *pathname, mode_t mode); diff --git a/src/rrd_utils.c b/src/rrd_utils.c index 3936cff..6853c66 100644 --- a/src/rrd_utils.c +++ b/src/rrd_utils.c @@ -50,29 +50,53 @@ long rrd_random(void) return random(); } -/* rrd_add_ptr: add a pointer to a dynamically sized array of pointers, - * realloc as necessary. returns 1 on success, 0 on failure. +/* rrd_add_ptr_chunk: add a pointer to a dynamically sized array of + * pointers, realloc as necessary in multiples of "chunk". + * + * "alloc" is the number of pointers allocated + * "dest_size" is the number of valid pointers + * + * returns 1 on success, 0 on failure. */ -int rrd_add_ptr(void ***dest, size_t *dest_size, void *src) +int rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, + size_t *alloc, size_t chunk) { void **temp; assert(dest != NULL); + assert(alloc != NULL); + assert(*alloc >= *dest_size); - temp = (void **) rrd_realloc(*dest, (*dest_size+1) * sizeof(*dest)); - if (!temp) - return 0; + if (*alloc == *dest_size) + { + temp = (void **) rrd_realloc(*dest, (*alloc+chunk) * sizeof(*dest)); + if (!temp) + return 0; - *dest = temp; - temp[*dest_size] = src; + *dest = temp; + *alloc += chunk; + } + + (*dest)[*dest_size] = src; (*dest_size)++; return 1; } -/* like rrd_add_ptr, but calls strdup() on a string first. */ -int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) +/* rrd_add_ptr: add a pointer to a dynamically sized array of pointers, + * realloc as necessary. returns 1 on success, 0 on failure. + */ +int rrd_add_ptr(void ***dest, size_t *dest_size, void *src) +{ + size_t alloc = *dest_size; + + return rrd_add_ptr_chunk(dest, dest_size, src, &alloc, 1); +} + +/* like rrd_add_ptr_chunk, but calls strdup() on a string first. */ +int rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, + size_t *alloc, size_t chunk) { char *dup_src; int add_ok; @@ -84,13 +108,20 @@ int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) if (!dup_src) return 0; - add_ok = rrd_add_ptr((void ***)dest, dest_size, (void *)dup_src); + add_ok = rrd_add_ptr_chunk((void ***)dest, dest_size, (void *)dup_src, alloc, chunk); if (!add_ok) free(dup_src); return add_ok; } +int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) +{ + size_t alloc = *dest_size; + + return rrd_add_strdup_chunk(dest, dest_size, src, &alloc, 1); +} + void rrd_free_ptrs(void ***src, size_t *cnt) { void **sp; -- 1.6.4 _______________________________________________ rrd-developers mailing list [hidden email] https://lists.oetiker.ch/cgi-bin/listinfo/rrd-developers |
||||||||||||||||
|
kevin brintnall
|
---
doc/rrdcached.pod | 9 +++++++++ src/rrd_daemon.c | 23 ++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/doc/rrdcached.pod b/doc/rrdcached.pod index 043e020..5e8728f 100644 --- a/doc/rrdcached.pod +++ b/doc/rrdcached.pod @@ -18,6 +18,7 @@ B<rrdcached> [-F] [-g] [B<-b>E<nbsp>I<base_dir>E<nbsp>[B<-B>]] +[B<-m>E<nbsp>I<alloc_size>] =head1 DESCRIPTION @@ -197,6 +198,14 @@ Only permit writes into the base directory specified in B<-b> (and any sub-directories). This does B<NOT> detect symbolic links. Paths containing C<../> will also be blocked. +=item B<-m> I<alloc_size> + +Allocate value pointers in chunks of I<alloc_size>. This may improve CPU +utilization on machines with slow C<realloc()> implementations, in +exchange for slightly higher memory utilization. The default isE<nbsp>1. +Do not set this more than the B<-w> value divided by your average RRD step +size. + =back =head1 AFFECTED RRDTOOL COMMANDS diff --git a/src/rrd_daemon.c b/src/rrd_daemon.c index 4c84f19..79ea17b 100644 --- a/src/rrd_daemon.c +++ b/src/rrd_daemon.c @@ -174,7 +174,8 @@ struct cache_item_s { char *file; char **values; - size_t values_num; + size_t values_num; /* number of valid pointers */ + size_t values_alloc; /* number of allocated pointers */ time_t last_flush_time; time_t last_update_stamp; #define CI_FLAGS_IN_TREE (1<<0) @@ -251,6 +252,7 @@ static char *config_pid_file = NULL; static char *config_base_dir = NULL; static size_t _config_base_dir_len = 0; static int config_write_base_only = 0; +static size_t config_alloc_chunk = 1; static listen_socket_t **config_listen_address_list = NULL; static size_t config_listen_address_list_len = 0; @@ -638,6 +640,7 @@ static void wipe_ci_values(cache_item_t *ci, time_t when) { ci->values = NULL; ci->values_num = 0; + ci->values_alloc = 0; ci->last_flush_time = when; if (config_write_jitter > 0) @@ -1434,7 +1437,8 @@ static int handle_request_update (HANDLER_PROTO) /* {{{ */ else ci->last_update_stamp = stamp; - if (!rrd_add_strdup(&ci->values, &ci->values_num, value)) + if (!rrd_add_strdup_chunk(&ci->values, &ci->values_num, value, + &ci->values_alloc, config_alloc_chunk)) { RRDD_LOG (LOG_ERR, "handle_request_update: rrd_add_strdup failed."); continue; @@ -2746,7 +2750,7 @@ static int read_options (int argc, char **argv) /* {{{ */ char **permissions = NULL; size_t permissions_len = 0; - while ((option = getopt(argc, argv, "gl:P:f:w:z:t:Bb:p:Fj:h?")) != -1) + while ((option = getopt(argc, argv, "gl:P:f:w:z:t:Bb:p:Fj:m:h?")) != -1) { switch (option) { @@ -3003,6 +3007,19 @@ static int read_options (int argc, char **argv) /* {{{ */ } break; + case 'm': + { + int temp = atoi(optarg); + if (temp > 0) + config_alloc_chunk = temp; + else + { + fprintf(stderr, "Invalid allocation size: %s\n", optarg); + status = 10; + } + } + break; + case 'h': case '?': printf ("RRDCacheD %s\n" -- 1.6.4 _______________________________________________ rrd-developers mailing list [hidden email] https://lists.oetiker.ch/cgi-bin/listinfo/rrd-developers |
||||||||||||||||
|
oetiker
|
In reply to this post
by kevin brintnall
Hi Kevin,
since 1.4.x is now out, I would like to hold off on adding new public functions to the 1.4 branche ... these would go into the new trunk. or is there also a bug-fix component to this change ? cheers tobi Oct 31 kevin brintnall wrote: > --- > doc/librrd.pod | 18 ++++++++++++++++++ > src/rrd.h | 4 ++++ > src/rrd_utils.c | 53 ++++++++++++++++++++++++++++++++++++++++++----------- > 3 files changed, 64 insertions(+), 11 deletions(-) > > diff --git a/doc/librrd.pod b/doc/librrd.pod > index 038746c..396e50c 100644 > --- a/doc/librrd.pod > +++ b/doc/librrd.pod > @@ -81,6 +81,16 @@ end of the new C<dest>. Returns 1 on success, 0 on failure. > if (!rrd_add_ptr(&arr, &arr_size, elem)) > handle_failure(); > > +=item B<rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, size_t *alloc, size_t chunk)> > + > +Like C<rrd_add_ptr>, except the destination is allocated in chunks of > +C<chunk>. C<alloc> points to the number of entries allocated, whereas > +C<dest_size> points to the number of valid pointers. If more pointers are > +needed, C<chunk> pointers are allocated and C<alloc> is increased > +accordingly. C<alloc> must be E<gt>= C<dest_size>. > + > +This method improves performance on hosts with expensive C<realloc()>. > + > =item B<rrd_add_strdup(char ***dest, size_t *dest_size, char *src)> > > Like C<rrd_add_ptr>, except adds a C<strdup> of the source string. > @@ -91,6 +101,14 @@ Like C<rrd_add_ptr>, except adds a C<strdup> of the source string. > if (!rrd_add_strdup(&arr, &arr_size, str)) > handle_failure(); > > +=item B<rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, size_t *alloc, size_t chunk)> > + > +Like C<rrd_add_strdup>, except the destination is allocated in chunks of > +C<chunk>. C<alloc> points to the number of entries allocated, whereas > +C<dest_size> points to the number of valid pointers. If more pointers are > +needed, C<chunk> pointers are allocated and C<alloc> is increased > +accordingly. C<alloc> must be E<gt>= C<dest_size>. > + > =item B<rrd_free_ptrs(void ***src, size_t *cnt)> > > Free an array of pointers allocated by C<rrd_add_ptr> or > diff --git a/src/rrd.h b/src/rrd.h > index ce7e749..6008794 100644 > --- a/src/rrd.h > +++ b/src/rrd.h > @@ -333,8 +333,12 @@ int rrd_proc_start_end( > > long rrd_random(void); > > + int rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, > + size_t *alloc, size_t chunk); > int rrd_add_ptr(void ***dest, size_t *dest_size, void *src); > int rrd_add_strdup(char ***dest, size_t *dest_size, char *src); > + int rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, > + size_t *alloc, size_t chunk); > void rrd_free_ptrs(void ***src, size_t *cnt); > > int rrd_mkdir_p(const char *pathname, mode_t mode); > diff --git a/src/rrd_utils.c b/src/rrd_utils.c > index 3936cff..6853c66 100644 > --- a/src/rrd_utils.c > +++ b/src/rrd_utils.c > @@ -50,29 +50,53 @@ long rrd_random(void) > return random(); > } > > -/* rrd_add_ptr: add a pointer to a dynamically sized array of pointers, > - * realloc as necessary. returns 1 on success, 0 on failure. > +/* rrd_add_ptr_chunk: add a pointer to a dynamically sized array of > + * pointers, realloc as necessary in multiples of "chunk". > + * > + * "alloc" is the number of pointers allocated > + * "dest_size" is the number of valid pointers > + * > + * returns 1 on success, 0 on failure. > */ > > -int rrd_add_ptr(void ***dest, size_t *dest_size, void *src) > +int rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, > + size_t *alloc, size_t chunk) > { > void **temp; > > assert(dest != NULL); > + assert(alloc != NULL); > + assert(*alloc >= *dest_size); > > - temp = (void **) rrd_realloc(*dest, (*dest_size+1) * sizeof(*dest)); > - if (!temp) > - return 0; > + if (*alloc == *dest_size) > + { > + temp = (void **) rrd_realloc(*dest, (*alloc+chunk) * sizeof(*dest)); > + if (!temp) > + return 0; > > - *dest = temp; > - temp[*dest_size] = src; > + *dest = temp; > + *alloc += chunk; > + } > + > + (*dest)[*dest_size] = src; > (*dest_size)++; > > return 1; > } > > -/* like rrd_add_ptr, but calls strdup() on a string first. */ > -int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) > +/* rrd_add_ptr: add a pointer to a dynamically sized array of pointers, > + * realloc as necessary. returns 1 on success, 0 on failure. > + */ > +int rrd_add_ptr(void ***dest, size_t *dest_size, void *src) > +{ > + size_t alloc = *dest_size; > + > + return rrd_add_ptr_chunk(dest, dest_size, src, &alloc, 1); > +} > + > +/* like rrd_add_ptr_chunk, but calls strdup() on a string first. */ > +int rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, > + size_t *alloc, size_t chunk) > { > char *dup_src; > int add_ok; > @@ -84,13 +108,20 @@ int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) > if (!dup_src) > return 0; > > - add_ok = rrd_add_ptr((void ***)dest, dest_size, (void *)dup_src); > + add_ok = rrd_add_ptr_chunk((void ***)dest, dest_size, (void *)dup_src, alloc, chunk); > if (!add_ok) > free(dup_src); > > return add_ok; > } > > +int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) > +{ > + size_t alloc = *dest_size; > + > + return rrd_add_strdup_chunk(dest, dest_size, src, &alloc, 1); > +} > + > void rrd_free_ptrs(void ***src, size_t *cnt) > { > void **sp; > -- Tobi Oetiker, OETIKER+PARTNER AG, Aarweg 15 CH-4600 Olten, Switzerland http://it.oetiker.ch [hidden email] ++41 62 775 9902 / sb: -9900 _______________________________________________ rrd-developers mailing list [hidden email] https://lists.oetiker.ch/cgi-bin/listinfo/rrd-developers |
||||||||||||||||
|
kevin brintnall
|
On Mon, Nov 09, 2009 at 09:57:43AM +0100, Tobias Oetiker wrote:
> Hi Kevin, > > since 1.4.x is now out, I would like to hold off on adding new > public functions to the 1.4 branche ... these would go into the new > trunk. or is there also a bug-fix component to this change ? Tobi, This improves performance for implementations where realloc() may be expensive.. It causes the realloc() for cache_item_t.values to be in larger chunks, and less frequent. This is in response to Thorsten's performance testing, where he sees CPU spikes at specific times after start-up, when his cache_item_t.values arrays all simultaneously cross some size threshold. Up to you. -- kevin brintnall =~ /[hidden email]/ > cheers > tobi > > Oct 31 kevin brintnall wrote: > > > --- > > doc/librrd.pod | 18 ++++++++++++++++++ > > src/rrd.h | 4 ++++ > > src/rrd_utils.c | 53 ++++++++++++++++++++++++++++++++++++++++++----------- > > 3 files changed, 64 insertions(+), 11 deletions(-) > > > > diff --git a/doc/librrd.pod b/doc/librrd.pod > > index 038746c..396e50c 100644 > > --- a/doc/librrd.pod > > +++ b/doc/librrd.pod > > @@ -81,6 +81,16 @@ end of the new C<dest>. Returns 1 on success, 0 on failure. > > if (!rrd_add_ptr(&arr, &arr_size, elem)) > > handle_failure(); > > > > +=item B<rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, size_t *alloc, size_t chunk)> > > + > > +Like C<rrd_add_ptr>, except the destination is allocated in chunks of > > +C<chunk>. C<alloc> points to the number of entries allocated, whereas > > +C<dest_size> points to the number of valid pointers. If more pointers are > > +needed, C<chunk> pointers are allocated and C<alloc> is increased > > +accordingly. C<alloc> must be E<gt>= C<dest_size>. > > + > > +This method improves performance on hosts with expensive C<realloc()>. > > + > > =item B<rrd_add_strdup(char ***dest, size_t *dest_size, char *src)> > > > > Like C<rrd_add_ptr>, except adds a C<strdup> of the source string. > > @@ -91,6 +101,14 @@ Like C<rrd_add_ptr>, except adds a C<strdup> of the source string. > > if (!rrd_add_strdup(&arr, &arr_size, str)) > > handle_failure(); > > > > +=item B<rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, size_t *alloc, size_t chunk)> > > + > > +Like C<rrd_add_strdup>, except the destination is allocated in chunks of > > +C<chunk>. C<alloc> points to the number of entries allocated, whereas > > +C<dest_size> points to the number of valid pointers. If more pointers are > > +needed, C<chunk> pointers are allocated and C<alloc> is increased > > +accordingly. C<alloc> must be E<gt>= C<dest_size>. > > + > > =item B<rrd_free_ptrs(void ***src, size_t *cnt)> > > > > Free an array of pointers allocated by C<rrd_add_ptr> or > > diff --git a/src/rrd.h b/src/rrd.h > > index ce7e749..6008794 100644 > > --- a/src/rrd.h > > +++ b/src/rrd.h > > @@ -333,8 +333,12 @@ int rrd_proc_start_end( > > > > long rrd_random(void); > > > > + int rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, > > + size_t *alloc, size_t chunk); > > int rrd_add_ptr(void ***dest, size_t *dest_size, void *src); > > int rrd_add_strdup(char ***dest, size_t *dest_size, char *src); > > + int rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, > > + size_t *alloc, size_t chunk); > > void rrd_free_ptrs(void ***src, size_t *cnt); > > > > int rrd_mkdir_p(const char *pathname, mode_t mode); > > diff --git a/src/rrd_utils.c b/src/rrd_utils.c > > index 3936cff..6853c66 100644 > > --- a/src/rrd_utils.c > > +++ b/src/rrd_utils.c > > @@ -50,29 +50,53 @@ long rrd_random(void) > > return random(); > > } > > > > -/* rrd_add_ptr: add a pointer to a dynamically sized array of pointers, > > - * realloc as necessary. returns 1 on success, 0 on failure. > > +/* rrd_add_ptr_chunk: add a pointer to a dynamically sized array of > > + * pointers, realloc as necessary in multiples of "chunk". > > + * > > + * "alloc" is the number of pointers allocated > > + * "dest_size" is the number of valid pointers > > + * > > + * returns 1 on success, 0 on failure. > > */ > > > > -int rrd_add_ptr(void ***dest, size_t *dest_size, void *src) > > +int rrd_add_ptr_chunk(void ***dest, size_t *dest_size, void *src, > > + size_t *alloc, size_t chunk) > > { > > void **temp; > > > > assert(dest != NULL); > > + assert(alloc != NULL); > > + assert(*alloc >= *dest_size); > > > > - temp = (void **) rrd_realloc(*dest, (*dest_size+1) * sizeof(*dest)); > > - if (!temp) > > - return 0; > > + if (*alloc == *dest_size) > > + { > > + temp = (void **) rrd_realloc(*dest, (*alloc+chunk) * sizeof(*dest)); > > + if (!temp) > > + return 0; > > > > - *dest = temp; > > - temp[*dest_size] = src; > > + *dest = temp; > > + *alloc += chunk; > > + } > > + > > + (*dest)[*dest_size] = src; > > (*dest_size)++; > > > > return 1; > > } > > > > -/* like rrd_add_ptr, but calls strdup() on a string first. */ > > -int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) > > +/* rrd_add_ptr: add a pointer to a dynamically sized array of pointers, > > + * realloc as necessary. returns 1 on success, 0 on failure. > > + */ > > +int rrd_add_ptr(void ***dest, size_t *dest_size, void *src) > > +{ > > + size_t alloc = *dest_size; > > + > > + return rrd_add_ptr_chunk(dest, dest_size, src, &alloc, 1); > > +} > > + > > +/* like rrd_add_ptr_chunk, but calls strdup() on a string first. */ > > +int rrd_add_strdup_chunk(char ***dest, size_t *dest_size, char *src, > > + size_t *alloc, size_t chunk) > > { > > char *dup_src; > > int add_ok; > > @@ -84,13 +108,20 @@ int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) > > if (!dup_src) > > return 0; > > > > - add_ok = rrd_add_ptr((void ***)dest, dest_size, (void *)dup_src); > > + add_ok = rrd_add_ptr_chunk((void ***)dest, dest_size, (void *)dup_src, alloc, chunk); > > if (!add_ok) > > free(dup_src); > > > > return add_ok; > > } > > > > +int rrd_add_strdup(char ***dest, size_t *dest_size, char *src) > > +{ > > + size_t alloc = *dest_size; > > + > > + return rrd_add_strdup_chunk(dest, dest_size, src, &alloc, 1); > > +} > > + > > void rrd_free_ptrs(void ***src, size_t *cnt) > > { > > void **sp; > > > > -- > Tobi Oetiker, OETIKER+PARTNER AG, Aarweg 15 CH-4600 Olten, Switzerland > http://it.oetiker.ch [hidden email] ++41 62 775 9902 / sb: -9900 _______________________________________________ rrd-developers mailing list [hidden email] https://lists.oetiker.ch/cgi-bin/listinfo/rrd-developers |
||||||||||||||||
| Free Embeddable Forum Powered by Nabble | Help |