[feature request] trackmax for all parameters (Was: [hobbit] [patch] split-ncv and maximum value tracking)

Gildas Le Nadan gn1 at sanger.ac.uk
Fri Jan 19 10:31:10 CET 2007


Hi,

The TRACKMAX feature is really interesting as the max values get 
"diluted" in the week/month/year views.

Alas, it seem to only work for NCV values at the moment (and I wish I 
could use it for cpu/memory/network/disk).

Charles, Henrik, do you think this is feasible?

Cheers,
Gildas

Charles Goyard wrote:
> Hi again,
> 
> of course I blew it up :)
> 
> The second hunk is a bit from another new feature, so here's another
> patch.
> 
> Sorry for the inconvenience.
> 
> 
> 
> ------------------------------------------------------------------------
> 
> diff -u -x .svn -x '*~' -x hobbitd.c -x Makefile -rw upstream-2006-10-03/hobbitd/do_rrd.c cvf/hobbitd/do_rrd.c
> --- upstream-2006-10-03/hobbitd/do_rrd.c	2006-11-24 17:18:09.000000000 +0100
> +++ cvf/hobbitd/do_rrd.c	2006-12-15 15:47:58.000000000 +0100
> @@ -29,6 +29,7 @@
>  #include "do_rrd.h"
>  
>  char *rrddir = NULL;
> +char *trackmax = NULL;
>  static char *exthandler = NULL;
>  static char **extids = NULL;
>  
> @@ -37,11 +38,16 @@
>  static char rra2[] = "RRA:AVERAGE:0.5:6:576";
>  static char rra3[] = "RRA:AVERAGE:0.5:24:576";
>  static char rra4[] = "RRA:AVERAGE:0.5:288:576";
> +static char rra5[] = "RRA:MAX:0.5:1:576";
> +static char rra6[] = "RRA:MAX:0.5:6:576";
> +static char rra7[] = "RRA:MAX:0.5:24:576";
> +static char rra8[] = "RRA:MAX:0.5:288:576";
>  
>  static char *senderip = NULL;
>  static char rrdfn[PATH_MAX];	/* This one used by the modules */
>  static char filedir[PATH_MAX];	/* This one used here */
>  
> +
>  void setup_exthandler(char *handlerpath, char *ids)
>  {
>  	char *p;
> @@ -100,6 +106,24 @@
>  	rrdfn[sizeof(rrdfn)-1] = '\0';
>  }
>  
> +static int has_trackmax(char *testname)
> +{
> +	char testnamecoma[strlen(testname)+3];
> +
> +	if(trackmax == NULL) {
> +		return 0;
> +	}
> +	
> +	sprintf(testnamecoma, ",%s,", testname);
> +	if(strstr(trackmax, testnamecoma)) {
> +		return 1;
> +	}
> +	else {
> +		return 0;
> +	}
> +}
> +
> +
>  static int create_and_update_rrd(char *hostname, char *fn, char *creparams[], char *template)
>  {
>  	struct stat st;
> diff -u -x .svn -x '*~' -x hobbitd.c -x Makefile -rw upstream-2006-10-03/hobbitd/do_rrd.h cvf/hobbitd/do_rrd.h
> --- upstream-2006-10-03/hobbitd/do_rrd.h	2006-11-24 17:18:09.000000000 +0100
> +++ cvf/hobbitd/do_rrd.h	2006-11-24 17:55:53.000000000 +0100
> @@ -16,6 +16,7 @@
>  #include "libbbgen.h"
>  
>  extern char *rrddir;
> +extern char *trackmax;
>  extern void setup_exthandler(char *handlerpath, char *ids);
>  extern void update_rrd(char *hostname, char *testname, char *restofmsg, time_t tstamp, char *sender, hobbitrrd_t *ldef);
>  
> Only in cvf/hobbitd/: hobbitd_cvfinventaire.c
> diff -u -x .svn -x '*~' -x hobbitd.c -x Makefile -rw upstream-2006-10-03/hobbitd/hobbitd_rrd.8 cvf/hobbitd/hobbitd_rrd.8
> --- upstream-2006-10-03/hobbitd/hobbitd_rrd.8	2006-11-24 17:18:09.000000000 +0100
> +++ cvf/hobbitd/hobbitd_rrd.8	2006-11-24 17:55:53.000000000 +0100
> @@ -65,6 +65,16 @@
>  Defines the types of data collected by the "ncv" module in hobbitd_rrd.
>  See below for more information.
>  
> +.IP SPLITNCV_testname
> +The same as NCV_testname, but keeps the data into separate files. That
> +is, it creates one rrd file per "NAME : value" line found in the
> +status message. It is useful when the list of NCV lines is varying.
> +
> +.IP TRACKMAX
> +Comma-separated list of columnname for which you want to keep the
> +maximum values along with the default average values. This only works
> + for the NCV backend.
> +
>  .SH COLLECTED DATA
>  The following RRD-file datasets are generated by hobbitd_rrd:
>  
> @@ -152,18 +162,22 @@
>  or "NAME = value". So a generic module in hobbitd_rrd allows for
>  easy tracking of this type of data.
>  
> -The "ncv" module will automatically detect all occurrences of
> -a "NAME : value" or "NAME = value" string in a status message, and
> -generate an RRD file holding all of the name/value data found in
> -the message. The colon- or equal-sign must be present - if there is
> -only whitespace, this module will fail.
> + The "ncv" module will automatically detect all occurrences of a "NAME
> + : value" or "NAME = value" string in a status message, and generate an
> + RRD file holding all of the name/value data found in the message
> + (unless you use SPLITNCV, see above). The colon- or equal-sign must be
> + present - if there is only whitespace, this module will fail.
>  
>  Only the valid letters (A-Z, a-z) and digits (0-9) are used in the 
>  dataset names; whitespace and other characters are stripped off 
> -automatically. Only the first 19 characters of a dataset name are
> -used (this is an RRD limitation). Underscore '_' is not allowed,
> -even though RRDtool permits this, and will be stripped from the
> -name.
> + automatically. Only the first 19 characters of a dataset name are used
> + (this is an RRD limitation). Underscore '_' is not allowed, even
> + though RRDtool permits this, and will be stripped from the name.
> +
> + When using the alternative SPLITNCV_testname, the dataset name is not
> + limited in length, and non-valid characters are changed to underscores
> + instead of being stripped off. The dataset inside the resulting rrd
> + file is always "lambda.
>  
>  Note that each "NAME : value" must be on a line by itself. If you have
>  a custom script generating the status- or data-message that is fed
> @@ -177,12 +191,13 @@
>  all status-messages for the column COLUMNNAME through the hobbitd_rrd
>  ncv-handler.
>  
> -The name of the RRD file will be COLUMNNAME.rrd.
> +The name of the RRD file will be COLUMNNAME.rrd. When using SPLITNCV,
> +the name of the RRD file will be COLUMNAME,DATASETNAME.rrd.
>  
>  By default, all of the datasets are generated as the RRD type "DERIVE"
>  which works for all types of monotonically increasing counters. If you 
>  have data that are of the type GAUGE, you can override the default via
> -an environment variable NCV_COLUMNNAME. 
> +an environment variable NCV_COLUMNNAME (or SPLITNCV_COLUMNAME). 
>  
>  E.g. if you are using the bb-mysqlstatus script from www.deadcat.net to 
>  collect data about your MySQL server, it generates a report in the column 
> diff -u -x .svn -x '*~' -x hobbitd.c -x Makefile -rw upstream-2006-10-03/hobbitd/hobbitd_rrd.c cvf/hobbitd/hobbitd_rrd.c
> --- upstream-2006-10-03/hobbitd/hobbitd_rrd.c	2006-11-24 17:18:09.000000000 +0100
> +++ cvf/hobbitd/hobbitd_rrd.c	2006-12-28 18:35:27.000000000 +0100
> @@ -74,6 +74,13 @@
>  		rrddir = strdup(xgetenv("BBRRDS"));
>  	}
>  
> +	/* trackmax initialization */
> +	if(getenv("TRACKMAX")) {
> +		trackmax = (char*)malloc(strlen(getenv("TRACKMAX"))+3);
> +		sprintf(trackmax, ",%s,", getenv("TRACKMAX"));
> +		dbgprintf("Will track max for: %s\n", trackmax);
> +	}
> +	
>  	save_errbuf = 0;
>  	setup_signalhandler("hobbitd_rrd");
>  	memset(&sa, 0, sizeof(sa));
> @@ -139,7 +146,7 @@
>  				break;
>  
>  			  default:
> -				/* Ignore reports with purple, blue or clear - they have no data we want. */
> +				/* Ignore reports with purple or clear - they have no data we want. */
>  				break;
>  			}
>  		}
> diff -u -x .svn -x '*~' -x hobbitd.c -x Makefile -rw upstream-2006-10-03/hobbitd/rrd/do_ncv.c cvf/hobbitd/rrd/do_ncv.c
> --- upstream-2006-10-03/hobbitd/rrd/do_ncv.c	2006-11-24 17:18:57.000000000 +0100
> +++ cvf/hobbitd/rrd/do_ncv.c	2006-12-28 18:42:58.000000000 +0100
> @@ -5,6 +5,7 @@
>  /*     NAME: VALUE                                                            */
>  /*                                                                            */
>  /* Copyright (C) 2004-2006 Henrik Storner <henrik at hswn.dk>                    */
> +/* split-ncv added by Charles Goyard November 2006                            */
>  /*                                                                            */
>  /* This program is released under the GNU General Public License (GPL),       */
>  /* version 2. See the file "COPYING" for details.                             */
> @@ -17,21 +18,30 @@
>  { 
>  	char **params = NULL;
>  	int paridx;
> -	char dsdef[1024];
> +	char dsdef[1024];     /* destination DS syntax for rrd engine */
>  	char *l, *name, *val;
>  	char *envnam;
> -	char *dstypes = NULL;
> -
> -	setupfn("%s.rrd", testname);
> +	char *dstypes = NULL; /* contain NCV_testname value*/
> +	int split_ncv = 0;
> +	int dslen;
>  	sprintf(rrdvalues, "%d", (int)tstamp);
> -
>  	params = (char **)calloc(8, sizeof(char *));
>  	params[0] = "rrdcreate";
> -	params[1] = rrdfn;
>  	paridx = 1;
>  
> -	envnam = (char *)malloc(4 + strlen(testname) + 1); sprintf(envnam, "NCV_%s", testname);
> +	envnam = (char *)malloc(9 + strlen(testname) + 1);
> +	sprintf(envnam, "SPLITNCV_%s", testname);
> +	l = getenv(envnam);
> +	if (l) {
> +		split_ncv = 1;
> +		dslen = 200;
> +	}
> +	else {
> +		dslen = 19;
> +		setupfn("%s.rrd", testname);
> +		sprintf(envnam, "NCV_%s", testname);
>  	l = getenv(envnam);
> +	}
>  	if (l) {
>  		dstypes = (char *)malloc(strlen(l)+3);
>  		sprintf(dstypes, ",%s,", l);
> @@ -84,62 +94,128 @@
>  
>  			strtod(val, &endptr);
>  			if (isspace((int)*endptr) || (*endptr == '\0')) {
> -				char dsname[20];
> -				char dskey[22];
> -				char *dstype = NULL;
> +				char dsname[250];    /* name of ncv in status message (with space and al) */
> +				char dskey[252];     /* name of final DS key (stripped)                   */
> +				char *dstype = NULL; /* type of final DS                                  */
>  				char *inp;
>  				int outidx;
> -
>  				/* val contains a valid number */
> -				/* rrdcreate(1) says: ds must be in the set [a-zA-Z0-9_] */
> -				for (inp=name,outidx=0; (*inp && (outidx < 19)); inp++) {
> +				/* rrdcreate(1) says: ds must be in the set [a-zA-Z0-9_] ... */
> +				for (inp=name,outidx=0; (*inp && (outidx < dslen)); inp++) {
>  					if ( ((*inp >= 'A') && (*inp <= 'Z')) ||
>  					     ((*inp >= 'a') && (*inp <= 'z')) ||
>  					     ((*inp >= '0') && (*inp <= '9'))    ) {
>  						dsname[outidx++] = *inp;
>  					}
> +					/* ... however, for split ncv, we replace anything else  */
> +					/* with an underscore, compacting successive invalid     */
> +					/* characters into a single one                          */
> +					else if (split_ncv && (dsname[outidx - 1] != '_')) {
> +						dsname[outidx++] = '_';
> +					}
>  				}
> +				if(dsname[outidx-1] == '_') {
> +					dsname[outidx-1] = '\0';
> +				}
> +				else {
>  				dsname[outidx] = '\0';
> +				}
>  				sprintf(dskey, ",%s:", dsname);
> +				if(split_ncv) {
> +					/* setupfn("%s,%s.rrd", testname, dsname); */
> +					snprintf(rrdfn, sizeof(rrdfn)-1, "%s,%s.rrd", testname,dsname);
> +					rrdfn[sizeof(rrdfn)-1] = '\0';
> +					
> +					params[1] = rrdfn;
> +					paridx = 1;
> +				}
>  
>  				if (dstypes) {
>  					dstype = strstr(dstypes, dskey);
> -					if (!dstype) { strcpy(dskey, ",*:"); dstype = strstr(dstypes, dskey); }
> +					if (!dstype) {
> +						strcpy(dskey, ",*:");
> +						dstype = strstr(dstypes, dskey);
> +					}
>  				}
>  
> -				if (dstype) {
> +				if (dstype) { /* if ds type is forced */
>  					char *p;
>  
>  					dstype += strlen(dskey);
>  					p = strchr(dstype, ','); if (p) *p = '\0';
> +					if(split_ncv) {
> +						sprintf(dsdef, "DS:lambda:%s:600:0:U", dstype);
> +					}
> +					else {
>  					sprintf(dsdef, "DS:%s:%s:600:0:U", dsname, dstype);
> +					}
>  					if (p) *p = ',';
>  				}
> +				else { /* nothing specified in the environnement, and no '*:' default */
> +					if(split_ncv) {
> +						strcpy(dsdef, "DS:lambda:DERIVE:600:0:U");
> +					}
>  				else {
>  					sprintf(dsdef, "DS:%s:DERIVE:600:0:U", dsname);
>  				}
> +				}
>  
> -				if (!dstype || (strncasecmp(dstype, "NONE", 4) != 0)) {
> +				if (!dstype || (strncasecmp(dstype, "NONE", 4) != 0)) { /* if we have something */
>  					paridx++;
>  					params = (char **)realloc(params, (7 + paridx)*sizeof(char *));
>  					params[paridx] = strdup(dsdef);
>  					params[paridx+1] = NULL;
> -
>  					sprintf(rrdvalues+strlen(rrdvalues), ":%s", val);
>  				}
>  			}
> +			
> +			if(split_ncv && (paridx > 1)) {
> +				int i;
> +				params[++paridx] = strdup(rra1);
> +				params[++paridx] = strdup(rra2);
> +				params[++paridx] = strdup(rra3);
> +				params[++paridx] = strdup(rra4);
> +
> +				if(has_trackmax(testname)) {
> +					params = (char **)realloc(params, (11 + paridx)*sizeof(char *));
> +					params[++paridx] = strdup(rra5);
> +					params[++paridx] = strdup(rra6);
> +					params[++paridx] = strdup(rra7);
> +					params[++paridx] = strdup(rra8);
> +				}
> +
> +				params[++paridx] = NULL;
> +				create_and_update_rrd(hostname, rrdfn, params, NULL);
> +				for(i = 2 ; i<paridx ; i++) {
> +					params[i] = NULL;
> +				}
> +				sprintf(rrdvalues, "%d", (int)tstamp);
>  		}
>  	}
>  
> -	if (paridx > 1) {
> +	} /* end of while */
> +
> +	if (split_ncv) {
> +		for (paridx=2; (params[paridx] != NULL); paridx++) {
> +			xfree(params[paridx]);
> +		}
> +	}
> +	else if(paridx > 1) {
>  		params[++paridx] = strdup(rra1);
>  		params[++paridx] = strdup(rra2);
>  		params[++paridx] = strdup(rra3);
>  		params[++paridx] = strdup(rra4);
> -		params[++paridx] = NULL;
>  
> -		create_and_update_rrd(hostname, rrdfn, params, NULL);
> +		if(has_trackmax(testname)) {
> +			params = (char **)realloc(params, (11 + paridx)*sizeof(char *));
> +			params[++paridx] = strdup(rra5);
> +			params[++paridx] = strdup(rra6);
> +			params[++paridx] = strdup(rra7);
> +			params[++paridx] = strdup(rra8);
> +		}
>  
> +		params[++paridx] = NULL;
> +		create_and_update_rrd(hostname, rrdfn, params, NULL);
>  		for (paridx=2; (params[paridx] != NULL); paridx++)
>  		xfree(params[paridx]);
>  	}
> 
> 
> 
> ------------------------------------------------------------------------
> 
> To unsubscribe from the hobbit list, send an e-mail to
> hobbit-unsubscribe at hswn.dk



More information about the Xymon mailing list