[Xymon] Capture and send data To Elasticsearch and Kibana
Mario
rower.master at gmail.com
Tue Sep 27 16:48:46 CEST 2022
Hi Francois,
I´m collecting data using your script and I have installed both
elasticsearch and kibana.
I´m not very familiar with kibana and elasticsearch yet.
Could you please help me to configure ELK and Kibana to read
the xymond_capture_php.log?
Thanks & regards,
Mario
On Sun, Dec 25, 2016 at 9:15 AM OUTTERYCK Francois <
francois.outteryck at soprasteria.com> wrote:
> Hi Peter,
>
>
>
> Examples of task.cfg configuration with status and stachg events :
>
>
>
>
>
> [capture_status]
>
> ENVFILE /home/xymon/server/etc/xymonserver.cfg
>
> NEED xymond
>
> CMD xymond_channel --channel=status
> /home/xymon/server/ext/capture.sh
> --log=$XYMONSERVERLOGS/xymond_task_capture.log
>
> LOGFILE /var/log/xymon/lunch_capture.log
>
>
>
> [capture_stachg]
>
> ENVFILE /home/xymon/server/etc/xymonserver.cfg
>
> NEED xymond
>
> CMD xymond_channel --channel=stachg
> /home/xymon/server/ext/capture.sh
> --log=$XYMONSERVERLOGS/xymond_task_capture_stachg.log
>
> LOGFILE /var/log/xymon/lunch_capture_stachg.log
>
>
>
>
>
> Capture.sh script simple made to call a php script :
>
>
>
> #!/bin/sh
>
> /usr/bin/php /var/www/kibana/capture.php $@ </dev/stdin
> >>/var/log/xymon/xymond_capture_php.log 2>&1
>
>
>
>
>
> And capture.php make all job :
>
>
>
> <?php
>
> date_default_timezone_set('Europe/Paris');
>
> $TODO=FALSE;
>
> $ARRAY_COLLECTOR=array();
>
> $COLLECTOR=null;
>
>
>
> while (FALSE !== ($line = fgets(STDIN))) if ($line!='') {
>
> $tab=reduce_array($line);
>
> if (sizeof($tab)==0) continue;
>
> if ($line=="@@\n") {
>
> if ($TODO) {
>
> $metadata['Message']=implode('',$ARRAY_COLLECTOR);
>
> SendData($id,$metadata,$COLLECTOR);
>
> $TODO=FALSE;
>
> $COLLECTOR=null;
>
> }
>
> } else if (substr($tab[0],0,8)=='@@stachg') {
>
> if ($TODO) {
>
> $metadata['Message']=implode('',$ARRAY_COLLECTOR);
>
> SendData($id,$metadata,$COLLECTOR);
>
> $TODO=FALSE;
>
> $COLLECTOR=null;
>
> }
>
> /* 0 1 2 3 4 5
> 6 7 8 9 10 11 12
> 13 14 15 16
>
> /*
> @@stachg#seq|timestamp|sender|origin|hostname|testname|expiretime|color|prevcolor|changetime|disabletime|disablemsg|downtimeactive|clienttstamp|modifiers
> */
>
>
>
> $COLLECTOR='stachg';
>
> $metadata=explode('|',$tab[0]);
>
> $sender=$metadata[2];
>
> $hostname=$metadata[3];
>
> $metadata=array(
>
> 'Collector'=>$COLLECTOR,
>
> 'Sender'=>$metadata[2],
>
> 'ServerName'=>$metadata[4],
>
> 'Type'=>$metadata[5],
>
> 'Status'=>$metadata[7],
>
> 'PrevStatus'=>$metadata[8],
>
> 'LogTime'=>$metadata[1],
>
> 'ValidTime'=>$metadata[6],
>
> 'LastChange'=>$metadata[9],
>
> 'DisableTime'=>$metadata[10],
>
> 'DisableMSg'=>$metadata[11],
>
> 'Message'=>''
>
> );
>
>
> $id=$metadata['ServerName'].'_'.$metadata['Type'].'_'.$meta['LastChange'];
>
> $ARRAY_COLLECTOR=null;
>
> $ARRAY_COLLECTOR=array();
>
> $TODO=TRUE;
>
> } else if (substr($tab[0],0,8)=='@@status') {
>
> if ($TODO) {
>
> $metadata['Message']=implode('',$ARRAY_COLLECTOR);
>
> SendData($id,$metadata,$COLLECTOR);
>
> $TODO=FALSE;
>
> $COLLECTOR=null;
>
> }
>
> /* 0 1 2 3 4 5
> 6 7 8 9 10 11 12
> 13 14 15 16
>
> /*
> @@status|timestamp|sender|origin|hostname|testname|expiretime|color|testflags|prevcolor|changetime|ackexpiretime|ackmessage|disableexpiretime|disablemessage|clientmsgtstamp|flapping
> */
>
> $COLLECTOR='status';
>
> $metadata=explode('|',$tab[0]);
>
> $sender=$metadata[2];
>
> $hostname=$metadata[3];
>
> $metadata=array(
>
> 'Collector'=>$COLLECTOR,
>
> 'Sender'=>$metadata[2],
>
> 'ServerName'=>$metadata[4],
>
> 'Type'=>$metadata[5],
>
> 'Status'=>$metadata[7],
>
> 'LogTime'=>$metadata[1],
>
> 'ValidTime'=>$metadata[6],
>
> 'LastChange'=>$metadata[10],
>
> 'DisableTime'=>$metadata[13],
>
> 'AckTime'=>$metadata[11],
>
> 'DisableMSg'=>$metadata[14],
>
> 'Message'=>''
>
> );
>
> $id=$metadata['ServerName'].'.'.$metadata['Type'];
>
> $ARRAY_COLLECTOR=null;
>
> $ARRAY_COLLECTOR=array();
>
> $TODO=TRUE;
>
> } else {
>
> if (substr($tab[0],0,2)=='@@') {
>
> if ($TODO) {
>
> $metadata['Message']=implode('',$ARRAY_COLLECTOR);
>
> SendData($id,$metadata,$COLLECTOR);
>
> $TODO=FALSE;
>
> $COLLECTOR=null;
>
> }
>
> }
>
> if ($TODO) $ARRAY_COLLECTOR[]=$line;
>
> }
>
> }
>
>
>
> function CallAPI($method, $url, $data = false)
>
> {
>
> $curl = curl_init();
>
> switch ($method)
>
> {
>
> case "POST":
>
> curl_setopt($curl, CURLOPT_POST, 1);
>
> if ($data)
>
> curl_setopt($curl, CURLOPT_POSTFIELDS, $data);
>
> break;
>
> case "PUT":
>
> curl_setopt($curl, CURLOPT_PUT, 1);
>
> break;
>
> default:
>
> if ($data)
>
> $url = sprintf("%s?%s", $url, http_build_query($data));
>
> }
>
> curl_setopt($curl, CURLOPT_URL, $url);
>
> curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
>
> $result = curl_exec($curl);
>
> curl_close($curl);
>
> return $result;
>
> }
>
>
>
> function reduce_array($line) {
>
> $tab=explode(' ',$line);
>
> $nr=array();
>
> for ($i=0;$i<sizeof($tab);$i++) if (trim($tab[$i])!='')
> $nr[]=trim($tab[$i]);
>
> return $nr;
>
> }
>
>
>
> function SendData($id,$cc,$COLLECTOR=null) {
>
> if ($COLLECTOR==null) return;
>
> $d=new DateTime();
>
> if ($cc['LastChange']!='')
> {$d->setTimestamp($cc['LastChange']);$cc['LastChange']=$d->format('c');}
> else {$cc['LastChange']='NaN';}
>
> if ($cc['LogTime']!='')
> {$d->setTimestamp($cc['LogTime']);$cc['LogTime']=$d->format('c');} else
> {$cc['LogTime']='NaN';}
>
> if ($cc['ValidTime']!='')
> {$d->setTimestamp($cc['ValidTime']);$cc['ValidTime']=$d->format('c');} else
> {$cc['ValidTime']='NaN';}
>
> if (isset($cc['AckTime'])) {
>
> if ($cc['AckTime']!='')
> {$d->setTimestamp($cc['AckTime']);$cc['AckTime']=$d->format('c');} else
> {$cc['AckTime']='NaN';}
>
> }
>
> if ($cc['DisableTime']!='')
> {$d->setTimestamp($cc['DisableTime']);$cc['DisableTime']=$d->format('c'); }
> else {$cc['DisableTime']='NaN';}
>
> $data = json_encode($cc);
>
> $data=str_replace('"NaN"','null',$data);
>
> echo $id.'=>'.$data."\n";
>
> $url="http://127.0.0.1:9200/xymon/".$COLLECTOR."/".$id;
>
> $res=CallAPI("POST",$url,$data);
>
> }
>
> ?>
>
>
>
> *De :* Peter Welter [mailto:peter.welter at gmail.com]
> *Envoyé :* vendredi 23 décembre 2016 14:45
> *À :* OUTTERYCK Francois
> *Cc :* xymon at xymon.com
> *Objet :* Re: [Xymon] Capture and send data To Elasticsearch and Kibana
>
>
>
> Hi Francois,
>
>
>
> Looks good, your sample. I am very interested in your integration efforts.
>
>
>
> How do you select what is going to be inserted in the ELK stack? All
> status messages that Xymon receives?
>
>
>
> And could you provide an example of the xymond_channel block in you
> task.cfg and the capture.sh?
>
>
>
> Tasks.cfg probably is something like:
>
>
>
> [es]
>
> ...
>
> CMD xymond_channel --channel=status /home/xymon/server/ext/capture.sh
>
> ...
>
>
>
> And the capture.sh contains some curl script to the ELK stack?
>
>
>
> Regards,
>
>
>
> Peter
>
>
>
> 2016-12-14 18:15 GMT+01:00 OUTTERYCK Francois <
> francois.outteryck at soprasteria.com>:
>
> Hello, I just do a small work with Xymon in order to send status data to
> Elasticsearch. And with Kibana reporting is easy.
>
>
>
> How it work ? Simple : With capture of status message (xymond_channel
> --channel=status /home/xymon/server/ext/capture.sh ) script encode status
> message in json and send as document in elasticSearch.
>
>
>
> Example of Mapping document status in elasticsearch :
>
>
>
> "status"=> array(
>
> "properties"=>array(
>
> "ServerName"=> array("type"=> "string"),
>
> "Type"=> array("type"=> "string"),
>
> "Status"=> array("type"=> "string"),
>
> "LastChange"=> array("type"=> "date","format"=>
> "strict_date_optional_time"),
>
> "LogTime"=> array("type"=> "date","format"=>
> "strict_date_optional_time"),
>
> "ValidTime"=> array("type"=> "date","format"=>
> "strict_date_optional_time"),
>
> "AckTime"=> array("type"=> "date","format"=>
> "strict_date_optional_time"),
>
> "DisableTime"=> array("type"=> "date","format"=>
> "strict_date_optional_time"),
>
> "Sender"=> array("type"=> "string"),
>
> "DisableMSg"=> array("type"=> "string"),
>
> "Message"=> array("type"=> "string")
>
> )
>
> )
>
>
>
> All is done with a 130 lines script. So simple to integrate in existing
> environment and so powerfull usage of Xymon/elasticsearch/Kibana
>
>
>
> For those who don’t know kibana : https://www.elastic.co/products/kibana
>
>
>
>
>
>
> _______________________________________________
> Xymon mailing list
> Xymon at xymon.com
> http://lists.xymon.com/mailman/listinfo/xymon
>
>
> _______________________________________________
> Xymon mailing list
> Xymon at xymon.com
> http://lists.xymon.com/mailman/listinfo/xymon
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.xymon.com/pipermail/xymon/attachments/20220927/990b4660/attachment.htm>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: image001.png
Type: image/png
Size: 52950 bytes
Desc: not available
URL: <http://lists.xymon.com/pipermail/xymon/attachments/20220927/990b4660/attachment.png>
More information about the Xymon
mailing list