From 9cfdddf49df7d5f47691ca467d9fbae51bfd71a0 Mon Sep 17 00:00:00 2001 From: Jonathan Prykop Date: Mon, 6 Apr 2015 22:01:05 -0500 Subject: [PATCH] RT#18834: Cacti integration [database storage] --- FS/FS/Cron/cacti_cleanup.pm | 19 +++ FS/FS/Schema.pm | 27 +++- FS/FS/cacti_page.pm | 135 +++++++++++++++++ FS/FS/part_export/cacti.pm | 251 ++++++++++++++++++------------- FS/bin/freeside-daily | 5 + httemplate/misc/cacti_graphs.html | 42 ++++-- httemplate/misc/process/cacti_graphs.cgi | 3 + 7 files changed, 356 insertions(+), 126 deletions(-) create mode 100644 FS/FS/Cron/cacti_cleanup.pm create mode 100644 FS/FS/cacti_page.pm diff --git a/FS/FS/Cron/cacti_cleanup.pm b/FS/FS/Cron/cacti_cleanup.pm new file mode 100644 index 000000000..f86262790 --- /dev/null +++ b/FS/FS/Cron/cacti_cleanup.pm @@ -0,0 +1,19 @@ +package FS::Cron::cacti_cleanup; +use base 'Exporter'; +use vars '@EXPORT_OK'; + +use FS::Record qw( qsearch ); +use Data::Dumper; + +@EXPORT_OK = qw( cacti_cleanup ); + +sub cacti_cleanup { + foreach my $export (qsearch({ + 'table' => 'part_export', + 'hashref' => { 'exporttype' => 'cacti' } + })) { + $export->cleanup; + } +} + +1; diff --git a/FS/FS/Schema.pm b/FS/FS/Schema.pm index 4bc3598fe..839a97176 100644 --- a/FS/FS/Schema.pm +++ b/FS/FS/Schema.pm @@ -203,6 +203,7 @@ sub dbdef_dist { && ! /^legacy_cust_history$/ && ( ! /^queue(_arg|_depend|_stat)?$/ || ! $opt->{'queue-no_history'} ) && ! $tables_hashref_torrus->{$_} + && ! /^cacti_graph$/ } $dbdef->tables ) { @@ -7007,9 +7008,29 @@ sub tables_hashref { ], }, - - - + 'cacti_page' => { + 'columns' => [ + 'cacti_pagenum', 'serial', '', '', '', '', + 'exportnum', 'int', 'NULL', '', '', '', + 'svcnum', 'int', 'NULL', '', '', '', + 'graphnum', 'int', 'NULL', '', '', '', + 'imported', @date_type, '', '', + 'content', 'text', 'NULL', '', '', '', + ], + 'primary_key' => 'cacti_pagenum', + 'unique' => [ ], + 'index' => [ ['svcnum'], ['imported'] ], + 'foreign_keys' => [ + { columns => [ 'svcnum' ], + table => 'cust_svc', + references => [ 'svcnum' ], + }, + { columns => [ 'exportnum' ], + table => 'part_export', + references => [ 'exportnum' ], + }, + ], + }, # name type nullability length default local diff --git a/FS/FS/cacti_page.pm b/FS/FS/cacti_page.pm new file mode 100644 index 000000000..febcae411 --- /dev/null +++ b/FS/FS/cacti_page.pm @@ -0,0 +1,135 @@ +package FS::cacti_page; +use base qw( FS::Record ); + +use strict; +use FS::Record qw( qsearch qsearchs ); + +=head1 NAME + +FS::cacti_page - Object methods for cacti_page records + +=head1 SYNOPSIS + + use FS::cacti_page; + + $record = new FS::cacti_page \%hash; + $record = new FS::table_name { + 'exportnum' => 3, #part_export associated with this page + 'svcnum' => 123, #svc_broadband associated with this page + 'graphnum' => 45, #blank for svcnum index + 'imported' => 1428358699, #date of import + 'content' => $htmlcontent, #html containing base64-encoded images + }; + + $error = $record->insert; + + $error = $new_record->replace($old_record); + + $error = $record->delete; + + $error = $record->check; + +=head1 DESCRIPTION + +An FS::cacti_page object represents an html page for viewing cacti graphs. +FS::cacti_page inherits from FS::Record. The following fields are currently supported: + +=over 4 + +=item cacti_pagenum - primary key + +=item exportnum - part_export exportnum for this page + +=item svcnum - svc_broadband svcnum for this page + +=item graphnum - cacti graphnum for this page (blank for overview page) + +=item imported - date this page was imported + +=item content - text/html content of page, should not include newlines + +=back + +=head1 METHODS + +=over 4 + +=item new HASHREF + +Creates a new object. To add the object to the database, see L<"insert">. + +Note that this stores the hash reference, not a distinct copy of the hash it +points to. You can ask the object for a copy with the I method. + +=cut + +# the new method can be inherited from FS::Record, if a table method is defined + +sub table { 'cacti_page'; } + +=item insert + +Adds this record to the database. If there is an error, returns the error, +otherwise returns false. + +=cut + +# the insert method can be inherited from FS::Record + +=item delete + +Delete this record from the database. + +=cut + +# the delete method can be inherited from FS::Record + +=item replace OLD_RECORD + +Replaces the OLD_RECORD with this one in the database. If there is an error, +returns the error, otherwise returns false. + +=cut + +# the replace method can be inherited from FS::Record + +=item check + +Checks all fields to make sure this is a valid example. If there is +an error, returns the error, otherwise returns false. Called by the insert +and replace methods. + +=cut + +# the check method should currently be supplied - FS::Record contains some +# data checking routines + +sub check { + my $self = shift; + + my $error = + $self->ut_numbern('cacti_pagenum', 'graphnum') + || $self->ut_foreign_key('exportnum','part_export','exportnum') + || $self->ut_foreign_key('svcnum','cust_svc','svcnum') + || $self->ut_number('imported') + || $self->ut_text('content') + ; + return $error if $error; + + $self->SUPER::check; +} + +=back + +=head1 BUGS + +Will be described here once found. + +=head1 SEE ALSO + +L + +=cut + +1; + diff --git a/FS/FS/part_export/cacti.pm b/FS/FS/part_export/cacti.pm index 1f5f64c2a..abeb5e4d7 100644 --- a/FS/FS/part_export/cacti.pm +++ b/FS/FS/part_export/cacti.pm @@ -1,13 +1,31 @@ package FS::part_export::cacti; +=pod + +=head1 NAME + +FS::part_export::cacti + +=head1 SYNOPSIS + +Cacti integration for Freeside + +=head1 DESCRIPTION + +This module in particular handles FS::part_export object creation for Cacti integration; +consult any existing L documentation for details on how that works. + +=cut + use strict; use base qw( FS::part_export ); -use FS::Record qw( qsearchs ); +use FS::Record qw( qsearchs qsearch ); use FS::UID qw( dbh ); +use FS::cacti_page; use File::Rsync; -use File::Slurp qw( append_file slurp write_file ); +use File::Slurp qw( slurp ); use File::stat; use MIME::Base64 qw( encode_base64 ); @@ -24,8 +42,8 @@ tie my %options, 'Tie::IxHash', default => '' }, 'tree_id' => { label => 'Graph Tree ID (optional)', default => '' }, - 'description' => { label => 'Description (can use $ip_addr and $description tokens)', - default => 'Freeside $description $ip_addr' }, + 'description' => { label => 'Description (can use tokens $contact, $ip_addr and $description)', + default => 'Freeside $contact $description $ip_addr' }, 'graphs_path' => { label => 'Graph Export Directory (user@host:/path/to/graphs/)', default => '' }, 'import_freq' => { label => 'Minimum minutes between graph imports', @@ -43,7 +61,7 @@ tie my %options, 'Tie::IxHash', 'options' => \%options, 'notes' => <<'END', Add service to cacti upon provisioning, for broadband services.
-See FS::part_export::cacti documentation for details. +See documentation for details. END ); @@ -113,6 +131,7 @@ sub _insert_queue { 'tree_id' => $self->option('tree_id'), 'description' => $self->option('description'), 'svc_desc' => $svc_broadband->description, + 'contact' => $svc_broadband->cust_main->contact, 'svcnum' => $svc_broadband->svcnum, ); return ($queue,$error); @@ -143,12 +162,13 @@ sub ssh_insert { die "Non-numerical Host Template ID, check export configuration\n" unless $opt{'template_id'} =~ /^\d+$/; die "Non-numerical Graph Tree ID, check export configuration\n" - unless $opt{'tree_id'} =~ /^\d+$/; + unless $opt{'tree_id'} =~ /^\d*$/; # Add host to cacti my $desc = $opt{'description'}; $desc =~ s/\$ip_addr/$opt{'hostname'}/g; $desc =~ s/\$description/$opt{'svc_desc'}/g; + $desc =~ s/\$contact/$opt{'contact'}/g; $desc =~ s/'/'\\''/g; my $cmd = $php . $opt{'script_path'} @@ -238,11 +258,24 @@ sub ssh_delete { return ''; } -# NOT A METHOD, run as an FS::queue job -# copies graphs for a single service from Cacti export directory to FS cache -# generates basic html pages for this service's graphs, and stores them in FS cache +=head1 SUBROUTINES + +=over 4 + +=item process_graphs JOB PARAM + +Intended to be run as an FS::queue job. + +Copies graphs for a single service from Cacti export directory to FS cache, +generates basic html pages for this service with base64-encoded graphs embedded, +and stores the generated pages in the database. + +=back + +=cut + sub process_graphs { - my ($job,$param) = @_; # + my ($job,$param) = @_; $job->update_statustext(10); my $cachedir = $FS::UID::cache_dir . '/cacti-graphs/'; @@ -259,23 +292,37 @@ sub process_graphs { $job->update_statustext(20); - # check for recent uploads, avoid doing this too often - my $svchtml = $cachedir.'svc_'.$svcnum.'.html'; - if (-e $svchtml) { - open(my $fh, "<$svchtml"); - my $firstline = <$fh>; - close($fh); - if ($firstline =~ /UPDATED (\d+)/) { - if ($1 > time - 60 * ($self->option('import_freq') || 5)) { - $job->update_statustext(100); - return ''; + my $oldAutoCommit = $FS::UID::AutoCommit; + local $FS::UID::AutoCommit = 0; + my $dbh = dbh; + + # check for existing pages + my $now = time; + my @oldpages = qsearch({ + 'table' => 'cacti_page', + 'hashref' => { 'svcnum' => $svcnum, 'exportnum' => $self->exportnum }, + 'select' => 'cacti_pagenum, exportnum, svcnum, graphnum, imported', #no need to load old content + 'order_by' => 'ORDER BY graphnum', + }); + if (@oldpages) { + #if pages are recent enough, do nothing and return + if ($oldpages[0]->imported > $self->exptime($now)) { + $job->update_statustext(100); + return ''; + } + #delete old pages + foreach my $oldpage (@oldpages) { + my $error = $oldpage->delete; + if ($error) { + $dbh->rollback if $oldAutoCommit; + die $error; } } } $job->update_statustext(30); - # get list of graphs for this svc + # get list of graphs for this svc from cacti server my $cmd = $php . $self->option('script_path') . q(freeside_cacti.php --get-graphs --ip=') @@ -290,7 +337,7 @@ sub process_graphs { $job->update_statustext(40); - # copy graphs to cache + # copy graphs from cacti server to cache # requires version 2.6.4 of rsync, released March 2005 my $rsync = File::Rsync->new({ 'rsh' => 'ssh', @@ -311,12 +358,11 @@ sub process_graphs { $job->update_statustext(50); - # create html files in cache - my $now = time; - my $svchead = q(\n) - . '

Service #' . $svcnum . '

' . "\n" - . q(

Last updated ) . scalar(localtime($now)) . q(

) . "\n"; - write_file($svchtml,$svchead); + # create html file contents + my $svchead = q() + . '

Service #' . $svcnum . '

' + . q(

Last updated ) . scalar(localtime($now)) . q(

); + my $svchtml = $svchead; my $maxgraph = 1024 * 1024 * ($self->options('max_graph_size') || 5); my $nographs = 1; for (my $i = 0; $i <= $#graphs; $i++) { @@ -328,31 +374,53 @@ sub process_graphs { ) { $nographs = 0; # add graph to main file - my $graphhead = q(

) . $$graph[1] . q(

) . "\n"; - append_file( $svchtml, $graphhead, - anchor_tag( - $svcnum, $$graph[0], img_tag($thumbfile) - ) - ); + my $graphhead = q(

) . $$graph[1] . q(

); + $svchtml .= $graphhead; + $svchtml .= anchor_tag( $svcnum, $$graph[0], img_tag($thumbfile) ); # create graph details file - my $graphhtml = $cachedir . 'svc_' . $svcnum . '_graph_' . $$graph[0] . '.html'; - write_file($graphhtml,$svchead,$graphhead); + my $graphhtml = $svchead . $graphhead; my $nodetail = 1; my $j = 1; while (-e (my $graphfile = $cachedir.'graphs/graph_'.$$graph[0].'_'.$j.'.png')) { if ( stat($graphfile)->size() < $maxgraph ) { $nodetail = 0; - append_file( $graphhtml, img_tag($graphfile) ); + $graphhtml .= img_tag($graphfile); } $j++; } - append_file($graphhtml, '

No detail graphs to display for this graph

') + $graphhtml .= '

No detail graphs to display for this graph

' if $nodetail; + my $newobj = new FS::cacti_page { + 'exportnum' => $self->exportnum, + 'svcnum' => $svcnum, + 'graphnum' => $$graph[0], + 'imported' => $now, + 'content' => $graphhtml, + }; + $error = $newobj->insert; + if ($error) { + $dbh->rollback if $oldAutoCommit; + die $error; + } } - $job->update_statustext(50 + ($i / $#graphs) * 50); + $job->update_statustext(49 + int($i / $#graphs) * 50); } - append_file($svchtml,'

No graphs to display for this service

') + $svchtml .= '

No graphs to display for this service

' if $nographs; + my $newobj = new FS::cacti_page { + 'exportnum' => $self->exportnum, + 'svcnum' => $svcnum, + 'graphnum' => '', + 'imported' => $now, + 'content' => $svchtml, + }; + $error = $newobj->insert; + if ($error) { + $dbh->rollback if $oldAutoCommit; + die $error; + } + + $dbh->commit or die $dbh->errstr if $oldAutoCommit; $job->update_statustext(100); return ''; @@ -361,8 +429,8 @@ sub process_graphs { sub img_tag { my $somefile = shift; return q(
\n); + . encode_base64(slurp($somefile,binmode=>':raw'),'') + . qq(" STYLE="margin-bottom: 1em;">
); } sub anchor_tag { @@ -389,82 +457,51 @@ sub ssh_cmd { return $output; } -=pod - -=head1 NAME - -FS::part_export::cacti - -=head1 SYNOPSIS - -Cacti integration for Freeside - -=head1 DESCRIPTION - -This module in particular handles FS::part_export object creation for Cacti integration; -consult any existing L documentation for details on how that works. -What follows is more general instructions for connecting your Cacti installation -to your Freeside installation. - -=head2 Connecting Cacti To Freeside - -Copy the freeside_cacti.php script from the bin directory of your Freeside -installation to the cli directory of your Cacti installation. Give this file -the same permissions as the other files in that directory, and create -(or choose an existing) user with sufficient permission to read these scripts. - -In the regular Cacti interface, create a Host Template to be used by -devices exported by Freeside, and note the template's id number. Optionally, -create a Graph Tree for these devices to be automatically added to, and note -the tree's id number. Configure a Graph Export (under Settings) and note -the Export Directory. - -In Freeside, go to Configuration->Services->Provisioning exports to -add a new export. From the Add Export page, select cacti for Export then enter... - -* the Hostname or IP address of your Cacti server - -* the User Name with permission to run scripts in the cli directory - -* the full Script Path to that directory (eg /usr/share/cacti/cli/) +=head1 METHODS -* the Host Template ID for adding new devices +=over 4 -* the Graph Tree ID for adding new devices (optional) +=item cleanup -* the Description for new devices; you can use the tokens - $ip_addr and $description to include the equivalent fields - from the broadband service definition +Removes all expired graphs for this export from the database. -* the Graph Export Directory, including connection information - if necessary (user@host:/path/to/graphs/) +=cut -* the minimum minutes between graph imports to Freeside (graphs will - otherwise be imported into Freeside as needed.) This should be at least - as long as the minumum time between graph exports configured in Cacti. - Defaults to 5 if unspecified. +sub cleanup { + my $self = shift; + my $oldAutoCommit = $FS::UID::AutoCommit; + local $FS::UID::AutoCommit = 0; + my $dbh = dbh; + my $sth = $dbh->prepare('DELETE FROM cacti_page WHERE exportnum = ? and imported <= ?') + or do { + $dbh->rollback if $oldAutoCommit; + return $dbh->errstr; + }; + $sth->execute($self->exportnum,$self->exptime) + or do { + $dbh->rollback if $oldAutoCommit; + return $dbh->errstr; + }; + $dbh->commit or return $dbh->errstr if $oldAutoCommit; + return ''; +} -* the maximum size per graph, in MB; individual graphs that exceed this size - will be quietly ignored by Freeside. Defaults to 5 if unspecified. +=item exptime [ TIME ] -After adding the export, go to Configuration->Services->Service definitions. -The export you just created will be available for selection when adding or -editing broadband service definitions; check the box to activate it for -a given service. Note that you should only have one cacti export per -broadband service definition. +Accepts optional current time, defaults to actual current time. -When properly configured broadband services are provisioned, they will now -be added to Cacti using the Host Template you specified. If you also specified -a Graph Tree, the created device will also be added to that. +Returns timestamp for the oldest possible non-expired graph import, +based on the import_freq option. -Once added, a link to the graphs for this host will be available when viewing -the details of the provisioned service in Freeside. +=cut -Devices will be deleted from Cacti when the service is unprovisioned in Freeside, -and they will be deleted and re-added if the ip address changes. +sub exptime { + my $self = shift; + my $now = shift || time; + return $now - 60 * ($self->option('import_freq') || 5); +} -Currently, graphs themselves must still be added in Cacti by hand or some -other form of automation tailored to your specific graph inputs and data sources. +=back =head1 AUTHOR diff --git a/FS/bin/freeside-daily b/FS/bin/freeside-daily index af48ec0cb..bf9f17728 100755 --- a/FS/bin/freeside-daily +++ b/FS/bin/freeside-daily @@ -83,6 +83,11 @@ export_batch_submit(%opt); use FS::Cron::agent_email qw(agent_email); agent_email(%opt); +#does nothing unless there are cacti imports +#should run before backup, no need to backup cacti imports +use FS::Cron::cacti_cleanup qw(cacti_cleanup); +cacti_cleanup(); + my $deldir = "$FS::UID::cache_dir/cache.$FS::UID::datasrc/"; unlink <${deldir}.invoice*>; unlink <${deldir}.letter*>; diff --git a/httemplate/misc/cacti_graphs.html b/httemplate/misc/cacti_graphs.html index 9cc5e2494..90a435091 100644 --- a/httemplate/misc/cacti_graphs.html +++ b/httemplate/misc/cacti_graphs.html @@ -21,33 +21,43 @@ process(); -% } else { -% if ($error) { +% } elsif ($error) { -

<% $error %>

+

<% emt($error) %>

+
+ + + +
-% } else { +% } else { -<% slurp($htmlfile) %> +<% $content %> -% } % } <%init> use File::Slurp qw( slurp ); +die "access denied" + unless $FS::CurrentUser::CurrentUser->access_right('View customer services'); + my $svcnum = $cgi->param('svcnum') or die 'Illegal svcnum'; my $load = $cgi->param('load'); -my $graphnum = $cgi->param('graphnum'); - -my $htmlfile = $FS::UID::cache_dir - . '/cacti-graphs/' - . 'svc_' - . $svcnum; -$htmlfile .= '_graph_' . $graphnum - if $graphnum; -$htmlfile .= '.html'; +my $graphnum = $cgi->param('graphnum') || ''; + +my ($content,$error); +unless ($load) { + my $page = qsearchs({ + 'table' => 'cacti_page', + 'hashref' => { 'svcnum' => $svcnum, 'graphnum' => $graphnum }, + }); + if ($page) { + $content = $page->content; + } else { + $error = 'No graphs found in import cache. Click below to retry import.'; + } +} -my $error = (-e $htmlfile) ? '' : 'File not found'; diff --git a/httemplate/misc/process/cacti_graphs.cgi b/httemplate/misc/process/cacti_graphs.cgi index 160b1ad85..f2baeb455 100644 --- a/httemplate/misc/process/cacti_graphs.cgi +++ b/httemplate/misc/process/cacti_graphs.cgi @@ -1,6 +1,9 @@ <% $server->process %> <%init> +die "access denied" + unless $FS::CurrentUser::CurrentUser->access_right('View customer services'); + my $server = FS::UI::Web::JSRPC->new('FS::part_export::cacti::process_graphs', $cgi); -- 2.11.0