From: Jeremy Davis Date: Mon, 16 Feb 2015 15:59:56 +0000 (-0500) Subject: CDR import scripts converted to real commands X-Git-Url: http://git.freeside.biz/gitweb/?p=freeside.git;a=commitdiff_plain;h=fa0e442fa889a49a0b7517567cb20beedc31c5aa CDR import scripts converted to real commands --- diff --git a/FS/bin/freeside-vitelity-cdrimport b/FS/bin/freeside-vitelity-cdrimport new file mode 100755 index 000000000..83bbdd96e --- /dev/null +++ b/FS/bin/freeside-vitelity-cdrimport @@ -0,0 +1,132 @@ +#!/usr/bin/perl + +=pod + +freeside-vitelity-cdrimport [ -v ] [ -k ] + -s date -e date + username + [ exportnum ] + +Download CDRs using the Vitelity API. + +-v: Be verbose. + +-k: Keep the .csv file for debugging purposes, instead of deleting it. + +-s date: Import only records on or after 'date' Now required as the Vitelity +API has changed. + +-e date: Import only records before 'date'. Now required as the Vitelity API +has changed. + +username: a Freeside user + +exportnum: Run only for that export. If not specified, this will run for +all Vitelity exports. + +=cut + +use strict; +use FS::UID qw(adminsuidsetup dbh); +use FS::Record qw(qsearchs qsearch); +use FS::cdr; +use FS::part_export; +use Getopt::Std; +use File::Temp; +use Date::Format 'time2str'; +use Date::Parse 'str2time'; + +my %opt; +getopts('vks:e:', \%opt); + +my $user = shift or die &usage; +my $exportnum = shift; +my $dbh = adminsuidsetup $user; + +my $start = $opt{'s'} ? str2time($opt{'s'}) : die &usage('-s is now required'); +my $end = $opt{'e'} ? str2time($opt{'e'}) : die &usage('-e is now required'); + +local $FS::UID::AutoCommit = 0; + +my @part_exports; +if ( $exportnum ) { + @part_exports = ( qsearchs('part_export', { 'exportnum' => $exportnum }) ) + or die "exportnum $exportnum not found\n"; +} +else { + @part_exports = qsearch('part_export', { 'exporttype' => 'vitelity' }) + or die "no Vitelity exports found\n"; +} + +foreach my $export (@part_exports) { + my $exportnum = $export->exportnum; + print "Processing exportnum $exportnum.\n" if $opt{'v'}; + $export->isa('FS::part_export::vitelity') + or die "exportnum $exportnum is not a Vitelity export\n"; + + my $dir = $FS::UID::cache_dir . "/cache.". $FS::UID::datasrc; + my $temp = new File::Temp ( TEMPLATE => 'download.XXXXXXXX', + SUFFIX => '.csv', + DIR => $dir, + UNLINK => !$opt{'k'} ) + or die "can't open temporary file to store download: $!\n"; + print "Downloading to ".$temp->filename."\n" if $opt{'v'}; + + print "Sending API request..." if $opt{'v'}; + + my $s = time2str('%m-%d-%Y', $start); + my $e = time2str('%m-%d-%Y', $end); + + my @records = eval { $export->vitelity_command('getcdr', + 'startdate' => $s, + 'enddate' => $e, + ); + }; + if ( $@ ) { + print "download error: $@\n"; + exit(-1); + } + + print "received ".scalar(@records)." records\n" if $opt{'v'}; + if ( !@records ) { + print "No records to process.\n" if $opt{'v'}; + exit(1); + } + + print $temp "Date,Source,Destination,Seconds,CallerID,Disposition,Cost\n"; + + while (my $rec = shift @records) { + print $temp $rec, "\n"; + } + close $temp; + + my $format = 'vitelity'; + my $batchname = "vitelity-$exportnum-".time2str('%Y/%m/%d-%T',time); + + print "Importing batch..." if $opt{'v'}; + my $error = FS::cdr::batch_import( { + 'file' => $temp->filename, + 'format' => $format, + 'batch_namevalue' => $batchname, + } ); + + if ( $error ) { + $dbh->rollback; + print "error: $error"; + exit(-2); + } +} +$dbh->commit; +print "done.\n"; +exit(0); + +sub usage { + my $err = @_ ? shift."\n\n" : ''; +$err."Usage: +freeside-vitelity-cdrimport [ -v ] [ -k ] + -s date -e date + username + [ exportnum ] +"; +} + diff --git a/FS/bin/freeside-voipinnovations-cdrimport b/FS/bin/freeside-voipinnovations-cdrimport new file mode 100755 index 000000000..484b330f6 --- /dev/null +++ b/FS/bin/freeside-voipinnovations-cdrimport @@ -0,0 +1,129 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; +use Date::Format; +use File::Temp 'tempdir'; +use Net::FTP; +use FS::UID qw(adminsuidsetup datasrc dbh); +use FS::cdr; +use FS::cdr_batch; +use FS::Record qw(qsearch qsearchs); +use Date::Format 'time2str'; +use Date::Parse 'str2time'; + + +### +# parse command line +### + +use vars qw( $opt_d $opt_v $opt_c $opt_s $opt_e $opt_a ); +getopts('dvc:s:e:a'); + +my ($user, $login, $password) = @ARGV; +$user and $login and $password or die &usage; + +my $dbh = adminsuidsetup $user; +$FS::UID::AutoCommit = 0; + +# index already-downloaded batches +my @previous = qsearch({ + 'table' => 'cdr_batch', + 'hashref' => { 'cdrbatch' => {op=>'like', value=>'voip_innovations%'} }, + 'order_by' => 'ORDER BY cdrbatch DESC', +}); +my %exists = map {$_->cdrbatch => 1} @previous; + +my $tempdir = tempdir( CLEANUP => !$opt_v ); + +my $format = 'voip_innovations'; +my $hostname = 'cdrs.globalpopsvoip.com'; + +my $ftp = Net::FTP->new($hostname, Debug => $opt_d) + or die "Can't connect to $hostname: $@\n"; + +$ftp->login($login, $password) + or die "Login failed: ".$ftp->message."\n"; + +### +# get the file list +### + +warn "Retrieving directory listing\n" if $opt_v; + +$ftp->cwd('/'); +my @dirs = $ftp->ls(); +warn scalar(@dirs)." directories found.\n" if $opt_v; +# apply date range +if ( $opt_a ) { + my $most_recent = $previous[0]; + if ($most_recent) { + if ($most_recent->cdrbatch =~ /^voip_innovations-(\d+)/) { + my $date = $1; + warn "limiting to dates > $date (from most recent batch)\n" if $opt_v; + @dirs = grep {$_ > $date} @dirs; + } + } # else download them all +} +if ( $opt_s ) { + # start date + # normalize date format + $opt_s = time2str('%Y%m%d', str2time($opt_s)) if $opt_s =~ /\D/; + warn "limiting to dates > $opt_s\n" if $opt_v; + @dirs = grep {$_ > $opt_s} @dirs; +} +if ( $opt_e ) { + # end date + $opt_e = time2str('%Y%m%d', str2time($opt_e)) if $opt_e =~ /\D/; + warn "limiting to dates < $opt_e\n" if $opt_v; + @dirs = grep {$_ < $opt_e} @dirs; +} +warn scalar(@dirs) ." to be downloaded\n" if $opt_v; +foreach my $dir (@dirs) { + $ftp->cwd($dir); + foreach my $file ($ftp->ls) { + warn "downloading $file\n" if $opt_v; + $ftp->get($file, "$tempdir/$file"); + warn "processing $file\n" if $opt_v; + + # "voip_innovations-20130628/20130628_20130628.CDR" + my $batchname = "$format-$dir/$file"; + if ($exists{$batchname}) { + warn "already imported $file\n"; + next; + } + my $import_options = { + 'file' => "$tempdir/$file", + 'format' => $format, + 'batch_namevalue' => $batchname, + 'empty_ok' => 1, + }; + $import_options->{'cdrtypenum'} = $opt_c if $opt_c; + + my $error = FS::cdr::batch_import($import_options); + + if ( $error ) { + die "error processing $dir/$file: $error\n"; + } + } + $ftp->cwd('..'); +} +warn "finished\n"; +$dbh->commit; + +### +# subs +### + +sub usage { + "Usage: \n freeside-voip_innovations-cdrimport user login password\n [ options ] + Options: + -v: be verbose + -d: enable FTP debugging (very noisy) + -c num: apply a cdrtypenum to the imported CDRs + -s date: start date + -e date: end date + -a: automatically choose start date from most recently downloaded batch + "; +} +