X-Git-Url: http://git.freeside.biz/gitweb/?a=blobdiff_plain;f=FS%2FFS%2FMisc%2FGeo.pm;h=e7b1fa877530318cb456c524713f4a754fbb60ce;hb=8cbdba85db636a87bcd133ed01fd14d9fc7754e7;hp=aa755643f68647b2dfe1f543f54da52b6914250a;hpb=e269e80233f594ce1387bdfcdba7dbf4861345c2;p=freeside.git diff --git a/FS/FS/Misc/Geo.pm b/FS/FS/Misc/Geo.pm index aa755643f..e7b1fa877 100644 --- a/FS/FS/Misc/Geo.pm +++ b/FS/FS/Misc/Geo.pm @@ -6,8 +6,7 @@ use vars qw( $DEBUG @EXPORT_OK $conf ); use LWP::UserAgent; use HTTP::Request; use HTTP::Request::Common qw( GET POST ); -use HTTP::Cookies; -use HTML::TokeParser; +use JSON; use URI::Escape 3.31; use Data::Dumper; use FS::Conf; @@ -29,7 +28,7 @@ FS::Misc::Geo - routines to fetch geographic information =over 4 -=item get_censustract LOCATION YEAR +=item get_censustract_ffiec LOCATION YEAR Given a location hash (see L) and a census map year, returns a census tract code (consisting of state, county, and tract @@ -41,105 +40,65 @@ sub get_censustract_ffiec { my $class = shift; my $location = shift; my $year = shift; + $year ||= 2012; - warn Dumper($location, $year) if $DEBUG; + if ( length($location->{country}) and uc($location->{country}) ne 'US' ) { + return ''; + } - my $url = 'http://www.ffiec.gov/Geocode/default.aspx'; + warn Dumper($location, $year) if $DEBUG; - my $return = {}; - my $error = ''; + # the old FFIEC geocoding service was shut down December 1, 2014. + # welcome to the future. + my $url = 'https://geomap.ffiec.gov/FFIECGeocMap/GeocodeMap1.aspx/GetGeocodeData'; + # build the single-line query + my $single_line = join(', ', $location->{address1}, + $location->{city}, + $location->{state} + ); + my $hashref = { sSingleLine => $single_line, iCensusYear => $year }; + my $request = POST( $url, + 'Content-Type' => 'application/json; charset=utf-8', + 'Accept' => 'application/json', + 'Content' => encode_json($hashref) + ); - my $ua = new LWP::UserAgent('cookie_jar' => HTTP::Cookies->new); - my $res = $ua->request( GET( $url ) ); + my $ua = new LWP::UserAgent; + my $res = $ua->request( $request ); warn $res->as_string if $DEBUG > 2; if (!$res->is_success) { - $error = $res->message; - - } else { - - my $content = $res->content; - - my $p = new HTML::TokeParser \$content; - my $viewstate; - my $eventvalidation; - while (my $token = $p->get_tag('input') ) { - if ($token->[1]->{name} eq '__VIEWSTATE') { - $viewstate = $token->[1]->{value}; - } - if ($token->[1]->{name} eq '__EVENTVALIDATION') { - $eventvalidation = $token->[1]->{value}; - } - last if $viewstate && $eventvalidation; - } - - if (!$viewstate or !$eventvalidation ) { - - $error = "either no __VIEWSTATE or __EVENTVALIDATION found"; - - } else { - - my($zip5, $zip4) = split('-',$location->{zip}); - - $year ||= '2012'; - my @ffiec_args = ( - __VIEWSTATE => $viewstate, - __EVENTVALIDATION => $eventvalidation, - __VIEWSTATEENCRYPTED => '', - ddlbYear => $year, - txtAddress => $location->{address1}, - txtCity => $location->{city}, - ddlbState => $location->{state}, - txtZipCode => $zip5, - btnSearch => 'Search', - ); - warn join("\n", @ffiec_args ) - if $DEBUG > 1; - - push @{ $ua->requests_redirectable }, 'POST'; - $res = $ua->request( POST( $url, \@ffiec_args ) ); - warn $res->as_string - if $DEBUG > 2; - - unless ($res->code eq '200') { - - $error = $res->message; - - } else { - - my @id = qw( MSACode StateCode CountyCode TractCode ); - $content = $res->content; - warn $res->content if $DEBUG > 2; - $p = new HTML::TokeParser \$content; - my $prefix = 'UcGeoResult11_lb'; - my $compare = - sub { my $t=shift; scalar( grep { lc($t) eq lc("$prefix$_")} @id ) }; - - while (my $token = $p->get_tag('span') ) { - next unless ( $token->[1]->{id} && &$compare( $token->[1]->{id} ) ); - $token->[1]->{id} =~ /^$prefix(\w+)$/; - $return->{lc($1)} = $p->get_trimmed_text("/span"); - } + die "Census tract lookup error: ".$res->message; - unless ( $return->{tractcode} ) { - warn "$error: $content ". Dumper($return) if $DEBUG; - $error = "No census tract found"; - } - $return->{tractcode} .= ' ' - unless $error || $JSON::VERSION >= 2; #broken JSON 1 workaround + } - } #unless ($res->code eq '200') + local $@; + my $content = eval { decode_json($res->content) }; + die "Census tract JSON error: $@\n" if $@; - } #unless ($viewstate) + if ( !exists $content->{d}->{sStatus} ) { + die "Census tract response is missing a status indicator.\nThis is an FFIEC problem.\n"; + } + if ( $content->{d}->{sStatus} eq 'Y' ) { + # success + # this also contains the (partial) standardized address, correct zip + # code, coordinates, etc., and we could get all of them, but right now + # we only want the census tract + my $tract = join('', $content->{d}->{sStateCode}, + $content->{d}->{sCountyCode}, + $content->{d}->{sTractCode}); + return $tract; - } #unless ($res->code eq '200') + } else { - die "FFIEC Geocoding error: $error\n" if $error; + my $error = $content->{d}->{sMsg} + || 'FFIEC lookup failed, but with no status message.'; + die "$error\n"; - $return->{'statecode'} . $return->{'countycode'} . $return->{'tractcode'}; + } } #sub get_district_methods { @@ -652,6 +611,57 @@ sub subloc_address2 { ($subloc, $addr2); } +sub standardize_melissa { + my $class = shift; + my $location = shift; + + local $@; + eval "use Geo::Melissa::WebSmart"; + die $@ if $@; + + my $id = $conf->config('melissa-userid') + or die "no melissa-userid configured\n"; + my $geocode = $conf->exists('melissa-enable_geocoding') ? 1 : 0; + + my $request = { + id => $id, + a1 => $location->{address1}, + a2 => $location->{address2}, + city => $location->{city}, + state => $location->{state}, + ctry => $location->{country}, + zip => $location->{zip}, + geocode => $geocode, + }; + my $result = Geo::Melissa::WebSmart->query($request); + if ( $result->code =~ /AS01/ ) { # always present on success + my $addr = $result->address; + warn Dumper $addr if $DEBUG > 1; + my $out = { + address1 => $addr->{Address1}, + address2 => $addr->{Address2}, + city => $addr->{City}->{Name}, + state => $addr->{State}->{Abbreviation}, + country => $addr->{Country}->{Abbreviation}, + zip => $addr->{Zip}, + latitude => $addr->{Latitude}, + longitude => $addr->{Longitude}, + addr_clean => 'Y', + }; + if ( $addr->{Census}->{Tract} ) { + my $censustract = $addr->{County}->{Fips} . $addr->{Census}->{Tract}; + # insert decimal point two digits from the end + $censustract =~ s/(\d\d)$/\.$1/; + $out->{censustract} = $censustract; + $out->{censusyear} = $conf->config('census_year'); + } + # we could do a lot more nuanced reporting of the warning/status codes, + # but the UI doesn't support that yet. + return $out; + } else { + die $result->status_message; + } +} =back