Add code formatter, format all code
This commit is contained in:
parent
602a59f1c3
commit
47a55f6322
120 changed files with 8061 additions and 6967 deletions
|
@ -3,21 +3,22 @@ use Moo;
|
|||
use Pear::LocalLoop::Error;
|
||||
|
||||
has external_name => (
|
||||
is => 'ro',
|
||||
default => 'LCC CSV',
|
||||
is => 'ro',
|
||||
default => 'LCC CSV',
|
||||
);
|
||||
|
||||
has csv_required_columns => (
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
Pear::LocalLoop::ImplementationError->throw("Must be implemented by child class");
|
||||
},
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
Pear::LocalLoop::ImplementationError->throw(
|
||||
"Must be implemented by child class");
|
||||
},
|
||||
);
|
||||
|
||||
with qw/
|
||||
Pear::LocalLoop::Import::Role::ExternalName
|
||||
Pear::LocalLoop::Import::Role::Schema
|
||||
Pear::LocalLoop::Import::Role::CSV
|
||||
/;
|
||||
/;
|
||||
|
||||
1;
|
||||
|
|
|
@ -6,38 +6,45 @@ use Geo::UK::Postcode::Regex;
|
|||
extends qw/Pear::LocalLoop::Import::LCCCsv/;
|
||||
|
||||
has '+csv_required_columns' => (
|
||||
builder => sub { return [ qw/
|
||||
postcode
|
||||
ward
|
||||
/ ]},
|
||||
builder => sub {
|
||||
return [
|
||||
qw/
|
||||
postcode
|
||||
ward
|
||||
/
|
||||
];
|
||||
},
|
||||
);
|
||||
|
||||
sub import_csv {
|
||||
my ($self) = @_;
|
||||
my ($self) = @_;
|
||||
|
||||
$self->check_headers;
|
||||
$self->check_headers;
|
||||
|
||||
while ( my $row = $self->get_csv_line ) {
|
||||
$self->_row_to_result($row);
|
||||
}
|
||||
while ( my $row = $self->get_csv_line ) {
|
||||
$self->_row_to_result($row);
|
||||
}
|
||||
}
|
||||
|
||||
sub _row_to_result {
|
||||
my ( $self, $row ) = @_;
|
||||
my ( $self, $row ) = @_;
|
||||
|
||||
my $postcode_obj = Geo::UK::Postcode::Regex->parse( $row->{postcode} );
|
||||
my $postcode_obj = Geo::UK::Postcode::Regex->parse( $row->{postcode} );
|
||||
|
||||
my $ward = $self->schema->resultset('GbWard')->find_or_create(ward => $row->{ward});
|
||||
my $ward = $self->schema->resultset('GbWard')
|
||||
->find_or_create( ward => $row->{ward} );
|
||||
|
||||
my $postcode_r = $self->schema->resultset('GbPostcode')->find({
|
||||
outcode => $postcode_obj->{outcode},
|
||||
incode => $postcode_obj->{incode},
|
||||
});
|
||||
my $postcode_r = $self->schema->resultset('GbPostcode')->find(
|
||||
{
|
||||
outcode => $postcode_obj->{outcode},
|
||||
incode => $postcode_obj->{incode},
|
||||
}
|
||||
);
|
||||
|
||||
return unless $postcode_r;
|
||||
return if $postcode_r->ward;
|
||||
return unless $postcode_r;
|
||||
return if $postcode_r->ward;
|
||||
|
||||
$postcode_r->update({ ward_id => $ward->id });
|
||||
$postcode_r->update( { ward_id => $ward->id } );
|
||||
}
|
||||
|
||||
1;
|
||||
|
|
|
@ -4,45 +4,58 @@ use Moo;
|
|||
extends qw/Pear::LocalLoop::Import::LCCCsv/;
|
||||
|
||||
has '+csv_required_columns' => (
|
||||
builder => sub { return [ qw/
|
||||
supplier_id
|
||||
name
|
||||
/ ]},
|
||||
builder => sub {
|
||||
return [
|
||||
qw/
|
||||
supplier_id
|
||||
name
|
||||
/
|
||||
];
|
||||
},
|
||||
);
|
||||
|
||||
sub import_csv {
|
||||
my ($self) = @_;
|
||||
my ($self) = @_;
|
||||
|
||||
$self->check_headers;
|
||||
$self->check_headers;
|
||||
|
||||
while ( my $row = $self->get_csv_line ) {
|
||||
$self->_row_to_result($row);
|
||||
}
|
||||
while ( my $row = $self->get_csv_line ) {
|
||||
$self->_row_to_result($row);
|
||||
}
|
||||
}
|
||||
|
||||
sub _row_to_result {
|
||||
my ( $self, $row ) = @_;
|
||||
my ( $self, $row ) = @_;
|
||||
|
||||
my $addr2 = $row->{post_town};
|
||||
my $addr2 = $row->{post_town};
|
||||
|
||||
my $address = ( defined $addr2 ? ( $row->{"address line 2"} . ' ' . $addr2) : $row->{"address line 2"} );
|
||||
my $address = (
|
||||
defined $addr2
|
||||
? ( $row->{"address line 2"} . ' ' . $addr2 )
|
||||
: $row->{"address line 2"} );
|
||||
|
||||
return if $self->external_result->organisations->find({external_id => $row->{supplier_id}});
|
||||
return
|
||||
if $self->external_result->organisations->find(
|
||||
{ external_id => $row->{supplier_id} } );
|
||||
|
||||
$self->schema->resultset('Entity')->create({
|
||||
type => 'organisation',
|
||||
organisation => {
|
||||
name => $row->{name},
|
||||
street_name => $row->{"address line 1"},
|
||||
town => $address,
|
||||
postcode => $row->{post_code},
|
||||
country => $row->{country_code},
|
||||
external_reference => [ {
|
||||
external_reference => $self->external_result,
|
||||
external_id => $row->{supplier_id},
|
||||
} ],
|
||||
}
|
||||
});
|
||||
$self->schema->resultset('Entity')->create(
|
||||
{
|
||||
type => 'organisation',
|
||||
organisation => {
|
||||
name => $row->{name},
|
||||
street_name => $row->{"address line 1"},
|
||||
town => $address,
|
||||
postcode => $row->{post_code},
|
||||
country => $row->{country_code},
|
||||
external_reference => [
|
||||
{
|
||||
external_reference => $self->external_result,
|
||||
external_id => $row->{supplier_id},
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
1;
|
||||
|
|
|
@ -8,121 +8,179 @@ use Geo::UK::Postcode::Regex;
|
|||
extends qw/Pear::LocalLoop::Import::LCCCsv/;
|
||||
|
||||
has target_entity_id => (
|
||||
is => 'ro',
|
||||
required => 1,
|
||||
is => 'ro',
|
||||
required => 1,
|
||||
);
|
||||
|
||||
has target_entity => (
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
my $entity = $self->schema->resultset('Entity')->find($self->target_entity_id);
|
||||
Pear::LocalLoop::Error->throw("Cannot find LCC Entity, did you pass the right id?") unless $entity;
|
||||
return $entity;
|
||||
},
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
my $entity =
|
||||
$self->schema->resultset('Entity')->find( $self->target_entity_id );
|
||||
Pear::LocalLoop::Error->throw(
|
||||
"Cannot find LCC Entity, did you pass the right id?")
|
||||
unless $entity;
|
||||
return $entity;
|
||||
},
|
||||
);
|
||||
|
||||
has '+csv_required_columns' => (
|
||||
builder => sub {return [ (
|
||||
'transaction_id',
|
||||
'supplier_id',
|
||||
'net_amount',
|
||||
'vat amount',
|
||||
'gross_amount',
|
||||
) ]},
|
||||
builder => sub {
|
||||
return [
|
||||
(
|
||||
'transaction_id', 'supplier_id',
|
||||
'net_amount', 'vat amount',
|
||||
'gross_amount',
|
||||
)
|
||||
];
|
||||
},
|
||||
);
|
||||
|
||||
sub import_csv {
|
||||
my ($self) = @_;
|
||||
my ($self) = @_;
|
||||
|
||||
$self->check_headers;
|
||||
my $lcc_org = $self->target_entity;
|
||||
$self->check_headers;
|
||||
my $lcc_org = $self->target_entity;
|
||||
|
||||
while ( my $row = $self->get_csv_line ) {
|
||||
$self->_row_to_result($row, $lcc_org);
|
||||
}
|
||||
while ( my $row = $self->get_csv_line ) {
|
||||
$self->_row_to_result( $row, $lcc_org );
|
||||
}
|
||||
}
|
||||
|
||||
sub _row_to_result {
|
||||
my ($self, $row, $lcc_org) = @_;
|
||||
my ( $self, $row, $lcc_org ) = @_;
|
||||
|
||||
my $supplier_id = $row->{supplier_id};
|
||||
my $supplier_id = $row->{supplier_id};
|
||||
|
||||
my $organisation = $self->schema->resultset('Organisation')->find({
|
||||
'external_reference.external_id' => $supplier_id
|
||||
}, { join => 'external_reference' });
|
||||
my $organisation = $self->schema->resultset('Organisation')->find(
|
||||
{
|
||||
'external_reference.external_id' => $supplier_id
|
||||
},
|
||||
{ join => 'external_reference' }
|
||||
);
|
||||
|
||||
unless ($organisation) {
|
||||
# Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
|
||||
unless ($organisation) {
|
||||
|
||||
return unless $row->{'Company Name (WHO)'};
|
||||
# Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
|
||||
|
||||
my $town = $row->{post_town};
|
||||
return unless $row->{'Company Name (WHO)'};
|
||||
|
||||
unless ($town) {
|
||||
my $postcode_obj = Geo::UK::Postcode::Regex->parse( $row->{post_code} );
|
||||
$town = Geo::UK::Postcode::Regex->outcode_to_posttowns($postcode_obj->{outcode});
|
||||
$town = $town->[0];
|
||||
my $town = $row->{post_town};
|
||||
|
||||
unless ($town) {
|
||||
my $postcode_obj =
|
||||
Geo::UK::Postcode::Regex->parse( $row->{post_code} );
|
||||
$town = Geo::UK::Postcode::Regex->outcode_to_posttowns(
|
||||
$postcode_obj->{outcode} );
|
||||
$town = $town->[0];
|
||||
}
|
||||
|
||||
return
|
||||
if $self->external_result->organisations->find(
|
||||
{ external_id => $row->{supplier_id} } );
|
||||
|
||||
$organisation = $self->schema->resultset('Entity')->create(
|
||||
{
|
||||
type => 'organisation',
|
||||
organisation => {
|
||||
name => $row->{'Company Name (WHO)'},
|
||||
street_name => $row->{"address line 1"},
|
||||
town => $town,
|
||||
postcode => $row->{post_code},
|
||||
country => $row->{country_code},
|
||||
external_reference => [
|
||||
{
|
||||
external_reference => $self->external_result,
|
||||
external_id => $row->{supplier_id},
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return if $self->external_result->organisations->find({external_id => $row->{supplier_id}});
|
||||
my $date_formatter = DateTime::Format::Strptime->new(
|
||||
pattern => '%m/%d/%Y',
|
||||
time_zone => 'Europe/London'
|
||||
);
|
||||
|
||||
$organisation = $self->schema->resultset('Entity')->create({
|
||||
type => 'organisation',
|
||||
organisation => {
|
||||
name => $row->{'Company Name (WHO)'},
|
||||
street_name => $row->{"address line 1"},
|
||||
town => $town,
|
||||
postcode => $row->{post_code},
|
||||
country => $row->{country_code},
|
||||
external_reference => [ {
|
||||
external_reference => $self->external_result,
|
||||
external_id => $row->{supplier_id},
|
||||
} ],
|
||||
}
|
||||
});
|
||||
}
|
||||
my $paid_date = (
|
||||
$row->{paid_date}
|
||||
? $date_formatter->parse_datetime( $row->{paid_date} )
|
||||
: $date_formatter->parse_datetime( $row->{invoice_date} )
|
||||
);
|
||||
|
||||
my $date_formatter = DateTime::Format::Strptime->new(
|
||||
pattern => '%m/%d/%Y',
|
||||
time_zone => 'Europe/London'
|
||||
);
|
||||
|
||||
my $paid_date = ( $row->{paid_date} ?
|
||||
$date_formatter->parse_datetime($row->{paid_date}) :
|
||||
$date_formatter->parse_datetime($row->{invoice_date}) );
|
||||
my $gross_value = $row->{gross_amount};
|
||||
$gross_value =~ s/,//g;
|
||||
my $sales_tax_value = $row->{"vat amount"};
|
||||
$sales_tax_value =~ s/,//g;
|
||||
my $net_value = $row->{net_amount};
|
||||
$net_value =~ s/,//g;
|
||||
|
||||
my $gross_value = $row->{gross_amount};
|
||||
$gross_value =~ s/,//g;
|
||||
my $sales_tax_value = $row->{"vat amount"};
|
||||
$sales_tax_value =~ s/,//g;
|
||||
my $net_value = $row->{net_amount};
|
||||
$net_value =~ s/,//g;
|
||||
# TODO negative values are sometimes present
|
||||
my $external_transaction = $self->external_result->update_or_create_related(
|
||||
'transactions',
|
||||
{ # This is a TransactionExternal result
|
||||
external_id => $row->{transaction_id},
|
||||
}
|
||||
);
|
||||
|
||||
# TODO negative values are sometimes present
|
||||
my $external_transaction = $self->external_result->update_or_create_related('transactions', { # This is a TransactionExternal result
|
||||
external_id => $row->{transaction_id},
|
||||
});
|
||||
my $transaction_result = $external_transaction->update_or_create_related(
|
||||
'transaction',
|
||||
{
|
||||
seller => $organisation->entity,
|
||||
buyer => $lcc_org,
|
||||
purchase_time => $paid_date,
|
||||
value => $gross_value * 100000,
|
||||
}
|
||||
);
|
||||
|
||||
my $transaction_result = $external_transaction->update_or_create_related( 'transaction', {
|
||||
seller => $organisation->entity,
|
||||
buyer => $lcc_org,
|
||||
purchase_time => $paid_date,
|
||||
value => $gross_value * 100000,
|
||||
});
|
||||
|
||||
my $meta_result = $transaction_result->update_or_create_related('meta', {
|
||||
gross_value => $gross_value * 100000,
|
||||
sales_tax_value => $sales_tax_value * 100000,
|
||||
net_value => $net_value * 100000,
|
||||
($row->{"local service"} ? (local_service => $row->{"local service"}) : ()),
|
||||
($row->{"regional service"} ? (regional_service => $row->{"regional service"}) : ()),
|
||||
($row->{"national service"} ? (national_service => $row->{"national service"}) : ()),
|
||||
($row->{"private household rebate"} ? (private_household_rebate => $row->{"private household rebate"}) : ()),
|
||||
($row->{"business tax and rebate"} ? (business_tax_and_rebate => $row->{"business tax and rebate"}) : ()),
|
||||
($row->{"stat loc gov"} ? (stat_loc_gov => $row->{"stat loc gov"}) : ()),
|
||||
($row->{"central loc gov"} ? (central_loc_gov => $row->{"central loc gov"}) : ()),
|
||||
});
|
||||
my $meta_result = $transaction_result->update_or_create_related(
|
||||
'meta',
|
||||
{
|
||||
gross_value => $gross_value * 100000,
|
||||
sales_tax_value => $sales_tax_value * 100000,
|
||||
net_value => $net_value * 100000,
|
||||
(
|
||||
$row->{"local service"}
|
||||
? ( local_service => $row->{"local service"} )
|
||||
: ()
|
||||
),
|
||||
(
|
||||
$row->{"regional service"}
|
||||
? ( regional_service => $row->{"regional service"} )
|
||||
: ()
|
||||
),
|
||||
(
|
||||
$row->{"national service"}
|
||||
? ( national_service => $row->{"national service"} )
|
||||
: ()
|
||||
),
|
||||
(
|
||||
$row->{"private household rebate"}
|
||||
? ( private_household_rebate =>
|
||||
$row->{"private household rebate"} )
|
||||
: ()
|
||||
),
|
||||
(
|
||||
$row->{"business tax and rebate"}
|
||||
? ( business_tax_and_rebate =>
|
||||
$row->{"business tax and rebate"} )
|
||||
: ()
|
||||
),
|
||||
(
|
||||
$row->{"stat loc gov"}
|
||||
? ( stat_loc_gov => $row->{"stat loc gov"} )
|
||||
: ()
|
||||
),
|
||||
(
|
||||
$row->{"central loc gov"}
|
||||
? ( central_loc_gov => $row->{"central loc gov"} )
|
||||
: ()
|
||||
),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
1;
|
||||
|
|
|
@ -9,80 +9,82 @@ use Pear::LocalLoop::Error;
|
|||
requires 'csv_required_columns';
|
||||
|
||||
has csv_file => (
|
||||
is => 'ro',
|
||||
predicate => 1,
|
||||
is => 'ro',
|
||||
predicate => 1,
|
||||
);
|
||||
|
||||
has csv_string => (
|
||||
is => 'ro',
|
||||
predicate => 1,
|
||||
is => 'ro',
|
||||
predicate => 1,
|
||||
);
|
||||
|
||||
has csv_error => (
|
||||
is => 'ro',
|
||||
predicate => 1,
|
||||
is => 'ro',
|
||||
predicate => 1,
|
||||
);
|
||||
|
||||
has _csv_filehandle => (
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
my $fh;
|
||||
if ( $self->has_csv_file ) {
|
||||
open $fh, '<', $self->csv_file;
|
||||
} elsif ( $self->has_csv_string ) {
|
||||
my $string = $self->csv_string;
|
||||
open $fh, '<', \$string;
|
||||
} else {
|
||||
die "Must provide csv_file or csv_string"
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
my $fh;
|
||||
if ( $self->has_csv_file ) {
|
||||
open $fh, '<', $self->csv_file;
|
||||
}
|
||||
elsif ( $self->has_csv_string ) {
|
||||
my $string = $self->csv_string;
|
||||
open $fh, '<', \$string;
|
||||
}
|
||||
else {
|
||||
die "Must provide csv_file or csv_string";
|
||||
}
|
||||
return $fh;
|
||||
}
|
||||
return $fh;
|
||||
}
|
||||
);
|
||||
|
||||
has text_csv_options => (
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
return {
|
||||
binary => 1,
|
||||
allow_whitespace => 1,
|
||||
};
|
||||
}
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
return {
|
||||
binary => 1,
|
||||
allow_whitespace => 1,
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
has _text_csv => (
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
return Text::CSV->new(shift->text_csv_options);
|
||||
}
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
return Text::CSV->new( shift->text_csv_options );
|
||||
}
|
||||
);
|
||||
|
||||
has csv_data => (
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
my $header_check = $self->check_headers;
|
||||
return 0 unless $header_check;
|
||||
return $self->_text_csv->getline_hr_all( $self->_csv_filehandle );
|
||||
}
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
my $header_check = $self->check_headers;
|
||||
return 0 unless $header_check;
|
||||
return $self->_text_csv->getline_hr_all( $self->_csv_filehandle );
|
||||
}
|
||||
);
|
||||
|
||||
sub get_csv_line {
|
||||
my $self = shift;
|
||||
return $self->_text_csv->getline_hr( $self->_csv_filehandle );
|
||||
my $self = shift;
|
||||
return $self->_text_csv->getline_hr( $self->_csv_filehandle );
|
||||
}
|
||||
|
||||
sub check_headers {
|
||||
my $self = shift;
|
||||
my $req_headers = $self->csv_required_columns;
|
||||
my @headers;
|
||||
@headers = $self->_text_csv->header( $self->_csv_filehandle );
|
||||
my %header_map = ( map { $_ => 1 } @headers );
|
||||
for my $req_header ( @$req_headers ) {
|
||||
next if $header_map{$req_header};
|
||||
die "Require header [" . $req_header . "]";
|
||||
}
|
||||
return 1;
|
||||
my $self = shift;
|
||||
my $req_headers = $self->csv_required_columns;
|
||||
my @headers;
|
||||
@headers = $self->_text_csv->header( $self->_csv_filehandle );
|
||||
my %header_map = ( map { $_ => 1 } @headers );
|
||||
for my $req_header (@$req_headers) {
|
||||
next if $header_map{$req_header};
|
||||
die "Require header [" . $req_header . "]";
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
1;
|
||||
|
|
|
@ -6,14 +6,15 @@ use Moo::Role;
|
|||
requires qw/
|
||||
external_name
|
||||
schema
|
||||
/;
|
||||
/;
|
||||
|
||||
has external_result => (
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
return $self->schema->resultset('ExternalReference')->find_or_create({ name => $self->external_name });
|
||||
}
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
return $self->schema->resultset('ExternalReference')
|
||||
->find_or_create( { name => $self->external_name } );
|
||||
}
|
||||
);
|
||||
|
||||
1;
|
||||
|
|
|
@ -4,8 +4,8 @@ use warnings;
|
|||
use Moo::Role;
|
||||
|
||||
has schema => (
|
||||
is => 'ro',
|
||||
required => 1,
|
||||
is => 'ro',
|
||||
required => 1,
|
||||
);
|
||||
|
||||
1;
|
||||
1;
|
||||
|
|
Reference in a new issue