Pear-Trading/Foodloop-Server
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix various bits for import

Browse source
This commit is contained in:
Tom Bloor 2019-07-14 15:15:14 +01:00
parent a45c354834
commit 71189d18fc
No known key found for this signature in database
GPG key ID: 4657C7EBE42CC5CC
8 changed files with 111 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

7
lib/Pear/LocalLoop/Command/codepoint_open.pm
View file

@ -22,7 +22,12 @@ sub run {
unless ( -d $output_dir ) {
print "Unzipping code-point-open data\n" unless $quiet_mode;
system( 'unzip', '-q', $zip_file, '-d', $output_dir );
eval { system( 'unzip', '-q', $zip_file, '-d', $output_dir ) };
if ( my $err = $@ ) {
print "Error extracting zip: " . $err . "\n";
print "Manually create etc/code-point-open/codepo_gb directory and extract zip into it";
die;
}
}
my $cpo = Geo::UK::Postcode::CodePointOpen->new( path => $output_dir );

40
lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm
View file

@ -2,10 +2,15 @@ package Pear::LocalLoop::Controller::Admin::ImportFrom;
use Mojo::Base 'Mojolicious::Controller';
use Moo;
use Try::Tiny;
use Mojo::File qw/ path /;
use Mojo::File qw/path/;
sub index {
my $c = shift;
$c->stash->{org_entities} = [
map {
{ id => $_->entity_id, name => $_->name }
} $c->schema->resultset('Organisation')->all
];
$c->app->max_request_size(104857600);
}
@ -14,56 +19,61 @@ sub post_suppliers {
my $c = shift;
unless ($c->param('suppliers_csv')) {
$c->flash( error => "No CSV file given" );
return $c->redirect_to( '/admin/import_from' );
$c->flash(error => "No CSV file given");
return $c->redirect_to('/admin/import_from');
}
# Check file size
if ($c->req->is_limit_exceeded) {
$c->flash( error => "CSV file size is too large" );
return $c->redirect_to( '/admin/import_from' );
$c->flash(error => "CSV file size is too large");
return $c->redirect_to('/admin/import_from');
}
my $file = $c->param('suppliers_csv');
my $filename = path($c->app->config->{upload_path}, time.'suppliers.csv' );
my $filename = path($c->app->config->{upload_path}, time . 'suppliers.csv');
$file->move_to($filename);
my $job_id = $c->minion->enqueue('csv_supplier_import' => [$filename] );
my $job_id = $c->minion->enqueue('csv_supplier_import' => [ $filename ]);
my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
$c->flash(success => "CSV import started, see status of minion job at: $job_url");
return $c->redirect_to( '/admin/import_from' );
return $c->redirect_to('/admin/import_from');
}
sub post_transactions {
my $c = shift;
unless ($c->param('entity_id') ne '') {
$c->flash(error => "Please Choose an organisation");
return $c->redirect_to('/admin/import_from');
}
unless ($c->param('transactions_csv')) {
$c->flash( error => "No CSV file given" );
return $c->redirect_to( '/admin/import_from' );
$c->flash(error => "No CSV file given");
return $c->redirect_to('/admin/import_from');
}
# Check file size
if ($c->req->is_limit_exceeded) {
$c->flash( error => "CSV file size is too large" );
return $c->redirect_to( '/admin/import_from' );
$c->flash(error => "CSV file size is too large");
return $c->redirect_to('/admin/import_from');
}
my $file = $c->param('transactions_csv');
my $filename = path($c->app->config->{upload_path}, time.'transactions.csv' );
my $filename = path($c->app->config->{upload_path}, time . 'transactions.csv');
$file->move_to($filename);
my $job_id = $c->minion->enqueue('csv_transaction_import' => [$filename] );
my $job_id = $c->minion->enqueue('csv_transaction_import' => [ $filename, $c->param('entity_id') ]);
my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
$c->flash(success => "CSV import started, see status of minion job at: $job_url");
return $c->redirect_to( '/admin/import_from' );
return $c->redirect_to('/admin/import_from');
}
1;

106
lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm
View file

@ -5,76 +5,86 @@ use DateTime::Format::Strptime;
extends qw/Pear::LocalLoop::Import::LCCCsv/;
has target_entity_id => (
is => 'ro',
required => 1,
);
has target_entity => (
is => 'lazy',
builder => sub {
my $self = shift;
my $entity = $self->schema->resultset('Entity')->find($self->target_entity_id);
Pear::LocalLoop::Error->throw("Cannot find LCC Entity, did you pass the right id?") unless $entity;
return $entity;
},
);
has '+csv_required_columns' => (
builder => sub { return [ (
'transaction_id',
'supplier_id',
'net_amount',
'vat amount',
'gross_amount',
)]},
builder => sub {return [ (
'transaction_id',
'supplier_id',
'net_amount',
'vat amount',
'gross_amount',
) ]},
);
sub import_csv {
my ($self) = @_;
my $rows = $self->csv_data;
my $lcc_org = $self->schema->resultset('Organisation')->find({
name => "Lancashire County Council",
street_name => "County Hall"
});
unless ($lcc_org) {
Pear::LocalLoop::Error->throw("Cannot find LCC Organisation, please contact an admin");
}
foreach my $row ( @{$rows} ) {
my $lcc_org = $self->target_entity;
foreach my $row (@{$rows}) {
$self->_row_to_result($row, $lcc_org);
}
}
sub _row_to_result {
my ( $self, $row, $lcc_org ) = @_;
my ($self, $row, $lcc_org) = @_;
my $supplier_id = $row->{supplier_id};
my $supplier_id = $row->{supplier_id};
my $organisation = $self->schema->resultset('Organisation')->find({
'external_reference.external_id' => $supplier_id
}, { join => 'external_reference' });
my $organisation = $self->schema->resultset('Organisation')->find({
'external_reference.external_id' => $supplier_id
}, { join => 'external_reference' });
unless ($organisation) {
Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
}
unless ($organisation) {
Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
}
my $date_formatter = DateTime::Format::Strptime->new(
pattern => '%m/%d/%Y',
time_zone => 'Europe/London'
);
my $date_formatter = DateTime::Format::Strptime->new(
pattern => '%m/%d/%Y',
time_zone => 'Europe/London'
);
my $paid_date = ( $row->{paid_date} ?
$date_formatter->parse_datetime($row->{paid_date}) :
$date_formatter->parse_datetime($row->{invoice_date}) );
my $gross_value = $row->{gross_amount};
$gross_value =~ s/,//g;
my $sales_tax_value = $row->{"vat amount"};
$sales_tax_value =~ s/,//g;
my $net_value = $row->{net_amount};
$net_value =~ s/,//g;
my $gross_value = $row->{gross_amount};
$gross_value =~ s/,//g;
my $sales_tax_value = $row->{"vat amount"};
$sales_tax_value =~ s/,//g;
my $net_value = $row->{net_amount};
$net_value =~ s/,//g;
# TODO negative values are sometimes present
$self->external_result->find_or_create_related('transactions', {
external_id => $row->{transaction_id},
transaction => {
seller => $organisation->entity,
buyer => $lcc_org->entity,
purchase_time => $paid_date,
value => $gross_value * 100000,
meta => {
gross_value => $gross_value * 100000,
sales_tax_value => $sales_tax_value * 100000,
net_value => $net_value * 100000,
},
}
});
# TODO negative values are sometimes present
$self->external_result->find_or_create_related('transactions', {
external_id => $row->{transaction_id},
transaction => {
seller => $organisation->entity,
buyer => $lcc_org,
purchase_time => $paid_date,
value => $gross_value * 100000,
meta => {
gross_value => $gross_value * 100000,
sales_tax_value => $sales_tax_value * 100000,
net_value => $net_value * 100000,
},
}
});
}
1;

9
lib/Pear/LocalLoop/Plugin/Minion/Job/csv_transaction_import.pm
View file

@ -4,11 +4,12 @@ use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
use Pear::LocalLoop::Import::LCCCsv::Transactions;
sub run {
my ( $self, $filename ) = @_;
my ($self, $filename, $entity_id) = @_;
my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new(
csv_file => $filename,
schema => $self->app->schema
Pear::LocalLoop::Import::LCCCsv::Transactions->new(
csv_file => $filename,
schema => $self->app->schema,
target_entity_id => $entity_id,
)->import_csv;
}