From a29c09c9bf03a34b370773cc7ef7bd7bc5b2ceff Mon Sep 17 00:00:00 2001 From: Tom Bloor Date: Sun, 14 Jul 2019 15:15:14 +0100 Subject: [PATCH] Fix various bits for import --- .idea/dataSources.xml | 6 + cpanfile | 2 + lib/Pear/LocalLoop/Command/codepoint_open.pm | 7 +- .../LocalLoop/Controller/Admin/ImportFrom.pm | 40 ++++--- .../LocalLoop/Import/LCCCsv/Transactions.pm | 106 ++++++++++-------- .../Minion/Job/csv_transaction_import.pm | 9 +- pear-local_loop.development.conf | 3 + templates/admin/import_from/index.html.ep | 7 +- 8 files changed, 111 insertions(+), 69 deletions(-) diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml index d60800f..11e39c8 100644 --- a/.idea/dataSources.xml +++ b/.idea/dataSources.xml @@ -10,5 +10,11 @@ + + postgresql + true + org.postgresql.Driver + jdbc:postgresql://localhost:5432/ + \ No newline at end of file diff --git a/cpanfile b/cpanfile index 9aed545..c50ecd7 100644 --- a/cpanfile +++ b/cpanfile @@ -24,6 +24,7 @@ requires 'GIS::Distance'; requires 'Text::CSV'; requires 'Try::Tiny'; requires 'Throwable::Error'; +requires 'Minion'; on 'test' => sub { requires 'Test::More'; @@ -38,6 +39,7 @@ feature 'schema-graph', 'Draw diagrams of Schema' => sub { feature 'postgres', 'PostgreSQL Support' => sub { requires 'DBD::Pg'; requires 'Test::PostgreSQL'; + requires 'Mojo::Pg'; }; feature 'sqlite', 'SQLite Support' => sub { diff --git a/lib/Pear/LocalLoop/Command/codepoint_open.pm b/lib/Pear/LocalLoop/Command/codepoint_open.pm index bfdedae..5f286aa 100644 --- a/lib/Pear/LocalLoop/Command/codepoint_open.pm +++ b/lib/Pear/LocalLoop/Command/codepoint_open.pm @@ -22,7 +22,12 @@ sub run { unless ( -d $output_dir ) { print "Unzipping code-point-open data\n" unless $quiet_mode; - system( 'unzip', '-q', $zip_file, '-d', $output_dir ); + eval { system( 'unzip', '-q', $zip_file, '-d', $output_dir ) }; + if ( my $err = $@ ) { + print "Error extracting zip: " . $err . "\n"; + print "Manually create etc/code-point-open/codepo_gb directory and extract zip into it"; + die; + } } my $cpo = Geo::UK::Postcode::CodePointOpen->new( path => $output_dir ); diff --git a/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm index aa54a81..bf0b3de 100644 --- a/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm +++ b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm @@ -2,10 +2,15 @@ package Pear::LocalLoop::Controller::Admin::ImportFrom; use Mojo::Base 'Mojolicious::Controller'; use Moo; use Try::Tiny; -use Mojo::File qw/ path /; +use Mojo::File qw/path/; sub index { my $c = shift; + $c->stash->{org_entities} = [ + map { + { id => $_->entity_id, name => $_->name } + } $c->schema->resultset('Organisation')->all + ]; $c->app->max_request_size(104857600); } @@ -14,56 +19,61 @@ sub post_suppliers { my $c = shift; unless ($c->param('suppliers_csv')) { - $c->flash( error => "No CSV file given" ); - return $c->redirect_to( '/admin/import_from' ); + $c->flash(error => "No CSV file given"); + return $c->redirect_to('/admin/import_from'); } # Check file size if ($c->req->is_limit_exceeded) { - $c->flash( error => "CSV file size is too large" ); - return $c->redirect_to( '/admin/import_from' ); + $c->flash(error => "CSV file size is too large"); + return $c->redirect_to('/admin/import_from'); } my $file = $c->param('suppliers_csv'); - my $filename = path($c->app->config->{upload_path}, time.'suppliers.csv' ); + my $filename = path($c->app->config->{upload_path}, time . 'suppliers.csv'); $file->move_to($filename); - my $job_id = $c->minion->enqueue('csv_supplier_import' => [$filename] ); + my $job_id = $c->minion->enqueue('csv_supplier_import' => [ $filename ]); my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs; $c->flash(success => "CSV import started, see status of minion job at: $job_url"); - return $c->redirect_to( '/admin/import_from' ); + return $c->redirect_to('/admin/import_from'); } sub post_transactions { my $c = shift; + unless ($c->param('entity_id') ne '') { + $c->flash(error => "Please Choose an organisation"); + return $c->redirect_to('/admin/import_from'); + } + unless ($c->param('transactions_csv')) { - $c->flash( error => "No CSV file given" ); - return $c->redirect_to( '/admin/import_from' ); + $c->flash(error => "No CSV file given"); + return $c->redirect_to('/admin/import_from'); } # Check file size if ($c->req->is_limit_exceeded) { - $c->flash( error => "CSV file size is too large" ); - return $c->redirect_to( '/admin/import_from' ); + $c->flash(error => "CSV file size is too large"); + return $c->redirect_to('/admin/import_from'); } my $file = $c->param('transactions_csv'); - my $filename = path($c->app->config->{upload_path}, time.'transactions.csv' ); + my $filename = path($c->app->config->{upload_path}, time . 'transactions.csv'); $file->move_to($filename); - my $job_id = $c->minion->enqueue('csv_transaction_import' => [$filename] ); + my $job_id = $c->minion->enqueue('csv_transaction_import' => [ $filename, $c->param('entity_id') ]); my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs; $c->flash(success => "CSV import started, see status of minion job at: $job_url"); - return $c->redirect_to( '/admin/import_from' ); + return $c->redirect_to('/admin/import_from'); } 1; diff --git a/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm b/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm index 2f68484..4c20d69 100644 --- a/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm +++ b/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm @@ -5,76 +5,86 @@ use DateTime::Format::Strptime; extends qw/Pear::LocalLoop::Import::LCCCsv/; +has target_entity_id => ( + is => 'ro', + required => 1, +); + +has target_entity => ( + is => 'lazy', + builder => sub { + my $self = shift; + my $entity = $self->schema->resultset('Entity')->find($self->target_entity_id); + Pear::LocalLoop::Error->throw("Cannot find LCC Entity, did you pass the right id?") unless $entity; + return $entity; + }, +); + has '+csv_required_columns' => ( - builder => sub { return [ ( - 'transaction_id', - 'supplier_id', - 'net_amount', - 'vat amount', - 'gross_amount', - )]}, + builder => sub {return [ ( + 'transaction_id', + 'supplier_id', + 'net_amount', + 'vat amount', + 'gross_amount', + ) ]}, ); sub import_csv { my ($self) = @_; my $rows = $self->csv_data; - my $lcc_org = $self->schema->resultset('Organisation')->find({ - name => "Lancashire County Council", - street_name => "County Hall" - }); - unless ($lcc_org) { - Pear::LocalLoop::Error->throw("Cannot find LCC Organisation, please contact an admin"); - } - foreach my $row ( @{$rows} ) { + my $lcc_org = $self->target_entity; + + foreach my $row (@{$rows}) { $self->_row_to_result($row, $lcc_org); } } sub _row_to_result { - my ( $self, $row, $lcc_org ) = @_; + my ($self, $row, $lcc_org) = @_; - my $supplier_id = $row->{supplier_id}; + my $supplier_id = $row->{supplier_id}; - my $organisation = $self->schema->resultset('Organisation')->find({ - 'external_reference.external_id' => $supplier_id - }, { join => 'external_reference' }); + my $organisation = $self->schema->resultset('Organisation')->find({ + 'external_reference.external_id' => $supplier_id + }, { join => 'external_reference' }); - unless ($organisation) { - Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id"); - } + unless ($organisation) { + Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id"); + } - my $date_formatter = DateTime::Format::Strptime->new( - pattern => '%m/%d/%Y', - time_zone => 'Europe/London' - ); + my $date_formatter = DateTime::Format::Strptime->new( + pattern => '%m/%d/%Y', + time_zone => 'Europe/London' + ); my $paid_date = ( $row->{paid_date} ? $date_formatter->parse_datetime($row->{paid_date}) : $date_formatter->parse_datetime($row->{invoice_date}) ); - my $gross_value = $row->{gross_amount}; - $gross_value =~ s/,//g; - my $sales_tax_value = $row->{"vat amount"}; - $sales_tax_value =~ s/,//g; - my $net_value = $row->{net_amount}; - $net_value =~ s/,//g; + my $gross_value = $row->{gross_amount}; + $gross_value =~ s/,//g; + my $sales_tax_value = $row->{"vat amount"}; + $sales_tax_value =~ s/,//g; + my $net_value = $row->{net_amount}; + $net_value =~ s/,//g; - # TODO negative values are sometimes present - $self->external_result->find_or_create_related('transactions', { - external_id => $row->{transaction_id}, - transaction => { - seller => $organisation->entity, - buyer => $lcc_org->entity, - purchase_time => $paid_date, - value => $gross_value * 100000, - meta => { - gross_value => $gross_value * 100000, - sales_tax_value => $sales_tax_value * 100000, - net_value => $net_value * 100000, - }, - } - }); + # TODO negative values are sometimes present + $self->external_result->find_or_create_related('transactions', { + external_id => $row->{transaction_id}, + transaction => { + seller => $organisation->entity, + buyer => $lcc_org, + purchase_time => $paid_date, + value => $gross_value * 100000, + meta => { + gross_value => $gross_value * 100000, + sales_tax_value => $sales_tax_value * 100000, + net_value => $net_value * 100000, + }, + } + }); } 1; diff --git a/lib/Pear/LocalLoop/Plugin/Minion/Job/csv_transaction_import.pm b/lib/Pear/LocalLoop/Plugin/Minion/Job/csv_transaction_import.pm index 4d6db82..e77b831 100644 --- a/lib/Pear/LocalLoop/Plugin/Minion/Job/csv_transaction_import.pm +++ b/lib/Pear/LocalLoop/Plugin/Minion/Job/csv_transaction_import.pm @@ -4,11 +4,12 @@ use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job'; use Pear::LocalLoop::Import::LCCCsv::Transactions; sub run { - my ( $self, $filename ) = @_; + my ($self, $filename, $entity_id) = @_; - my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new( - csv_file => $filename, - schema => $self->app->schema + Pear::LocalLoop::Import::LCCCsv::Transactions->new( + csv_file => $filename, + schema => $self->app->schema, + target_entity_id => $entity_id, )->import_csv; } diff --git a/pear-local_loop.development.conf b/pear-local_loop.development.conf index cfb1416..a0d3c92 100644 --- a/pear-local_loop.development.conf +++ b/pear-local_loop.development.conf @@ -4,4 +4,7 @@ user => undef, pass => undef, key => "a", + minion => { + SQLite => 'sqlite:minion.db', + }, }; diff --git a/templates/admin/import_from/index.html.ep b/templates/admin/import_from/index.html.ep index 51b9781..62ff66d 100644 --- a/templates/admin/import_from/index.html.ep +++ b/templates/admin/import_from/index.html.ep @@ -21,7 +21,6 @@

NOTE: Files must be in CSV format.
Redundant columns and rows above and to the left of the table in the CSV must be deleted.
There can be no duplicate header columns, and there can be no fields with values outside the table.

-

Warning: Large files will take a long time to process, just leave the tab open until the Success message appears.

@@ -39,6 +38,12 @@

LCC Procurement Import - Transactions

Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".

+