From afc635fdb465da4f2202e737ed15c73e430e895d Mon Sep 17 00:00:00 2001 From: Finn Date: Fri, 5 Jul 2019 18:44:46 +0100 Subject: [PATCH] Added nicer explosions via try::tiny --- .../LocalLoop/Controller/Admin/ImportFrom.pm | 21 +++++++++++-------- lib/Pear/LocalLoop/Import/LCCCsv/Suppliers.pm | 2 -- .../LocalLoop/Import/LCCCsv/Transactions.pm | 20 +++++++----------- lib/Pear/LocalLoop/Import/Role/CSV.pm | 11 +--------- templates/admin/import/index.html.ep | 1 + templates/admin/import_from/index.html.ep | 5 +++++ 6 files changed, 27 insertions(+), 33 deletions(-) diff --git a/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm index 484ea6d..dfb710d 100644 --- a/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm +++ b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm @@ -1,7 +1,7 @@ package Pear::LocalLoop::Controller::Admin::ImportFrom; use Mojo::Base 'Mojolicious::Controller'; use Moo; -use Devel::Dwarn; +use Try::Tiny; use Pear::LocalLoop::Import::LCCCsv::Suppliers; use Pear::LocalLoop::Import::LCCCsv::Transactions; @@ -47,19 +47,22 @@ sub post_transactions { $c->flash( error => "CSV file size is too large" ); return $c->redirect_to( '/admin/import_from' ); } - my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new( - csv_string => $c->param('transactions_csv')->slurp, - schema => $c->app->schema - )->import_csv; - - if ($csv_import->csv_error) { - $c->flash( error => $csv_import->csv_error ); + my $csv_error; + try { + Pear::LocalLoop::Import::LCCCsv::Transactions->new( + csv_string => $c->param('transactions_csv')->slurp, + schema => $c->app->schema + )->import_csv; + } catch { + $csv_error = $_; + }; + if ( $csv_error ) { + $c->flash( error => $csv_error ); return $c->redirect_to( '/admin/import_from' ); } else { $c->flash( success => "CSV imported" ); return $c->redirect_to( '/admin/import_from' ); } - } 1; diff --git a/lib/Pear/LocalLoop/Import/LCCCsv/Suppliers.pm b/lib/Pear/LocalLoop/Import/LCCCsv/Suppliers.pm index 5ffe96a..ca87344 100644 --- a/lib/Pear/LocalLoop/Import/LCCCsv/Suppliers.pm +++ b/lib/Pear/LocalLoop/Import/LCCCsv/Suppliers.pm @@ -1,6 +1,5 @@ package Pear::LocalLoop::Import::LCCCsv::Suppliers; use Moo; -use Devel::Dwarn; extends qw/Pear::LocalLoop::Import::LCCCsv/; @@ -15,7 +14,6 @@ sub import_csv { my ($self) = @_; my $rows = $self->csv_data; - return $self unless $rows; foreach my $row ( @{$rows} ) { $self->_row_to_result($row); } diff --git a/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm b/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm index ce671da..89e574c 100644 --- a/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm +++ b/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm @@ -6,19 +6,19 @@ use DateTime::Format::Strptime; extends qw/Pear::LocalLoop::Import::LCCCsv/; has '+csv_required_columns' => ( - builder => sub { return [ qw/ - transaction_id - supplier_id - net_amount - gross_amount - / ]}, + builder => sub { return [ ( + 'transaction_id', + 'supplier_id', + 'net_amount', + 'vat amount', + 'gross_amount', + )]}, ); sub import_csv { my ($self) = @_; my $rows = $self->csv_data; - return 0 unless $rows; my $lcc_org = $self->schema->resultset('Organisation')->find({ name => "Lancashire County Council" }); foreach my $row ( @{$rows} ) { $self->_row_to_result($row, $lcc_org); @@ -29,10 +29,6 @@ sub import_csv { sub _row_to_result { my ( $self, $row, $lcc_org ) = @_; - use Devel::Dwarn; - - Dwarn $row; - my $supplier_id = $row->{supplier_id}; my $organisation = $self->schema->resultset('Organisation')->find({ @@ -49,7 +45,7 @@ sub _row_to_result { my $paid_date = ( $row->{paid_date} ? $date_formatter->parse_datetime($row->{paid_date}) : DateTime->today ); - # TODO negative values + # TODO negative values are sometimes present $self->external_result->find_or_create_related('transactions', { external_id => $row->{transaction_id}, transaction => { diff --git a/lib/Pear/LocalLoop/Import/Role/CSV.pm b/lib/Pear/LocalLoop/Import/Role/CSV.pm index 018ea05..b4315cb 100644 --- a/lib/Pear/LocalLoop/Import/Role/CSV.pm +++ b/lib/Pear/LocalLoop/Import/Role/CSV.pm @@ -70,17 +70,8 @@ has csv_data => ( sub check_headers { my $self = shift; my $req_headers = $self->csv_required_columns; - use Devel::Dwarn; - Dwarn $req_headers; - # TODO catch the boom my @headers; - try { - @headers = $self->_text_csv->header( $self->_csv_filehandle ); - } catch { - $self->csv_error = $_->[1]; - }; - return 0 unless @headers; - Dwarn \@headers; + @headers = $self->_text_csv->header( $self->_csv_filehandle ); my %header_map = ( map { $_ => 1 } @headers ); for my $req_header ( @$req_headers ) { next if $header_map{$req_header}; diff --git a/templates/admin/import/index.html.ep b/templates/admin/import/index.html.ep index 379dbdb..675ab6c 100644 --- a/templates/admin/import/index.html.ep +++ b/templates/admin/import/index.html.ep @@ -15,6 +15,7 @@

CSV Import

Import Data + Import Suppliers/Transactions Data
% for my $import ( $import_rs->all ) { % my $total = $import_rs->get_values( $import->id, 1, 1 )->count; diff --git a/templates/admin/import_from/index.html.ep b/templates/admin/import_from/index.html.ep index 2e1ef1c..51b9781 100644 --- a/templates/admin/import_from/index.html.ep +++ b/templates/admin/import_from/index.html.ep @@ -18,12 +18,16 @@

Various import options depending on source. Each one is custom. For Creditor reports from LCC.

+

NOTE: Files must be in CSV format.
+ Redundant columns and rows above and to the left of the table in the CSV must be deleted.
+ There can be no duplicate header columns, and there can be no fields with values outside the table.

Warning: Large files will take a long time to process, just leave the tab open until the Success message appears.

LCC Procurement Import - Suppliers

+

Expected headers at very least: "supplier_id", "name", "post_code".

@@ -33,6 +37,7 @@

LCC Procurement Import - Transactions

+

Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".