Added properly working imports with minions and status
This commit is contained in:
parent
22e6001362
commit
d0506c2a95
7 changed files with 49 additions and 59 deletions
|
@ -26,6 +26,7 @@ sub startup {
|
||||||
$self->plugin('Config', {
|
$self->plugin('Config', {
|
||||||
default => {
|
default => {
|
||||||
storage_path => tempdir,
|
storage_path => tempdir,
|
||||||
|
upload_path => $self->home->child('upload'),
|
||||||
sessionTimeSeconds => 60 * 60 * 24 * 7,
|
sessionTimeSeconds => 60 * 60 * 24 * 7,
|
||||||
sessionTokenJsonName => 'session_key',
|
sessionTokenJsonName => 'session_key',
|
||||||
sessionExpiresJsonName => 'sessionExpires',
|
sessionExpiresJsonName => 'sessionExpires',
|
||||||
|
|
|
@ -2,9 +2,7 @@ package Pear::LocalLoop::Controller::Admin::ImportFrom;
|
||||||
use Mojo::Base 'Mojolicious::Controller';
|
use Mojo::Base 'Mojolicious::Controller';
|
||||||
use Moo;
|
use Moo;
|
||||||
use Try::Tiny;
|
use Try::Tiny;
|
||||||
|
use Mojo::File qw/ path /;
|
||||||
use Pear::LocalLoop::Import::LCCCsv::Suppliers;
|
|
||||||
use Pear::LocalLoop::Import::LCCCsv::Transactions;
|
|
||||||
|
|
||||||
sub index {
|
sub index {
|
||||||
my $c = shift;
|
my $c = shift;
|
||||||
|
@ -26,20 +24,17 @@ sub post_suppliers {
|
||||||
return $c->redirect_to( '/admin/import_from' );
|
return $c->redirect_to( '/admin/import_from' );
|
||||||
}
|
}
|
||||||
|
|
||||||
my $filename;
|
my $file = $c->param('suppliers_csv');
|
||||||
|
|
||||||
$c->minion->enqueue('csv_supplier_import' => $filename );
|
my $filename = path($c->app->config->{upload_path}, time.'suppliers.csv' );
|
||||||
|
|
||||||
# my $csv_import = Pear::LocalLoop::Import::LCCCsv::Suppliers->new(
|
$file->move_to($filename);
|
||||||
# csv_string => $c->param('suppliers_csv')->slurp,
|
|
||||||
# schema => $c->app->schema
|
|
||||||
# )->import_csv;
|
|
||||||
#
|
|
||||||
# my $job_id = $c->minion->enqueue('csv_supplier_import' => [$csv_import] );
|
|
||||||
|
|
||||||
|
my $job_id = $c->minion->enqueue('csv_supplier_import' => [$filename] );
|
||||||
|
|
||||||
|
my $job_url = $c->url_for("/admin/minionjobs?id=$job_id")->to_abs;
|
||||||
|
|
||||||
$c->flash( success => "CSV imported" );
|
$c->flash(success => "CSV import started, see status of minion job at: $job_url");
|
||||||
return $c->redirect_to( '/admin/import_from' );
|
return $c->redirect_to( '/admin/import_from' );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -56,22 +51,19 @@ sub post_transactions {
|
||||||
$c->flash( error => "CSV file size is too large" );
|
$c->flash( error => "CSV file size is too large" );
|
||||||
return $c->redirect_to( '/admin/import_from' );
|
return $c->redirect_to( '/admin/import_from' );
|
||||||
}
|
}
|
||||||
my $csv_error;
|
|
||||||
try {
|
my $file = $c->param('transactions_csv');
|
||||||
Pear::LocalLoop::Import::LCCCsv::Transactions->new(
|
|
||||||
csv_string => $c->param('transactions_csv')->slurp,
|
my $filename = path($c->app->config->{upload_path}, time.'transactions.csv' );
|
||||||
schema => $c->app->schema
|
|
||||||
)->import_csv;
|
$file->move_to($filename);
|
||||||
} catch {
|
|
||||||
$csv_error = $_;
|
my $job_id = $c->minion->enqueue('csv_transaction_import' => [$filename] );
|
||||||
};
|
|
||||||
if ( $csv_error ) {
|
my $job_url = $c->url_for("/admin/minionjobs?id=$job_id")->to_abs;
|
||||||
$c->flash( error => $csv_error );
|
|
||||||
return $c->redirect_to( '/admin/import_from' );
|
$c->flash(success => "CSV import started, see status of minion job at: $job_url");
|
||||||
} else {
|
return $c->redirect_to( '/admin/import_from' );
|
||||||
$c->flash( success => "CSV imported" );
|
|
||||||
return $c->redirect_to( '/admin/import_from' );
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
1;
|
1;
|
||||||
|
|
|
@ -15,7 +15,9 @@ sub import_csv {
|
||||||
|
|
||||||
my $rows = $self->csv_data;
|
my $rows = $self->csv_data;
|
||||||
|
|
||||||
return $rows;
|
foreach my $row ( @{$rows} ) {
|
||||||
|
$self->_row_to_result($row);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sub _row_to_result {
|
sub _row_to_result {
|
||||||
|
|
|
@ -19,11 +19,10 @@ sub import_csv {
|
||||||
my ($self) = @_;
|
my ($self) = @_;
|
||||||
|
|
||||||
my $rows = $self->csv_data;
|
my $rows = $self->csv_data;
|
||||||
my $lcc_org = $self->schema->resultset('Organisation')->find({ name => "Lancashire County Council" });
|
|
||||||
foreach my $row ( @{$rows} ) {
|
foreach my $row ( @{$rows} ) {
|
||||||
$self->_row_to_result($row, $lcc_org);
|
$self->_row_to_result($row);
|
||||||
}
|
}
|
||||||
return 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sub _row_to_result {
|
sub _row_to_result {
|
||||||
|
|
|
@ -29,7 +29,7 @@ has _csv_filehandle => (
|
||||||
my $self = shift;
|
my $self = shift;
|
||||||
my $fh;
|
my $fh;
|
||||||
if ( $self->has_csv_file ) {
|
if ( $self->has_csv_file ) {
|
||||||
open $fh, '<', \${$self->csv_file};
|
open $fh, '<', $self->csv_file;
|
||||||
} elsif ( $self->has_csv_string ) {
|
} elsif ( $self->has_csv_string ) {
|
||||||
my $string = $self->csv_string;
|
my $string = $self->csv_string;
|
||||||
open $fh, '<', \$string;
|
open $fh, '<', \$string;
|
||||||
|
|
|
@ -1,34 +1,15 @@
|
||||||
package Pear::LocalLoop::Plugin::Minion::Job::csv_supplier_import;
|
package Pear::LocalLoop::Plugin::Minion::Job::csv_supplier_import;
|
||||||
use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
|
use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
|
||||||
use Devel::Dwarn;
|
|
||||||
|
use Pear::LocalLoop::Import::LCCCsv::Suppliers;
|
||||||
|
|
||||||
sub run {
|
sub run {
|
||||||
my ( $self, $rows ) = @_;
|
my ( $self, $filename ) = @_;
|
||||||
|
|
||||||
foreach my $row ( @{$rows} ) {
|
my $csv_import = Pear::LocalLoop::Import::LCCCsv::Suppliers->new(
|
||||||
$self->_row_to_result($row);
|
csv_file => $filename,
|
||||||
}
|
schema => $self->app->schema
|
||||||
}
|
)->import_csv;
|
||||||
|
|
||||||
sub _row_to_result {
|
|
||||||
my ( $self, $row ) = @_;
|
|
||||||
# Dwarn $row->{supplier_id};
|
|
||||||
my $addr2 = $row->{post_town};
|
|
||||||
|
|
||||||
my $address = ( defined $addr2 ? ( $row->{"address line 2"} . ' ' . $addr2) : $row->{"address line 2"} );
|
|
||||||
|
|
||||||
$self->external_result->find_or_create_related('organisations', {
|
|
||||||
external_id => $row->{supplier_id},
|
|
||||||
organisation => {
|
|
||||||
name => $row->{name},
|
|
||||||
street_name => $row->{"address line 1"},
|
|
||||||
town => $address,
|
|
||||||
postcode => $row->{post_code},
|
|
||||||
country => $row->{country_code},
|
|
||||||
entity => { type => 'organisation' },
|
|
||||||
}
|
|
||||||
});
|
|
||||||
$self->app->log->debug('Imported the CSV fully!');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
1;
|
1;
|
||||||
|
|
|
@ -0,0 +1,15 @@
|
||||||
|
package Pear::LocalLoop::Plugin::Minion::Job::csv_transaction_import;
|
||||||
|
use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
|
||||||
|
|
||||||
|
use Pear::LocalLoop::Import::LCCCsv::Transactions;
|
||||||
|
|
||||||
|
sub run {
|
||||||
|
my ( $self, $filename ) = @_;
|
||||||
|
|
||||||
|
my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new(
|
||||||
|
csv_file => $filename,
|
||||||
|
schema => $self->app->schema
|
||||||
|
)->import_csv;
|
||||||
|
}
|
||||||
|
|
||||||
|
1;
|
Reference in a new issue