Fix various bits for import
This commit is contained in:
parent
a45c354834
commit
71189d18fc
8 changed files with 111 additions and 69 deletions
|
@ -10,5 +10,11 @@
|
|||
<property name="enable_load_extension" value="true" />
|
||||
</driver-properties>
|
||||
</data-source>
|
||||
<data-source source="LOCAL" name="PostgreSQL foodloop@localhost" uuid="8161e393-4db4-4e03-aa8b-7a961f14a591">
|
||||
<driver-ref>postgresql</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<jdbc-driver>org.postgresql.Driver</jdbc-driver>
|
||||
<jdbc-url>jdbc:postgresql://localhost:5432/</jdbc-url>
|
||||
</data-source>
|
||||
</component>
|
||||
</project>
|
2
cpanfile
2
cpanfile
|
@ -24,6 +24,7 @@ requires 'GIS::Distance';
|
|||
requires 'Text::CSV';
|
||||
requires 'Try::Tiny';
|
||||
requires 'Throwable::Error';
|
||||
requires 'Minion';
|
||||
|
||||
on 'test' => sub {
|
||||
requires 'Test::More';
|
||||
|
@ -38,6 +39,7 @@ feature 'schema-graph', 'Draw diagrams of Schema' => sub {
|
|||
feature 'postgres', 'PostgreSQL Support' => sub {
|
||||
requires 'DBD::Pg';
|
||||
requires 'Test::PostgreSQL';
|
||||
requires 'Mojo::Pg';
|
||||
};
|
||||
|
||||
feature 'sqlite', 'SQLite Support' => sub {
|
||||
|
|
|
@ -22,7 +22,12 @@ sub run {
|
|||
|
||||
unless ( -d $output_dir ) {
|
||||
print "Unzipping code-point-open data\n" unless $quiet_mode;
|
||||
system( 'unzip', '-q', $zip_file, '-d', $output_dir );
|
||||
eval { system( 'unzip', '-q', $zip_file, '-d', $output_dir ) };
|
||||
if ( my $err = $@ ) {
|
||||
print "Error extracting zip: " . $err . "\n";
|
||||
print "Manually create etc/code-point-open/codepo_gb directory and extract zip into it";
|
||||
die;
|
||||
}
|
||||
}
|
||||
|
||||
my $cpo = Geo::UK::Postcode::CodePointOpen->new( path => $output_dir );
|
||||
|
|
|
@ -2,10 +2,15 @@ package Pear::LocalLoop::Controller::Admin::ImportFrom;
|
|||
use Mojo::Base 'Mojolicious::Controller';
|
||||
use Moo;
|
||||
use Try::Tiny;
|
||||
use Mojo::File qw/ path /;
|
||||
use Mojo::File qw/path/;
|
||||
|
||||
sub index {
|
||||
my $c = shift;
|
||||
$c->stash->{org_entities} = [
|
||||
map {
|
||||
{ id => $_->entity_id, name => $_->name }
|
||||
} $c->schema->resultset('Organisation')->all
|
||||
];
|
||||
|
||||
$c->app->max_request_size(104857600);
|
||||
}
|
||||
|
@ -14,56 +19,61 @@ sub post_suppliers {
|
|||
my $c = shift;
|
||||
|
||||
unless ($c->param('suppliers_csv')) {
|
||||
$c->flash( error => "No CSV file given" );
|
||||
return $c->redirect_to( '/admin/import_from' );
|
||||
$c->flash(error => "No CSV file given");
|
||||
return $c->redirect_to('/admin/import_from');
|
||||
}
|
||||
|
||||
# Check file size
|
||||
if ($c->req->is_limit_exceeded) {
|
||||
$c->flash( error => "CSV file size is too large" );
|
||||
return $c->redirect_to( '/admin/import_from' );
|
||||
$c->flash(error => "CSV file size is too large");
|
||||
return $c->redirect_to('/admin/import_from');
|
||||
}
|
||||
|
||||
my $file = $c->param('suppliers_csv');
|
||||
|
||||
my $filename = path($c->app->config->{upload_path}, time.'suppliers.csv' );
|
||||
my $filename = path($c->app->config->{upload_path}, time . 'suppliers.csv');
|
||||
|
||||
$file->move_to($filename);
|
||||
|
||||
my $job_id = $c->minion->enqueue('csv_supplier_import' => [$filename] );
|
||||
my $job_id = $c->minion->enqueue('csv_supplier_import' => [ $filename ]);
|
||||
|
||||
my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
|
||||
|
||||
$c->flash(success => "CSV import started, see status of minion job at: $job_url");
|
||||
return $c->redirect_to( '/admin/import_from' );
|
||||
return $c->redirect_to('/admin/import_from');
|
||||
}
|
||||
|
||||
sub post_transactions {
|
||||
my $c = shift;
|
||||
|
||||
unless ($c->param('entity_id') ne '') {
|
||||
$c->flash(error => "Please Choose an organisation");
|
||||
return $c->redirect_to('/admin/import_from');
|
||||
}
|
||||
|
||||
unless ($c->param('transactions_csv')) {
|
||||
$c->flash( error => "No CSV file given" );
|
||||
return $c->redirect_to( '/admin/import_from' );
|
||||
$c->flash(error => "No CSV file given");
|
||||
return $c->redirect_to('/admin/import_from');
|
||||
}
|
||||
|
||||
# Check file size
|
||||
if ($c->req->is_limit_exceeded) {
|
||||
$c->flash( error => "CSV file size is too large" );
|
||||
return $c->redirect_to( '/admin/import_from' );
|
||||
$c->flash(error => "CSV file size is too large");
|
||||
return $c->redirect_to('/admin/import_from');
|
||||
}
|
||||
|
||||
my $file = $c->param('transactions_csv');
|
||||
|
||||
my $filename = path($c->app->config->{upload_path}, time.'transactions.csv' );
|
||||
my $filename = path($c->app->config->{upload_path}, time . 'transactions.csv');
|
||||
|
||||
$file->move_to($filename);
|
||||
|
||||
my $job_id = $c->minion->enqueue('csv_transaction_import' => [$filename] );
|
||||
my $job_id = $c->minion->enqueue('csv_transaction_import' => [ $filename, $c->param('entity_id') ]);
|
||||
|
||||
my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
|
||||
|
||||
$c->flash(success => "CSV import started, see status of minion job at: $job_url");
|
||||
return $c->redirect_to( '/admin/import_from' );
|
||||
return $c->redirect_to('/admin/import_from');
|
||||
}
|
||||
|
||||
1;
|
||||
|
|
|
@ -5,76 +5,86 @@ use DateTime::Format::Strptime;
|
|||
|
||||
extends qw/Pear::LocalLoop::Import::LCCCsv/;
|
||||
|
||||
has target_entity_id => (
|
||||
is => 'ro',
|
||||
required => 1,
|
||||
);
|
||||
|
||||
has target_entity => (
|
||||
is => 'lazy',
|
||||
builder => sub {
|
||||
my $self = shift;
|
||||
my $entity = $self->schema->resultset('Entity')->find($self->target_entity_id);
|
||||
Pear::LocalLoop::Error->throw("Cannot find LCC Entity, did you pass the right id?") unless $entity;
|
||||
return $entity;
|
||||
},
|
||||
);
|
||||
|
||||
has '+csv_required_columns' => (
|
||||
builder => sub { return [ (
|
||||
'transaction_id',
|
||||
'supplier_id',
|
||||
'net_amount',
|
||||
'vat amount',
|
||||
'gross_amount',
|
||||
)]},
|
||||
builder => sub {return [ (
|
||||
'transaction_id',
|
||||
'supplier_id',
|
||||
'net_amount',
|
||||
'vat amount',
|
||||
'gross_amount',
|
||||
) ]},
|
||||
);
|
||||
|
||||
sub import_csv {
|
||||
my ($self) = @_;
|
||||
|
||||
my $rows = $self->csv_data;
|
||||
my $lcc_org = $self->schema->resultset('Organisation')->find({
|
||||
name => "Lancashire County Council",
|
||||
street_name => "County Hall"
|
||||
});
|
||||
unless ($lcc_org) {
|
||||
Pear::LocalLoop::Error->throw("Cannot find LCC Organisation, please contact an admin");
|
||||
}
|
||||
foreach my $row ( @{$rows} ) {
|
||||
my $lcc_org = $self->target_entity;
|
||||
|
||||
foreach my $row (@{$rows}) {
|
||||
$self->_row_to_result($row, $lcc_org);
|
||||
}
|
||||
}
|
||||
|
||||
sub _row_to_result {
|
||||
my ( $self, $row, $lcc_org ) = @_;
|
||||
my ($self, $row, $lcc_org) = @_;
|
||||
|
||||
my $supplier_id = $row->{supplier_id};
|
||||
my $supplier_id = $row->{supplier_id};
|
||||
|
||||
my $organisation = $self->schema->resultset('Organisation')->find({
|
||||
'external_reference.external_id' => $supplier_id
|
||||
}, { join => 'external_reference' });
|
||||
my $organisation = $self->schema->resultset('Organisation')->find({
|
||||
'external_reference.external_id' => $supplier_id
|
||||
}, { join => 'external_reference' });
|
||||
|
||||
unless ($organisation) {
|
||||
Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
|
||||
}
|
||||
unless ($organisation) {
|
||||
Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
|
||||
}
|
||||
|
||||
my $date_formatter = DateTime::Format::Strptime->new(
|
||||
pattern => '%m/%d/%Y',
|
||||
time_zone => 'Europe/London'
|
||||
);
|
||||
my $date_formatter = DateTime::Format::Strptime->new(
|
||||
pattern => '%m/%d/%Y',
|
||||
time_zone => 'Europe/London'
|
||||
);
|
||||
|
||||
my $paid_date = ( $row->{paid_date} ?
|
||||
$date_formatter->parse_datetime($row->{paid_date}) :
|
||||
$date_formatter->parse_datetime($row->{invoice_date}) );
|
||||
|
||||
my $gross_value = $row->{gross_amount};
|
||||
$gross_value =~ s/,//g;
|
||||
my $sales_tax_value = $row->{"vat amount"};
|
||||
$sales_tax_value =~ s/,//g;
|
||||
my $net_value = $row->{net_amount};
|
||||
$net_value =~ s/,//g;
|
||||
my $gross_value = $row->{gross_amount};
|
||||
$gross_value =~ s/,//g;
|
||||
my $sales_tax_value = $row->{"vat amount"};
|
||||
$sales_tax_value =~ s/,//g;
|
||||
my $net_value = $row->{net_amount};
|
||||
$net_value =~ s/,//g;
|
||||
|
||||
# TODO negative values are sometimes present
|
||||
$self->external_result->find_or_create_related('transactions', {
|
||||
external_id => $row->{transaction_id},
|
||||
transaction => {
|
||||
seller => $organisation->entity,
|
||||
buyer => $lcc_org->entity,
|
||||
purchase_time => $paid_date,
|
||||
value => $gross_value * 100000,
|
||||
meta => {
|
||||
gross_value => $gross_value * 100000,
|
||||
sales_tax_value => $sales_tax_value * 100000,
|
||||
net_value => $net_value * 100000,
|
||||
},
|
||||
}
|
||||
});
|
||||
# TODO negative values are sometimes present
|
||||
$self->external_result->find_or_create_related('transactions', {
|
||||
external_id => $row->{transaction_id},
|
||||
transaction => {
|
||||
seller => $organisation->entity,
|
||||
buyer => $lcc_org,
|
||||
purchase_time => $paid_date,
|
||||
value => $gross_value * 100000,
|
||||
meta => {
|
||||
gross_value => $gross_value * 100000,
|
||||
sales_tax_value => $sales_tax_value * 100000,
|
||||
net_value => $net_value * 100000,
|
||||
},
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
1;
|
||||
|
|
|
@ -4,11 +4,12 @@ use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
|
|||
use Pear::LocalLoop::Import::LCCCsv::Transactions;
|
||||
|
||||
sub run {
|
||||
my ( $self, $filename ) = @_;
|
||||
my ($self, $filename, $entity_id) = @_;
|
||||
|
||||
my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new(
|
||||
csv_file => $filename,
|
||||
schema => $self->app->schema
|
||||
Pear::LocalLoop::Import::LCCCsv::Transactions->new(
|
||||
csv_file => $filename,
|
||||
schema => $self->app->schema,
|
||||
target_entity_id => $entity_id,
|
||||
)->import_csv;
|
||||
}
|
||||
|
||||
|
|
|
@ -4,4 +4,7 @@
|
|||
user => undef,
|
||||
pass => undef,
|
||||
key => "a",
|
||||
minion => {
|
||||
SQLite => 'sqlite:minion.db',
|
||||
},
|
||||
};
|
||||
|
|
|
@ -21,7 +21,6 @@
|
|||
<p><strong>NOTE:</strong> Files must be in CSV format.<br>
|
||||
Redundant columns and rows above and to the left of the table in the CSV must be deleted.<br>
|
||||
There can be no duplicate header columns, and there can be no fields with values outside the table.</p>
|
||||
<p>Warning: Large files will take a long time to process, just leave the tab open until the Success message appears.</p>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="card col-md-6 m-3">
|
||||
|
@ -39,6 +38,12 @@
|
|||
<h4 class="card-title">LCC Procurement Import - Transactions</h4>
|
||||
<p>Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".</p>
|
||||
<form action="/admin/import_from/transactions" method="POST" enctype="multipart/form-data">
|
||||
<select name="entity_id">
|
||||
<option>Select an Organisation</option>
|
||||
<% for my $org ( @$org_entities ) { %>
|
||||
<option value="<%= $org->{id}; %>"><%= $org->{name}; %></option>
|
||||
<% } %>
|
||||
</select><br/>
|
||||
<input type="file" name="transactions_csv" accept="text/csv">
|
||||
<input type="submit" value="Upload Transactions CSV">
|
||||
</form>
|
||||
|
|
Reference in a new issue