Pear-Trading
/
Foodloop-Server
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix various bits for import

This commit is contained in:
Tom Bloor 2019-07-14 15:15:14 +01:00
parent a45c354834
commit 71189d18fc
No known key found for this signature in database
GPG Key ID: 4657C7EBE42CC5CC
8 changed files with 111 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.idea/dataSources.xml
View File

@ -10,5 +10,11 @@
<property name="enable_load_extension" value="true" />
</driver-properties>
</data-source>
<data-source source="LOCAL" name="PostgreSQL foodloop@localhost" uuid="8161e393-4db4-4e03-aa8b-7a961f14a591">
<driver-ref>postgresql</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>org.postgresql.Driver</jdbc-driver>
<jdbc-url>jdbc:postgresql://localhost:5432/</jdbc-url>
</data-source>
</component>
</project>

2
cpanfile
View File

@ -24,6 +24,7 @@ requires 'GIS::Distance';
requires 'Text::CSV';
requires 'Try::Tiny';
requires 'Throwable::Error';
requires 'Minion';
on 'test' => sub {
requires 'Test::More';
@ -38,6 +39,7 @@ feature 'schema-graph', 'Draw diagrams of Schema' => sub {
feature 'postgres', 'PostgreSQL Support' => sub {
requires 'DBD::Pg';
requires 'Test::PostgreSQL';
requires 'Mojo::Pg';
};
feature 'sqlite', 'SQLite Support' => sub {

7
lib/Pear/LocalLoop/Command/codepoint_open.pm
View File

@ -22,7 +22,12 @@ sub run {
unless ( -d $output_dir ) {
print "Unzipping code-point-open data\n" unless $quiet_mode;
system( 'unzip', '-q', $zip_file, '-d', $output_dir );
eval { system( 'unzip', '-q', $zip_file, '-d', $output_dir ) };
if ( my $err = $@ ) {
print "Error extracting zip: " . $err . "\n";
print "Manually create etc/code-point-open/codepo_gb directory and extract zip into it";
die;
}
}
my $cpo = Geo::UK::Postcode::CodePointOpen->new( path => $output_dir );

40
lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm
View File

@ -2,10 +2,15 @@ package Pear::LocalLoop::Controller::Admin::ImportFrom;
use Mojo::Base 'Mojolicious::Controller';
use Moo;
use Try::Tiny;
use Mojo::File qw/ path /;
use Mojo::File qw/path/;
sub index {
my $c = shift;
$c->stash->{org_entities} = [
map {
{ id => $_->entity_id, name => $_->name }
} $c->schema->resultset('Organisation')->all
];
$c->app->max_request_size(104857600);
}
@ -14,56 +19,61 @@ sub post_suppliers {
my $c = shift;
unless ($c->param('suppliers_csv')) {
$c->flash( error => "No CSV file given" );
return $c->redirect_to( '/admin/import_from' );
$c->flash(error => "No CSV file given");
return $c->redirect_to('/admin/import_from');
}
# Check file size
if ($c->req->is_limit_exceeded) {
$c->flash( error => "CSV file size is too large" );
return $c->redirect_to( '/admin/import_from' );
$c->flash(error => "CSV file size is too large");
return $c->redirect_to('/admin/import_from');
}
my $file = $c->param('suppliers_csv');
my $filename = path($c->app->config->{upload_path}, time.'suppliers.csv' );
my $filename = path($c->app->config->{upload_path}, time . 'suppliers.csv');
$file->move_to($filename);
my $job_id = $c->minion->enqueue('csv_supplier_import' => [$filename] );
my $job_id = $c->minion->enqueue('csv_supplier_import' => [ $filename ]);
my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
$c->flash(success => "CSV import started, see status of minion job at: $job_url");
return $c->redirect_to( '/admin/import_from' );
return $c->redirect_to('/admin/import_from');
}
sub post_transactions {
my $c = shift;
unless ($c->param('entity_id') ne '') {
$c->flash(error => "Please Choose an organisation");
return $c->redirect_to('/admin/import_from');
}
unless ($c->param('transactions_csv')) {
$c->flash( error => "No CSV file given" );
return $c->redirect_to( '/admin/import_from' );
$c->flash(error => "No CSV file given");
return $c->redirect_to('/admin/import_from');
}
# Check file size
if ($c->req->is_limit_exceeded) {
$c->flash( error => "CSV file size is too large" );
return $c->redirect_to( '/admin/import_from' );
$c->flash(error => "CSV file size is too large");
return $c->redirect_to('/admin/import_from');
}
my $file = $c->param('transactions_csv');
my $filename = path($c->app->config->{upload_path}, time.'transactions.csv' );
my $filename = path($c->app->config->{upload_path}, time . 'transactions.csv');
$file->move_to($filename);
my $job_id = $c->minion->enqueue('csv_transaction_import' => [$filename] );
my $job_id = $c->minion->enqueue('csv_transaction_import' => [ $filename, $c->param('entity_id') ]);
my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
$c->flash(success => "CSV import started, see status of minion job at: $job_url");
return $c->redirect_to( '/admin/import_from' );
return $c->redirect_to('/admin/import_from');
}
1;

106
lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm
View File

@ -5,76 +5,86 @@ use DateTime::Format::Strptime;
extends qw/Pear::LocalLoop::Import::LCCCsv/;
has target_entity_id => (
is => 'ro',
required => 1,
);
has target_entity => (
is => 'lazy',
builder => sub {
my $self = shift;
my $entity = $self->schema->resultset('Entity')->find($self->target_entity_id);
Pear::LocalLoop::Error->throw("Cannot find LCC Entity, did you pass the right id?") unless $entity;
return $entity;
},
);
has '+csv_required_columns' => (
builder => sub { return [ (
'transaction_id',
'supplier_id',
'net_amount',
'vat amount',
'gross_amount',
)]},
builder => sub {return [ (
'transaction_id',
'supplier_id',
'net_amount',
'vat amount',
'gross_amount',
) ]},
);
sub import_csv {
my ($self) = @_;
my $rows = $self->csv_data;
my $lcc_org = $self->schema->resultset('Organisation')->find({
name => "Lancashire County Council",
street_name => "County Hall"
});
unless ($lcc_org) {
Pear::LocalLoop::Error->throw("Cannot find LCC Organisation, please contact an admin");
}
foreach my $row ( @{$rows} ) {
my $lcc_org = $self->target_entity;
foreach my $row (@{$rows}) {
$self->_row_to_result($row, $lcc_org);
}
}
sub _row_to_result {
my ( $self, $row, $lcc_org ) = @_;
my ($self, $row, $lcc_org) = @_;
my $supplier_id = $row->{supplier_id};
my $supplier_id = $row->{supplier_id};
my $organisation = $self->schema->resultset('Organisation')->find({
'external_reference.external_id' => $supplier_id
}, { join => 'external_reference' });
my $organisation = $self->schema->resultset('Organisation')->find({
'external_reference.external_id' => $supplier_id
}, { join => 'external_reference' });
unless ($organisation) {
Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
}
unless ($organisation) {
Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
}
my $date_formatter = DateTime::Format::Strptime->new(
pattern => '%m/%d/%Y',
time_zone => 'Europe/London'
);
my $date_formatter = DateTime::Format::Strptime->new(
pattern => '%m/%d/%Y',
time_zone => 'Europe/London'
);
my $paid_date = ( $row->{paid_date} ?
$date_formatter->parse_datetime($row->{paid_date}) :
$date_formatter->parse_datetime($row->{invoice_date}) );
my $gross_value = $row->{gross_amount};
$gross_value =~ s/,//g;
my $sales_tax_value = $row->{"vat amount"};
$sales_tax_value =~ s/,//g;
my $net_value = $row->{net_amount};
$net_value =~ s/,//g;
my $gross_value = $row->{gross_amount};
$gross_value =~ s/,//g;
my $sales_tax_value = $row->{"vat amount"};
$sales_tax_value =~ s/,//g;
my $net_value = $row->{net_amount};
$net_value =~ s/,//g;
# TODO negative values are sometimes present
$self->external_result->find_or_create_related('transactions', {
external_id => $row->{transaction_id},
transaction => {
seller => $organisation->entity,
buyer => $lcc_org->entity,
purchase_time => $paid_date,
value => $gross_value * 100000,
meta => {
gross_value => $gross_value * 100000,
sales_tax_value => $sales_tax_value * 100000,
net_value => $net_value * 100000,
},
}
});
# TODO negative values are sometimes present
$self->external_result->find_or_create_related('transactions', {
external_id => $row->{transaction_id},
transaction => {
seller => $organisation->entity,
buyer => $lcc_org,
purchase_time => $paid_date,
value => $gross_value * 100000,
meta => {
gross_value => $gross_value * 100000,
sales_tax_value => $sales_tax_value * 100000,
net_value => $net_value * 100000,
},
}
});
}
1;

9
lib/Pear/LocalLoop/Plugin/Minion/Job/csv_transaction_import.pm
View File

@ -4,11 +4,12 @@ use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
use Pear::LocalLoop::Import::LCCCsv::Transactions;
sub run {
my ( $self, $filename ) = @_;
my ($self, $filename, $entity_id) = @_;
my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new(
csv_file => $filename,
schema => $self->app->schema
Pear::LocalLoop::Import::LCCCsv::Transactions->new(
csv_file => $filename,
schema => $self->app->schema,
target_entity_id => $entity_id,
)->import_csv;
}

3
pear-local_loop.development.conf
View File

@ -4,4 +4,7 @@
user => undef,
pass => undef,
key => "a",
minion => {
SQLite => 'sqlite:minion.db',
},
};

7
templates/admin/import_from/index.html.ep
View File

@ -21,7 +21,6 @@
<p><strong>NOTE:</strong> Files must be in CSV format.<br>
Redundant columns and rows above and to the left of the table in the CSV must be deleted.<br>
There can be no duplicate header columns, and there can be no fields with values outside the table.</p>
<p>Warning: Large files will take a long time to process, just leave the tab open until the Success message appears.</p>
</div>
<div class="row">
<div class="card col-md-6 m-3">
@ -39,6 +38,12 @@
<h4 class="card-title">LCC Procurement Import - Transactions</h4>
<p>Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".</p>
<form action="/admin/import_from/transactions" method="POST" enctype="multipart/form-data">
<select name="entity_id">
<option>Select an Organisation</option>
<% for my $org ( @$org_entities ) { %>
<option value="<%= $org->{id}; %>"><%= $org->{name}; %></option>
<% } %>
</select><br/>
<input type="file" name="transactions_csv" accept="text/csv">
<input type="submit" value="Upload Transactions CSV">
</form>