Fix various bits for import

This commit is contained in:
Tom Bloor 2019-07-14 15:15:14 +01:00
parent a45c354834
commit 71189d18fc
No known key found for this signature in database
GPG Key ID: 4657C7EBE42CC5CC
8 changed files with 111 additions and 69 deletions

View File

@ -10,5 +10,11 @@
<property name="enable_load_extension" value="true" />
</driver-properties>
</data-source>
<data-source source="LOCAL" name="PostgreSQL foodloop@localhost" uuid="8161e393-4db4-4e03-aa8b-7a961f14a591">
<driver-ref>postgresql</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>org.postgresql.Driver</jdbc-driver>
<jdbc-url>jdbc:postgresql://localhost:5432/</jdbc-url>
</data-source>
</component>
</project>

View File

@ -24,6 +24,7 @@ requires 'GIS::Distance';
requires 'Text::CSV';
requires 'Try::Tiny';
requires 'Throwable::Error';
requires 'Minion';
on 'test' => sub {
requires 'Test::More';
@ -38,6 +39,7 @@ feature 'schema-graph', 'Draw diagrams of Schema' => sub {
feature 'postgres', 'PostgreSQL Support' => sub {
requires 'DBD::Pg';
requires 'Test::PostgreSQL';
requires 'Mojo::Pg';
};
feature 'sqlite', 'SQLite Support' => sub {

View File

@ -22,7 +22,12 @@ sub run {
unless ( -d $output_dir ) {
print "Unzipping code-point-open data\n" unless $quiet_mode;
system( 'unzip', '-q', $zip_file, '-d', $output_dir );
eval { system( 'unzip', '-q', $zip_file, '-d', $output_dir ) };
if ( my $err = $@ ) {
print "Error extracting zip: " . $err . "\n";
print "Manually create etc/code-point-open/codepo_gb directory and extract zip into it";
die;
}
}
my $cpo = Geo::UK::Postcode::CodePointOpen->new( path => $output_dir );

View File

@ -6,6 +6,11 @@ use Mojo::File qw/ path /;
sub index {
my $c = shift;
$c->stash->{org_entities} = [
map {
{ id => $_->entity_id, name => $_->name }
} $c->schema->resultset('Organisation')->all
];
$c->app->max_request_size(104857600);
}
@ -41,6 +46,11 @@ sub post_suppliers {
sub post_transactions {
my $c = shift;
unless ($c->param('entity_id') ne '') {
$c->flash(error => "Please Choose an organisation");
return $c->redirect_to('/admin/import_from');
}
unless ($c->param('transactions_csv')) {
$c->flash(error => "No CSV file given");
return $c->redirect_to('/admin/import_from');
@ -58,7 +68,7 @@ sub post_transactions {
$file->move_to($filename);
my $job_id = $c->minion->enqueue('csv_transaction_import' => [$filename] );
my $job_id = $c->minion->enqueue('csv_transaction_import' => [ $filename, $c->param('entity_id') ]);
my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;

View File

@ -5,6 +5,21 @@ use DateTime::Format::Strptime;
extends qw/Pear::LocalLoop::Import::LCCCsv/;
has target_entity_id => (
is => 'ro',
required => 1,
);
has target_entity => (
is => 'lazy',
builder => sub {
my $self = shift;
my $entity = $self->schema->resultset('Entity')->find($self->target_entity_id);
Pear::LocalLoop::Error->throw("Cannot find LCC Entity, did you pass the right id?") unless $entity;
return $entity;
},
);
has '+csv_required_columns' => (
builder => sub {return [ (
'transaction_id',
@ -19,13 +34,8 @@ sub import_csv {
my ($self) = @_;
my $rows = $self->csv_data;
my $lcc_org = $self->schema->resultset('Organisation')->find({
name => "Lancashire County Council",
street_name => "County Hall"
});
unless ($lcc_org) {
Pear::LocalLoop::Error->throw("Cannot find LCC Organisation, please contact an admin");
}
my $lcc_org = $self->target_entity;
foreach my $row (@{$rows}) {
$self->_row_to_result($row, $lcc_org);
}
@ -65,7 +75,7 @@ sub _row_to_result {
external_id => $row->{transaction_id},
transaction => {
seller => $organisation->entity,
buyer => $lcc_org->entity,
buyer => $lcc_org,
purchase_time => $paid_date,
value => $gross_value * 100000,
meta => {

View File

@ -4,11 +4,12 @@ use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
use Pear::LocalLoop::Import::LCCCsv::Transactions;
sub run {
my ( $self, $filename ) = @_;
my ($self, $filename, $entity_id) = @_;
my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new(
Pear::LocalLoop::Import::LCCCsv::Transactions->new(
csv_file => $filename,
schema => $self->app->schema
schema => $self->app->schema,
target_entity_id => $entity_id,
)->import_csv;
}

View File

@ -4,4 +4,7 @@
user => undef,
pass => undef,
key => "a",
minion => {
SQLite => 'sqlite:minion.db',
},
};

View File

@ -21,7 +21,6 @@
<p><strong>NOTE:</strong> Files must be in CSV format.<br>
Redundant columns and rows above and to the left of the table in the CSV must be deleted.<br>
There can be no duplicate header columns, and there can be no fields with values outside the table.</p>
<p>Warning: Large files will take a long time to process, just leave the tab open until the Success message appears.</p>
</div>
<div class="row">
<div class="card col-md-6 m-3">
@ -39,6 +38,12 @@
<h4 class="card-title">LCC Procurement Import - Transactions</h4>
<p>Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".</p>
<form action="/admin/import_from/transactions" method="POST" enctype="multipart/form-data">
<select name="entity_id">
<option>Select an Organisation</option>
<% for my $org ( @$org_entities ) { %>
<option value="<%= $org->{id}; %>"><%= $org->{name}; %></option>
<% } %>
</select><br/>
<input type="file" name="transactions_csv" accept="text/csv">
<input type="submit" value="Upload Transactions CSV">
</form>