Fix various bits for import

.idea/dataSources.xml

@@ -10,5 +10,11 @@
         <property name="enable_load_extension" value="true" />
       </driver-properties>
     </data-source>
+    <data-source source="LOCAL" name="PostgreSQL foodloop@localhost" uuid="8161e393-4db4-4e03-aa8b-7a961f14a591">
+      <driver-ref>postgresql</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.postgresql.Driver</jdbc-driver>
+      <jdbc-url>jdbc:postgresql://localhost:5432/</jdbc-url>
+    </data-source>
   </component>
 </project>

cpanfile

@@ -24,6 +24,7 @@ requires 'GIS::Distance';
 requires 'Text::CSV';
 requires 'Try::Tiny';
 requires 'Throwable::Error';
+requires 'Minion';
 
 on 'test' => sub {
   requires 'Test::More';
@@ -38,6 +39,7 @@ feature 'schema-graph', 'Draw diagrams of Schema' => sub {
 feature 'postgres', 'PostgreSQL Support' => sub {
   requires 'DBD::Pg';
   requires 'Test::PostgreSQL';
+  requires 'Mojo::Pg';
 };
 
 feature 'sqlite', 'SQLite Support' => sub {

lib/Pear/LocalLoop/Command/codepoint_open.pm

@@ -22,7 +22,12 @@ sub run {
 
   unless ( -d $output_dir ) {
     print "Unzipping code-point-open data\n" unless $quiet_mode;
-    system( 'unzip', '-q', $zip_file, '-d', $output_dir );
+    eval { system( 'unzip', '-q', $zip_file, '-d', $output_dir ) };
+    if ( my $err = $@ ) {
+        print "Error extracting zip: " . $err . "\n";
+        print "Manually create etc/code-point-open/codepo_gb directory and extract zip into it";
+        die;
+    }
   }
 
   my $cpo = Geo::UK::Postcode::CodePointOpen->new( path => $output_dir );

lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm

@@ -2,10 +2,15 @@ package Pear::LocalLoop::Controller::Admin::ImportFrom;
 use Mojo::Base 'Mojolicious::Controller';
 use Moo;
 use Try::Tiny;
-use Mojo::File qw/ path /;
+use Mojo::File qw/path/;
 
 sub index {
   my $c = shift;
+  $c->stash->{org_entities} = [
+    map {
+      { id => $_->entity_id, name => $_->name }
+    } $c->schema->resultset('Organisation')->all
+  ];
 
   $c->app->max_request_size(104857600);
 }
@@ -14,56 +19,61 @@ sub post_suppliers {
   my $c = shift;
 
   unless ($c->param('suppliers_csv')) {
-    $c->flash( error => "No CSV file given" );
-    return $c->redirect_to( '/admin/import_from' );
+    $c->flash(error => "No CSV file given");
+    return $c->redirect_to('/admin/import_from');
   }
 
   # Check file size
   if ($c->req->is_limit_exceeded) {
-    $c->flash( error => "CSV file size is too large" );
-    return $c->redirect_to( '/admin/import_from' );
+    $c->flash(error => "CSV file size is too large");
+    return $c->redirect_to('/admin/import_from');
   }
 
   my $file = $c->param('suppliers_csv');
 
-  my $filename = path($c->app->config->{upload_path}, time.'suppliers.csv' );
+  my $filename = path($c->app->config->{upload_path}, time . 'suppliers.csv');
 
   $file->move_to($filename);
 
-  my $job_id = $c->minion->enqueue('csv_supplier_import' => [$filename] );
+  my $job_id = $c->minion->enqueue('csv_supplier_import' => [ $filename ]);
 
   my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
 
   $c->flash(success => "CSV import started, see status of minion job at: $job_url");
-  return $c->redirect_to( '/admin/import_from' );
+  return $c->redirect_to('/admin/import_from');
 }
 
 sub post_transactions {
   my $c = shift;
 
+  unless ($c->param('entity_id') ne '') {
+    $c->flash(error => "Please Choose an organisation");
+    return $c->redirect_to('/admin/import_from');
+  }
+
   unless ($c->param('transactions_csv')) {
-    $c->flash( error => "No CSV file given" );
-    return $c->redirect_to( '/admin/import_from' );
+    $c->flash(error => "No CSV file given");
+    return $c->redirect_to('/admin/import_from');
   }
 
   # Check file size
   if ($c->req->is_limit_exceeded) {
-    $c->flash( error => "CSV file size is too large" );
-    return $c->redirect_to( '/admin/import_from' );
+    $c->flash(error => "CSV file size is too large");
+    return $c->redirect_to('/admin/import_from');
   }
 
   my $file = $c->param('transactions_csv');
 
-  my $filename = path($c->app->config->{upload_path}, time.'transactions.csv' );
+  my $filename = path($c->app->config->{upload_path}, time . 'transactions.csv');
 
   $file->move_to($filename);
 
-  my $job_id = $c->minion->enqueue('csv_transaction_import' => [$filename] );
+  my $job_id = $c->minion->enqueue('csv_transaction_import' => [ $filename, $c->param('entity_id') ]);
 
   my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
 
   $c->flash(success => "CSV import started, see status of minion job at: $job_url");
-  return $c->redirect_to( '/admin/import_from' );
+  return $c->redirect_to('/admin/import_from');
 }
 
 1;

lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm

@@ -5,34 +5,44 @@ use DateTime::Format::Strptime;
 
 extends qw/Pear::LocalLoop::Import::LCCCsv/;
 
+has target_entity_id => (
+  is       => 'ro',
+  required => 1,
+);
+
+has target_entity => (
+  is      => 'lazy',
+  builder => sub {
+    my $self = shift;
+    my $entity = $self->schema->resultset('Entity')->find($self->target_entity_id);
+    Pear::LocalLoop::Error->throw("Cannot find LCC Entity, did you pass the right id?") unless $entity;
+    return $entity;
+  },
+);
+
 has '+csv_required_columns' => (
-  builder => sub { return [ (
+  builder => sub {return [ (
     'transaction_id',
     'supplier_id',
     'net_amount',
     'vat amount',
     'gross_amount',
-  )]},
+  ) ]},
 );
 
 sub import_csv {
   my ($self) = @_;
 
   my $rows = $self->csv_data;
-  my $lcc_org = $self->schema->resultset('Organisation')->find({
-    name => "Lancashire County Council",
-    street_name => "County Hall"
-  });
-  unless ($lcc_org) {
-    Pear::LocalLoop::Error->throw("Cannot find LCC Organisation, please contact an admin");
-  }
-  foreach my $row ( @{$rows} ) {
+  my $lcc_org = $self->target_entity;
+
+  foreach my $row (@{$rows}) {
     $self->_row_to_result($row, $lcc_org);
   }
 }
 
 sub _row_to_result {
-  my ( $self, $row, $lcc_org ) = @_;
+  my ($self, $row, $lcc_org) = @_;
 
   my $supplier_id = $row->{supplier_id};
 
@@ -65,7 +75,7 @@ sub _row_to_result {
     external_id => $row->{transaction_id},
     transaction => {
       seller        => $organisation->entity,
-        buyer => $lcc_org->entity,
+      buyer         => $lcc_org,
       purchase_time => $paid_date,
       value         => $gross_value * 100000,
       meta          => {

lib/Pear/LocalLoop/Plugin/Minion/Job/csv_transaction_import.pm

@@ -4,11 +4,12 @@ use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
 use Pear::LocalLoop::Import::LCCCsv::Transactions;
 
 sub run {
-  my ( $self, $filename ) = @_;
+  my ($self, $filename, $entity_id) = @_;
 
-  my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new(
+  Pear::LocalLoop::Import::LCCCsv::Transactions->new(
     csv_file         => $filename,
-    schema => $self->app->schema
+    schema           => $self->app->schema,
+    target_entity_id => $entity_id,
   )->import_csv;
 }
 

pear-local_loop.development.conf

@@ -4,4 +4,7 @@
   user => undef,
   pass => undef,
   key => "a",
+  minion => {
+    SQLite => 'sqlite:minion.db',
+  },
 }; 

templates/admin/import_from/index.html.ep

@@ -21,7 +21,6 @@
     <p><strong>NOTE:</strong> Files must be in CSV format.<br>
     Redundant columns and rows above and to the left of the table in the CSV must be deleted.<br>
     There can be no duplicate header columns, and there can be no fields with values outside the table.</p>
-    <p>Warning: Large files will take a long time to process, just leave the tab open until the Success message appears.</p>
   </div>
   <div class="row">
     <div class="card col-md-6 m-3">
@@ -39,6 +38,12 @@
         <h4 class="card-title">LCC Procurement Import - Transactions</h4>
         <p>Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".</p>
         <form action="/admin/import_from/transactions" method="POST" enctype="multipart/form-data">
+          <select name="entity_id">
+            <option>Select an Organisation</option>
+            <% for my $org ( @$org_entities ) { %>
+            <option value="<%= $org->{id}; %>"><%= $org->{name}; %></option>
+            <% } %>
+          </select><br/>
           <input type="file" name="transactions_csv" accept="text/csv">
           <input type="submit" value="Upload Transactions CSV">
         </form>