Fix various bits for import

.idea/dataSources.xml

@@ -10,5 +10,11 @@
         <property name="enable_load_extension" value="true" />
       </driver-properties>
     </data-source>
+    <data-source source="LOCAL" name="PostgreSQL foodloop@localhost" uuid="8161e393-4db4-4e03-aa8b-7a961f14a591">
+      <driver-ref>postgresql</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.postgresql.Driver</jdbc-driver>
+      <jdbc-url>jdbc:postgresql://localhost:5432/</jdbc-url>
+    </data-source>
   </component>
 </project>

cpanfile

@@ -24,6 +24,7 @@ requires 'GIS::Distance';
 requires 'Text::CSV';
 requires 'Try::Tiny';
 requires 'Throwable::Error';
+requires 'Minion';
 
 on 'test' => sub {
   requires 'Test::More';
@@ -38,6 +39,7 @@ feature 'schema-graph', 'Draw diagrams of Schema' => sub {
 feature 'postgres', 'PostgreSQL Support' => sub {
   requires 'DBD::Pg';
   requires 'Test::PostgreSQL';
+  requires 'Mojo::Pg';
 };
 
 feature 'sqlite', 'SQLite Support' => sub {

lib/Pear/LocalLoop/Command/codepoint_open.pm

@@ -22,7 +22,12 @@ sub run {
 
   unless ( -d $output_dir ) {
     print "Unzipping code-point-open data\n" unless $quiet_mode;
-    system( 'unzip', '-q', $zip_file, '-d', $output_dir );
+    eval { system( 'unzip', '-q', $zip_file, '-d', $output_dir ) };
+    if ( my $err = $@ ) {
+        print "Error extracting zip: " . $err . "\n";
+        print "Manually create etc/code-point-open/codepo_gb directory and extract zip into it";
+        die;
+    }
   }
 
   my $cpo = Geo::UK::Postcode::CodePointOpen->new( path => $output_dir );

lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm

@@ -2,10 +2,15 @@ package Pear::LocalLoop::Controller::Admin::ImportFrom;
 use Mojo::Base 'Mojolicious::Controller';
 use Moo;
 use Try::Tiny;
-use Mojo::File qw/ path /;
+use Mojo::File qw/path/;
 
 sub index {
   my $c = shift;
+  $c->stash->{org_entities} = [
+    map {
+      { id => $_->entity_id, name => $_->name }
+    } $c->schema->resultset('Organisation')->all
+  ];
 
   $c->app->max_request_size(104857600);
 }
@@ -14,56 +19,61 @@ sub post_suppliers {
   my $c = shift;
 
   unless ($c->param('suppliers_csv')) {
-    $c->flash( error => "No CSV file given" );
+    $c->flash(error => "No CSV file given");
-    return $c->redirect_to( '/admin/import_from' );
+    return $c->redirect_to('/admin/import_from');
   }
 
   # Check file size
   if ($c->req->is_limit_exceeded) {
-    $c->flash( error => "CSV file size is too large" );
+    $c->flash(error => "CSV file size is too large");
-    return $c->redirect_to( '/admin/import_from' );
+    return $c->redirect_to('/admin/import_from');
   }
 
   my $file = $c->param('suppliers_csv');
 
-  my $filename = path($c->app->config->{upload_path}, time.'suppliers.csv' );
+  my $filename = path($c->app->config->{upload_path}, time . 'suppliers.csv');
 
   $file->move_to($filename);
 
-  my $job_id = $c->minion->enqueue('csv_supplier_import' => [$filename] );
+  my $job_id = $c->minion->enqueue('csv_supplier_import' => [ $filename ]);
 
   my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
 
   $c->flash(success => "CSV import started, see status of minion job at: $job_url");
-  return $c->redirect_to( '/admin/import_from' );
+  return $c->redirect_to('/admin/import_from');
 }
 
 sub post_transactions {
   my $c = shift;
 
+  unless ($c->param('entity_id') ne '') {
+    $c->flash(error => "Please Choose an organisation");
+    return $c->redirect_to('/admin/import_from');
+  }
+
   unless ($c->param('transactions_csv')) {
-    $c->flash( error => "No CSV file given" );
+    $c->flash(error => "No CSV file given");
-    return $c->redirect_to( '/admin/import_from' );
+    return $c->redirect_to('/admin/import_from');
   }
 
   # Check file size
   if ($c->req->is_limit_exceeded) {
-    $c->flash( error => "CSV file size is too large" );
+    $c->flash(error => "CSV file size is too large");
-    return $c->redirect_to( '/admin/import_from' );
+    return $c->redirect_to('/admin/import_from');
   }
 
   my $file = $c->param('transactions_csv');
 
-  my $filename = path($c->app->config->{upload_path}, time.'transactions.csv' );
+  my $filename = path($c->app->config->{upload_path}, time . 'transactions.csv');
 
   $file->move_to($filename);
 
-  my $job_id = $c->minion->enqueue('csv_transaction_import' => [$filename] );
+  my $job_id = $c->minion->enqueue('csv_transaction_import' => [ $filename, $c->param('entity_id') ]);
 
   my $job_url = $c->url_for("/admin/minion/jobs?id=$job_id")->to_abs;
 
   $c->flash(success => "CSV import started, see status of minion job at: $job_url");
-  return $c->redirect_to( '/admin/import_from' );
+  return $c->redirect_to('/admin/import_from');
 }
 
 1;

lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm

@@ -5,76 +5,86 @@ use DateTime::Format::Strptime;
 
 extends qw/Pear::LocalLoop::Import::LCCCsv/;
 
+has target_entity_id => (
+  is       => 'ro',
+  required => 1,
+);
+
+has target_entity => (
+  is      => 'lazy',
+  builder => sub {
+    my $self = shift;
+    my $entity = $self->schema->resultset('Entity')->find($self->target_entity_id);
+    Pear::LocalLoop::Error->throw("Cannot find LCC Entity, did you pass the right id?") unless $entity;
+    return $entity;
+  },
+);
+
 has '+csv_required_columns' => (
-  builder => sub { return [ (
+  builder => sub {return [ (
-  'transaction_id',
+    'transaction_id',
-  'supplier_id',
+    'supplier_id',
-  'net_amount',
+    'net_amount',
-  'vat amount',
+    'vat amount',
-  'gross_amount',
+    'gross_amount',
-  )]},
+  ) ]},
 );
 
 sub import_csv {
   my ($self) = @_;
 
   my $rows = $self->csv_data;
-  my $lcc_org = $self->schema->resultset('Organisation')->find({
+  my $lcc_org = $self->target_entity;
-    name => "Lancashire County Council",
+
-    street_name => "County Hall"
+  foreach my $row (@{$rows}) {
-  });
-  unless ($lcc_org) {
-    Pear::LocalLoop::Error->throw("Cannot find LCC Organisation, please contact an admin");
-  }
-  foreach my $row ( @{$rows} ) {
     $self->_row_to_result($row, $lcc_org);
   }
 }
 
 sub _row_to_result {
-  my ( $self, $row, $lcc_org ) = @_;
+  my ($self, $row, $lcc_org) = @_;
 
-    my $supplier_id = $row->{supplier_id};
+  my $supplier_id = $row->{supplier_id};
 
-    my $organisation = $self->schema->resultset('Organisation')->find({
+  my $organisation = $self->schema->resultset('Organisation')->find({
-      'external_reference.external_id' => $supplier_id
+    'external_reference.external_id' => $supplier_id
-    }, { join => 'external_reference' });
+  }, { join => 'external_reference' });
 
-    unless ($organisation) {
+  unless ($organisation) {
-      Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
+    Pear::LocalLoop::Error->throw("Cannot find an organisation with supplier_id $supplier_id");
-    }
+  }
 
-    my $date_formatter = DateTime::Format::Strptime->new(
+  my $date_formatter = DateTime::Format::Strptime->new(
-      pattern => '%m/%d/%Y',
+    pattern   => '%m/%d/%Y',
-      time_zone => 'Europe/London'
+    time_zone => 'Europe/London'
-    );
+  );
 
     my $paid_date = ( $row->{paid_date} ?
       $date_formatter->parse_datetime($row->{paid_date}) :
       $date_formatter->parse_datetime($row->{invoice_date}) );
 
-    my $gross_value = $row->{gross_amount};
+  my $gross_value = $row->{gross_amount};
-    $gross_value =~ s/,//g;
+  $gross_value =~ s/,//g;
-    my $sales_tax_value = $row->{"vat amount"};
+  my $sales_tax_value = $row->{"vat amount"};
-    $sales_tax_value =~ s/,//g;
+  $sales_tax_value =~ s/,//g;
-    my $net_value = $row->{net_amount};
+  my $net_value = $row->{net_amount};
-    $net_value =~ s/,//g;
+  $net_value =~ s/,//g;
 
-    # TODO negative values are sometimes present
+  # TODO negative values are sometimes present
-    $self->external_result->find_or_create_related('transactions', {
+  $self->external_result->find_or_create_related('transactions', {
-      external_id => $row->{transaction_id},
+    external_id => $row->{transaction_id},
-      transaction => {
+    transaction => {
-        seller => $organisation->entity,
+      seller        => $organisation->entity,
-        buyer => $lcc_org->entity,
+      buyer         => $lcc_org,
-        purchase_time => $paid_date,
+      purchase_time => $paid_date,
-        value => $gross_value * 100000,
+      value         => $gross_value * 100000,
-        meta => {
+      meta          => {
-          gross_value => $gross_value * 100000,
+        gross_value     => $gross_value * 100000,
-          sales_tax_value => $sales_tax_value * 100000,
+        sales_tax_value => $sales_tax_value * 100000,
-          net_value => $net_value * 100000,
+        net_value       => $net_value * 100000,
-        },
+      },
-      }
+    }
-    });
+  });
 }
 
 1;

lib/Pear/LocalLoop/Plugin/Minion/Job/csv_transaction_import.pm

@@ -4,11 +4,12 @@ use Mojo::Base 'Pear::LocalLoop::Plugin::Minion::Job';
 use Pear::LocalLoop::Import::LCCCsv::Transactions;
 
 sub run {
-  my ( $self, $filename ) = @_;
+  my ($self, $filename, $entity_id) = @_;
 
-  my $csv_import = Pear::LocalLoop::Import::LCCCsv::Transactions->new(
+  Pear::LocalLoop::Import::LCCCsv::Transactions->new(
-    csv_file => $filename,
+    csv_file         => $filename,
-    schema => $self->app->schema
+    schema           => $self->app->schema,
+    target_entity_id => $entity_id,
   )->import_csv;
 }
 

pear-local_loop.development.conf

@@ -4,4 +4,7 @@
   user => undef,
   pass => undef,
   key => "a",
+  minion => {
+    SQLite => 'sqlite:minion.db',
+  },
 }; 

templates/admin/import_from/index.html.ep

@@ -21,7 +21,6 @@
     <p><strong>NOTE:</strong> Files must be in CSV format.<br>
     Redundant columns and rows above and to the left of the table in the CSV must be deleted.<br>
     There can be no duplicate header columns, and there can be no fields with values outside the table.</p>
-    <p>Warning: Large files will take a long time to process, just leave the tab open until the Success message appears.</p>
   </div>
   <div class="row">
     <div class="card col-md-6 m-3">
@@ -39,6 +38,12 @@
         <h4 class="card-title">LCC Procurement Import - Transactions</h4>
         <p>Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".</p>
         <form action="/admin/import_from/transactions" method="POST" enctype="multipart/form-data">
+          <select name="entity_id">
+            <option>Select an Organisation</option>
+            <% for my $org ( @$org_entities ) { %>
+            <option value="<%= $org->{id}; %>"><%= $org->{name}; %></option>
+            <% } %>
+          </select><br/>
           <input type="file" name="transactions_csv" accept="text/csv">
           <input type="submit" value="Upload Transactions CSV">
         </form>