From 9096bef00dacdde7c4ce348f93f101da342c85d2 Mon Sep 17 00:00:00 2001 From: Tom Bloor Date: Mon, 13 Nov 2017 13:30:33 +0000 Subject: [PATCH] Further work on import functions --- cpanfile | 2 + lib/Pear/LocalLoop.pm | 6 + lib/Pear/LocalLoop/Controller/Admin/Import.pm | 138 ++++++++++++++++++ .../LocalLoop/Schema/ResultSet/ImportSet.pm | 43 ++++++ templates/admin/import/get_add.html.ep | 43 ++++++ templates/admin/import/get_value.html.ep | 0 templates/admin/import/index.html.ep | 45 ++++++ templates/admin/import/list.html.ep | 102 +++++++++++++ templates/layouts/admin.html.ep | 3 + 9 files changed, 382 insertions(+) create mode 100644 lib/Pear/LocalLoop/Schema/ResultSet/ImportSet.pm create mode 100644 templates/admin/import/get_add.html.ep create mode 100644 templates/admin/import/get_value.html.ep create mode 100644 templates/admin/import/index.html.ep create mode 100644 templates/admin/import/list.html.ep diff --git a/cpanfile b/cpanfile index f3ab731..1916f82 100644 --- a/cpanfile +++ b/cpanfile @@ -22,6 +22,8 @@ requires 'Module::Runtime'; requires 'DBIx::Class::DeploymentHandler'; requires 'DBIx::Class::Fixtures'; requires 'GIS::Distance'; +requires 'Text::CSV'; +requires 'Try::Tiny'; feature 'schema-graph', 'Draw diagrams of Schema' => sub { requires 'GraphViz'; diff --git a/lib/Pear/LocalLoop.pm b/lib/Pear/LocalLoop.pm index 5a2bcfe..0457f2e 100644 --- a/lib/Pear/LocalLoop.pm +++ b/lib/Pear/LocalLoop.pm @@ -200,6 +200,12 @@ sub startup { $admin_routes->get('/reports/transactions')->to('admin-reports#transaction_data'); + $admin_routes->get('/import')->to('admin-import#index'); + $admin_routes->get('/import/add')->to('admin-import#get_add'); + $admin_routes->post('/import/add')->to('admin-import#post_add'); + $admin_routes->get('/import/:set_id')->to('admin-import#list'); + $admin_routes->get('/import/:set_id/:value_id')->to('admin-import#get_value'); + $admin_routes->post('/import/:set_id/:value_id')->to('admin-import#post_value'); # my $user_routes = $r->under('/')->to('root#under'); # $user_routes->get('/home')->to('root#home'); diff --git a/lib/Pear/LocalLoop/Controller/Admin/Import.pm b/lib/Pear/LocalLoop/Controller/Admin/Import.pm index 310c66a..87fb963 100644 --- a/lib/Pear/LocalLoop/Controller/Admin/Import.pm +++ b/lib/Pear/LocalLoop/Controller/Admin/Import.pm @@ -1,9 +1,147 @@ package Pear::LocalLoop::Controller::Admin::Import; use Mojo::Base 'Mojolicious::Controller'; +use Text::CSV; +use Try::Tiny; + +has result_set => sub { + my $c = shift; + return $c->schema->resultset('ImportSet'); +}; + sub index { my $c = shift; + my $import_rs = $c->result_set->search( + undef, + { + page => $c->param('page') || 1, + rows => 10, + order_by => { -desc => 'date' }, + }, + ); + $c->stash( import_rs => $import_rs ); +} + +sub list { + my $c = shift; + my $set_id = $c->param('set_id'); + + my $import_set = $c->result_set->find($set_id); + my $import_value_rs = $c->result_set->get_values($set_id); + my $import_users_rs = $c->result_set->get_users($set_id); + my $import_org_rs = $c->result_set->get_orgs($set_id); + + $c->stash( + import_set => $import_set, + import_value_rs => $import_value_rs, + import_users_rs => $import_users_rs, + import_org_rs => $import_org_rs, + ); +} + +sub get_add { + my $c = shift; +} + +sub post_add { + my $c = shift; + + my $csv_data = $c->param('csv'); + my $date_format = $c->param('date_format'); + + my $csv = Text::CSV->new({ + binary => 1, + allow_whitespace => 1, + }); + + open my $fh, '<', \$csv_data; + + # List context returns the actual headers + my @csv_headers; + my $error; + try { + @csv_headers = $csv->header( $fh ); + } catch { + $error = $_; + }; + + if ( defined $error ) { + $c->flash( error => $error, csv_data => $csv_data, date_format => $date_format ); + $c->redirect_to( '/admin/import/add' ); + return; + } + + # Text::CSV Already errors on duplicate columns, so this is fine + my @required = grep {/^user$|^value$|^date$|^organisation$/} @csv_headers; + + unless ( scalar( @required ) == 4 ) { + $c->flash( error => 'Required columns not available', csv_data => $csv_data, date_format => $date_format ); + $c->redirect_to( '/admin/import/add' ); + return; + } + + my $csv_output = $csv->getline_hr_all( $fh ); + + unless ( scalar( @$csv_output ) ) { + $c->flash( error => "No data found", csv_data => $csv_data, date_format => $date_format ); + $c->redirect_to( '/admin/import/add' ); + return; + } + + for my $data ( @$csv_output ) { + Dwarn $data; + for my $key ( qw/ user value organisation / ) { + unless ( defined $data->{$key} ) { + $c->flash( error => "Undefined [$key] data found", csv_data => $csv_data, date_format => $date_format ); + $c->redirect_to( '/admin/import/add' ); + return; + } + } + if ( defined $data->{date} ) { + my $dtp = DateTime::Format::Strptime->new( pattern => $date_format ); + my $dt_obj = $dtp->parse_datetime($data->{date}); + unless ( defined $dt_obj ) { + $c->flash( error => "Undefined or incorrect format for [date] data found", csv_data => $csv_data, date_format => $date_format ); + $c->redirect_to( '/admin/import/add' ); + return; + } + $data->{date} = $dt_obj; + } + } + + my $value_set; + $c->schema->txn_do( + sub { + $value_set = $c->result_set->create({}); + + $value_set->values->populate( + [ + [ qw/ user_name purchase_value purchase_date org_name / ], + ( map { [ @{$_}{qw/ user value date organisation /} ] } @$csv_output ), + ] + ); + } + ); + + unless ( defined $value_set ) { + $c->flash( error => 'Error creating new Value Set', csv_data => $csv_data, date_format => $date_format ); + $c->redirect_to( '/admin/import/add' ); + return; + } + + $c->flash( success => 'Created Value Set' ); + $c->redirect_to( '/admin/import/' . $value_set->id ); +} + +sub get_value { + my $c = shift; + my $set_id = $c->param('set_id'); +} + +sub post_value { + my $c = shift; + my $set_id = $c->param('set_id'); } 1; diff --git a/lib/Pear/LocalLoop/Schema/ResultSet/ImportSet.pm b/lib/Pear/LocalLoop/Schema/ResultSet/ImportSet.pm new file mode 100644 index 0000000..614110a --- /dev/null +++ b/lib/Pear/LocalLoop/Schema/ResultSet/ImportSet.pm @@ -0,0 +1,43 @@ +package Pear::LocalLoop::Schema::ResultSet::ImportSet; + +use strict; +use warnings; + +use base 'DBIx::Class::ResultSet'; + +sub get_values { + my $self = shift; + my $id = shift; + + return $self->find($id)->search_related( + 'values', + undef, + { + order_by => { -asc => 'id' }, + }, + ); +} + +sub get_users { + my $self = shift; + my $id = shift; + + return $self->get_values($id)->search({}, + { + group_by => 'user_name', + }, + ); +} + +sub get_orgs { + my $self = shift; + my $id = shift; + + return $self->get_values($id)->search({}, + { + group_by => 'org_name', + }, + ); +} + +1; diff --git a/templates/admin/import/get_add.html.ep b/templates/admin/import/get_add.html.ep new file mode 100644 index 0000000..4b5359e --- /dev/null +++ b/templates/admin/import/get_add.html.ep @@ -0,0 +1,43 @@ +% layout 'admin'; +% title 'Import'; +% content_for javascript => begin +% end +% if ( my $error = flash 'error' ) { + +% } elsif ( my $success = flash 'success' ) { + +% } +
+
+

Add CSV Import

+

+ Copy and paste a CSV in to import it. The first line should contain the + headers The following headers are recognised: +

+
    +
  • user
    • +
  • value
    • +
  • date
    • +
  • organisation
    • +
+

+ Any columns which are not in the list above will be ignored and can be + called anything you want, as long as they are not in the list above. They + must also be unique. +

+

+ For date format, consult here for the patterns +

+
+
+
+ + + +
+
+
diff --git a/templates/admin/import/get_value.html.ep b/templates/admin/import/get_value.html.ep new file mode 100644 index 0000000..e69de29 diff --git a/templates/admin/import/index.html.ep b/templates/admin/import/index.html.ep new file mode 100644 index 0000000..7d83b86 --- /dev/null +++ b/templates/admin/import/index.html.ep @@ -0,0 +1,45 @@ +% layout 'admin'; +% title 'Import'; +% content_for javascript => begin +% end +% if ( my $error = flash 'error' ) { + +% } elsif ( my $success = flash 'success' ) { + +% } +
+
+
+

+ CSV Import + Import Data +

+
+
+ % for my $import ( $import_rs->all ) { +
+
+
+ <%= $import->id %> + %= format_human_datetime $import->date; +
+
+
+
+ + Edit Import + +
+
+
+ % } +
+
+
+ %= bootstrap_pagination( $c->param('page') || 1, $import_rs->pager->last_page, { class => 'justify-content-center' } ); +
+
diff --git a/templates/admin/import/list.html.ep b/templates/admin/import/list.html.ep new file mode 100644 index 0000000..cc0f1e4 --- /dev/null +++ b/templates/admin/import/list.html.ep @@ -0,0 +1,102 @@ +% layout 'admin'; +% title 'Import'; +% content_for javascript => begin +% end +% if ( my $error = flash 'error' ) { + +% } elsif ( my $success = flash 'success' ) { + +% } +
+
+
+

+ User Assignments +

+
+ Unique users in this Import, and their assigned entity +
+
+ % for my $user ( $import_users_rs->all ) { +
+
+
+ %= $user->user_name +
+
+ Unassigned +
+
+ Select +
+
+
+ % } +
+
+
+
+
+

+ Org Assignments +

+
+ Unique orgs in this Import, and their assigned entity +
+
+ % for my $org ( $import_org_rs->all ) { +
+
+
+ %= $org->org_name +
+
+ Unassigned +
+
+ Select +
+
+
+ % } +
+
+
+
+
+

+ %= format_human_datetime $import_set->date; +

+
+ Content listed in original order of import +
+
+ % for my $import_value ( $import_value_rs->all ) { +
+
+
+ <%= $import_value->user_name %> +
+
+ <%= format_human_datetime $import_value->purchase_date %> +
+
+ <%= $import_value->purchase_value %> +
+
+ <%= $import_value->org_name %> +
+
+ Ignore +
+
+
+ % } +
+
+
+
diff --git a/templates/layouts/admin.html.ep b/templates/layouts/admin.html.ep index 265f51e..c546b04 100644 --- a/templates/layouts/admin.html.ep +++ b/templates/layouts/admin.html.ep @@ -50,6 +50,9 @@
  • Organisations
    • +
  • + Import +
  • Logout