From 962cf972dafc40608f5a1d4ea2809085332300be Mon Sep 17 00:00:00 2001 From: Thomas Bloor Date: Mon, 9 Sep 2019 15:37:26 +0100 Subject: [PATCH] Hopefully fix speed issue on external data --- .gitignore | 1 + .idea/.gitignore | 7 ++ .idea/Foodloop-Server.iml | 1 + .idea/sqldialects.xml | 7 ++ README.md | 57 +++++++++++- cpanfile | 3 +- lib/Pear/LocalLoop.pm | 1 + .../LocalLoop/Controller/Admin/ImportFrom.pm | 20 +++++ .../Controller/Admin/Transactions.pm | 17 ++-- lib/Pear/LocalLoop/Controller/Api/External.pm | 36 +++----- .../Result/ViewQuantisedTransactionPg.pm | 14 +++ templates/admin/import/get_org.html.ep | 2 +- templates/admin/import/get_user.html.ep | 2 +- templates/admin/import_from/index.html.ep | 32 +++---- .../admin/organisations/merge_detail.html.ep | 2 +- .../admin/organisations/merge_list.html.ep | 2 +- templates/layouts/admin.html.ep | 41 +++++++-- templates/layouts/admin_errors.html.ep | 89 ------------------- templates/root/index.html.ep | 4 +- 19 files changed, 182 insertions(+), 156 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/sqldialects.xml delete mode 100644 templates/layouts/admin_errors.html.ep diff --git a/.gitignore b/.gitignore index 4f7c461..3dd59f9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ hypnotoad.pid *.db *.db-wal *.db-shm +*.db-journal *~ /images *.swp diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..1519111 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,7 @@ +# Default ignored files +/workspace.xml +/perl5local.xml + +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml \ No newline at end of file diff --git a/.idea/Foodloop-Server.iml b/.idea/Foodloop-Server.iml index 9e66f1a..d83ec66 100644 --- a/.idea/Foodloop-Server.iml +++ b/.idea/Foodloop-Server.iml @@ -3,6 +3,7 @@ + diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml new file mode 100644 index 0000000..3738403 --- /dev/null +++ b/.idea/sqldialects.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index 065845c..513afa1 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,22 @@ And then add the following to your configuration file: }, ``` -This will then use an SQLite db for the minion backend, at minion.db +This will then use an SQLite db for the minion backend, using `minion.db` as +the database file. To start the minion itself, run: +``` +./script/pear-local_loop minion worker +``` + +# Importing Ward Data + +To import ward data, get the ward data csv and then run the following command: + +```shell script +./script/pear-local_loop minion job \ + --enqueue 'csv_postcode_import' \ + --args '[ "/path/to/ward/csv" ]' +``` ## Example PostgreSQL setup @@ -57,7 +71,46 @@ psql=# alter user minion with encrypted password 'abc123'; psql=# grant all privileges on database localloop_minion to minion; ``` -# Dev notes +# Development + +There are a couple of setup steps to getting a development environment ready. +Use the corresponding instructions depending on what state your current setup +is in. + +## First Time Setup + +First, decide if you're using SQLite or PostgreSQL locally. Development supports +both, however production uses PostgreSQL. For this example we will use SQLite. +As the default config is set up for this, no configuration changes are +needed initially. So, first off, install dependencies: + +```shell script +cpanm --installdeps . --with-feature=sqlite +``` + +Then install the database: + +```shell script +./script/deploy_db install -c 'dbi:SQLite:dbname=foodloop.db' +``` + +Then set up the development users: + +```shell script +./script/pear-local_loop dev_data --force +``` + +***Note: do NOT run that script on production.*** + +Then you can start the application: + +```shell script +morbo script/pear-local_loop -l http://*:3000 +``` + +You can modify the host and port for listening as needed. + +# Old Docs ## Local test database diff --git a/cpanfile b/cpanfile index c50ecd7..4ea8baf 100644 --- a/cpanfile +++ b/cpanfile @@ -14,7 +14,6 @@ requires 'DBIx::Class::Schema::Loader'; requires 'SQL::Translator'; requires 'DateTime'; requires 'DateTime::Format::Strptime', "1.73"; -requires 'DateTime::Format::SQLite'; requires 'Try::Tiny'; requires 'MooX::Options::Actions'; requires 'Module::Runtime'; @@ -40,10 +39,12 @@ feature 'postgres', 'PostgreSQL Support' => sub { requires 'DBD::Pg'; requires 'Test::PostgreSQL'; requires 'Mojo::Pg'; + requires 'DateTime::Format::Pg'; }; feature 'sqlite', 'SQLite Support' => sub { requires 'Minion::Backend::SQLite'; + requires 'DateTime::Format::SQLite'; }; feature 'codepoint-open', 'Code Point Open manipulation' => sub { diff --git a/lib/Pear/LocalLoop.pm b/lib/Pear/LocalLoop.pm index 6697cd7..caa62ae 100644 --- a/lib/Pear/LocalLoop.pm +++ b/lib/Pear/LocalLoop.pm @@ -267,6 +267,7 @@ sub startup { $admin_routes->post('/import_from/suppliers')->to('admin-import_from#post_suppliers'); $admin_routes->post('/import_from/transactions')->to('admin-import_from#post_transactions'); $admin_routes->post('/import_from/postcodes')->to('admin-import_from#post_postcodes'); + $admin_routes->get('/import_from/org_search')->to('admin-import_from#org_search'); # my $user_routes = $r->under('/')->to('root#under'); diff --git a/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm index a01924e..aa7a6b4 100644 --- a/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm +++ b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm @@ -98,4 +98,24 @@ sub post_transactions { return $c->redirect_to('/admin/import_from'); } +sub org_search { + my $c = shift; + my $term = $c->param('term'); + + my $rs = $c->schema->resultset('Organisation')->search( + { name => { like => $term . '%' } }, + { + join => 'entity', + columns => [ qw/ me.name entity.id / ] + }, + ); + + my @results = ( map { { + label => $_->name, + value => $_->entity->id, + } } $rs->all); + + $c->render( json => \@results ); +} + 1; diff --git a/lib/Pear/LocalLoop/Controller/Admin/Transactions.pm b/lib/Pear/LocalLoop/Controller/Admin/Transactions.pm index 05bcd70..c73697f 100644 --- a/lib/Pear/LocalLoop/Controller/Admin/Transactions.pm +++ b/lib/Pear/LocalLoop/Controller/Admin/Transactions.pm @@ -17,15 +17,10 @@ sub index { my $week_transaction_rs = $c->schema->resultset('ViewQuantisedTransaction' . $driver)->search( {}, { - columns => [ - { - quantised => 'quantised_weeks', - count => \"COUNT(*)", - sum_value => $c->pg_or_sqlite( - 'SUM("me"."value")', - 'SUM("me"."value")', - ), - } + select => [ + { count => 'id', '-as' => 'count' }, + { sum => 'value', '-as' => 'sum_value' }, + 'quantised_weeks', ], group_by => 'quantised_weeks', order_by => { '-asc' => 'quantised_weeks' }, @@ -33,8 +28,8 @@ sub index { ); my @all_weeks = $week_transaction_rs->all; - my $first_week_count = $all_weeks[0]->get_column('count') || 0; - my $first_week_value = $all_weeks[0]->get_column('sum_value') / 100000 || 0; + my $first_week_count = defined $all_weeks[0] ? $all_weeks[0]->get_column('count') || 0 : 0; + my $first_week_value = defined $all_weeks[0] ? $all_weeks[0]->get_column('sum_value') / 100000 || 0 : 0; my $second_week_count = defined $all_weeks[1] ? $all_weeks[1]->get_column('count') || 0 : 0; my $second_week_value = defined $all_weeks[1] ? $all_weeks[1]->get_column('sum_value') / 100000 || 0 : 0; diff --git a/lib/Pear/LocalLoop/Controller/Api/External.pm b/lib/Pear/LocalLoop/Controller/Api/External.pm index 43a3087..a33c5f2 100644 --- a/lib/Pear/LocalLoop/Controller/Api/External.pm +++ b/lib/Pear/LocalLoop/Controller/Api/External.pm @@ -199,40 +199,24 @@ sub post_supplier_count { buyer_id => $user->entity->id, }, { - columns => [ - 'seller_id', - { - quantised => 'quantised_days', - count => \"COUNT(*)", - total_spend => { sum => 'value' }, - } + prefetch => { 'seller' => 'organisation' }, + select => [ + { count => 'me.id', '-as' => 'count' }, + { sum => 'me.value', '-as' => 'total_spend' }, + 'organisation.name', + 'me.quantised_days', ], - group_by => [ 'quantised_days', 'seller_id' ], - order_by => { '-asc' => 'quantised_days' }, + group_by => [ 'me.quantised_days', 'seller.id' ], + order_by => { '-asc' => 'me.quantised_days' }, } ); - my $name_rs = $c->schema->resultset('Transaction')->search( - { - 'me.buyer_id' => $user->entity->id, - }, - { - join => { seller => 'organisation' }, - } - ); - - my %name_map = ( - map { - $_->seller->id => $_->seller->organisation->name, - } $name_rs->all - ); - my @graph_data = ( map {{ count => $_->get_column('count'), value => ($_->get_column('total_spend') / 100000) // 0, - date => $_->get_column('quantised'), - seller => $name_map{ $_->get_column('seller_id') }, + date => $_->get_column('quantised_days'), + seller => $_->seller->organisation->name, }} $spend_rs->all, ); diff --git a/lib/Pear/LocalLoop/Schema/Result/ViewQuantisedTransactionPg.pm b/lib/Pear/LocalLoop/Schema/Result/ViewQuantisedTransactionPg.pm index a524424..c710908 100644 --- a/lib/Pear/LocalLoop/Schema/Result/ViewQuantisedTransactionPg.pm +++ b/lib/Pear/LocalLoop/Schema/Result/ViewQuantisedTransactionPg.pm @@ -22,4 +22,18 @@ SELECT "value", FROM "transactions" /); +__PACKAGE__->belongs_to( + "buyer", + "Pear::LocalLoop::Schema::Result::Entity", + { id => "buyer_id" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" }, +); + +__PACKAGE__->belongs_to( + "seller", + "Pear::LocalLoop::Schema::Result::Entity", + { id => "seller_id" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" }, +); + 1; diff --git a/templates/admin/import/get_org.html.ep b/templates/admin/import/get_org.html.ep index b6bc8a1..56bfeaf 100644 --- a/templates/admin/import/get_org.html.ep +++ b/templates/admin/import/get_org.html.ep @@ -1,4 +1,4 @@ -% layout 'admin_errors'; +% layout 'admin'; % title 'Import';
diff --git a/templates/admin/import/get_user.html.ep b/templates/admin/import/get_user.html.ep index d0ead9d..b24736d 100644 --- a/templates/admin/import/get_user.html.ep +++ b/templates/admin/import/get_user.html.ep @@ -1,4 +1,4 @@ -% layout 'admin_errors'; +% layout 'admin'; % title 'Import';
diff --git a/templates/admin/import_from/index.html.ep b/templates/admin/import_from/index.html.ep index f649564..068c0d5 100644 --- a/templates/admin/import_from/index.html.ep +++ b/templates/admin/import_from/index.html.ep @@ -1,6 +1,20 @@ % layout 'admin'; % title 'Import From'; % content_for javascript => begin + % end % if (my $error = flash 'error') {
-
-
-

Postcode Data

-

Expected headers at very least: "postcode", "ward".

-
- - -
-
-

LCC Procurement Import - Transactions

Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".

-
+ +
diff --git a/templates/admin/organisations/merge_detail.html.ep b/templates/admin/organisations/merge_detail.html.ep index 4b3afce..a0f7395 100644 --- a/templates/admin/organisations/merge_detail.html.ep +++ b/templates/admin/organisations/merge_detail.html.ep @@ -1,4 +1,4 @@ -% layout 'admin_errors'; +% layout 'admin'; % title 'Organisations';
diff --git a/templates/admin/organisations/merge_list.html.ep b/templates/admin/organisations/merge_list.html.ep index 0a0dc49..b177c81 100644 --- a/templates/admin/organisations/merge_list.html.ep +++ b/templates/admin/organisations/merge_list.html.ep @@ -1,4 +1,4 @@ -% layout 'admin_errors'; +% layout 'admin'; % title 'Organisations';
diff --git a/templates/layouts/admin.html.ep b/templates/layouts/admin.html.ep index 4e2405c..68c084f 100644 --- a/templates/layouts/admin.html.ep +++ b/templates/layouts/admin.html.ep @@ -6,8 +6,13 @@ LocalLoop Admin - <%= title %> - - + + %= stylesheet '/static/admin/css/main.css'; @@ -59,6 +64,9 @@
  • Import
    • +
  • + Minion +
  • Logout
    • @@ -66,6 +74,19 @@
    + % if ( my $f_error = flash 'error' ) { + + % } elsif ( my $s_error = stash 'error' ) { + + % } elsif ( my $success = flash 'success' ) { + + % } <%= content %>
    @@ -73,10 +94,18 @@ Version: <%= $c->config->{version} %>
    - - - - + + + + %= content_for 'javascript'; diff --git a/templates/layouts/admin_errors.html.ep b/templates/layouts/admin_errors.html.ep deleted file mode 100644 index 808a5b9..0000000 --- a/templates/layouts/admin_errors.html.ep +++ /dev/null @@ -1,89 +0,0 @@ - - - - - - LocalLoop Admin - <%= title %> - - - - - - %= stylesheet '/static/admin/css/main.css'; - - -
    - -
    - % if ( my $f_error = flash 'error' ) { - - % } elsif ( my $s_error = stash 'error' ) { - - % } elsif ( my $success = flash 'success' ) { - - % } - <%= content %> -
    -
    - - Version: <%= $c->config->{version} %> - -
    - - - - - %= content_for 'javascript'; - - diff --git a/templates/root/index.html.ep b/templates/root/index.html.ep index 9a2d6e4..c529771 100644 --- a/templates/root/index.html.ep +++ b/templates/root/index.html.ep @@ -4,5 +4,7 @@ %= javascript '/static/user/js/home.js'; % end
    -

    App currently in development, please come back later!

    +

    Local Loop API Server

    +

    If you have arrived here, you're either a developer or something has gone wrong! Oops!

    + Go to Admin Login