Hopefully fix speed issue on external data

2019-09-09 15:37:26 +01:00 · 2019-09-09 15:37:26 +01:00 · 962cf972da
parent 3b8b5b97f4
commit 962cf972da
19 changed files with 182 additions and 156 deletions
--- a/.gitignore
+++ b/.gitignore
@ -4,6 +4,7 @@ hypnotoad.pid
 *.db
 *.db-wal
 *.db-shm
+*.db-journal
 *~
 /images
 *.swp
--- a/.idea/.gitignore
+++ b/.idea/.gitignore
@ -0,0 +1,7 @@
+# Default ignored files
+/workspace.xml
+/perl5local.xml
+
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
--- a/.idea/Foodloop-Server.iml
+++ b/.idea/Foodloop-Server.iml
@ -3,6 +3,7 @@
  <component name="NewModuleRootManager" inherit-compiler-output="true">
    <perl5>
      <path value="$MODULE_DIR$/lib" type="perl-library" />
+      <path value="$MODULE_DIR$/templates" type="mojo-template" />
    </perl5>
    <exclude-output />
    <content url="file://$MODULE_DIR$" />
--- a/.idea/sqldialects.xml
+++ b/.idea/sqldialects.xml
@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/share/ddl/PostgreSQL" dialect="PostgreSQL" />
+    <file url="file://$PROJECT_DIR$/share/ddl/SQLite" dialect="SQLite" />
+  </component>
+</project>
--- a/README.md
+++ b/README.md
@ -42,8 +42,22 @@ And then add the following to your configuration file:
  },
 ```

-This will then use an SQLite db for the minion backend, at minion.db
+This will then use an SQLite db for the minion backend, using `minion.db` as
+the database file. To start the minion itself, run:

+```
+./script/pear-local_loop minion worker
+```
+
+# Importing Ward Data
+
+To import ward data, get the ward data csv and then run the following command:
+
+```shell script
+./script/pear-local_loop minion job \
+  --enqueue 'csv_postcode_import' \
+  --args '[ "/path/to/ward/csv" ]'
+```

 ## Example PostgreSQL setup

@ -57,7 +71,46 @@ psql=# alter user minion with encrypted password 'abc123';
 psql=# grant all privileges on database localloop_minion to minion;
 ```

-# Dev notes
+# Development
+
+There are a couple of setup steps to getting a development environment ready.
+Use the corresponding instructions depending on what state your current setup
+is in.
+
+## First Time Setup
+
+First, decide if you're using SQLite or PostgreSQL locally. Development supports
+both, however production uses PostgreSQL. For this example we will use SQLite.
+As the default config is set up for this, no configuration changes are
+needed initially. So, first off, install dependencies:
+
+```shell script
+cpanm --installdeps . --with-feature=sqlite
+```
+
+Then install the database:
+
+```shell script
+./script/deploy_db install -c 'dbi:SQLite:dbname=foodloop.db'
+```
+
+Then set up the development users:
+
+```shell script
+./script/pear-local_loop dev_data --force
+```
+
+***Note: do NOT run that script on production.***
+
+Then you can start the application:
+
+```shell script
+morbo script/pear-local_loop -l http://*:3000
+```
+
+You can modify the host and port for listening as needed.
+
+# Old Docs

 ## Local test database

--- a/3
+++ b/3
@ -14,7 +14,6 @@ requires 'DBIx::Class::Schema::Loader';
 requires 'SQL::Translator';
 requires 'DateTime';
 requires 'DateTime::Format::Strptime', "1.73";
-requires 'DateTime::Format::SQLite';
 requires 'Try::Tiny';
 requires 'MooX::Options::Actions';
 requires 'Module::Runtime';
@ -40,10 +39,12 @@ feature 'postgres', 'PostgreSQL Support' => sub {
  requires 'DBD::Pg';
  requires 'Test::PostgreSQL';
  requires 'Mojo::Pg';
+  requires 'DateTime::Format::Pg';
 };

 feature 'sqlite', 'SQLite Support' => sub {
  requires 'Minion::Backend::SQLite';
+  requires 'DateTime::Format::SQLite';
 };

 feature 'codepoint-open', 'Code Point Open manipulation' => sub {
--- a/lib/Pear/LocalLoop.pm
+++ b/lib/Pear/LocalLoop.pm
@ -267,6 +267,7 @@ sub startup {
  $admin_routes->post('/import_from/suppliers')->to('admin-import_from#post_suppliers');
  $admin_routes->post('/import_from/transactions')->to('admin-import_from#post_transactions');
  $admin_routes->post('/import_from/postcodes')->to('admin-import_from#post_postcodes');
+  $admin_routes->get('/import_from/org_search')->to('admin-import_from#org_search');

 #  my $user_routes = $r->under('/')->to('root#under');

--- a/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm
+++ b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm
@ -98,4 +98,24 @@ sub post_transactions {
  return $c->redirect_to('/admin/import_from');
 }

+sub org_search {
+  my $c = shift;
+  my $term = $c->param('term');
+
+  my $rs = $c->schema->resultset('Organisation')->search(
+    { name => { like => $term . '%' } },
+    {
+      join    => 'entity',
+      columns => [ qw/ me.name entity.id / ]
+    },
+  );
+
+  my @results = ( map { {
+    label => $_->name,
+    value => $_->entity->id,
+  } } $rs->all);
+
+  $c->render( json => \@results );
+}
+
 1;
--- a/lib/Pear/LocalLoop/Controller/Admin/Transactions.pm
+++ b/lib/Pear/LocalLoop/Controller/Admin/Transactions.pm
@ -17,15 +17,10 @@ sub index {
  my $week_transaction_rs = $c->schema->resultset('ViewQuantisedTransaction' . $driver)->search(
    {},
    {
-      columns => [
-        {
-          quantised        => 'quantised_weeks',
-          count            => \"COUNT(*)",
-          sum_value        => $c->pg_or_sqlite(
-                                'SUM("me"."value")',
-                                'SUM("me"."value")',
-                              ),
-        }
+      select => [
+        { count => 'id', '-as' => 'count' },
+        { sum => 'value', '-as' => 'sum_value' },
+        'quantised_weeks',
      ],
      group_by => 'quantised_weeks',
      order_by => { '-asc' => 'quantised_weeks' },
@ -33,8 +28,8 @@ sub index {
  );

  my @all_weeks = $week_transaction_rs->all;
-  my $first_week_count = $all_weeks[0]->get_column('count') || 0;
-  my $first_week_value = $all_weeks[0]->get_column('sum_value') / 100000 || 0;
+  my $first_week_count = defined $all_weeks[0] ? $all_weeks[0]->get_column('count') || 0 : 0;
+  my $first_week_value = defined $all_weeks[0] ? $all_weeks[0]->get_column('sum_value') / 100000 || 0 : 0;
  my $second_week_count = defined $all_weeks[1] ? $all_weeks[1]->get_column('count') || 0 : 0;
  my $second_week_value = defined $all_weeks[1] ? $all_weeks[1]->get_column('sum_value') / 100000 || 0 : 0;

--- a/lib/Pear/LocalLoop/Controller/Api/External.pm
+++ b/lib/Pear/LocalLoop/Controller/Api/External.pm
@ -199,40 +199,24 @@ sub post_supplier_count {
      buyer_id      => $user->entity->id,
    },
    {
-      columns  => [
-        'seller_id',
-        {
-          quantised   => 'quantised_days',
-          count       => \"COUNT(*)",
-          total_spend => { sum => 'value' },
-        }
+      prefetch => { 'seller' => 'organisation' },
+      select => [
+        { count => 'me.id', '-as' => 'count' },
+        { sum => 'me.value', '-as' => 'total_spend' },
+        'organisation.name',
+        'me.quantised_days',
      ],
-      group_by => [ 'quantised_days', 'seller_id' ],
-      order_by => { '-asc' => 'quantised_days' },
+      group_by => [ 'me.quantised_days', 'seller.id' ],
+      order_by => { '-asc' => 'me.quantised_days' },
    }
  );

-  my $name_rs = $c->schema->resultset('Transaction')->search(
-    {
-      'me.buyer_id' => $user->entity->id,
-    },
-    {
-      join => { seller => 'organisation' },
-    }
-  );
-
-  my %name_map = (
-    map {
-      $_->seller->id => $_->seller->organisation->name,
-    } $name_rs->all
-  );
-
  my @graph_data = (
    map {{
      count  => $_->get_column('count'),
      value  => ($_->get_column('total_spend') / 100000) // 0,
-      date   => $_->get_column('quantised'),
-      seller => $name_map{ $_->get_column('seller_id') },
+      date   => $_->get_column('quantised_days'),
+      seller => $_->seller->organisation->name,
    }} $spend_rs->all,
  );

--- a/lib/Pear/LocalLoop/Schema/Result/ViewQuantisedTransactionPg.pm
+++ b/lib/Pear/LocalLoop/Schema/Result/ViewQuantisedTransactionPg.pm
@ -22,4 +22,18 @@ SELECT "value",
  FROM "transactions"
 /);

+__PACKAGE__->belongs_to(
+  "buyer",
+  "Pear::LocalLoop::Schema::Result::Entity",
+  { id => "buyer_id" },
+  { is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" },
+);
+
+__PACKAGE__->belongs_to(
+  "seller",
+  "Pear::LocalLoop::Schema::Result::Entity",
+  { id => "seller_id" },
+  { is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" },
+);
+
 1;
--- a/templates/admin/import/get_org.html.ep
+++ b/templates/admin/import/get_org.html.ep
@ -1,4 +1,4 @@
-% layout 'admin_errors';
+% layout 'admin';
 % title 'Import';
 <div class="row">
  <div class="col-8">
--- a/templates/admin/import/get_user.html.ep
+++ b/templates/admin/import/get_user.html.ep
@ -1,4 +1,4 @@
-% layout 'admin_errors';
+% layout 'admin';
 % title 'Import';
 <div class="row">
  <div class="col-8">
--- a/templates/admin/import_from/index.html.ep
+++ b/templates/admin/import_from/index.html.ep
@ -1,6 +1,20 @@
 % layout 'admin';
 % title 'Import From';
 % content_for javascript => begin
+<script>
+$(function() {
+    $('#select-org').autocomplete({
+      source: '<%= url_for '/admin/import_from/org_search' %>',
+      minLength: 2,
+      select: function( event, ui ) {
+          console.log(ui);
+          $('#select-org').val(ui.item.label);
+          $('#select-org-id').val(ui.item.value);
+          return false;
+      }
+    });
+})
+</script>
 % end
 % if (my $error = flash 'error') {
 <div class="alert alert-danger" role="alert">
@ -33,27 +47,13 @@
        </form>
      </div>
    </div>
-    <div class="card col-md-6 m-3">
-      <div class="card-body">
-        <h4 class="card-title">Postcode Data</h4>
-        <p>Expected headers at very least: "postcode", "ward".</p>
-        <form action="/admin/import_from/postcodes" method="POST" enctype="multipart/form-data">
-          <input type="file" name="postcodes_csv" accept="text/csv">
-          <input type="submit" value="Upload Postcode CSV">
-        </form>
-      </div>
-    </div>
    <div class="card col-md-6 m-3">
      <div class="card-body">
        <h4 class="card-title">LCC Procurement Import - Transactions</h4>
        <p>Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".</p>
        <form action="/admin/import_from/transactions" method="POST" enctype="multipart/form-data">
-          <select name="entity_id">
-            <option>Select an Organisation</option>
-            <% for my $org ( @$org_entities ) { %>
-            <option value="<%= $org->{id}; %>"><%= $org->{name}; %></option>
-            <% } %>
-          </select><br/>
+          <input id="select-org" type="text">
+          <input id="select-org-id" name="entity_id" type="hidden">
          <input type="file" name="transactions_csv" accept="text/csv">
          <input type="submit" value="Upload Transactions CSV">
        </form>
--- a/templates/admin/organisations/merge_detail.html.ep
+++ b/templates/admin/organisations/merge_detail.html.ep
@ -1,4 +1,4 @@
-% layout 'admin_errors';
+% layout 'admin';
 % title 'Organisations';
 <div class="row">
  <div class="col-12">
--- a/templates/admin/organisations/merge_list.html.ep
+++ b/templates/admin/organisations/merge_list.html.ep
@ -1,4 +1,4 @@
-% layout 'admin_errors';
+% layout 'admin';
 % title 'Organisations';
 <div class="row">
  <div class="col-12">
--- a/templates/layouts/admin.html.ep
+++ b/templates/layouts/admin.html.ep
@ -6,8 +6,13 @@
    <title>LocalLoop Admin - <%= title %></title>

    <!-- Bootstrap and jQuery js -->
-    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/css/bootstrap.min.css" integrity="sha384-/Y6pD6FV/Vv2HJnA6t+vslU6fwYXjCFtcEpHbNJ0lyAFsXTsjBbfaDjzALeQsN6M" crossorigin="anonymous">
-
+    <link rel="stylesheet"
+          href="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.css"
+          integrity="sha256-rByPlHULObEjJ6XQxW/flG2r+22R5dKiAoef+aXWfik=" crossorigin="anonymous" />
+    <link rel="stylesheet"
+          href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
+          integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T"
+          crossorigin="anonymous">

    %= stylesheet '/static/admin/css/main.css';
  </head>
@ -59,6 +64,9 @@
          <li class="nav-item">
            <a class="nav-link<%= title eq 'Import' ? ' active' : '' %>" href="<%= url_for '/admin/import' %>">Import</a>
          </li>
+          <li class="nav-item">
+            <a class="nav-link" href="<%= url_for '/admin/minion' %>">Minion</a>
+          </li>
          <li class="nav-item">
            <a class="nav-link" href="<%= url_for '/admin/logout' %>">Logout</a>
          </li>
@ -66,6 +74,19 @@
      </div>
    </nav>
    <div class="container">
+      % if ( my $f_error = flash 'error' ) {
+      <div class="alert alert-danger" role="alert">
+        <strong>Error!</strong> <%= $f_error %>
+      </div>
+      % } elsif ( my $s_error = stash 'error' ) {
+      <div class="alert alert-danger" role="alert">
+        <strong>Error!</strong> <%= $s_error %>
+      </div>
+      % } elsif ( my $success = flash 'success' ) {
+      <div class="alert alert-success" role="alert">
+        <strong>Success!</strong> <%= $success %>
+      </div>
+      % }
      <%= content %>
    </div>
    <div class="navbar bg-dark fixed-bottom">
@ -73,10 +94,18 @@
        Version: <%= $c->config->{version} %>
      </span>
    </div>
-    <script src="https://code.jquery.com/jquery-3.1.1.slim.min.js" integrity="sha384-A7FZj7v+d/sdmMqp/nOQwliLvUsJfDHW+k9Omg/a/EheAdgtzNs3hpfag6Ed950n" crossorigin="anonymous"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.5/umd/popper.min.js" integrity="sha256-jpW4gXAhFvqGDD5B7366rIPD7PDbAmqq4CO0ZnHbdM4=" crossorigin="anonymous"></script>
-    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/js/bootstrap.min.js" integrity="sha384-h0AbiXch4ZDo7tp9hKZ4TsHbi047NrKGLO3SEJAg45jXxnGIfYzk4Si90RDIqNm1" crossorigin="anonymous"></script>
-
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"
+            integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8="
+            crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"
+            integrity="sha256-KM512VNnjElC30ehFwehXjx1YCHPiQkOPmqnrWtpccM="
+            crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"
+            integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1"
+            crossorigin="anonymous"></script>
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"
+            integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM"
+            crossorigin="anonymous"></script>
    %= content_for 'javascript';
  </body>
 </html>
--- a/templates/layouts/admin_errors.html.ep
+++ b/templates/layouts/admin_errors.html.ep
@ -1,89 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>LocalLoop Admin - <%= title %></title>
-
-    <!-- Bootstrap and jQuery js -->
-    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/css/bootstrap.min.css" integrity="sha384-/Y6pD6FV/Vv2HJnA6t+vslU6fwYXjCFtcEpHbNJ0lyAFsXTsjBbfaDjzALeQsN6M" crossorigin="anonymous">
-
-
-    %= stylesheet '/static/admin/css/main.css';
-  </head>
-  <body>
-    <nav class="navbar navbar-expand-md fixed-top navbar-dark bg-danger">
-      <a class="navbar-brand" href="<%= url_for '/admin/home' %>">LocalLoop Admin</a>
-      <button class="navbar-toggler navbar-toggler-right"
-              type="button"
-              data-toggle="collapse"
-              data-target="#navbarNav"
-              aria-controls="navbarNav"
-              aria-expanded="false"
-              aria-label="Toggle navigation">
-        <span class="navbar-toggler-icon"></span>
-      </button>
-
-      <div class="collapse navbar-collapse" id="navbarNav">
-        <ul class="navbar-nav ml-auto">
-          <li class="nav-item dropdown">
-            <a class="nav-link dropdown-toggle" data-toggle="dropdown" href="#">
-              Reports
-            </a>
-            <div class="dropdown-menu">
-              <a class="dropdown-item" href="<%= url_for '/admin/reports/transactions' %>">Transactions (Hourly)</a>
-              <a class="dropdown-item" href="<%= url_for('/admin/reports/transactions')->query(scale =>'days') %>">Transactions (Daily)</a>
-            </div>
-          </li>
-          <li class="nav-item">
-            <a class="nav-link<%= title eq 'Feedback' ? ' active' : '' %>" href="<%= url_for '/admin/feedback' %>">Feedback</a>
-          </li>
-          <li class="nav-item">
-            <a class="nav-link<%= title eq 'Tokens' ? ' active' : '' %>" href="<%= url_for '/admin/tokens' %>">Tokens</a>
-          </li>
-          <li class="nav-item">
-            <a class="nav-link<%= title eq 'Transactions' ? ' active' : '' %>" href="<%= url_for '/admin/transactions' %>">Transactions</a>
-          </li>
-          <li class="nav-item">
-            <a class="nav-link<%= title eq 'Users' ? ' active' : '' %>" href="<%= url_for '/admin/users' %>">Users</a>
-          </li>
-          <li class="nav-item">
-            <a class="nav-link<%= title eq 'Organisations' ? ' active' : '' %>" href="<%= url_for '/admin/organisations' %>">Organisations</a>
-          </li>
-          <li class="nav-item">
-            <a class="nav-link<%= title eq 'Import' ? ' active' : '' %>" href="<%= url_for '/admin/import' %>">Import</a>
-          </li>
-          <li class="nav-item">
-            <a class="nav-link" href="<%= url_for '/admin/logout' %>">Logout</a>
-          </li>
-        </div>
-      </div>
-    </nav>
-    <div class="container">
-      % if ( my $f_error = flash 'error' ) {
-        <div class="alert alert-danger" role="alert">
-          <strong>Error!</strong> <%= $f_error %>
-        </div>
-        % } elsif ( my $s_error = stash 'error' ) {
-        <div class="alert alert-danger" role="alert">
-          <strong>Error!</strong> <%= $s_error %>
-        </div>
-      % } elsif ( my $success = flash 'success' ) {
-        <div class="alert alert-success" role="alert">
-          <strong>Success!</strong> <%= $success %>
-        </div>
-      % }
-      <%= content %>
-    </div>
-    <div class="navbar bg-dark fixed-bottom">
-      <span class="navbar-text ml-auto text-muted">
-        Version: <%= $c->config->{version} %>
-      </span>
-    </div>
-    <script src="https://code.jquery.com/jquery-3.1.1.slim.min.js" integrity="sha384-A7FZj7v+d/sdmMqp/nOQwliLvUsJfDHW+k9Omg/a/EheAdgtzNs3hpfag6Ed950n" crossorigin="anonymous"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.5/umd/popper.min.js" integrity="sha256-jpW4gXAhFvqGDD5B7366rIPD7PDbAmqq4CO0ZnHbdM4=" crossorigin="anonymous"></script>
-    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/js/bootstrap.min.js" integrity="sha384-h0AbiXch4ZDo7tp9hKZ4TsHbi047NrKGLO3SEJAg45jXxnGIfYzk4Si90RDIqNm1" crossorigin="anonymous"></script>
-
-    %= content_for 'javascript';
-  </body>
-</html>
--- a/templates/root/index.html.ep
+++ b/templates/root/index.html.ep
@ -4,5 +4,7 @@
 %= javascript '/static/user/js/home.js';
 % end
 <div>
-  <h1>App currently in development, please come back later!</h1>
+  <h1>Local Loop API Server</h1>
+  <p>If you have arrived here, you're either a developer or something has gone wrong! Oops!</p>
+  <a href="/admin" class="btn btn-primary">Go to Admin Login</a>
 </div>