Hopefully fix speed issue on external data

This commit is contained in:
Thomas Bloor 2019-09-09 15:37:26 +01:00
parent 3b8b5b97f4
commit 962cf972da
No known key found for this signature in database
GPG key ID: 4657C7EBE42CC5CC
19 changed files with 182 additions and 156 deletions

1
.gitignore vendored
View file

@ -4,6 +4,7 @@ hypnotoad.pid
*.db
*.db-wal
*.db-shm
*.db-journal
*~
/images
*.swp

7
.idea/.gitignore vendored Normal file
View file

@ -0,0 +1,7 @@
# Default ignored files
/workspace.xml
/perl5local.xml
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

View file

@ -3,6 +3,7 @@
<component name="NewModuleRootManager" inherit-compiler-output="true">
<perl5>
<path value="$MODULE_DIR$/lib" type="perl-library" />
<path value="$MODULE_DIR$/templates" type="mojo-template" />
</perl5>
<exclude-output />
<content url="file://$MODULE_DIR$" />

7
.idea/sqldialects.xml Normal file
View file

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$/share/ddl/PostgreSQL" dialect="PostgreSQL" />
<file url="file://$PROJECT_DIR$/share/ddl/SQLite" dialect="SQLite" />
</component>
</project>

View file

@ -42,8 +42,22 @@ And then add the following to your configuration file:
},
```
This will then use an SQLite db for the minion backend, at minion.db
This will then use an SQLite db for the minion backend, using `minion.db` as
the database file. To start the minion itself, run:
```
./script/pear-local_loop minion worker
```
# Importing Ward Data
To import ward data, get the ward data csv and then run the following command:
```shell script
./script/pear-local_loop minion job \
--enqueue 'csv_postcode_import' \
--args '[ "/path/to/ward/csv" ]'
```
## Example PostgreSQL setup
@ -57,7 +71,46 @@ psql=# alter user minion with encrypted password 'abc123';
psql=# grant all privileges on database localloop_minion to minion;
```
# Dev notes
# Development
There are a couple of setup steps to getting a development environment ready.
Use the corresponding instructions depending on what state your current setup
is in.
## First Time Setup
First, decide if you're using SQLite or PostgreSQL locally. Development supports
both, however production uses PostgreSQL. For this example we will use SQLite.
As the default config is set up for this, no configuration changes are
needed initially. So, first off, install dependencies:
```shell script
cpanm --installdeps . --with-feature=sqlite
```
Then install the database:
```shell script
./script/deploy_db install -c 'dbi:SQLite:dbname=foodloop.db'
```
Then set up the development users:
```shell script
./script/pear-local_loop dev_data --force
```
***Note: do NOT run that script on production.***
Then you can start the application:
```shell script
morbo script/pear-local_loop -l http://*:3000
```
You can modify the host and port for listening as needed.
# Old Docs
## Local test database

View file

@ -14,7 +14,6 @@ requires 'DBIx::Class::Schema::Loader';
requires 'SQL::Translator';
requires 'DateTime';
requires 'DateTime::Format::Strptime', "1.73";
requires 'DateTime::Format::SQLite';
requires 'Try::Tiny';
requires 'MooX::Options::Actions';
requires 'Module::Runtime';
@ -40,10 +39,12 @@ feature 'postgres', 'PostgreSQL Support' => sub {
requires 'DBD::Pg';
requires 'Test::PostgreSQL';
requires 'Mojo::Pg';
requires 'DateTime::Format::Pg';
};
feature 'sqlite', 'SQLite Support' => sub {
requires 'Minion::Backend::SQLite';
requires 'DateTime::Format::SQLite';
};
feature 'codepoint-open', 'Code Point Open manipulation' => sub {

View file

@ -267,6 +267,7 @@ sub startup {
$admin_routes->post('/import_from/suppliers')->to('admin-import_from#post_suppliers');
$admin_routes->post('/import_from/transactions')->to('admin-import_from#post_transactions');
$admin_routes->post('/import_from/postcodes')->to('admin-import_from#post_postcodes');
$admin_routes->get('/import_from/org_search')->to('admin-import_from#org_search');
# my $user_routes = $r->under('/')->to('root#under');

View file

@ -98,4 +98,24 @@ sub post_transactions {
return $c->redirect_to('/admin/import_from');
}
sub org_search {
my $c = shift;
my $term = $c->param('term');
my $rs = $c->schema->resultset('Organisation')->search(
{ name => { like => $term . '%' } },
{
join => 'entity',
columns => [ qw/ me.name entity.id / ]
},
);
my @results = ( map { {
label => $_->name,
value => $_->entity->id,
} } $rs->all);
$c->render( json => \@results );
}
1;

View file

@ -17,15 +17,10 @@ sub index {
my $week_transaction_rs = $c->schema->resultset('ViewQuantisedTransaction' . $driver)->search(
{},
{
columns => [
{
quantised => 'quantised_weeks',
count => \"COUNT(*)",
sum_value => $c->pg_or_sqlite(
'SUM("me"."value")',
'SUM("me"."value")',
),
}
select => [
{ count => 'id', '-as' => 'count' },
{ sum => 'value', '-as' => 'sum_value' },
'quantised_weeks',
],
group_by => 'quantised_weeks',
order_by => { '-asc' => 'quantised_weeks' },
@ -33,8 +28,8 @@ sub index {
);
my @all_weeks = $week_transaction_rs->all;
my $first_week_count = $all_weeks[0]->get_column('count') || 0;
my $first_week_value = $all_weeks[0]->get_column('sum_value') / 100000 || 0;
my $first_week_count = defined $all_weeks[0] ? $all_weeks[0]->get_column('count') || 0 : 0;
my $first_week_value = defined $all_weeks[0] ? $all_weeks[0]->get_column('sum_value') / 100000 || 0 : 0;
my $second_week_count = defined $all_weeks[1] ? $all_weeks[1]->get_column('count') || 0 : 0;
my $second_week_value = defined $all_weeks[1] ? $all_weeks[1]->get_column('sum_value') / 100000 || 0 : 0;

View file

@ -199,40 +199,24 @@ sub post_supplier_count {
buyer_id => $user->entity->id,
},
{
columns => [
'seller_id',
{
quantised => 'quantised_days',
count => \"COUNT(*)",
total_spend => { sum => 'value' },
}
prefetch => { 'seller' => 'organisation' },
select => [
{ count => 'me.id', '-as' => 'count' },
{ sum => 'me.value', '-as' => 'total_spend' },
'organisation.name',
'me.quantised_days',
],
group_by => [ 'quantised_days', 'seller_id' ],
order_by => { '-asc' => 'quantised_days' },
group_by => [ 'me.quantised_days', 'seller.id' ],
order_by => { '-asc' => 'me.quantised_days' },
}
);
my $name_rs = $c->schema->resultset('Transaction')->search(
{
'me.buyer_id' => $user->entity->id,
},
{
join => { seller => 'organisation' },
}
);
my %name_map = (
map {
$_->seller->id => $_->seller->organisation->name,
} $name_rs->all
);
my @graph_data = (
map {{
count => $_->get_column('count'),
value => ($_->get_column('total_spend') / 100000) // 0,
date => $_->get_column('quantised'),
seller => $name_map{ $_->get_column('seller_id') },
date => $_->get_column('quantised_days'),
seller => $_->seller->organisation->name,
}} $spend_rs->all,
);

View file

@ -22,4 +22,18 @@ SELECT "value",
FROM "transactions"
/);
__PACKAGE__->belongs_to(
"buyer",
"Pear::LocalLoop::Schema::Result::Entity",
{ id => "buyer_id" },
{ is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" },
);
__PACKAGE__->belongs_to(
"seller",
"Pear::LocalLoop::Schema::Result::Entity",
{ id => "seller_id" },
{ is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" },
);
1;

View file

@ -1,4 +1,4 @@
% layout 'admin_errors';
% layout 'admin';
% title 'Import';
<div class="row">
<div class="col-8">

View file

@ -1,4 +1,4 @@
% layout 'admin_errors';
% layout 'admin';
% title 'Import';
<div class="row">
<div class="col-8">

View file

@ -1,6 +1,20 @@
% layout 'admin';
% title 'Import From';
% content_for javascript => begin
<script>
$(function() {
$('#select-org').autocomplete({
source: '<%= url_for '/admin/import_from/org_search' %>',
minLength: 2,
select: function( event, ui ) {
console.log(ui);
$('#select-org').val(ui.item.label);
$('#select-org-id').val(ui.item.value);
return false;
}
});
})
</script>
% end
% if (my $error = flash 'error') {
<div class="alert alert-danger" role="alert">
@ -33,27 +47,13 @@
</form>
</div>
</div>
<div class="card col-md-6 m-3">
<div class="card-body">
<h4 class="card-title">Postcode Data</h4>
<p>Expected headers at very least: "postcode", "ward".</p>
<form action="/admin/import_from/postcodes" method="POST" enctype="multipart/form-data">
<input type="file" name="postcodes_csv" accept="text/csv">
<input type="submit" value="Upload Postcode CSV">
</form>
</div>
</div>
<div class="card col-md-6 m-3">
<div class="card-body">
<h4 class="card-title">LCC Procurement Import - Transactions</h4>
<p>Expected headers at very least: "supplier_id", "transaction_id", "net_amount", "vat amount" , "gross_amount".</p>
<form action="/admin/import_from/transactions" method="POST" enctype="multipart/form-data">
<select name="entity_id">
<option>Select an Organisation</option>
<% for my $org ( @$org_entities ) { %>
<option value="<%= $org->{id}; %>"><%= $org->{name}; %></option>
<% } %>
</select><br/>
<input id="select-org" type="text">
<input id="select-org-id" name="entity_id" type="hidden">
<input type="file" name="transactions_csv" accept="text/csv">
<input type="submit" value="Upload Transactions CSV">
</form>

View file

@ -1,4 +1,4 @@
% layout 'admin_errors';
% layout 'admin';
% title 'Organisations';
<div class="row">
<div class="col-12">

View file

@ -1,4 +1,4 @@
% layout 'admin_errors';
% layout 'admin';
% title 'Organisations';
<div class="row">
<div class="col-12">

View file

@ -6,8 +6,13 @@
<title>LocalLoop Admin - <%= title %></title>
<!-- Bootstrap and jQuery js -->
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/css/bootstrap.min.css" integrity="sha384-/Y6pD6FV/Vv2HJnA6t+vslU6fwYXjCFtcEpHbNJ0lyAFsXTsjBbfaDjzALeQsN6M" crossorigin="anonymous">
<link rel="stylesheet"
href="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.css"
integrity="sha256-rByPlHULObEjJ6XQxW/flG2r+22R5dKiAoef+aXWfik=" crossorigin="anonymous" />
<link rel="stylesheet"
href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T"
crossorigin="anonymous">
%= stylesheet '/static/admin/css/main.css';
</head>
@ -59,6 +64,9 @@
<li class="nav-item">
<a class="nav-link<%= title eq 'Import' ? ' active' : '' %>" href="<%= url_for '/admin/import' %>">Import</a>
</li>
<li class="nav-item">
<a class="nav-link" href="<%= url_for '/admin/minion' %>">Minion</a>
</li>
<li class="nav-item">
<a class="nav-link" href="<%= url_for '/admin/logout' %>">Logout</a>
</li>
@ -66,6 +74,19 @@
</div>
</nav>
<div class="container">
% if ( my $f_error = flash 'error' ) {
<div class="alert alert-danger" role="alert">
<strong>Error!</strong> <%= $f_error %>
</div>
% } elsif ( my $s_error = stash 'error' ) {
<div class="alert alert-danger" role="alert">
<strong>Error!</strong> <%= $s_error %>
</div>
% } elsif ( my $success = flash 'success' ) {
<div class="alert alert-success" role="alert">
<strong>Success!</strong> <%= $success %>
</div>
% }
<%= content %>
</div>
<div class="navbar bg-dark fixed-bottom">
@ -73,10 +94,18 @@
Version: <%= $c->config->{version} %>
</span>
</div>
<script src="https://code.jquery.com/jquery-3.1.1.slim.min.js" integrity="sha384-A7FZj7v+d/sdmMqp/nOQwliLvUsJfDHW+k9Omg/a/EheAdgtzNs3hpfag6Ed950n" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.5/umd/popper.min.js" integrity="sha256-jpW4gXAhFvqGDD5B7366rIPD7PDbAmqq4CO0ZnHbdM4=" crossorigin="anonymous"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/js/bootstrap.min.js" integrity="sha384-h0AbiXch4ZDo7tp9hKZ4TsHbi047NrKGLO3SEJAg45jXxnGIfYzk4Si90RDIqNm1" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"
integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8="
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"
integrity="sha256-KM512VNnjElC30ehFwehXjx1YCHPiQkOPmqnrWtpccM="
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"
integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"
integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM"
crossorigin="anonymous"></script>
%= content_for 'javascript';
</body>
</html>

View file

@ -1,89 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LocalLoop Admin - <%= title %></title>
<!-- Bootstrap and jQuery js -->
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/css/bootstrap.min.css" integrity="sha384-/Y6pD6FV/Vv2HJnA6t+vslU6fwYXjCFtcEpHbNJ0lyAFsXTsjBbfaDjzALeQsN6M" crossorigin="anonymous">
%= stylesheet '/static/admin/css/main.css';
</head>
<body>
<nav class="navbar navbar-expand-md fixed-top navbar-dark bg-danger">
<a class="navbar-brand" href="<%= url_for '/admin/home' %>">LocalLoop Admin</a>
<button class="navbar-toggler navbar-toggler-right"
type="button"
data-toggle="collapse"
data-target="#navbarNav"
aria-controls="navbarNav"
aria-expanded="false"
aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarNav">
<ul class="navbar-nav ml-auto">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" data-toggle="dropdown" href="#">
Reports
</a>
<div class="dropdown-menu">
<a class="dropdown-item" href="<%= url_for '/admin/reports/transactions' %>">Transactions (Hourly)</a>
<a class="dropdown-item" href="<%= url_for('/admin/reports/transactions')->query(scale =>'days') %>">Transactions (Daily)</a>
</div>
</li>
<li class="nav-item">
<a class="nav-link<%= title eq 'Feedback' ? ' active' : '' %>" href="<%= url_for '/admin/feedback' %>">Feedback</a>
</li>
<li class="nav-item">
<a class="nav-link<%= title eq 'Tokens' ? ' active' : '' %>" href="<%= url_for '/admin/tokens' %>">Tokens</a>
</li>
<li class="nav-item">
<a class="nav-link<%= title eq 'Transactions' ? ' active' : '' %>" href="<%= url_for '/admin/transactions' %>">Transactions</a>
</li>
<li class="nav-item">
<a class="nav-link<%= title eq 'Users' ? ' active' : '' %>" href="<%= url_for '/admin/users' %>">Users</a>
</li>
<li class="nav-item">
<a class="nav-link<%= title eq 'Organisations' ? ' active' : '' %>" href="<%= url_for '/admin/organisations' %>">Organisations</a>
</li>
<li class="nav-item">
<a class="nav-link<%= title eq 'Import' ? ' active' : '' %>" href="<%= url_for '/admin/import' %>">Import</a>
</li>
<li class="nav-item">
<a class="nav-link" href="<%= url_for '/admin/logout' %>">Logout</a>
</li>
</div>
</div>
</nav>
<div class="container">
% if ( my $f_error = flash 'error' ) {
<div class="alert alert-danger" role="alert">
<strong>Error!</strong> <%= $f_error %>
</div>
% } elsif ( my $s_error = stash 'error' ) {
<div class="alert alert-danger" role="alert">
<strong>Error!</strong> <%= $s_error %>
</div>
% } elsif ( my $success = flash 'success' ) {
<div class="alert alert-success" role="alert">
<strong>Success!</strong> <%= $success %>
</div>
% }
<%= content %>
</div>
<div class="navbar bg-dark fixed-bottom">
<span class="navbar-text ml-auto text-muted">
Version: <%= $c->config->{version} %>
</span>
</div>
<script src="https://code.jquery.com/jquery-3.1.1.slim.min.js" integrity="sha384-A7FZj7v+d/sdmMqp/nOQwliLvUsJfDHW+k9Omg/a/EheAdgtzNs3hpfag6Ed950n" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.5/umd/popper.min.js" integrity="sha256-jpW4gXAhFvqGDD5B7366rIPD7PDbAmqq4CO0ZnHbdM4=" crossorigin="anonymous"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/js/bootstrap.min.js" integrity="sha384-h0AbiXch4ZDo7tp9hKZ4TsHbi047NrKGLO3SEJAg45jXxnGIfYzk4Si90RDIqNm1" crossorigin="anonymous"></script>
%= content_for 'javascript';
</body>
</html>

View file

@ -4,5 +4,7 @@
%= javascript '/static/user/js/home.js';
% end
<div>
<h1>App currently in development, please come back later!</h1>
<h1>Local Loop API Server</h1>
<p>If you have arrived here, you're either a developer or something has gone wrong! Oops!</p>
<a href="/admin" class="btn btn-primary">Go to Admin Login</a>
</div>