Merge pull request #116 from Pear-Trading/v0.10.7

v0.10.7 Release

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 # Next Release
 
+# v.10.7
+
+* Added `cron_daily` script for holding all daily cronjobs
+* **Admin Fix** Parse currency without a currency symbol on import
+* **Admin Fix** Fix large CSV issue on import
+* Use custom secrets for encryption
+* Made purchase categories easier to pull
+* Added dashboard data for getting essential for all purchases along with
+weekly and monthly view of category purchases
+* Amended tests where relevant
+
 # v0.10.6
 
 * Fixed organisation submission

lib/Pear/LocalLoop.pm

@@ -34,6 +34,13 @@ sub startup {
   });
   my $config = $self->config;
 
+  if ( defined $config->{secret} ) {
+    $self->secrets([ $config->{secret} ]);
+  } elsif ( $self->mode eq 'production' ) {
+    # Just incase we end up in production and it hasnt been set!
+    $self->secrets([ Data::UUID->new->create() ]);
+  }
+
   push @{ $self->commands->namespaces }, __PACKAGE__ . '::Command';
 
   $self->plugin('Pear::LocalLoop::Plugin::BootstrapPagination', { bootstrap4 => 1 } );
@@ -247,9 +254,9 @@ sub startup {
 #  $portal_api->post('/search')->to('api-upload#post_search');
 
   $self->hook( before_dispatch => sub {
-    my $self = shift;
+    my $c = shift;
 
-    $self->res->headers->header('Access-Control-Allow-Origin' => '*') if $self->app->mode eq 'development';
+    $c->res->headers->header('Access-Control-Allow-Origin' => '*') if $c->app->mode eq 'development';
   });
 
   $self->helper( copy_transactions_and_delete => sub {

lib/Pear/LocalLoop/Controller/Admin/Import.pm

@@ -144,7 +144,8 @@ sub _csv_flash_error {
 
   $c->flash(
     error => $error,
-    csv_data => $c->param('csv'),
+    # If csv info is huge, this fails epically
+    #csv_data => $c->param('csv'),
     date_format => $c->param('date_format'),
   );
 }

lib/Pear/LocalLoop/Controller/Api/Categories.pm

@@ -33,30 +33,31 @@ sub post_category_list {
           essential        => 'essential',
         },
       ],
-      group_by => [ qw/ category_id quantised_weeks essential value / ],
+      group_by => [ qw/ category_id quantised_weeks essential / ],
-      order_by => { '-desc' => 'value' },
     }
   );
 
   my $data = { categories => {}, essentials => {} };
 
+  my $category_list = $c->schema->resultset('Category')->as_hash;
+
   for my $cat_trans ( $month_transaction_category_rs->all ) {
     my $quantised = $c->db_datetime_parser->parse_datetime($cat_trans->get_column('quantised'));
     my $days = $c->format_iso_date( $quantised ) || 0;
-    my $category = $cat_trans->get_column('category_id') || undef;
+    my $category = $cat_trans->get_column('category_id') || 0;
     my $value = ($cat_trans->get_column('value') || 0) / 100000;
-    $data->{categories}->{$days} = [] unless exists $data->{categories}->{$days};
+    $data->{categories}->{$days}->{$category_list->{$category}} += $value;
-    push @{ $data->{categories}->{$days} }, {
-      days => $days,
-      value => $value,
-      category => $category,
-    };
     next unless $cat_trans->get_column('essential');
     $data->{essentials}->{$days}->{value} += $value;
   }
 
   for my $day ( keys %{ $data->{categories} } ) {
-    $data->{categories}->{$day} = [ sort { $b->{value} <=> $a->{value} } @{ $data->{categories}->{$day} } ];
+    my @days = ( map{ {
+      days => $day,
+      value => $data->{categories}->{$day}->{$_},
+      category => $_,
+    } } keys %{ $data->{categories}->{$day} } );
+    $data->{categories}->{$day} = [ sort { $b->{value} <=> $a->{value} } @days ];
   }
 
   return $c->render(

lib/Pear/LocalLoop/Controller/Api/Stats.pm

@@ -63,9 +63,9 @@ sub post_customer {
 
   my $entity = $c->stash->{api_user}->entity;
 
-  my $duration = DateTime::Duration->new( weeks => 7 );
+  my $duration_weeks = DateTime::Duration->new( weeks => 7 );
   my $end = DateTime->today;
-  my $start = $end->clone->subtract_duration( $duration );
+  my $start_weeks = $end->clone->subtract_duration( $duration_weeks );
 
   my $dtf = $c->schema->storage->datetime_parser;
   my $driver = $c->schema->storage->dbh->{Driver}->{Name};
@@ -73,7 +73,7 @@ sub post_customer {
     {
       purchase_time => {
         -between => [
-          $dtf->format_datetime($start),
+          $dtf->format_datetime($start_weeks),
           $dtf->format_datetime($end),
         ],
       },
@@ -92,7 +92,7 @@ sub post_customer {
   );
 
   my @all_weeks = $week_transaction_rs->all;
-  my $first = $all_weeks[0]->get_column('count') || 0;
+  my $first = defined $all_weeks[0] ? $all_weeks[0]->get_column('count') || 0 : 0;
   my $second = defined $all_weeks[1] ? $all_weeks[1]->get_column('count') || 0 : 0;
   my $max = max( map { $_->get_column('count') } @all_weeks );
   my $sum = sum( map { $_->get_column('count') } @all_weeks );
@@ -125,8 +125,69 @@ sub post_customer {
     push @{ $sectors->{ purchases } }, ($_->get_column('count') || 0);
   }
 
+  my $data = { cat_total => {}, categories => {}, essentials => {} };
+
+  my $purchase_rs = $entity->purchases;
+
+  my $purchase_no_essential_rs = $purchase_rs->search({
+    "me.essential" => 1,
+  });
+
+  $data->{essentials} = {
+    purchase_no_total => $purchase_rs->count,
+    purchase_no_essential_total => $purchase_no_essential_rs->count,
+  };
+
+  my $duration_month = DateTime::Duration->new( days => 28 );
+  my $start_month = $end->clone->subtract_duration( $duration_month );
+  my $month_transaction_category_rs = $c->schema->resultset('ViewQuantisedTransactionCategory' . $driver)->search(
+    {
+      purchase_time => {
+        -between => [
+          $dtf->format_datetime($start_month),
+          $dtf->format_datetime($end),
+        ],
+      },
+      buyer_id => $entity->id,
+    },
+    {
+      columns => [
+        {
+          quantised        => 'quantised_weeks',
+          value            => { sum => 'value' },
+          category_id      => 'category_id',
+          essential        => 'essential',
+        },
+      ],
+      group_by => [ qw/ category_id quantised_weeks essential / ],
+    }
+  );
+
+  my $category_list = $c->schema->resultset('Category')->as_hash;
+
+  for my $cat_trans ( $month_transaction_category_rs->all ) {
+    my $quantised = $c->db_datetime_parser->parse_datetime($cat_trans->get_column('quantised'));
+    my $days = $c->format_iso_date( $quantised ) || 0;
+    my $category = $cat_trans->get_column('category_id') || 0;
+    my $value = ($cat_trans->get_column('value') || 0) / 100000;
+    $data->{cat_total}->{$category_list->{$category}} += $value;
+    $data->{categories}->{$days}->{$category_list->{$category}} += $value;
+    next unless $cat_trans->get_column('essential');
+    $data->{essentials}->{$days}->{value} += $value;
+  }
+
+  for my $day ( keys %{ $data->{categories} } ) {
+    my @days = ( map{ {
+      days => $day,
+      value => $data->{categories}->{$day}->{$_},
+      category => $_,
+    } } keys %{ $data->{categories}->{$day} } );
+    $data->{categories}->{$day} = [ sort { $b->{value} <=> $a->{value} } @days ];
+  }
+
   return $c->render( json => {
     success => Mojo::JSON->true,
+    data => $data,
     weeks => $weeks,
     sectors => $sectors,
   });

lib/Pear/LocalLoop/Controller/Api/Upload.pm

@@ -243,21 +243,12 @@ sub post_category {
   my $c = shift;
   my $self = $c;
 
-  my $category_rs = $c->schema->resultset('Category');
+  my $category_list = $c->schema->resultset('Category')->as_hash;
-
+  delete $category_list->{0};
-  # for ( $category_rs->all ) {
-  #   push @{ $categories->{ ids } }, $_->get_column('id');
-  #   push @{ $categories->{ names } }, $_->get_column('name');
-  # }
-  my %category_list = (
-    map {
-      $_->id => $_->name,
-    } $category_rs->all
-  );
 
   return $self->render( json => {
     success => Mojo::JSON->true,
-    categories => \%category_list,
+    categories => $category_list,
   });
 }
 

lib/Pear/LocalLoop/Controller/Api/V1/Customer/Pies.pm

@@ -7,6 +7,7 @@ sub index {
   my $entity = $c->stash->{api_user}->entity;
 
   my $purchase_rs = $entity->purchases;
+
   my $local_org_local_purchase = $purchase_rs->search({
     "me.distance" => { '<', 20000 },
     'organisation.is_local' => 1,
@@ -43,7 +44,7 @@ sub index {
   }
   );
 
-  my $data = {
+  my $local_all = {
     'Local shop local purchaser' => $local_org_local_purchase->count,
     'Local shop non-local purchaser' => $local_org_non_local_purchase->count,
     'Non-local shop local purchaser' => $non_local_org_local_purchase->count,
@@ -53,7 +54,7 @@ sub index {
   return $c->render(
     json => {
       success => Mojo::JSON->true,
-      pie => $data,
+      local_all => $local_all,
     }
   );
 

lib/Pear/LocalLoop/Plugin/Currency.pm

@@ -9,6 +9,8 @@ sub register {
     my $value;
     if ( $currency_string =~ /^£([\d.]+)/ ) {
       $value = $1 * 1;
+    } elsif ( $currency_string =~ /^([\d.]+)/ ) {
+      $value = $1 * 1;
     }
     return $value;
   });

lib/Pear/LocalLoop/Schema/ResultSet/Category.pm

@@ -0,0 +1,22 @@
+package Pear::LocalLoop::Schema::ResultSet::Category;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::ResultSet';
+
+sub as_hash {
+  my ( $self ) = @_;
+
+  my %category_list = (
+    (
+      map {
+        $_->id => $_->name,
+      } $self->all
+    ),
+    0 => 'Uncategorised',
+  );
+  return \%category_list;
+}
+
+1;

script/cron_daily

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+# Scripts to run daily.
+# This will be run sometime between 2 & 3AM every morning.
+# If order matters, make sure they are in the right place.
+
+eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib)
+
+MOJO_MODE=production ./script/pear-local_loop recur_transactions --force
+MOJO_MODE=production ./script/pear-local_loop recalc_leaderboards

script/recalc_leaderboards

@@ -1,5 +0,0 @@
-#! /bin/bash
-
-eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib)
-
-MOJO_MODE=production ./script/pear-local_loop recalc_leaderboards

t/api/categories.t

@@ -24,7 +24,7 @@ $schema->resultset('Category')->create({
   name => 'test',
 });
 
-set_absolute_time('2017-01-02T00:00:00Z');
+set_absolute_time('2018-01-08T00:00:00Z');
 
 my $start = DateTime->today->subtract( hours => 12 );
 
@@ -55,38 +55,38 @@ $t->post_ok('/api/stats/category' => json => {
   ->status_is(200)->or($framework->dump_error)
   ->json_is('/data', {
     categories => {
-      "2016-12-05" => [{
+      "2017-12-11" => [{
-        days => "2016-12-05",
+        days => "2017-12-11",
         value => 210,
-        category => 1,
+        category => 'test',
       }],
-      "2016-12-12" => [{
+      "2017-12-18" => [{
-        days => "2016-12-12",
+        days => "2017-12-18",
         value => 200,
-        category => 1,
+        category => 'test',
       }],
-      "2016-12-19" => [{
+      "2017-12-25" => [{
-        days => "2016-12-19",
+        days => "2017-12-25",
         value => 210,
-        category => 1,
+        category => 'test',
       }],
-      "2016-12-26" => [{
+      "2018-01-01" => [{
-        days => "2016-12-26",
+        days => "2018-01-01",
         value => 190,
-        category => 1,
+        category => 'test',
       }],
     },
     essentials => {
-      "2016-12-05" => {
+      "2017-12-11" => {
         value => 210,
       },
-      "2016-12-12" => {
+      "2017-12-18" => {
         value => 200,
       },
-      "2016-12-19" => {
+      "2017-12-25" => {
         value => 210,
       },
-      "2016-12-26" => {
+      "2018-01-01" => {
         value => 190,
       },
     }

t/api/stats.t

@@ -19,7 +19,7 @@ $framework->install_fixtures('users');
 my $t = $framework->framework;
 my $schema = $t->app->schema;
 
-set_absolute_time('2017-01-01T00:00:00Z');
+set_absolute_time('2018-01-08T00:00:00Z');
 
 my $start = DateTime->today->subtract( hours => 12 );
 
@@ -49,17 +49,50 @@ $t->post_ok('/api/stats/customer' => json => {
   })
   ->status_is(200)->or($framework->dump_error)
   ->json_is('/weeks', {
-    first => 20,
+    first => 17,
     second => 20,
-    max => 22,
+    max => 21,
     sum => 118,
     count => 6,
     })
-  ->or($framework->dump_error)
   ->json_is('/sectors', {
     sectors => ['A'],
     purchases => [118],
-  });
+  })
+  ->json_is('/data', {
+    cat_total => {
+      Uncategorised => 810,
+    },
+    categories => {
+      "2017-12-11" => [{
+        days => "2017-12-11",
+        value => 210,
+        category => 'Uncategorised',
+      }],
+      "2017-12-18" => [{
+        days => "2017-12-18",
+        value => 200,
+        category => 'Uncategorised',
+      }],
+      "2017-12-25" => [{
+        days => "2017-12-25",
+        value => 210,
+        category => 'Uncategorised',
+      }],
+      "2018-01-01" => [{
+        days => "2018-01-01",
+        value => 190,
+        category => 'Uncategorised',
+      }],
+    },
+    cat_total => {
+      Uncategorised => 810,
+    },
+    essentials => {
+      purchase_no_essential_total => 0,
+      purchase_no_total => 118,
+    },
+  })->or($framework->dump_error);
 
 sub create_random_transaction {
   my $buyer = shift;

t/api/v1/customer/pies.t

@@ -42,7 +42,7 @@ $t->post_ok('/api/v1/customer/pies' => json => {
     session_key => $session_key,
   })
   ->status_is(200)->or($framework->dump_error)
-  ->json_is('/pie', {
+  ->json_is('/local_all', {
     'Local shop local purchaser' => 0,
     'Local shop non-local purchaser' => 0,
     'Non-local shop local purchaser' => 0,