diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/cpanfile b/cpanfile index 5ef4b1f..4d7839d 100644 --- a/cpanfile +++ b/cpanfile @@ -23,6 +23,7 @@ requires 'DBIx::Class::Fixtures'; requires 'GIS::Distance'; requires 'Text::CSV'; requires 'Try::Tiny'; +requires 'Throwable::Error'; on 'test' => sub { requires 'Test::More'; diff --git a/lib/Pear/LocalLoop.pm b/lib/Pear/LocalLoop.pm index db7bba8..fe1d6f4 100644 --- a/lib/Pear/LocalLoop.pm +++ b/lib/Pear/LocalLoop.pm @@ -244,6 +244,9 @@ sub startup { $admin_routes->get('/import/:set_id/ignore/:value_id')->to('admin-import#ignore_value'); $admin_routes->get('/import/:set_id/import')->to('admin-import#run_import'); + + $admin_routes->get('/import_from')->to('admin-import_from#index'); + # my $user_routes = $r->under('/')->to('root#under'); # $user_routes->get('/home')->to('root#home'); diff --git a/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm new file mode 100644 index 0000000..66b1129 --- /dev/null +++ b/lib/Pear/LocalLoop/Controller/Admin/ImportFrom.pm @@ -0,0 +1,4 @@ +package Pear::LocalLoop::Controller::Admin::ImportFrom; +use Mojo::Base 'Mojolicious::Controller'; + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Error.pm b/lib/Pear/LocalLoop/Error.pm new file mode 100644 index 0000000..7adf103 --- /dev/null +++ b/lib/Pear/LocalLoop/Error.pm @@ -0,0 +1,10 @@ +package Pear::LocalLoop::Error; +use Moo; +extends 'Throwable::Error'; + +package Pear::LocalLoop::ImplementationError; +use Moo; +use namespace::clean; +extends Pear::LocalLoop::Error; + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Import/LCCCsv.pm b/lib/Pear/LocalLoop/Import/LCCCsv.pm new file mode 100644 index 0000000..da9b64c --- /dev/null +++ b/lib/Pear/LocalLoop/Import/LCCCsv.pm @@ -0,0 +1,23 @@ +package Pear::LocalLoop::Import::LCCCsv; +use Moo; +use Pear::LocalLoop::Error; + +with qw/ + Pear::LocalLoop::Import::Role::ExternalName + Pear::LocalLoop::Import::Role::Schema + Pear::LocalLoop::Import::Role::CSV +/; + +has external_name => ( + is => 'ro', + default => 'LCC CSV', +); + +has csv_required_columns => ( + is => 'lazy', + builder => sub { + Pear::LocalLoop::ImplementationError->throw("Must be implemented by child class"); + }, +); + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Import/LCCCsv/Suppliers.pm b/lib/Pear/LocalLoop/Import/LCCCsv/Suppliers.pm new file mode 100644 index 0000000..a9d2635 --- /dev/null +++ b/lib/Pear/LocalLoop/Import/LCCCsv/Suppliers.pm @@ -0,0 +1,16 @@ +package Pear::LocalLoop::Import::LCCCsv::Suppliers; +use Moo; + +extends qw/Pear::LocalLoop::Import::LCCCsv/; + +sub import { + my $self = shift; +} + +sub _row_to_result { + my ( $self, $row ) = @_; + + +} + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm b/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm new file mode 100644 index 0000000..e784c41 --- /dev/null +++ b/lib/Pear/LocalLoop/Import/LCCCsv/Transactions.pm @@ -0,0 +1,10 @@ +package Pear::LocalLoop::Import::LCCCsv::Transactions; +use Moo; + +extends qw/Pear::LocalLoop::Import::LCCCsv/; + +sub import { + my $self = shift; +} + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Import/Role/CSV.pm b/lib/Pear/LocalLoop/Import/Role/CSV.pm new file mode 100644 index 0000000..5b20fef --- /dev/null +++ b/lib/Pear/LocalLoop/Import/Role/CSV.pm @@ -0,0 +1,46 @@ +package Pear::LocalLoop::Import::Role::CSV; +use strict; +use warnings; +use Moo::Role; +use Text::CSV; + +requires 'csv_required_columns'; + +has csv_file => ( + is => 'ro', + required => 1, +); + +has _csv_filehandle => ( + is => 'lazy', + builder => sub { + open my $fh, '<', $self->csv_file; + return $fh; + } +); + +has text_csv_options => ( + is => 'lazy', + builder => sub { + return { + binary => 1, + allow_whitespace => 1, + }; + } +); + +has _text_csv => ( + is => 'lazy', + builder => sub { + return Text::CSV->new(shift->text_csv_options); + } +); + +has csv_data => ( + is => 'lazy', + builder => sub { + my $self = shift; + } +); + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Import/Role/ExternalName.pm b/lib/Pear/LocalLoop/Import/Role/ExternalName.pm new file mode 100644 index 0000000..9fe79d6 --- /dev/null +++ b/lib/Pear/LocalLoop/Import/Role/ExternalName.pm @@ -0,0 +1,19 @@ +package Pear::LocalLoop::Import::Role::ExternalName; +use strict; +use warnings; +use Moo::Role; + +requires qw/ + external_name + schema +/; + +has external_result => ( + is => 'lazy', + builder => sub { + my $self = shift; + return $self->resultset('ExternalReference')->find_or_create({ name => $self->external_name }); + } +); + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Import/Role/Schema.pm b/lib/Pear/LocalLoop/Import/Role/Schema.pm new file mode 100644 index 0000000..71bae0c --- /dev/null +++ b/lib/Pear/LocalLoop/Import/Role/Schema.pm @@ -0,0 +1,11 @@ +package Pear::LocalLoop::Import::Role::Schema; +use strict; +use warnings; +use Moo::Role; + +has schema => ( + is => 'ro', + required => 1, +); + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Schema/Result/ExternalReference.pm b/lib/Pear/LocalLoop/Schema/Result/ExternalReference.pm new file mode 100644 index 0000000..15d5060 --- /dev/null +++ b/lib/Pear/LocalLoop/Schema/Result/ExternalReference.pm @@ -0,0 +1,39 @@ +package Pear::LocalLoop::Schema::Result::ExternalReference; + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +__PACKAGE__->table("external_references"); + +__PACKAGE__->add_columns( + "id" => { + data_type => "integer", + is_auto_increment => 1, + is_nullable => 0, + }, + "name" => { + data_type => "varchar", + size => 255, + is_nullable => 0, + }, +); + +__PACKAGE__->set_primary_key("id"); + +__PACKAGE__->add_unique_constraint([ qw/name/ ]); + +__PACKAGE__->has_many( + 'transactions', + "Pear::LocalLoop::Schema::Result::TransactionExternal", + { 'foreign.external_reference_id' => 'self.id' }, +); + +__PACKAGE__->has_many( + 'organisations', + "Pear::LocalLoop::Schema::Result::OrganisationExternal", + { 'foreign.external_reference_id' => 'self.id' }, +); + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Schema/Result/Organisation.pm b/lib/Pear/LocalLoop/Schema/Result/Organisation.pm index cdd71db..f9b5a94 100644 --- a/lib/Pear/LocalLoop/Schema/Result/Organisation.pm +++ b/lib/Pear/LocalLoop/Schema/Result/Organisation.pm @@ -10,76 +10,91 @@ __PACKAGE__->load_components("InflateColumn::DateTime", "FilterColumn"); __PACKAGE__->table("organisations"); __PACKAGE__->add_columns( - id => { - data_type => 'integer', + id => { + data_type => 'integer', is_auto_increment => 1, - is_nullable => 0, + is_nullable => 0, }, - entity_id => { - data_type => 'integer', - is_nullable => 0, + entity_id => { + data_type => 'integer', + is_nullable => 0, is_foreign_key => 1, }, - name => { - data_type => 'varchar', - size => 255, + name => { + data_type => 'varchar', + size => 255, is_nullable => 0, }, - street_name => { - data_type => 'text', + street_name => { + data_type => 'text', is_nullable => 1, }, - town => { - data_type => 'varchar', - size => 255, + town => { + data_type => 'varchar', + size => 255, is_nullable => 0, }, - postcode => { - data_type => 'varchar', - size => 16, + postcode => { + data_type => 'varchar', + size => 16, is_nullable => 1, }, - country => { - data_type => 'varchar', - size => 255, + country => { + data_type => 'varchar', + size => 255, is_nullable => 1, }, - sector => { - data_type => 'varchar', - size => 1, + sector => { + data_type => 'varchar', + size => 1, is_nullable => 1, }, - pending => { - data_type => 'boolean', - default => \"false", + pending => { + data_type => 'boolean', + default => \"false", is_nullable => 0, }, - is_local => { - data_type => 'boolean', - default => undef, + is_local => { + data_type => 'boolean', + default => undef, is_nullable => 1, }, - is_fair => { - data_type => 'boolean', - default => undef, + is_fair => { + data_type => 'boolean', + default => undef, is_nullable => 1, }, submitted_by_id => { - data_type => 'integer', + data_type => 'integer', is_nullable => 1, }, - latitude => { - data_type => 'decimal', - size => [8,5], - is_nullable => 1, + latitude => { + data_type => 'decimal', + size => [ 8, 5 ], + is_nullable => 1, default_value => undef, }, - longitude => { - data_type => 'decimal', - size => [8,5], - is_nullable => 1, + longitude => { + data_type => 'decimal', + size => [ 8, 5 ], + is_nullable => 1, default_value => undef, }, + type_id => { + data_type => 'integer', + is_nullable => 1, + is_foreign_key => 1, + }, + social_type_id => { + data_type => 'integer', + is_nullable => 1, + is_foreign_key => 1, + }, + is_anchor => { + data_type => 'boolean', + is_nullable => 0, + default_value => \'FALSE', + } ); __PACKAGE__->set_primary_key('id'); @@ -98,32 +113,37 @@ __PACKAGE__->has_many( ); __PACKAGE__->filter_column( - pending => { + pending => { filter_to_storage => 'to_bool', }, - is_local => { + is_local => { + filter_to_storage => 'to_bool', + }, + is_anchor => { filter_to_storage => 'to_bool', } ); # Only works when calling ->deploy, but atleast helps for tests sub sqlt_deploy_hook { - my ( $source_instance, $sqlt_table ) = @_; + my ($source_instance, $sqlt_table) = @_; my $pending_field = $sqlt_table->get_field('pending'); - if ( $sqlt_table->schema->translator->producer_type =~ /SQLite$/ ) { + if ($sqlt_table->schema->translator->producer_type =~ /SQLite$/) { $pending_field->{default_value} = 0; - } else { + } + else { $pending_field->{default_value} = \"false"; } } sub to_bool { - my ( $self, $val ) = @_; - return if ! defined $val; + my ($self, $val) = @_; + return if !defined $val; my $driver_name = $self->result_source->schema->storage->dbh->{Driver}->{Name}; - if ( $driver_name eq 'SQLite' ) { + if ($driver_name eq 'SQLite') { return $val ? 1 : 0; - } else { + } + else { return $val ? 'true' : 'false'; } } diff --git a/lib/Pear/LocalLoop/Schema/Result/OrganisationExternal.pm b/lib/Pear/LocalLoop/Schema/Result/OrganisationExternal.pm new file mode 100644 index 0000000..aadbec5 --- /dev/null +++ b/lib/Pear/LocalLoop/Schema/Result/OrganisationExternal.pm @@ -0,0 +1,49 @@ +package Pear::LocalLoop::Schema::Result::OrganisationExternal; + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +__PACKAGE__->table("organisations_external"); + +__PACKAGE__->add_columns( + "id" => { + data_type => "integer", + is_auto_increment => 1, + is_nullable => 0, + }, + "org_id" => { + data_type => "integer", + is_foreign_key => 1, + is_nullable => 0, + }, + "external_reference_id" => { + data_type => "integer", + is_foreign_key => 1, + is_nullable => 0, + }, + "external_id" => { + data_type => "varchar", + size => 255, + is_nullable => 0, + } +); + +__PACKAGE__->set_primary_key("id"); + +__PACKAGE__->add_unique_constraint([ qw/external_reference_id external_id/ ]); + +__PACKAGE__->belongs_to( + "organisation", + "Pear::LocalLoop::Schema::Result::Organisation", + { 'foreign.id' => 'self.org_id' }, +); + +__PACKAGE__->belongs_to( + "external_reference", + "Pear::LocalLoop::Schema::Result::ExternalReference", + { 'foreign.id' => 'self.external_reference_id' }, +); + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Schema/Result/OrganisationSocialType.pm b/lib/Pear/LocalLoop/Schema/Result/OrganisationSocialType.pm new file mode 100644 index 0000000..e0d1645 --- /dev/null +++ b/lib/Pear/LocalLoop/Schema/Result/OrganisationSocialType.pm @@ -0,0 +1,39 @@ +package Pear::LocalLoop::Schema::Result::OrganisationSocialType; + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +__PACKAGE__->table("organisation_social_types"); + +__PACKAGE__->add_columns( + "id" => { + data_type => "integer", + is_auto_increment => 1, + is_nullable => 0, + }, + "key" => { + data_type => "varchar", + size => 255, + is_nullable => 0, + }, + "name" => { + data_type => "varchar", + size => 255, + is_nullable => 0, + } +); + +__PACKAGE__->set_primary_key("id"); + +__PACKAGE__->add_unique_constraint([ qw/key/ ]); + +__PACKAGE__->has_many( + "organisations", + "Pear::LocalLoop::Schema::Result::Organisation", + { 'foreign.social_type_id' => 'self.id' }, +); + +1; +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Schema/Result/OrganisationType.pm b/lib/Pear/LocalLoop/Schema/Result/OrganisationType.pm new file mode 100644 index 0000000..f7967d6 --- /dev/null +++ b/lib/Pear/LocalLoop/Schema/Result/OrganisationType.pm @@ -0,0 +1,38 @@ +package Pear::LocalLoop::Schema::Result::OrganisationType; + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +__PACKAGE__->table("organisation_types"); + +__PACKAGE__->add_columns( + "id" => { + data_type => "integer", + is_auto_increment => 1, + is_nullable => 0, + }, + "key" => { + data_type => "varchar", + size => 255, + is_nullable => 0, + }, + "name" => { + data_type => "varchar", + size => 255, + is_nullable => 0, + } +); + +__PACKAGE__->set_primary_key("id"); + +__PACKAGE__->add_unique_constraint([ qw/key/ ]); + +__PACKAGE__->has_many( + "organisations", + "Pear::LocalLoop::Schema::Result::Organisation", + { 'foreign.type_id' => 'self.id' }, +); + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Schema/Result/TransactionExternal.pm b/lib/Pear/LocalLoop/Schema/Result/TransactionExternal.pm new file mode 100644 index 0000000..13d61ca --- /dev/null +++ b/lib/Pear/LocalLoop/Schema/Result/TransactionExternal.pm @@ -0,0 +1,49 @@ +package Pear::LocalLoop::Schema::Result::TransactionExternal; + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +__PACKAGE__->table("transactions_external"); + +__PACKAGE__->add_columns( + "id" => { + data_type => "integer", + is_auto_increment => 1, + is_nullable => 0, + }, + "transaction_id" => { + data_type => "integer", + is_foreign_key => 1, + is_nullable => 0, + }, + "external_reference_id" => { + data_type => "integer", + is_foreign_key => 1, + is_nullable => 0, + }, + "external_id" => { + data_type => "varchar", + size => 255, + is_nullable => 0, + } +); + +__PACKAGE__->set_primary_key("id"); + +__PACKAGE__->add_unique_constraint([ qw/external_reference_id external_id/ ]); + +__PACKAGE__->belongs_to( + "transaction", + "Pear::LocalLoop::Schema::Result::Transaction", + { 'foreign.id' => 'self.transaction_id' }, +); + +__PACKAGE__->belongs_to( + "external_reference", + "Pear::LocalLoop::Schema::Result::ExternalReference", + { 'foreign.id' => 'self.external_reference_id' }, +); + +1; \ No newline at end of file diff --git a/lib/Pear/LocalLoop/Schema/Result/TransactionMeta.pm b/lib/Pear/LocalLoop/Schema/Result/TransactionMeta.pm index b986067..8c16900 100644 --- a/lib/Pear/LocalLoop/Schema/Result/TransactionMeta.pm +++ b/lib/Pear/LocalLoop/Schema/Result/TransactionMeta.pm @@ -8,29 +8,29 @@ use base 'DBIx::Class::Core'; __PACKAGE__->table("transactions_meta"); __PACKAGE__->add_columns( - "id" => { - data_type => "integer", + "id" => { + data_type => "integer", is_auto_increment => 1, - is_nullable => 0, + is_nullable => 0, }, - "transaction_id" => { - data_type => "integer", + "transaction_id" => { + data_type => "integer", is_foreign_key => 1, - is_nullable => 0, + is_nullable => 0, }, - "net_value" => { - data_type => "numeric", - size => [ 100, 0 ], + "net_value" => { + data_type => "numeric", + size => [ 100, 0 ], is_nullable => 0, }, "sales_tax_value" => { - data_type => "numeric", - size => [ 100, 0 ], + data_type => "numeric", + size => [ 100, 0 ], is_nullable => 0, }, - "gross_value" => { - data_type => "numeric", - size => [ 100, 0 ], + "gross_value" => { + data_type => "numeric", + size => [ 100, 0 ], is_nullable => 0, }, ); @@ -43,19 +43,4 @@ __PACKAGE__->belongs_to( { 'foreign.id' => 'self.transaction_id' }, ); -__PACKAGE__->might_have( - "category", - "Pear::LocalLoop::Schema::Result::TransactionCategory" => "transaction_id", -); - -sub sqlt_deploy_hook { - my ( $source_instance, $sqlt_table ) = @_; - my $pending_field = $sqlt_table->get_field('essential'); - if ( $sqlt_table->schema->translator->producer_type =~ /SQLite$/ ) { - $pending_field->{default_value} = 0; - } else { - $pending_field->{default_value} = \"false"; - } -} - 1; diff --git a/templates/admin/import_from/index.html.ep b/templates/admin/import_from/index.html.ep new file mode 100644 index 0000000..e9b45f1 --- /dev/null +++ b/templates/admin/import_from/index.html.ep @@ -0,0 +1,32 @@ +% layout 'admin'; +% title 'Import From'; +% content_for javascript => begin +% end +% if (my $error = flash 'error') { + +% } +% elsif (my $success = flash 'success') { + +% } +
+
+

Import From

+
+
+

Various import options depending on source. Each one is custom.

+
+
+
+
+
LCC Procurement Import
+
Creditor reports from LCC
+ Suppliers + Transactions +
+
+
+