Improved data import from other services

Addresses #3

Added people associated with orgs.

Import jobs now create new orgs and people as needed.

Author: jdudley1123

app/jobs/design_hops_job.rb

@@ -1,26 +1,48 @@
-class DesignHopsJob < ActiveJob::Base
+# frozen_string_literal: true
+
+class DesignHopsJob < ApplicationJob
   queue_as :default
 
-  def perform()
-    @hopAttendees = Airrecord.table(ENV['AIRTABLE_API_KEY'], "appchfb5dUtfrlVSZ", "Attendee sign-up data")
+  def perform
+    @hop_attendees = Airrecord.table(ENV['AIRTABLE_API_KEY'], 'appchfb5dUtfrlVSZ', 'Attendee sign-up data')
 
-    @hopAttendees.all.each do |record|
-      domain = record['Email'].split("@").last.strip
+    @hop_attendees.all.each do |record|
+      domain = record['Email'].split('@').last.strip
       org_name = record['Organisation name'].strip
 
-      @org = Organisation.find_by({name: org_name}) || Organisation.find_by({domain: domain]})
-      if @org
-        Action.create(
-          potential_action: PotentialAction.find(1), 
-          organisation: @org,
-          details: {
-            person_name: record['First name'] + " " + record['Last name'], 
-            hop: record['Which design hop are you applying to join?'].first
-          }, 
-          start_time: record.created_at, 
-          end_time: record.created_at
+      # Create new org
+      @org = Organisation.find_by({ name: org_name }) || Organisation.find_by({ domain: domain })
+      if @org.nil?
+        @org = Organisation.create!(
+          name: org_name,
+          domain: domain
         )
+        @org.get_charity_number
       end
+
+      # Create new org
+      @person = Person.find_by({ email: record['Email'] })
+      if @person.nil?
+        @person = Person.create!(
+          first_name: record['First name'],
+          last_name: record['Last name'],
+          email: record['Email'],
+          organisation: @org
+        )
+      end
+
+      # Record action
+      Action.create!(
+        potential_action: PotentialAction.find(1),
+        organisation_id: @org.id,
+        person_id: @person.id,
+        details: {
+          person_name: record['First name'] + ' ' + record['Last name'],
+          hop: record['Which design hop are you applying to join?'].first
+        },
+        start_time: record.created_at,
+        end_time: record.created_at
+      )
     end
   end
-end
+end

app/jobs/digital_candle_job.rb

@@ -1,39 +1,56 @@
-class DigitalCandleJob < ActiveJob::Base
+# frozen_string_literal: true
+
+class DigitalCandleJob < ApplicationJob
   queue_as :default
 
-  def perform()
-    
+  def perform
     @rows = GoogleSheetsImport.new.extract(ENV['DIGITAL_CANDLE_CONFIG'])
-    
-    @hopAttendees = Airrecord.table(ENV['AIRTABLE_API_KEY'], "appchfb5dUtfrlVSZ", "Attendee sign-up data")
 
     @rows.each do |row|
-      domain = row[:email].split("@").last.strip
+      domain = row[:email].split('@').last.strip
+      first_name = row[:name].match('^(\w+) ')[1].strip
+      last_name = row[:name].match(' (.*)$')[1].strip
+
+      # Find org
+      @org = Organisation.find_by({ domain: domain })
 
-      @org = Organisation.find_by({domain: domain})
-      if @org
-        Action.create(
-          potential_action: PotentialAction.find(3), 
-          organisation: @org,
+      unless @org.nil?
+        # Create new person
+        @person = Person.find_by({ email: row[:email] })
+        if @person.nil?
+          @person = Person.create!(
+            first_name: first_name,
+            last_name: last_name,
+            email: row[:email],
+            organisation: @org
+          )
+        end
+
+        # Record actions
+        Action.create!(
+          potential_action: PotentialAction.find(3),
+          organisation_id: @org.id,
+          person_id: @person.id,
           details: {
             person_name: row[:name]
-          }, 
-          start_time: row[:timestamp], 
+          },
+          start_time: row[:timestamp],
           end_time: row[:timestamp]
         )
-        
-        Action.create(
-          potential_action: PotentialAction.find(4), 
-          organisation: @org,
+
+        Action.create!(
+          potential_action: PotentialAction.find(4),
+          organisation_id: @org.id,
+          person_id: @person.id,
           details: {
             person_name: row[:name],
             intro: row[:intro],
             intro_date: row[:intro_date]
-          }, 
-          start_time: row[:intro_date], 
+          },
+          start_time: row[:intro_date],
           end_time: row[:intro_date]
         )
       end
     end
   end
-end
+end

app/jobs/service_recipes_job.rb

@@ -1,24 +1,34 @@
-class ServiceRecipesJob < ActiveJob::Base
+# frozen_string_literal: true
+
+class ServiceRecipesJob < ApplicationJob
   queue_as :default
-  
-  def perform()
-    @hopAttendees = Airrecord.table(ENV['AIRTABLE_API_KEY'], "appL7gqKhmfsrP0Ex", "Recipes to go online")
 
-    @hopAttendees.all(filter: "{Recipe progress} = 'Step 10: Published'").each do |record|
+  def perform
+    @service_recipes = Airrecord.table(ENV['AIRTABLE_API_KEY'], 'appL7gqKhmfsrP0Ex', 'Recipes to go online')
+
+    @service_recipes.all(filter: "{Recipe progress} = 'Step 10: Published'").each do |record|
       org_name = record['Charity / org'].strip
-      
-      @org = Organisation.find_by({name: org_name})
-      if @org
-        Action.create(
-          potential_action: PotentialAction.find(2), 
-          organisation: @org,
-          start_time: record.created_at, 
-          end_time: record.created_at,
-          details: {
-             recipe_name: record['Name of the thing']
-          }
+
+      # Create new org
+      @org = Organisation.find_by({ name: org_name })
+      if @org.nil?
+        @org = Organisation.create(
+          name: org_name
         )
+        @org.get_charity_number
       end
+
+      # Record action
+      Action.create(
+        potential_action: PotentialAction.find(2),
+        organisation_id: @org.id,
+        person_id: nil,
+        start_time: record.created_at,
+        end_time: record.created_at,
+        details: {
+          recipe_name: record['Name of the thing']
+        }
+      )
     end
   end
-end
+end

app/models/action.rb

@@ -1,18 +1,16 @@
+# frozen_string_literal: true
+
 class Action < ApplicationRecord
   belongs_to :potential_action
   belongs_to :organisation
-  
-  def getString()
-    
-    baseString = self.potential_action.format
-    
-    unless self.details.nil? 
-      self.details.each do |key, val|
-        baseString = baseString.sub('{{'+key+'}}', val)
-      end
+
+  def get_string
+    base_string = potential_action.format
+
+    details&.each do |key, val|
+      base_string = base_string.sub('{{' + key + '}}', val)
     end
-    
-    return baseString
+
+    base_string
   end
-  
 end

app/models/person.rb

@@ -0,0 +1,3 @@
+class Person < ApplicationRecord
+  belongs_to :organisation
+end

app/models/platform.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 class Platform < ApplicationRecord
 end

app/models/potential_action.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 class PotentialAction < ApplicationRecord
 end

app/services/google_sheets_import.rb

@@ -47,17 +47,17 @@ def extract(config, row_start: 2)
 
     worksheet = @session.spreadsheet_by_key(config.file_id)
                         .worksheet_by_sheet_id(config.sheet_id)
-    
+
     ret = []
-    
+
     (row_start..worksheet.num_rows).each do |row|
       unless worksheet[row, 1].empty?
         ret_row = {}
         config.fields.to_h.each { |k, v| ret_row[k] = worksheet[row, v] }
         ret << ret_row
       end
     end
-    
-    return ret
+
+    ret
   end
 end

db/migrate/20200610154932_make_organisation_optional.rb

@@ -0,0 +1,6 @@
+class MakeOrganisationOptional < ActiveRecord::Migration[6.0]
+  def change
+    change_column :actions, :person_id, :bigint, null: true
+    change_column :actions, :organisation_id, :bigint, null: true
+  end
+end

db/schema.rb

@@ -10,14 +10,14 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_06_10_140241) do
+ActiveRecord::Schema.define(version: 2020_06_10_154932) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
   create_table "actions", force: :cascade do |t|
     t.bigint "potential_action_id", null: false
-    t.bigint "organisation_id", null: false
+    t.bigint "organisation_id"
     t.jsonb "details"
     t.datetime "start_time"
     t.datetime "end_time"
@@ -83,7 +83,7 @@
     t.string "first_name"
     t.string "last_name"
     t.string "email"
-    t.bigint "organisation_id", null: false
+    t.bigint "organisation_id"
     t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
     t.index ["organisation_id"], name: "index_people_on_organisation_id"