Update importer (#160)

* Importer now displays validation errors

This will help people debug errors before submitting their PR.

* Fix errors in the outstanding updates

db/seed/updates/20250113-characters-016.csv

@@ -1,2 +1,2 @@
 name_en,name_jp,granblue_id,rarity,element,proficiency1,proficiency2,gender,race1,race2,flb,min_hp,max_hp,max_hp_flb,min_atk,max_atk,max_atk_flb,base_da,base_ta,ougi_ratio,ougi_ratio_flb,special,ulb,max_hp_ulb,max_atk_ulb,character_id,wiki_en,release_date,flb_date,ulb_date,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp
-,,3040254000,,,,,,,,true,230,1285,1515,1880,9800,11680,,,,,,,,,,Lucius (SSR),,2024-12-29,,ルシウス (SSR),182086,SSRルシウス,,
+,,3040254000,,,,,,,,true,230,1285,1515,1880,9800,11680,,,,,false,false,,,,Lucius (SSR),,2024-12-29,,ルシウス (SSR),182086,SSRルシウス,,

db/seed/updates/20250113-characters-019.csv

@@ -1,2 +1,2 @@
 name_en,name_jp,granblue_id,rarity,element,proficiency1,proficiency2,gender,race1,race2,flb,min_hp,max_hp,max_hp_flb,min_atk,max_atk,max_atk_flb,base_da,base_ta,ougi_ratio,ougi_ratio_flb,special,ulb,max_hp_ulb,max_atk_ulb,character_id,wiki_en,release_date,flb_date,ulb_date,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp
-,,3040377000,,,,,,,,true,136,852,988,2022,10140,12162,12,6,,,,,,,,,,2025-01-10,,シンダラ (SSR),311656,SSRシンダラ,,
+,,3040377000,,,,,,,,true,136,852,988,2022,10140,12162,12,6,,,false,false,,,,,,2025-01-10,,シンダラ (SSR),311656,SSRシンダラ,,

db/seed/updates/20250113-characters-020.csv

@@ -1,2 +1,2 @@
 name_en,name_jp,granblue_id,rarity,element,proficiency1,proficiency2,gender,race1,race2,flb,min_hp,max_hp,max_hp_flb,min_atk,max_atk,max_atk_flb,base_da,base_ta,ougi_ratio,ougi_ratio_flb,special,ulb,max_hp_ulb,max_atk_ulb,character_id,wiki_en,release_date,flb_date,ulb_date,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp
-,,3040158000,,,,,,,,true,368,1960,2328,1310,7000,8310,7,3,,,,,,,,,,2024-08-27,,ブローディア (SSR),99533,SSRブローディア,{brodia},
+,,3040158000,,,,,,,,true,368,1960,2328,1310,7000,8310,7,3,,,false,false,,,,,,2024-08-27,,ブローディア (SSR),99533,SSRブローディア,{brodia},

db/seed/updates/20250113-weapons-003.csv

@@ -1,6 +1,6 @@
 name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date
-Amaranthine,アマラントス,1040423600,3,5,6,36,false,false,100,10,47,289,,,364,2143,,,,,,,3040543000,,2024-07-31,,,Amaranthine,アマラントス (SSR),458411,アマラントス,,,false,
+Amaranthine,アマラントス,1040423600,3,5,6,36,false,false,100,10,47,289,,,364,2143,,,false,,false,false,3040543000,,2024-07-31,,,Amaranthine,アマラントス (SSR),458411,アマラントス,,,false,
-Sole Intenso,ソーレ・インテンソ,1040619100,3,6,7,36,false,false,100,10,35,421,,,235,2414,,,,,,,3040542000,,2024-07-31,,,Sole Intenso,ソーレ・インテンソ (SSR),458412,ソーレ・インテンソ,,,false,
+Sole Intenso,ソーレ・インテンソ,1040619100,3,6,7,36,false,false,100,10,35,421,,,235,2414,,,false,,false,false,3040542000,,2024-07-31,,,Sole Intenso,ソーレ・インテンソ (SSR),458412,ソーレ・インテンソ,,,false,
-Shimmering Bloom,燦花,1040713700,3,2,5,36,false,false,100,10,35,223,,,432,2525,,,,,,,3040541000,,2024-07-31,,,Shimmering Bloom,燦花 (SSR),458413,燦花,,,false,
+Shimmering Bloom,燦花,1040713700,3,2,5,36,false,false,100,10,35,223,,,432,2525,,,false,,false,false,3040541000,,2024-07-31,,,Shimmering Bloom,燦花 (SSR),458413,燦花,,,false,
-Albacore Body,アルバコアボディ,1040423500,3,5,6,35,false,false,100,10,32,224,,,303,1759,,,,,,,,,2024-07-29,,,Albacore Body,アルバコアボディ (SSR),458282,アルバコアボディ,,,false,
+Albacore Body,アルバコアボディ,1040423500,3,5,6,35,false,false,100,10,32,224,,,303,1759,,,false,,false,false,,,2024-07-29,,,Albacore Body,アルバコアボディ (SSR),458282,アルバコアボディ,,,false,
-Rhothion Harp,ロティオンハープ,1030804600,2,5,8,35,false,false,75,10,20,146,,,211,1225,,,,,,,,,2024-07-29,,,Rhothion Harp,ロティオンハープ (SR),,ロティオンハープ,,,false,
+Rhothion Harp,ロティオンハープ,1030804600,2,5,8,35,false,false,75,10,20,146,,,211,1225,,,false,,false,false,,,2024-07-29,,,Rhothion Harp,ロティオンハープ (SR),,ロティオンハープ,,,false,

db/seed/updates/20250113-weapons-004.csv

@@ -1,2 +1,2 @@
 name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date
-Exo Ashavan,神刃エクス・アシャワン,1040119900,3,4,2,33,true,,150,15,36,222,269,,359,2221,2687,,false,,false,false,,10,2024-08-07,2024-08-07,,Exo Ashavan,神刃エクス・アシャワン (SSR),459089,神刃エクス・アシャワン,,,false,
+Exo Ashavan,神刃エクス・アシャワン,1040119900,3,4,2,33,true,false,150,15,36,222,269,,359,2221,2687,,false,,false,false,,10,2024-08-07,2024-08-07,,Exo Ashavan,神刃エクス・アシャワン (SSR),459089,神刃エクス・アシャワン,,,false,

db/seed/updates/20250114-weapons-006.csv

@@ -1,4 +1,4 @@
 name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date
-Frostbloom Fan,白氷碧扇,1040120000,3,2,2,36,false,false,100,10,34,231,,,429,2432,,,false,,,false,3040545000,,2024-08-16,,,Frostbloom_Fan,白氷碧扇 (SSR),460531,白氷碧扇,,,false,
+Frostbloom Fan,白氷碧扇,1040120000,3,2,2,36,false,false,100,10,34,231,,,429,2432,,,false,,false,false,3040545000,,2024-08-16,,,Frostbloom_Fan,白氷碧扇 (SSR),460531,白氷碧扇,,,false,
-Passion Parasol,愛々傘,1040219300,3,3,4,36,false,false,100,10,49,272,,,353,2226,,,false,,,false,3040546000,,2024-08-16,,,Passion_Parasol,愛々傘 (SSR),460532,愛々傘,,,false,
+Passion Parasol,愛々傘,1040219300,3,3,4,36,false,false,100,10,49,272,,,353,2226,,,false,,false,false,3040546000,,2024-08-16,,,Passion_Parasol,愛々傘 (SSR),460532,愛々傘,,,false,
-Galleon Float,ガレヲン・フロート,1040816600,3,1,8,36,true,false,150,15,42,266,322,,389,2258,2725,,false,,,false,3040544000,,2024-08-16,2024-08-16,,Galleon_Float,ガレヲン・フロート (SSR),460530,ガレヲン・フロート,,,false,
+Galleon Float,ガレヲン・フロート,1040816600,3,1,8,36,true,false,150,15,42,266,322,,389,2258,2725,,false,,false,false,3040544000,,2024-08-16,2024-08-16,,Galleon_Float,ガレヲン・フロート (SSR),460530,ガレヲン・フロート,,,false,

db/seed/updates/20250114-weapons-014.csv

@@ -1,6 +1,6 @@
 name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date, , , , , , , , , , , , , ,
-Royal Deliverance,ロイヤル・ブリンガー,1040027000,3,0,1,24,true,true,200,20,40,240,290,340,400,2490,3013,3536,,,false,false,,15,2024-10-29,2024-10-29,2024-10-29,Royal_Deliverance,ロイヤル・ブリンガー (SSR),470415,ロイヤル・ブリンガー,,,false,,,,,,,,,,,,,,,
+Royal Deliverance,ロイヤル・ブリンガー,1040027000,3,0,1,24,true,true,200,20,40,240,290,340,400,2490,3013,3536,false,,false,false,,15,2024-10-29,2024-10-29,2024-10-29,Royal_Deliverance,ロイヤル・ブリンガー (SSR),470415,ロイヤル・ブリンガー,,,false,,,,,,,,,,,,,,,
-Diadem Moulinet,ダイアデム・ムーリネ,1040120300,3,0,2,24,true,true,200,20,35,240,292,344,425,2490,3007,3524,,,false,false,,15,2024-10-29,2024-10-29,2024-10-29,Diadem_Moulinet,ダイアデム・ムーリネ (SSR),470412,ダイアデム・ムーリネ,,,false,,,,,,,,,,,,,,,
+Diadem Moulinet,ダイアデム・ムーリネ,1040120300,3,0,2,24,true,true,200,20,35,240,292,344,425,2490,3007,3524,false,,false,false,,15,2024-10-29,2024-10-29,2024-10-29,Diadem_Moulinet,ダイアデム・ムーリネ (SSR),470412,ダイアデム・ムーリネ,,,false,,,,,,,,,,,,,,,
-Norden Labrys,ノルデン・ラブリュス,1040318500,3,0,3,24,true,true,200,20,27,180,219,258,465,2790,3372,3954,,,false,false,,15,2024-10-29,2024-10-29,2024-10-29,Norden_Labrys,ノルデン・ラブリュス (SSR),470413,ノルデン・ラブリュス,,,false,,,,,,,,,,,,,,,
+Norden Labrys,ノルデン・ラブリュス,1040318500,3,0,3,24,true,true,200,20,27,180,219,258,465,2790,3372,3954,false,,false,false,,15,2024-10-29,2024-10-29,2024-10-29,Norden_Labrys,ノルデン・ラブリュス (SSR),470413,ノルデン・ラブリュス,,,false,,,,,,,,,,,,,,,
-Agarthan Hunter's Dagger,アガルタ猟団制式短剣,1040120900,3,4,2,35,false,false,100,10,28,185,,,324,1956,,,,,false,false,,,2024-10-29,,,Agarthan_Hunter%27s_Dagger,アガルタ猟団制式短剣 (SSR),470438,アガルタ猟団制式短剣,,,false,,,,,,,,,,,,,,,
+Agarthan Hunter's Dagger,アガルタ猟団制式短剣,1040120900,3,4,2,35,false,false,100,10,28,185,,,324,1956,,,false,,false,false,,,2024-10-29,,,Agarthan_Hunter%27s_Dagger,アガルタ猟団制式短剣 (SSR),470438,アガルタ猟団制式短剣,,,false,,,,,,,,,,,,,,,
-Tracker's Gauntlet,狩穫者の手甲,1030609700,2,4,7,35,false,false,75,10,20,130,,,207,1302,,,,,false,false,,,2024-10-29,,,Tracker%27s_Gauntlet,狩穫者の手甲 (SR),,狩穫者の手甲,,,false,,,,,,,,,,,,,,,
+Tracker's Gauntlet,狩穫者の手甲,1030609700,2,4,7,35,false,false,75,10,20,130,,,207,1302,,,false,,false,false,,,2024-10-29,,,Tracker%27s_Gauntlet,狩穫者の手甲 (SR),,狩穫者の手甲,,,false,,,,,,,,,,,,,,,

lib/granblue/importers/base_importer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'import_error'
+
 module Granblue
   module Importers
     class BaseImporter
@@ -28,42 +30,27 @@ module Granblue
 
         CSV.foreach(@file_path, headers: true) do |row|
           attributes = build_attributes(row)
+
+          # Check if record exists before doing any validation
           existing_record = model_class.find_by(granblue_id: attributes[:granblue_id])
 
           if existing_record
-            # For updates, only include non-nil attributes
+            simulate_update(existing_record, attributes, simulated_updated, type)
-            update_attributes = attributes.compact
-            would_update = update_attributes.any? { |key, value| existing_record[key] != value }
-
-            if would_update
-              log_test_update(existing_record, attributes)
-              simulated_updated[type] << {
-                granblue_id: attributes[:granblue_id],
-                name_en: attributes[:name_en] || existing_record.name_en,
-                attributes: update_attributes,
-                operation: :update
-              }
-            end
           else
-            log_test_creation(attributes)
+            validate_required_attributes(attributes)
-            simulated_new[type] << {
+            simulate_create(attributes, simulated_new, type)
-              granblue_id: attributes[:granblue_id],
-              name_en: attributes[:name_en],
-              attributes: attributes,
-              operation: :create
-            }
           end
         end
 
         { new: simulated_new, updated: simulated_updated }
+      rescue StandardError => e
+        handle_error(e)
       end
 
       private
 
       def import_row(row)
         attributes = build_attributes(row)
-        # Remove nil values from attributes hash for updates
-        # Keep them for new records to ensure proper defaults
         record = find_or_create_record(attributes)
         track_record(record) if record
       end
@@ -76,7 +63,6 @@ module Granblue
             log_test_update(existing_record, attributes)
             nil
           else
-            # For updates, only include non-nil attributes
             update_attributes = attributes.compact
             was_updated = update_attributes.any? { |key, value| existing_record[key] != value }
             existing_record.update!(update_attributes) if was_updated
@@ -87,12 +73,138 @@ module Granblue
             log_test_creation(attributes)
             nil
           else
-            # For new records, use all attributes including nil values
             [model_class.create!(attributes), false]
           end
         end
       end
 
+      def simulate_create(attributes, simulated_new, type)
+        test_record = model_class.new(attributes)
+        validate_record(test_record)
+
+        log_test_creation(attributes)
+        simulated_new[type] << {
+          granblue_id: attributes[:granblue_id],
+          name_en: attributes[:name_en],
+          attributes: attributes,
+          operation: :create
+        }
+      end
+
+      def simulate_update(existing_record, attributes, simulated_updated, type)
+        update_attributes = attributes.compact
+        would_update = update_attributes.any? { |key, value| existing_record[key] != value }
+
+        if would_update
+          # Create a test record with existing data
+          test_record = existing_record.dup
+
+          # Validate only the columns being updated
+          validate_update_attributes(update_attributes)
+
+          # Apply the updates and validate the resulting record
+          test_record.assign_attributes(update_attributes)
+          validate_record(test_record)
+
+          log_test_update(existing_record, attributes)
+          simulated_updated[type] << {
+            granblue_id: attributes[:granblue_id],
+            name_en: attributes[:name_en] || existing_record.name_en,
+            attributes: update_attributes,
+            operation: :update
+          }
+        end
+      end
+
+      def validate_required_attributes(attributes)
+        required_columns = model_class.columns.select { |c| !c.null }.map(&:name)
+
+        missing_columns = required_columns.select do |column|
+          attributes[column.to_sym].nil? &&
+            !model_class.column_defaults[column] &&
+            !%w[id created_at updated_at].include?(column)
+        end
+
+        if missing_columns.any?
+          details = [
+            "Missing required columns:",
+            missing_columns.map { |col| "  • #{col}" },
+            "",
+            "Affected model: #{model_class.name}"
+          ].flatten.join("\n")
+
+          raise ImportError.new(
+            file_name: File.basename(@file_path),
+            details: details
+          )
+        end
+      end
+
+      def validate_update_attributes(update_attributes)
+        # Get the list of columns that cannot be null in the database
+        required_columns = model_class.columns.select { |c| !c.null }.map(&:name)
+
+        # For updates, we only need to validate the attributes that are being updated
+        update_columns = update_attributes.keys.map(&:to_s)
+
+        # Only check required columns that are included in the update
+        missing_columns = required_columns.select do |column|
+          update_columns.include?(column) &&
+            update_attributes[column.to_sym].nil? &&
+            !model_class.column_defaults[column] &&
+            !%w[id created_at updated_at].include?(column)
+        end
+
+        if missing_columns.any?
+          details = [
+            "Missing required values for update:",
+            missing_columns.map { |col| "  • #{col}" },
+            "",
+            "Affected model: #{model_class.name}"
+          ].flatten.join("\n")
+
+          raise ImportError.new(
+            file_name: File.basename(@file_path),
+            details: details
+          )
+        end
+      end
+
+      def validate_record(record)
+        unless record.valid?
+          raise ImportError.new(
+            file_name: File.basename(@file_path),
+            details: format_validation_error(ActiveRecord::RecordInvalid.new(record))
+          )
+        end
+
+        begin
+          ActiveRecord::Base.transaction(requires_new: true) do
+            record.save!
+            raise ActiveRecord::Rollback
+          end
+        rescue ActiveRecord::StatementInvalid => e
+          if e.message.include?('violates not-null constraint')
+            column = e.message.match(/column "([^"]+)"/)[1]
+            details = [
+              "Database constraint violation:",
+              "  • Column '#{column}' cannot be null",
+              "",
+              "Affected model: #{model_class.name}"
+            ].join("\n")
+
+            raise ImportError.new(
+              file_name: File.basename(@file_path),
+              details: details
+            )
+          end
+          raise ImportError.new(
+            file_name: File.basename(@file_path),
+            details: format_standard_error(e)
+          )
+        end
+      end
+
       def track_record(result)
         record, was_updated = result
         type = model_class.name.demodulize.downcase
@@ -119,17 +231,19 @@ module Granblue
       end
 
       def log_test_update(record, attributes)
-        # For test mode, show only the attributes that would be updated
         update_attributes = attributes.compact
-        @logger&.log_step("Updating #{model_class.name} #{record.granblue_id}...")
+        @logger&.log_step("\nUpdate #{model_class.name} #{record.granblue_id}:")
+        @logger&.log_verbose("Current values:")
+        @logger&.log_verbose(format_attributes(record.attributes.symbolize_keys))
+        @logger&.log_verbose("\nNew values:")
         @logger&.log_verbose(format_attributes(update_attributes))
-        @logger&.log_step("\n\n") if @verbose
+        @logger&.log_step("\n")
       end
 
       def log_test_creation(attributes)
-        @logger&.log_step("Creating #{model_class.name}...")
+        @logger&.log_step("\nCreate #{model_class.name}:")
         @logger&.log_verbose(format_attributes(attributes))
-        @logger&.log_step("\n\n") if @verbose
+        @logger&.log_step("\n")
       end
 
       def log_new_record(record)
@@ -187,6 +301,43 @@ module Granblue
       def build_attributes(row)
         raise NotImplementedError, 'Subclasses must define build_attributes'
       end
+
+      def handle_error(error)
+        details = case error
+                  when ActiveRecord::RecordInvalid
+                    format_validation_error(error)
+                  else
+                    format_standard_error(error)
+                  end
+
+        raise ImportError.new(
+          file_name: File.basename(@file_path),
+          details: details
+        )
+      end
+
+      def format_validation_error(error)
+        [
+          "Validation failed:",
+          error.record.errors.full_messages.map { |msg| "  • #{msg}" },
+          "",
+          "Record attributes:",
+          format_attributes(error.record.attributes.symbolize_keys)
+        ].flatten.join("\n")
+      end
+
+      def format_standard_error(error)
+        if @verbose && error.respond_to?(:backtrace)
+          [
+            error.message,
+            "",
+            "Backtrace:",
+            error.backtrace.take(3).map { |line| "  #{line}" }
+          ].flatten.join("\n")
+        else
+          error.message.to_s
+        end
+      end
     end
   end
 end

lib/granblue/importers/import_error.rb

@@ -0,0 +1,31 @@
+module Granblue
+  module Importers
+    class ImportError < StandardError
+      attr_reader :file_name, :details
+
+      def initialize(file_name:, details:)
+        @file_name = file_name
+        @details = details
+        super(build_message)
+      end
+
+      private
+
+      def build_message
+        "Error importing #{file_name}: #{details}"
+      end
+    end
+
+    def format_attributes(attributes)
+      attributes.map do |key, value|
+        formatted_value = case value
+                          when Array
+                            value.empty? ? '[]' : value.inspect
+                          else
+                            value.inspect
+                          end
+        "  #{key}: #{formatted_value}"
+      end.join("\n")
+    end
+  end
+end

lib/logging_helper.rb

@@ -10,7 +10,7 @@ module LoggingHelper
   end
 
   def log_error(message)
-    puts "❌ #{message}"
+    puts "#{message}"
   end
 
   def log_warning(message)

lib/post_deployment/manager.rb

@@ -127,10 +127,54 @@ module PostDeployment
     end
 
     def handle_error(error)
-      log_error("\nError during deployment: #{error.message}")
+      error_message = format_error_message(error)
-      log_error(error.backtrace.take(10).join("\n")) if @verbose
+      log_formatted_error(error_message)
       @test_transaction&.rollback
-      raise error
+      exit(1)
+    end
+
+    def format_error_message(error)
+      sections = []
+
+      # Add header section
+      sections << [
+        "═" * 60,
+        "❌ Error during deployment",
+        "═" * 60
+      ]
+
+      # Add main error message
+      sections << format_main_error(error)
+
+      # Add stack trace if verbose
+      if @verbose && error.respond_to?(:backtrace)
+        sections << [
+          "Stack trace:",
+          error.backtrace.take(5).map { |line| "  #{line}" }
+        ].flatten
+      end
+
+      sections.flatten.join("\n")
+    end
+
+    def format_main_error(error)
+      case error
+      when Granblue::Importers::ImportError
+        [
+          "File: #{error.file_name}",
+          "-" * 80,
+          error.details
+        ]
+      else
+        error.message.to_s
+      end
+    end
+
+    def log_formatted_error(message)
+      # Split message into lines and log each with error prefix
+      message.split("\n").each do |line|
+        log_error line
+      end
     end
 
     def all_records_empty?

lib/post_deployment/search_indexer.rb

@@ -55,7 +55,7 @@ module PostDeployment
     end
 
     def log_error(message)
-      puts "❌ #{message}"
+      puts "#{message}"
     end
   end
 end