Merge branch 'main' into jedmund/database-fast-forward-2
This commit is contained in:
commit
4b8f507c19
19 changed files with 713 additions and 195 deletions
|
|
@ -1 +1 @@
|
|||
granblue
|
||||
hensei
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
DNg3u1sscYjhg9KuSpOmx/E5ysJ89hktGTi2aslpe0R5DFzs/MAulFMJwZJPzjNKxXoyxJb4CCASGTUXUQFATasO/aUwws9ZWN/dVQS2CTd4guRICRqR4Kzip43XHgE9ctnOP9E2NMXfGnRIXmQohu+mjMJp0fQnJzr7L70o3cjtB1iQ/KwF3BithKQF/xPi+HCd4OZUxOhyixsG0OhNbCNsb7/tSqAs9JBrslRbN+XiRibzbWGD8rtNapU+IuBMWNK5B++8KpyUNWvUhTJup84L5FNHHysRlP0kAd8XM119EnMOs0rb0QwQsbZk2WfIGXgnKDzqr02XXsUjWtNZrbTM2zqiaLioYIvLxE6EMnFFEMNU+2Bpgj9xUu/x+WIw57xI9/6Iyr8Ck3PmFe5r0gpNLs2xXHkweCrXWDZjyNNzwNhSt3HTb5K+3QsU0JkB2wqGZZnez2CwfrvBfMFjKfAxAVGygeKFZsRY3XCVhs7r5NSHg6Wp/X+/jyYz8MCjlyw/yppyA4c/sAs1bJ1fmzo5K5reOzmpv1K7uqvX57o4--9yICsk5RvHZzyqdC--g3xaeflXn1y3Z/H5v6/oWw==
|
||||
7wcHmOJGd2lnyS5YYSo7OMJZlS+O/iNQqQJOHju+eZReCgKfV2PVji7MU6Bs0yymA3Z4SmTwsDRgPXfHnPb5ZiiLygGzsfWlPtcuwZA8U/9QFzerfz5/0ttgo2iAboJ8oY/NJzz73vVEzBwDv99CSvyMiy8Z9Y9QATnX9bE18pLll1A7/a+SpoH+JTO5zoDg/l/+RhLxaH/U+jc6u88sM1jjGbsA+5oH/RyNycjH2MA5suFvWMdrUUEu0fS90yv0IJaqHOB/XqpTxhkRd5aOjNbToNnVA5SHfBSdqQ9KpT4HCmOHhL2YSdGHhklkZP+Oo+Yh2je7Ve+siD0e5l9b/ckc9ojg8eb4D7A9NN8PwWtVtp6tEPGp7DovqpGVSK1MRtw1xtXhNuGr17aeRoz/fNVX19UjaTGYaiWngHGkbMt2s92jIP/XRvVrRNDgYlHiFRETwZepX83yyg1fkZRQ8rDwNBysowsfcnYyukh/C6ksAkV0wODT2FlZK7FA/OnmFG1c1cT+hRoMvddf+gxIO5MC--jogWWrqO3IqhQ8XD--4nqp+9AVerhwjXAn3xzs9w==
|
||||
30
db/data/20250115094623_populate_weapon_recruits.rb
Normal file
30
db/data/20250115094623_populate_weapon_recruits.rb
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class PopulateWeaponRecruits < ActiveRecord::Migration[7.0]
|
||||
def up
|
||||
# Get all character mappings and convert to hash properly
|
||||
results = execute(<<-SQL)
|
||||
SELECT id, granblue_id
|
||||
FROM characters
|
||||
WHERE granblue_id IS NOT NULL
|
||||
SQL
|
||||
|
||||
character_mapping = {}
|
||||
results.each do |row|
|
||||
character_mapping[row['id']] = row['granblue_id']
|
||||
end
|
||||
|
||||
# Update weapons table using the mapping
|
||||
character_mapping.each do |char_id, granblue_id|
|
||||
execute(<<-SQL)
|
||||
UPDATE weapons
|
||||
SET recruits = #{connection.quote(granblue_id)}
|
||||
WHERE recruits_id = #{connection.quote(char_id)}
|
||||
SQL
|
||||
end
|
||||
end
|
||||
|
||||
def down
|
||||
execute("UPDATE weapons SET recruits = NULL")
|
||||
end
|
||||
end
|
||||
|
|
@ -1 +1 @@
|
|||
DataMigrate::Data.define(version: 20231119051223)
|
||||
DataMigrate::Data.define(version: 20250115094623)
|
||||
|
|
|
|||
5
db/migrate/20250115094528_add_recruits_to_weapons.rb
Normal file
5
db/migrate/20250115094528_add_recruits_to_weapons.rb
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
class AddRecruitsToWeapons < ActiveRecord::Migration[7.0]
|
||||
def change
|
||||
add_column :weapons, :recruits, :string
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
class RemoveRecruitsIdFromWeapons < ActiveRecord::Migration[7.0]
|
||||
def change
|
||||
remove_column :weapons, :recruits_id, :uuid
|
||||
remove_index :weapons, :recruits_id if index_exists?(:weapons, :recruits_id)
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
class AddIndexToWeaponRecruits < ActiveRecord::Migration[7.0]
|
||||
def change
|
||||
add_index :weapons, :recruits
|
||||
end
|
||||
end
|
||||
|
|
@ -10,7 +10,7 @@
|
|||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[7.0].define(version: 2025_01_10_070255) do
|
||||
ActiveRecord::Schema[7.0].define(version: 2025_01_15_100356) do
|
||||
# These are extensions that must be enabled in order to support this database
|
||||
enable_extension "btree_gin"
|
||||
enable_extension "pg_trgm"
|
||||
|
|
@ -456,7 +456,6 @@ ActiveRecord::Schema[7.0].define(version: 2025_01_10_070255) do
|
|||
t.integer "ax_type"
|
||||
t.boolean "limit", default: false, null: false
|
||||
t.boolean "ax", default: false, null: false
|
||||
t.uuid "recruits_id"
|
||||
t.integer "max_awakening_level"
|
||||
t.date "release_date"
|
||||
t.date "flb_date"
|
||||
|
|
@ -469,8 +468,9 @@ ActiveRecord::Schema[7.0].define(version: 2025_01_10_070255) do
|
|||
t.string "nicknames_jp", default: [], null: false, array: true
|
||||
t.boolean "transcendence", default: false
|
||||
t.date "transcendence_date"
|
||||
t.string "recruits"
|
||||
t.index ["name_en"], name: "index_weapons_on_name_en", opclass: :gin_trgm_ops, using: :gin
|
||||
t.index ["recruits_id"], name: "index_weapons_on_recruits_id"
|
||||
t.index ["recruits"], name: "index_weapons_on_recruits"
|
||||
end
|
||||
|
||||
add_foreign_key "favorites", "parties"
|
||||
|
|
|
|||
|
|
@ -21,6 +21,43 @@ module Granblue
|
|||
{ new: @new_records, updated: @updated_records }
|
||||
end
|
||||
|
||||
def simulate_import
|
||||
simulated_new = Hash.new { |h, k| h[k] = [] }
|
||||
simulated_updated = Hash.new { |h, k| h[k] = [] }
|
||||
type = model_class.name.demodulize.downcase
|
||||
|
||||
CSV.foreach(@file_path, headers: true) do |row|
|
||||
attributes = build_attributes(row)
|
||||
existing_record = model_class.find_by(granblue_id: attributes[:granblue_id])
|
||||
|
||||
if existing_record
|
||||
# For updates, only include non-nil attributes
|
||||
update_attributes = attributes.compact
|
||||
would_update = update_attributes.any? { |key, value| existing_record[key] != value }
|
||||
|
||||
if would_update
|
||||
log_test_update(existing_record, attributes)
|
||||
simulated_updated[type] << {
|
||||
granblue_id: attributes[:granblue_id],
|
||||
name_en: attributes[:name_en] || existing_record.name_en,
|
||||
attributes: update_attributes,
|
||||
operation: :update
|
||||
}
|
||||
end
|
||||
else
|
||||
log_test_creation(attributes)
|
||||
simulated_new[type] << {
|
||||
granblue_id: attributes[:granblue_id],
|
||||
name_en: attributes[:name_en],
|
||||
attributes: attributes,
|
||||
operation: :create
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
{ new: simulated_new, updated: simulated_updated }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def import_row(row)
|
||||
|
|
@ -69,22 +106,38 @@ module Granblue
|
|||
end
|
||||
end
|
||||
|
||||
def format_attributes(attributes)
|
||||
attributes.map do |key, value|
|
||||
formatted_value = case value
|
||||
when Array
|
||||
value.empty? ? '[]' : value.inspect
|
||||
else
|
||||
value.inspect
|
||||
end
|
||||
" #{key}: #{formatted_value}"
|
||||
end.join("\n")
|
||||
end
|
||||
|
||||
def log_test_update(record, attributes)
|
||||
# For test mode, show only the attributes that would be updated
|
||||
update_attributes = attributes.compact
|
||||
@logger&.send(:log_operation, "Update #{model_class.name} #{record.granblue_id}: #{update_attributes.inspect}")
|
||||
@logger&.log_step("Updating #{model_class.name} #{record.granblue_id}...")
|
||||
@logger&.log_verbose(format_attributes(update_attributes))
|
||||
@logger&.log_step("\n\n") if @verbose
|
||||
end
|
||||
|
||||
def log_test_creation(attributes)
|
||||
@logger&.send(:log_operation, "Create #{model_class.name}: #{attributes.inspect}")
|
||||
@logger&.log_step("Creating #{model_class.name}...")
|
||||
@logger&.log_verbose(format_attributes(attributes))
|
||||
@logger&.log_step("\n\n") if @verbose
|
||||
end
|
||||
|
||||
def log_new_record(record)
|
||||
puts "Created #{model_class.name} with ID: #{record.granblue_id}"
|
||||
@logger&.log_verbose("Created #{model_class.name} with ID: #{record.granblue_id}")
|
||||
end
|
||||
|
||||
def log_updated_record(record)
|
||||
puts "Updated #{model_class.name} with ID: #{record.granblue_id}"
|
||||
@logger&.log_verbose("Updated #{model_class.name} with ID: #{record.granblue_id}")
|
||||
end
|
||||
|
||||
def parse_value(value)
|
||||
|
|
|
|||
|
|
@ -1,159 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../logging_helper'
|
||||
|
||||
class PostDeploymentManager
|
||||
include LoggingHelper
|
||||
|
||||
STORAGE_DESCRIPTIONS = {
|
||||
local: 'to local disk',
|
||||
s3: 'to S3',
|
||||
both: 'to local disk and S3'
|
||||
}.freeze
|
||||
|
||||
def initialize(options = {})
|
||||
@test_mode = options.fetch(:test_mode, false)
|
||||
@verbose = options.fetch(:verbose, false)
|
||||
@storage = options.fetch(:storage, :both)
|
||||
@new_records = Hash.new { |h, k| h[k] = [] }
|
||||
@updated_records = Hash.new { |h, k| h[k] = [] }
|
||||
end
|
||||
|
||||
def run
|
||||
migrate_database
|
||||
import_new_data
|
||||
display_import_summary
|
||||
download_images
|
||||
rebuild_search_indices
|
||||
display_completion_message
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def migrate_database
|
||||
log_header 'Running database migrations...', '-'
|
||||
puts "\n"
|
||||
if @test_mode
|
||||
log_step "TEST MODE: Would run pending migrations..."
|
||||
else
|
||||
ActiveRecord::Migration.verbose = @verbose
|
||||
version = ActiveRecord::Migrator.current_version
|
||||
ActiveRecord::Tasks::DatabaseTasks.migrate
|
||||
new_version = ActiveRecord::Migrator.current_version
|
||||
if version == new_version
|
||||
log_step "No pending migrations."
|
||||
else
|
||||
log_step "Migrated from version #{version} to #{new_version}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def import_new_data
|
||||
log_header 'Importing new data...'
|
||||
puts "\n"
|
||||
importer = Granblue::DataImporter.new(
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose
|
||||
)
|
||||
|
||||
process_imports(importer)
|
||||
end
|
||||
|
||||
def process_imports(importer)
|
||||
importer.process_all_files do |result|
|
||||
result[:new].each do |type, ids|
|
||||
@new_records[type].concat(ids)
|
||||
end
|
||||
result[:updated].each do |type, ids|
|
||||
@updated_records[type].concat(ids)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def rebuild_search_indices
|
||||
log_header 'Rebuilding search indices...', '-'
|
||||
puts "\n"
|
||||
[Character, Summon, Weapon, Job].each do |model|
|
||||
log_verbose "• #{model.name}... "
|
||||
PgSearch::Multisearch.rebuild(model)
|
||||
log_verbose "✅ done!\n"
|
||||
end
|
||||
end
|
||||
|
||||
def display_import_summary
|
||||
if @new_records.size > 0 || @updated_records.size > 0
|
||||
log_header 'Import Summary', '-'
|
||||
puts "\n"
|
||||
display_record_summary('New', @new_records)
|
||||
display_record_summary('Updated', @updated_records)
|
||||
else
|
||||
log_step "\nNo new records imported."
|
||||
end
|
||||
end
|
||||
|
||||
def display_record_summary(label, records)
|
||||
records.each do |type, ids|
|
||||
next if ids.empty?
|
||||
puts "#{type.capitalize}: #{ids.size} #{label.downcase} records"
|
||||
puts "IDs: #{ids.inspect}" if @verbose
|
||||
end
|
||||
end
|
||||
|
||||
def download_images
|
||||
return if all_records_empty?
|
||||
|
||||
if @test_mode
|
||||
log_step "\nTEST MODE: Would download images for new and updated records..."
|
||||
else
|
||||
log_header 'Downloading images...', '+'
|
||||
end
|
||||
|
||||
[@new_records, @updated_records].each do |records|
|
||||
records.each do |type, ids|
|
||||
next if ids.empty?
|
||||
download_type_images(type, ids)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def download_type_images(type, ids)
|
||||
log_step "\nProcessing new #{type.pluralize} (#{ids.size} records)..."
|
||||
download_options = {
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
storage: @storage
|
||||
}
|
||||
|
||||
ids.each do |id|
|
||||
download_single_image(type, id, download_options)
|
||||
end
|
||||
end
|
||||
|
||||
def download_single_image(type, id, options)
|
||||
action_text = @test_mode ? 'Would download' : 'Downloading'
|
||||
storage_text = STORAGE_DESCRIPTIONS[options[:storage]]
|
||||
log_verbose "\n#{action_text} images #{storage_text} for #{type} #{id}...\n"
|
||||
|
||||
Granblue::Downloader::DownloadManager.download_for_object(
|
||||
type,
|
||||
id,
|
||||
**options
|
||||
)
|
||||
rescue => e
|
||||
error_message = "Error #{@test_mode ? 'would occur' : 'occurred'} downloading images for #{type} #{id}: #{e.message}"
|
||||
puts error_message
|
||||
puts e.backtrace.take(5) if @verbose
|
||||
end
|
||||
|
||||
def display_completion_message
|
||||
if @test_mode
|
||||
log_step "\n✓ Test run completed successfully!"
|
||||
else
|
||||
log_step "\n✓ Post-deployment tasks completed successfully!"
|
||||
end
|
||||
end
|
||||
|
||||
def all_records_empty?
|
||||
@new_records.values.all?(&:empty?) && @updated_records.values.all?(&:empty?)
|
||||
end
|
||||
end
|
||||
|
|
@ -9,10 +9,18 @@ module LoggingHelper
|
|||
print message if @verbose
|
||||
end
|
||||
|
||||
def log_error(message)
|
||||
puts "❌ #{message}"
|
||||
end
|
||||
|
||||
def log_warning(message)
|
||||
puts "⚠️ #{message}"
|
||||
end
|
||||
|
||||
def log_divider(character = '+', leading_newline = true, trailing_newlines = 1)
|
||||
output = ""
|
||||
output += "\n" if leading_newline
|
||||
output += character * 35
|
||||
output += character * 60
|
||||
output += "\n" * trailing_newlines
|
||||
log_step output
|
||||
end
|
||||
|
|
|
|||
153
lib/post_deployment/data_importer.rb
Normal file
153
lib/post_deployment/data_importer.rb
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../logging_helper'
|
||||
|
||||
module PostDeployment
|
||||
class DataImporter
|
||||
include LoggingHelper
|
||||
|
||||
def initialize(test_mode:, verbose:, test_transaction: nil, force: false)
|
||||
@test_mode = test_mode
|
||||
@verbose = verbose
|
||||
@test_transaction = test_transaction
|
||||
@force = force
|
||||
@processed_files = []
|
||||
@total_changes = { new: {}, updated: {} }
|
||||
end
|
||||
|
||||
def process_all_files(&block)
|
||||
files = Dir.glob(Rails.root.join('db', 'seed', 'updates', '*.csv')).sort
|
||||
|
||||
files.each do |file|
|
||||
if (result = import_csv(file))
|
||||
merge_results(result)
|
||||
block.call(result) if block_given?
|
||||
end
|
||||
end
|
||||
|
||||
if @processed_files.any?
|
||||
print_summary
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def merge_results(result)
|
||||
result[:new].each do |type, records|
|
||||
@total_changes[:new][type] ||= []
|
||||
@total_changes[:new][type].concat(records)
|
||||
end
|
||||
|
||||
result[:updated].each do |type, records|
|
||||
@total_changes[:updated][type] ||= []
|
||||
@total_changes[:updated][type].concat(records)
|
||||
end
|
||||
end
|
||||
|
||||
def import_csv(file_path)
|
||||
filename = File.basename(file_path)
|
||||
if already_imported?(filename)
|
||||
log_verbose("Skipping #{filename} - already imported\n") if @verbose
|
||||
return
|
||||
end
|
||||
|
||||
importer = create_importer(filename, file_path)
|
||||
return unless importer
|
||||
|
||||
@processed_files << filename
|
||||
mode_text = @test_mode ? '🛠️ Testing' : 'Processing'
|
||||
force_text = @force ? ' (Force mode)' : ''
|
||||
|
||||
if @verbose
|
||||
log_header("#{mode_text}#{force_text}: #{filename}", "-")
|
||||
puts "\n"
|
||||
end
|
||||
|
||||
result = if @test_mode
|
||||
test_import(importer)
|
||||
else
|
||||
importer.import
|
||||
end
|
||||
|
||||
log_import(filename, result)
|
||||
result
|
||||
end
|
||||
|
||||
def test_import(importer)
|
||||
# In test mode, we simulate the import and record what would happen
|
||||
simulated_result = importer.simulate_import
|
||||
|
||||
if @test_transaction
|
||||
simulated_result.each do |operation, type_records|
|
||||
type_records.each do |type, records|
|
||||
records.each do |record_attrs|
|
||||
@test_transaction.add_change(
|
||||
model: type.to_s.classify.constantize,
|
||||
attributes: record_attrs,
|
||||
operation: operation
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
simulated_result
|
||||
end
|
||||
|
||||
def create_importer(filename, file_path)
|
||||
# This pattern matches both singular and plural: character(s), weapon(s), summon(s)
|
||||
match = filename.match(/\A\d{8}-(character(?:s)?|weapon(?:s)?|summon(?:s)?)-\d+\.csv\z/)
|
||||
return unless match
|
||||
|
||||
matched_type = match[1]
|
||||
singular_type = matched_type.sub(/s$/, '')
|
||||
importer_class = "Granblue::Importers::#{singular_type.capitalize}Importer".constantize
|
||||
|
||||
importer_class.new(
|
||||
file_path,
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
logger: self
|
||||
)
|
||||
rescue NameError
|
||||
log_warning "No importer found for type: #{singular_type}"
|
||||
nil
|
||||
end
|
||||
|
||||
def already_imported?(filename)
|
||||
return false if @force
|
||||
DataVersion.imported?(filename)
|
||||
end
|
||||
|
||||
def log_import(filename, result)
|
||||
return if @test_mode
|
||||
|
||||
DataVersion.mark_as_imported(filename)
|
||||
log_import_results(result) if @verbose
|
||||
end
|
||||
|
||||
def log_import_results(result)
|
||||
result[:new].each do |type, records|
|
||||
log_verbose "Created #{records.size} new #{type.pluralize}" if records.any?
|
||||
end
|
||||
result[:updated].each do |type, records|
|
||||
log_verbose "Updated #{records.size} existing #{type.pluralize}" if records.any?
|
||||
end
|
||||
end
|
||||
|
||||
def print_summary
|
||||
return if @processed_files.empty?
|
||||
|
||||
log_header("Processed files:")
|
||||
puts "\n"
|
||||
@processed_files.each { |file| log_step " • #{file}" }
|
||||
end
|
||||
|
||||
def print_change_summary(action, changes)
|
||||
changes.each do |type, records|
|
||||
next if records.empty?
|
||||
log_step " • #{action} #{records.size} #{type} #{records.size == 1 ? 'record' : 'records'}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
87
lib/post_deployment/database_migrator.rb
Normal file
87
lib/post_deployment/database_migrator.rb
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../logging_helper'
|
||||
|
||||
module PostDeployment
|
||||
class DatabaseMigrator
|
||||
include LoggingHelper
|
||||
|
||||
def initialize(test_mode:, verbose:)
|
||||
@test_mode = test_mode
|
||||
@verbose = verbose
|
||||
end
|
||||
|
||||
def run
|
||||
log_header 'Running database migrations...', '-'
|
||||
puts "\n"
|
||||
|
||||
if @test_mode
|
||||
simulate_migrations
|
||||
else
|
||||
perform_migrations
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def simulate_migrations
|
||||
log_step "TEST MODE: Would run pending migrations..."
|
||||
|
||||
# Check schema migrations
|
||||
pending_schema_migrations = ActiveRecord::Base.connection.migration_context.needs_migration?
|
||||
schema_migrations = ActiveRecord::Base.connection.migration_context.migrations
|
||||
|
||||
# Check data migrations
|
||||
data_migrations_path = DataMigrate.config.data_migrations_path
|
||||
data_migration_context = DataMigrate::MigrationContext.new(data_migrations_path)
|
||||
pending_data_migrations = data_migration_context.needs_migration?
|
||||
data_migrations = data_migration_context.migrations
|
||||
|
||||
if pending_schema_migrations || pending_data_migrations
|
||||
if schema_migrations.any?
|
||||
log_step "Would apply #{schema_migrations.size} pending schema migrations:"
|
||||
schema_migrations.each do |migration|
|
||||
log_step " • #{migration.name}"
|
||||
end
|
||||
end
|
||||
|
||||
if data_migrations.any?
|
||||
log_step "\nWould apply #{data_migrations.size} pending data migrations:"
|
||||
data_migrations.each do |migration|
|
||||
log_step " • #{migration.name}"
|
||||
end
|
||||
end
|
||||
else
|
||||
log_step "No pending migrations."
|
||||
end
|
||||
end
|
||||
|
||||
def perform_migrations
|
||||
ActiveRecord::Migration.verbose = @verbose
|
||||
|
||||
# Run schema migrations
|
||||
schema_version = ActiveRecord::Base.connection.migration_context.current_version
|
||||
ActiveRecord::Tasks::DatabaseTasks.migrate
|
||||
new_schema_version = ActiveRecord::Base.connection.migration_context.current_version
|
||||
|
||||
# Run data migrations
|
||||
data_migrations_path = DataMigrate.config.data_migrations_path
|
||||
data_migration_context = DataMigrate::MigrationContext.new(data_migrations_path)
|
||||
|
||||
data_version = data_migration_context.current_version
|
||||
data_migration_context.migrate
|
||||
new_data_version = data_migration_context.current_version
|
||||
|
||||
if schema_version == new_schema_version && data_version == new_data_version
|
||||
log_step "No pending migrations."
|
||||
else
|
||||
if schema_version != new_schema_version
|
||||
log_step "Migrated schema from version #{schema_version} to #{new_schema_version}"
|
||||
end
|
||||
if data_version != new_data_version
|
||||
log_step "Migrated data from version #{data_version} to #{new_data_version}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
73
lib/post_deployment/image_downloader.rb
Normal file
73
lib/post_deployment/image_downloader.rb
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../logging_helper'
|
||||
|
||||
module PostDeployment
|
||||
class ImageDownloader
|
||||
include LoggingHelper
|
||||
|
||||
STORAGE_DESCRIPTIONS = {
|
||||
local: 'to local disk',
|
||||
s3: 'to S3',
|
||||
both: 'to local disk and S3'
|
||||
}.freeze
|
||||
|
||||
def initialize(test_mode:, verbose:, storage:, new_records:, updated_records:)
|
||||
@test_mode = test_mode
|
||||
@verbose = verbose
|
||||
@storage = storage
|
||||
@new_records = new_records
|
||||
@updated_records = updated_records
|
||||
end
|
||||
|
||||
def run
|
||||
log_header 'Downloading images...', '+'
|
||||
|
||||
[@new_records, @updated_records].each do |records|
|
||||
records.each do |type, items|
|
||||
next if items.empty?
|
||||
download_type_images(type, items)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def download_type_images(type, items)
|
||||
if @verbose
|
||||
log_header "Processing #{type.pluralize} (#{items.size} records)...", "-"
|
||||
puts "\n"
|
||||
end
|
||||
|
||||
download_options = {
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
storage: @storage
|
||||
}
|
||||
|
||||
items.each do |item|
|
||||
id = @test_mode ? item[:granblue_id] : item.id
|
||||
download_single_image(type, id, download_options)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
def download_single_image(type, id, options)
|
||||
action_text = @test_mode ? 'Would download' : 'Downloading'
|
||||
storage_text = STORAGE_DESCRIPTIONS[options[:storage]]
|
||||
log_verbose "#{action_text} images #{storage_text} for #{type} #{id}...\n"
|
||||
|
||||
unless @test_mode
|
||||
Granblue::Downloader::DownloadManager.download_for_object(
|
||||
type,
|
||||
id,
|
||||
**options
|
||||
)
|
||||
end
|
||||
rescue => e
|
||||
error_message = "Error #{@test_mode ? 'would occur' : 'occurred'} downloading images for #{type} #{id}: #{e.message}"
|
||||
puts error_message
|
||||
puts e.backtrace.take(5) if @verbose
|
||||
end
|
||||
end
|
||||
end
|
||||
140
lib/post_deployment/manager.rb
Normal file
140
lib/post_deployment/manager.rb
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'test_mode_transaction'
|
||||
require_relative 'database_migrator'
|
||||
require_relative 'data_importer'
|
||||
require_relative 'image_downloader'
|
||||
require_relative 'search_indexer'
|
||||
require_relative '../logging_helper'
|
||||
|
||||
module PostDeployment
|
||||
class Manager
|
||||
include LoggingHelper
|
||||
|
||||
def initialize(options = {})
|
||||
@test_mode = options.fetch(:test_mode, false)
|
||||
@verbose = options.fetch(:verbose, false)
|
||||
@storage = options.fetch(:storage, :both)
|
||||
@force = options.fetch(:force, false)
|
||||
@new_records = Hash.new { |h, k| h[k] = [] }
|
||||
@updated_records = Hash.new { |h, k| h[k] = [] }
|
||||
@test_transaction = TestModeTransaction.new if @test_mode
|
||||
end
|
||||
|
||||
def run
|
||||
migrate_database
|
||||
import_new_data
|
||||
display_import_summary
|
||||
download_images
|
||||
rebuild_search_indices
|
||||
display_completion_message
|
||||
rescue => e
|
||||
handle_error(e)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def migrate_database
|
||||
DatabaseMigrator.new(
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose
|
||||
).run
|
||||
end
|
||||
|
||||
def import_new_data
|
||||
log_header 'Importing new data...'
|
||||
puts "\n"
|
||||
|
||||
importer = DataImporter.new(
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
test_transaction: @test_transaction,
|
||||
force: @force
|
||||
)
|
||||
|
||||
process_imports(importer)
|
||||
end
|
||||
|
||||
def process_imports(importer)
|
||||
importer.process_all_files do |result|
|
||||
merge_import_results(result)
|
||||
end
|
||||
end
|
||||
|
||||
def merge_import_results(result)
|
||||
result[:new].each do |type, records|
|
||||
@new_records[type].concat(records)
|
||||
end
|
||||
result[:updated].each do |type, records|
|
||||
@updated_records[type].concat(records)
|
||||
end
|
||||
end
|
||||
|
||||
def download_images
|
||||
return if all_records_empty?
|
||||
|
||||
ImageDownloader.new(
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
storage: @storage,
|
||||
new_records: @new_records,
|
||||
updated_records: @updated_records
|
||||
).run
|
||||
end
|
||||
|
||||
def rebuild_search_indices
|
||||
SearchIndexer.new(
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose
|
||||
).rebuild_all
|
||||
end
|
||||
|
||||
def display_import_summary
|
||||
if @new_records.size > 0 || @updated_records.size > 0
|
||||
log_header 'Import Summary', '-'
|
||||
display_record_summary('New', @new_records)
|
||||
display_record_summary('Updated', @updated_records)
|
||||
else
|
||||
log_step "\nNo new records imported."
|
||||
end
|
||||
end
|
||||
|
||||
def display_record_summary(label, records)
|
||||
records.each do |type, items|
|
||||
next if items.empty?
|
||||
count = items.size
|
||||
puts "\n#{type.capitalize}: #{count} #{label.downcase} #{count == 1 ? 'record' : 'records'}"
|
||||
items.each do |item|
|
||||
if @test_mode
|
||||
puts " • #{item[:name_en]} (ID: #{item[:granblue_id]})"
|
||||
else
|
||||
puts " • #{item.name_en} (ID: #{item.granblue_id})"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def display_completion_message
|
||||
if @test_mode
|
||||
log_header "✅ Test run completed successfully!", "-"
|
||||
puts "\n"
|
||||
log_step "#{@new_records.values.flatten.size} records would be created"
|
||||
log_step "#{@updated_records.values.flatten.size} records would be updated"
|
||||
puts "\n"
|
||||
else
|
||||
log_header "✅ Post-deployment tasks completed successfully!"
|
||||
end
|
||||
end
|
||||
|
||||
def handle_error(error)
|
||||
log_error("\nError during deployment: #{error.message}")
|
||||
log_error(error.backtrace.take(10).join("\n")) if @verbose
|
||||
@test_transaction&.rollback
|
||||
raise error
|
||||
end
|
||||
|
||||
def all_records_empty?
|
||||
@new_records.values.all?(&:empty?) && @updated_records.values.all?(&:empty?)
|
||||
end
|
||||
end
|
||||
end
|
||||
61
lib/post_deployment/search_indexer.rb
Normal file
61
lib/post_deployment/search_indexer.rb
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../logging_helper'
|
||||
|
||||
module PostDeployment
|
||||
class SearchIndexer
|
||||
include LoggingHelper
|
||||
|
||||
def initialize(test_mode:, verbose:)
|
||||
@test_mode = test_mode
|
||||
@verbose = verbose
|
||||
end
|
||||
|
||||
def rebuild_all
|
||||
log_header 'Rebuilding search indices...', '-'
|
||||
puts "\n"
|
||||
|
||||
ensure_models_loaded
|
||||
rebuild_indices
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def ensure_models_loaded
|
||||
Rails.application.eager_load! if Rails.application.config.eager_load
|
||||
end
|
||||
|
||||
def rebuild_indices
|
||||
searchable_models.each do |model_name|
|
||||
begin
|
||||
model = model_name.constantize
|
||||
rebuild_index_for(model)
|
||||
rescue NameError => e
|
||||
log_error("Could not load model: #{model_name}")
|
||||
log_error(e.message) if @verbose
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def rebuild_index_for(model)
|
||||
if @test_mode
|
||||
log_step "Would rebuild search index for #{model.name}"
|
||||
else
|
||||
log_verbose "• #{model.name}... "
|
||||
PgSearch::Multisearch.rebuild(model)
|
||||
log_verbose "✅ done!\n"
|
||||
end
|
||||
rescue StandardError => e
|
||||
log_error("Failed to rebuild index for #{model.name}: #{e.message}")
|
||||
log_error(e.backtrace.take(5).join("\n")) if @verbose
|
||||
end
|
||||
|
||||
def searchable_models
|
||||
%w[Character Summon Weapon Job]
|
||||
end
|
||||
|
||||
def log_error(message)
|
||||
puts "❌ #{message}"
|
||||
end
|
||||
end
|
||||
end
|
||||
25
lib/post_deployment/test_mode_transaction.rb
Normal file
25
lib/post_deployment/test_mode_transaction.rb
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module PostDeployment
|
||||
class TestModeTransaction
|
||||
def initialize
|
||||
@changes = []
|
||||
end
|
||||
|
||||
def add_change(model:, attributes:, operation:)
|
||||
@changes << {
|
||||
model: model,
|
||||
attributes: attributes,
|
||||
operation: operation
|
||||
}
|
||||
end
|
||||
|
||||
def rollback
|
||||
@changes.clear
|
||||
end
|
||||
|
||||
def committed_changes
|
||||
@changes
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,52 +1,83 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../granblue/downloaders/base_downloader'
|
||||
require_relative '../post_deployment/manager'
|
||||
require_relative '../logging_helper'
|
||||
|
||||
namespace :deploy do
|
||||
desc 'Post-deployment tasks: Import new data and download related images. Options: TEST=true for test mode, VERBOSE=true for verbose output, STORAGE=local|s3|both'
|
||||
desc 'Post-deployment tasks: Run migrations, import data, download images, and rebuild search indices. Options: TEST=true for test mode, VERBOSE=true for verbose output, STORAGE=local|s3|both'
|
||||
task post_deployment: :environment do
|
||||
include LoggingHelper
|
||||
|
||||
# Load all required files
|
||||
Dir[Rails.root.join('lib', 'post_deployment', '**', '*.rb')].each { |file| require file }
|
||||
Dir[Rails.root.join('lib', 'granblue', '**', '*.rb')].each { |file| require file }
|
||||
|
||||
# Ensure Rails environment is loaded
|
||||
Rails.application.eager_load!
|
||||
|
||||
log_header('Starting post-deploy script...', '-', false)
|
||||
print "\n"
|
||||
begin
|
||||
display_startup_banner
|
||||
|
||||
# Parse and validate storage option
|
||||
storage = (ENV['STORAGE'] || 'both').to_sym
|
||||
unless [:local, :s3, :both].include?(storage)
|
||||
puts 'Invalid STORAGE option. Must be one of: local, s3, both'
|
||||
options = parse_and_validate_options
|
||||
display_configuration(options)
|
||||
|
||||
# Execute the deployment tasks
|
||||
manager = PostDeployment::Manager.new(options)
|
||||
manager.run
|
||||
|
||||
rescue StandardError => e
|
||||
display_error(e)
|
||||
exit 1
|
||||
end
|
||||
end
|
||||
|
||||
options = {
|
||||
private
|
||||
|
||||
def display_startup_banner
|
||||
puts "Starting deployment process...\n"
|
||||
end
|
||||
|
||||
def parse_and_validate_options
|
||||
storage = parse_storage_option
|
||||
|
||||
{
|
||||
test_mode: ENV['TEST'] == 'true',
|
||||
verbose: ENV['VERBOSE'] == 'true',
|
||||
storage: storage
|
||||
storage: storage,
|
||||
force: ENV['FORCE'] == 'true'
|
||||
}
|
||||
end
|
||||
|
||||
print "Test mode:\t"
|
||||
if options[:test_mode]
|
||||
print "✅ Enabled\n"
|
||||
else
|
||||
print "❌ Disabled\n"
|
||||
def parse_storage_option
|
||||
storage = (ENV['STORAGE'] || 'both').to_sym
|
||||
|
||||
unless [:local, :s3, :both].include?(storage)
|
||||
raise ArgumentError, 'Invalid STORAGE option. Must be one of: local, s3, both'
|
||||
end
|
||||
|
||||
print "Verbose output:\t"
|
||||
if options[:verbose]
|
||||
print "✅ Enabled\n"
|
||||
else
|
||||
print "❌ Disabled\n"
|
||||
end
|
||||
storage
|
||||
end
|
||||
|
||||
puts "Storage mode:\t#{storage}"
|
||||
def display_configuration(options)
|
||||
log_header('Configuration', '-')
|
||||
puts "\n"
|
||||
display_status("Test mode", options[:test_mode])
|
||||
display_status("Verbose output", options[:verbose])
|
||||
display_status("Process all", options[:force])
|
||||
puts "Storage mode:\t#{options[:storage]}"
|
||||
puts "\n"
|
||||
end
|
||||
|
||||
# Execute the task
|
||||
manager = PostDeploymentManager.new(options)
|
||||
manager.run
|
||||
def display_status(label, enabled)
|
||||
status = enabled ? "✅ Enabled" : "❌ Disabled"
|
||||
puts "#{label}:\t#{status}"
|
||||
end
|
||||
|
||||
def display_error(error)
|
||||
puts "\n❌ Error during deployment:"
|
||||
puts " #{error.class}: #{error.message}"
|
||||
puts "\nStack trace:" if ENV['VERBOSE'] == 'true'
|
||||
puts error.backtrace.take(10) if ENV['VERBOSE'] == 'true'
|
||||
puts "\nDeployment failed! Please check the logs for details."
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in a new issue