From d75e4a8448e4a27b874304aa11dc0020b24866b3 Mon Sep 17 00:00:00 2001 From: Justin Edmund Date: Wed, 15 Jan 2025 19:15:58 -0800 Subject: [PATCH] Downloader fixes (#162) * Fix summon data * Update logic for downloaders These are responsible for downloading variants (01, 02, etc) * Update puts statement * Simplify ImageDownloader * Simplify DownloadManager --- db/seed/updates/20250113-summons-001.csv | 2 +- db/seed/updates/20250113-summons-002.csv | 4 +- lib/granblue/download_manager.rb | 79 +++---------------- .../downloaders/character_downloader.rb | 40 ++++++++++ lib/granblue/downloaders/summon_downloader.rb | 42 ++++++++++ lib/granblue/downloaders/weapon_downloader.rb | 39 +++++++++ lib/post_deployment/image_downloader.rb | 53 ++++++++----- lib/post_deployment/manager.rb | 6 +- 8 files changed, 166 insertions(+), 99 deletions(-) diff --git a/db/seed/updates/20250113-summons-001.csv b/db/seed/updates/20250113-summons-001.csv index 051fc02..606bd91 100644 --- a/db/seed/updates/20250113-summons-001.csv +++ b/db/seed/updates/20250113-summons-001.csv @@ -1,2 +1,2 @@ name_en,name_jp,granblue_id,rarity,element,series,flb,ulb,max_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,subaura,limit,transcendence,max_atk_xlb,max_hp_xlb,summon_id,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,transcendence_date,nicknames_en,nicknames_jp -Mars (Summer),マーズ(水着ver),2040428000,3,6,,true,false,150,131,804,972,,362,2082,2512,,true,false,false,,,2040428000,2024-07-16,2024-07-16,,Mars_(Summer),マーズ (SSR)水着バージョン,457007,SSR/水着マーズ,,, +Mars (Summer),マーズ(水着ver),2040428000,3,6,,true,false,150,131,804,972,,362,2082,2512,,true,false,false,,,3337,2024-07-16,2024-07-16,,Mars_(Summer),マーズ (SSR)水着バージョン,457007,SSR/水着マーズ,,, diff --git a/db/seed/updates/20250113-summons-002.csv b/db/seed/updates/20250113-summons-002.csv index 899817b..35d00b2 100644 --- a/db/seed/updates/20250113-summons-002.csv +++ b/db/seed/updates/20250113-summons-002.csv @@ -1,3 +1,3 @@ name_en,name_jp,granblue_id,rarity,element,series,flb,ulb,max_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,subaura,limit,transcendence,max_atk_xlb,max_hp_xlb,summon_id,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,transcendence_date,nicknames_en,nicknames_jp -Beelzebub (Summer),ベルゼバブ(水着ver),2040429000,3,3,,true,false,150,115,793,1132,,442,2137,2985,,true,false,false,,,{3319},2024-07-31,2024-07-31,,Beelzebub_(Summer),ベルゼバブ (SSR)水着,458158,水着ベルゼバブ,,"{babu,bubz}", -Wedges of the Sky,蒼空の楔,2040430000,3,1,,false,false,100,109,668,,,277,1740,,,true,false,false,,,{3338},2024-08-09,,,Wedges_of_the_Sky,蒼空の楔 (SSR),458492,蒼空の楔,,"{boost,dragons}", +Beelzebub (Summer),ベルゼバブ(水着ver),2040429000,3,3,,true,false,150,115,793,1132,,442,2137,2985,,true,false,false,,,3319,2024-07-31,2024-07-31,,Beelzebub_(Summer),ベルゼバブ (SSR)水着,458158,水着ベルゼバブ,,"{babu,bubz}", +Wedges of the Sky,蒼空の楔,2040430000,3,1,,false,false,100,109,668,,,277,1740,,,true,false,false,,,3338,2024-08-09,,,Wedges_of_the_Sky,蒼空の楔 (SSR),458492,蒼空の楔,,"{boost,dragons}", diff --git a/lib/granblue/download_manager.rb b/lib/granblue/download_manager.rb index 44b0204..9c2b20b 100644 --- a/lib/granblue/download_manager.rb +++ b/lib/granblue/download_manager.rb @@ -5,84 +5,23 @@ module Granblue class DownloadManager class << self def download_for_object(type, granblue_id, test_mode: false, verbose: false, storage: :both) - @test_mode = test_mode - @verbose = verbose - @storage = storage + downloader_options = { + test_mode: test_mode, + verbose: verbose, + storage: storage + } case type when 'character' - download_character(granblue_id) + CharacterDownloader.new(granblue_id, **downloader_options).download when 'weapon' - download_weapon(granblue_id) + WeaponDownloader.new(granblue_id, **downloader_options).download when 'summon' - download_summon(granblue_id) + SummonDownloader.new(granblue_id, **downloader_options).download else - log_info "Unknown object type: #{type}" + log_info "Unknown object type: #{type}" if verbose || test_mode end end - - private - - def download_character(id) - character = Character.find_by(granblue_id: id) - return unless character - - downloader_options = { - test_mode: @test_mode, - verbose: @verbose, - storage: @storage - } - - %W[#{id}_01 #{id}_02].each do |variant_id| - CharacterDownloader.new(variant_id, **downloader_options).download - end - - CharacterDownloader.new("#{id}_03", **downloader_options).download if character.flb - CharacterDownloader.new("#{id}_04", **downloader_options).download if character.ulb - end - - def download_weapon(id) - weapon = Weapon.find_by(granblue_id: id) - return unless weapon - - downloader_options = { - test_mode: @test_mode, - verbose: @verbose, - storage: @storage - } - - WeaponDownloader.new(id, **downloader_options).download - - return unless weapon.transcendence - - WeaponDownloader.new("#{id}_02", **downloader_options).download - WeaponDownloader.new("#{id}_03", **downloader_options).download - - end - - def download_summon(id) - summon = Summon.find_by(granblue_id: id) - return unless summon - - downloader_options = { - test_mode: @test_mode, - verbose: @verbose, - storage: @storage - } - - SummonDownloader.new(id, **downloader_options).download - SummonDownloader.new("#{id}_02", **downloader_options).download if summon.ulb - - return unless summon.transcendence - - SummonDownloader.new("#{id}_03", **downloader_options).download - SummonDownloader.new("#{id}_04", **downloader_options).download - - end - - def log_info(message) - puts message if @verbose || @test_mode - end end end end diff --git a/lib/granblue/downloaders/character_downloader.rb b/lib/granblue/downloaders/character_downloader.rb index 34aa8a6..2ad492b 100644 --- a/lib/granblue/downloaders/character_downloader.rb +++ b/lib/granblue/downloaders/character_downloader.rb @@ -3,8 +3,48 @@ module Granblue module Downloader class CharacterDownloader < BaseDownloader + def download + character = Character.find_by(granblue_id: @id) + return unless character + + download_variants(character) + end + private + def download_variants(character) + # All characters have 01 and 02 variants + variants = ["#{@id}_01", "#{@id}_02"] + + # Add FLB variant if available + variants << "#{@id}_03" if character.flb + + # Add ULB variant if available + variants << "#{@id}_04" if character.ulb + + log_info "Downloading character variants: #{variants.join(', ')}" if @verbose + + variants.each do |variant_id| + download_variant(variant_id) + end + end + + def download_variant(variant_id) + log_info "-> #{variant_id}" if @verbose + return if @test_mode + + SIZES.each_with_index do |size, index| + path = download_path(size) + url = build_variant_url(variant_id, size) + process_download(url, size, path, last: index == SIZES.size - 1) + end + end + + def build_variant_url(variant_id, size) + directory = directory_for_size(size) + "#{@base_url}/#{directory}/#{variant_id}.jpg" + end + def object_type 'character' end diff --git a/lib/granblue/downloaders/summon_downloader.rb b/lib/granblue/downloaders/summon_downloader.rb index 2d9ba8d..5874f78 100644 --- a/lib/granblue/downloaders/summon_downloader.rb +++ b/lib/granblue/downloaders/summon_downloader.rb @@ -3,8 +3,50 @@ module Granblue module Downloader class SummonDownloader < BaseDownloader + def download + summon = Summon.find_by(granblue_id: @id) + return unless summon + + download_variants(summon) + end + private + def download_variants(summon) + # All summons have base variant + variants = [@id] + + # Add ULB variant if available + variants << "#{@id}_02" if summon.ulb + + # Add Transcendence variants if available + if summon.transcendence + variants.push("#{@id}_03", "#{@id}_04") + end + + log_info "Downloading summon variants: #{variants.join(', ')}" if @verbose + + variants.each do |variant_id| + download_variant(variant_id) + end + end + + def download_variant(variant_id) + log_info "-> #{variant_id}" if @verbose + return if @test_mode + + SIZES.each_with_index do |size, index| + path = download_path(size) + url = build_variant_url(variant_id, size) + process_download(url, size, path, last: index == SIZES.size - 1) + end + end + + def build_variant_url(variant_id, size) + directory = directory_for_size(size) + "#{@base_url}/#{directory}/#{variant_id}.jpg" + end + def object_type 'summon' end diff --git a/lib/granblue/downloaders/weapon_downloader.rb b/lib/granblue/downloaders/weapon_downloader.rb index 38f158f..5879ff0 100644 --- a/lib/granblue/downloaders/weapon_downloader.rb +++ b/lib/granblue/downloaders/weapon_downloader.rb @@ -3,8 +3,47 @@ module Granblue module Downloader class WeaponDownloader < BaseDownloader + def download + weapon = Weapon.find_by(granblue_id: @id) + return unless weapon + + download_variants(weapon) + end + private + def download_variants(weapon) + # All weapons have base variant + variants = [@id] + + # Add transcendence variants if available + if weapon.transcendence + variants.push("#{@id}_02", "#{@id}_03") + end + + log_info "Downloading weapon variants: #{variants.join(', ')}" if @verbose + + variants.each do |variant_id| + download_variant(variant_id) + end + end + + def download_variant(variant_id) + log_info "-> #{variant_id}" if @verbose + return if @test_mode + + SIZES.each_with_index do |size, index| + path = download_path(size) + url = build_variant_url(variant_id, size) + process_download(url, size, path, last: index == SIZES.size - 1) + end + end + + def build_variant_url(variant_id, size) + directory = directory_for_size(size) + "#{@base_url}/#{directory}/#{variant_id}.jpg" + end + def object_type 'weapon' end diff --git a/lib/post_deployment/image_downloader.rb b/lib/post_deployment/image_downloader.rb index d8db8a6..7027bf3 100644 --- a/lib/post_deployment/image_downloader.rb +++ b/lib/post_deployment/image_downloader.rb @@ -1,6 +1,12 @@ # frozen_string_literal: true require_relative '../logging_helper' +require_relative '../granblue/downloaders/base_downloader' +require_relative '../granblue/downloaders/character_downloader' +require_relative '../granblue/downloaders/weapon_downloader' +require_relative '../granblue/downloaders/summon_downloader' +require_relative '../granblue/downloaders/elemental_weapon_downloader' +require_relative '../granblue/download_manager' module PostDeployment class ImageDownloader @@ -12,6 +18,12 @@ module PostDeployment both: 'to local disk and S3' }.freeze + SUPPORTED_TYPES = { + 'character' => Granblue::Downloader::CharacterDownloader, + 'summon' => Granblue::Downloader::SummonDownloader, + 'weapon' => Granblue::Downloader::WeaponDownloader + }.freeze + def initialize(test_mode:, verbose:, storage:, new_records:, updated_records:) @test_mode = test_mode @verbose = verbose @@ -21,35 +33,34 @@ module PostDeployment end def run + return if @test_mode + log_header 'Downloading images...', '+' - [@new_records, @updated_records].each do |records| - records.each do |type, items| - next if items.empty? - download_type_images(type, items) - end + SUPPORTED_TYPES.each do |type, downloader_class| + download_type_images(type, downloader_class) end end private - def download_type_images(type, items) - if @verbose - log_header "Processing #{type.pluralize} (#{items.size} records)...", "-" - puts "\n" + def download_type_images(type, downloader_class) + records = (@new_records[type] || []) + (@updated_records[type] || []) + return if records.empty? + + puts "\nDownloading #{type} images..." if @verbose + records.each do |record| + # Get the ID either from the granblue_id hash key (test mode) or method (normal mode) + id = record.respond_to?(:granblue_id) ? record.granblue_id : record[:granblue_id] + + downloader = downloader_class.new( + id, + test_mode: @test_mode, + verbose: @verbose, + storage: @storage + ) + downloader.download end - - download_options = { - test_mode: @test_mode, - verbose: @verbose, - storage: @storage - } - - items.each do |item| - id = @test_mode ? item[:granblue_id] : item.id - download_single_image(type, id, download_options) - end - end def download_single_image(type, id, options) diff --git a/lib/post_deployment/manager.rb b/lib/post_deployment/manager.rb index f5ec3c1..64a24f8 100644 --- a/lib/post_deployment/manager.rb +++ b/lib/post_deployment/manager.rb @@ -105,11 +105,7 @@ module PostDeployment count = items.size puts "\n#{type.capitalize}: #{count} #{label.downcase} #{count == 1 ? 'record' : 'records'}" items.each do |item| - if @test_mode - puts " - #{item[:name_en]} (ID: #{item[:granblue_id]})" - else - puts " - #{item.name_en} (ID: #{item.granblue_id})" - end + puts " - #{item[:name_en]} (ID: #{item[:granblue_id]})" end end end