diff --git a/Gemfile b/Gemfile index 84a05ea..de7767b 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,9 @@ gem 'gemoji-parser' # An awesome replacement for acts_as_nested_set and better_nested_set. gem 'awesome_nested_set' +# Official AWS Ruby gem for Amazon Simple Storage Service (Amazon S3) +gem 'aws-sdk-s3' + # An email validator for Rails gem 'email_validator' diff --git a/Gemfile.lock b/Gemfile.lock index f02220c..abb4111 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,6 +79,22 @@ GEM ast (2.4.2) awesome_nested_set (3.5.0) activerecord (>= 4.0.0, < 7.1) + aws-eventstream (1.3.0) + aws-partitions (1.1035.0) + aws-sdk-core (3.215.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.96.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-s3 (1.177.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.11.0) + aws-eventstream (~> 1, >= 1.0.2) backport (1.2.0) bcrypt (3.1.18) benchmark (0.2.1) @@ -133,6 +149,7 @@ GEM i18n (1.12.0) concurrent-ruby (~> 1.0) jaro_winkler (1.5.4) + jmespath (1.6.2) json (2.6.3) kramdown (2.4.0) rexml @@ -331,6 +348,7 @@ DEPENDENCIES api_matchers apipie-rails awesome_nested_set + aws-sdk-s3 bcrypt blueprinter bootsnap @@ -372,4 +390,4 @@ RUBY VERSION ruby 3.0.0p0 BUNDLED WITH - 2.4.2 + 2.5.1 diff --git a/app/models/data_version.rb b/app/models/data_version.rb new file mode 100644 index 0000000..19ad81f --- /dev/null +++ b/app/models/data_version.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +class DataVersion < ActiveRecord::Base + validates :filename, presence: true, uniqueness: true + validates :imported_at, presence: true + + def self.mark_as_imported(filename) + create!(filename: filename, imported_at: Time.current) + end + + def self.imported?(filename) + exists?(filename: filename) + end +end diff --git a/app/services/aws_service.rb b/app/services/aws_service.rb new file mode 100644 index 0000000..332b3ec --- /dev/null +++ b/app/services/aws_service.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'aws-sdk-s3' + +class AwsService + class ConfigurationError < StandardError; end + + def initialize + validate_credentials! + + @s3_client = Aws::S3::Client.new( + region: Rails.application.credentials.dig(:aws, :region), + access_key_id: Rails.application.credentials.dig(:aws, :access_key_id), + secret_access_key: Rails.application.credentials.dig(:aws, :secret_access_key) + ) + @bucket = Rails.application.credentials.dig(:aws, :bucket_name) + rescue KeyError => e + raise ConfigurationError, "Missing AWS credential: #{e.message}" + end + + def upload_stream(io, key) + @s3_client.put_object( + bucket: @bucket, + key: key, + body: io + ) + end + + def file_exists?(key) + @s3_client.head_object( + bucket: @bucket, + key: key + ) + true + rescue Aws::S3::Errors::NotFound + false + end + + private + + def credentials + @credentials ||= begin + creds = Rails.application.credentials[:aws] + raise ConfigurationError, 'AWS credentials not found' unless creds + + { + region: creds[:region], + access_key_id: creds[:access_key_id], + secret_access_key: creds[:secret_access_key], + bucket_name: creds[:bucket_name] + } + end + end + + def validate_credentials! + missing = [] + creds = Rails.application.credentials[:aws] + + %i[region access_key_id secret_access_key bucket_name].each do |key| + missing << key unless creds&.dig(key) + end + + return unless missing.any? + + raise ConfigurationError, "Missing AWS credentials: #{missing.join(', ')}" + end +end diff --git a/config/credentials.yml.enc b/config/credentials.yml.enc index 719ac75..1259d8d 100644 --- a/config/credentials.yml.enc +++ b/config/credentials.yml.enc @@ -1 +1 @@ -Fxc8acnxWOFdt+zwWoACR/fskFH2+ZY5izq5cHf8pnGDKRSoI7QYm0h8RwevJtRUvUJQsJ+ja/xzbTYxNC4ABRSBe06lXwHJuCnt5YtR+4l+NiFnS76kGzSfhlfhmvPLtSdfTVRfhRib1vrz7E38jM1pcc2QBkzCxyaoZRu3X65U+gc7EqTjOsg8wpTjJwvfTXW9gkFNwFSen3nOSytewYDcivwUjr/3NUAONKHn4rNhBN3UJiNgOSCGj77Xx60E0Q95CidbkgExcyKAIMMsQgLKGhQRr9yUGxdshMuhA3JhVQSyvtd+jX8PmNX3FQusQIg7YUCh/WpiKo3aimZLQYY2n7lbfeSLpwuishjn138GAxe59Wgm1JhKN4xAkcAq54Q9d4AGFnu/IphMhv1TO03CqnwX1BbfY142--n8Fil7/q3W/SrENe--J31ORG+51iIo29fjiZU3Uw== \ No newline at end of file +DNg3u1sscYjhg9KuSpOmx/E5ysJ89hktGTi2aslpe0R5DFzs/MAulFMJwZJPzjNKxXoyxJb4CCASGTUXUQFATasO/aUwws9ZWN/dVQS2CTd4guRICRqR4Kzip43XHgE9ctnOP9E2NMXfGnRIXmQohu+mjMJp0fQnJzr7L70o3cjtB1iQ/KwF3BithKQF/xPi+HCd4OZUxOhyixsG0OhNbCNsb7/tSqAs9JBrslRbN+XiRibzbWGD8rtNapU+IuBMWNK5B++8KpyUNWvUhTJup84L5FNHHysRlP0kAd8XM119EnMOs0rb0QwQsbZk2WfIGXgnKDzqr02XXsUjWtNZrbTM2zqiaLioYIvLxE6EMnFFEMNU+2Bpgj9xUu/x+WIw57xI9/6Iyr8Ck3PmFe5r0gpNLs2xXHkweCrXWDZjyNNzwNhSt3HTb5K+3QsU0JkB2wqGZZnez2CwfrvBfMFjKfAxAVGygeKFZsRY3XCVhs7r5NSHg6Wp/X+/jyYz8MCjlyw/yppyA4c/sAs1bJ1fmzo5K5reOzmpv1K7uqvX57o4--9yICsk5RvHZzyqdC--g3xaeflXn1y3Z/H5v6/oWw== \ No newline at end of file diff --git a/db/migrate/20250110070255_create_data_versions.rb b/db/migrate/20250110070255_create_data_versions.rb new file mode 100644 index 0000000..3b330dc --- /dev/null +++ b/db/migrate/20250110070255_create_data_versions.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class CreateDataVersions < ActiveRecord::Migration[7.0] + def change + create_table :data_versions, id: :uuid, default: -> { 'gen_random_uuid()' } do |t| + t.string :filename, null: false + t.datetime :imported_at, null: false + t.index :filename, unique: true + end + end +end diff --git a/db/schema.rb b/db/schema.rb index d30d30c..c4468fd 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,12 +10,13 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_01_26_032358) do +ActiveRecord::Schema[7.0].define(version: 2025_01_10_070255) do # These are extensions that must be enabled in order to support this database enable_extension "btree_gin" enable_extension "pg_trgm" enable_extension "pgcrypto" enable_extension "plpgsql" + enable_extension "uuid-ossp" create_table "app_updates", primary_key: "updated_at", id: :datetime, force: :cascade do |t| t.string "update_type", null: false @@ -72,6 +73,12 @@ ActiveRecord::Schema[7.0].define(version: 2024_01_26_032358) do create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t| end + create_table "data_versions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.string "filename", null: false + t.datetime "imported_at", null: false + t.index ["filename"], name: "index_data_versions_on_filename", unique: true + end + create_table "favorites", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "user_id" t.uuid "party_id" diff --git a/db/seed/README.md b/db/seed/README.md new file mode 100644 index 0000000..db4d354 --- /dev/null +++ b/db/seed/README.md @@ -0,0 +1,179 @@ +# Adding items to the database + +Anyone can add data to the database via a Pull Request on Github. Fork the repository, then create a new CSV file in the +`updates` folder with the date, type of item you're updating, and the next number in the sequence. The most important +part is the type of item, which should be plural for convention. This is what tells the service how to process your +data. + +``` +20240618-characters-001.csv +20241231-weapons-010.csv +20250115-summons-025.csv +``` + +It's recommended to use a CSV editor to edit this data, but something like Microsoft Excel or Numbers should work fine +too. + +## Required values + +You are required to set values for the following fields: +TBA + +## Arrays + +When adding the `character_id`, `nicknames_en`, or `nicknames_jp`, make sure to wrap your values in brackets ({}). If +there are multiple values, you can separate them with a comma (,) with no spaces on either side. + +``` +# character_id, Zeta and Vaseraga (Halloween): +{3024,3025} + +# nicknames_en, Threo +{sarasa,cake,thalatha} +``` + +## Value tables + +Values for properties like `element` are pre-defined, so you only need to input the corresponding digit. + +#### Rarity + +| Rarity | Value | +|--------|-------| +| SSR | 3 | +| SR | 2 | +| R | 1 | + +#### Element + +| Element | Value | +|---------|-------| +| Wind | 1 | +| Fire | 2 | +| Water | 3 | +| Earth | 4 | +| Dark | 5 | +| Light | 6 | + +#### Proficiency + +| Proficiency | Value | +|-------------|-------| +| Sabre | 1 | +| Dagger | 2 | +| Axe | 3 | +| Spear | 4 | +| Bow | 5 | +| Staff | 6 | +| Melee | 7 | +| Harp | 8 | +| Gun | 9 | +| Katana | 10 | + +#### Race + +| Race | Value | +|---------|-------| +| Unknown | 0 | +| Human | 1 | +| Erune | 2 | +| Draph | 3 | +| Harvin | 4 | +| Primal | 5 | + +#### Gender + +| Gender | Value | +|-------------|-------| +| Other | 0 | +| Male | 1 | +| Female | 2 | +| Male/Female | 3 | + +#### Weapon Series (Needs to be cleaned) + +| Series Name | Value | Series Name | Value | +|--------------|-------|----------------------|-------| +| Seraphic | 0 | Cosmic | 20 | +| Grand | 1 | Draconic | 21 | +| Dark Opus | 2 | Superlative | 22 | +| Draconic | 3 | Vintage | 23 | +| Revenant | 4 | Class Champion | 24 | +| Beast | 5 | Proven | 25 | +| Primal | 6 | Malice | 26 | +| Beast | 7 | Menace | 27 | +| Regalia | 8 | Sephira | 28 | +| Omega | 9 | New World Foundation | 29 | +| Olden Primal | 10 | Revans | 30 | +| Militis | 11 | Illustrious | 31 | +| Hollowsky | 12 | World | 32 | +| Xeno | 13 | Exo | 33 | +| Astral | 14 | Event | 35 | +| Rose Crystal | 15 | Gacha | 36 | +| Bahamut | 16 | Celestial | 37 | +| Ultima | 17 | Omega Rebirth | 38 | +| Epic | 18 | Assorted | -1 | +| Ennead | 19 | | | + +#### Summon Series (Needs to be cleaned) + +| Series Name | Value | +|--------------|-------| +| Providence | 0 | +| Genesis | 1 | +| Omega | 2 | +| Optimus | 3 | +| Demi Optimus | 4 | +| Archangel | 5 | +| Arcarum | 6 | +| Epic | 7 | +| Dynamis | 9 | +| Cryptid | 11 | +| Six Dragons | 12 | + +### Wiki links + +You should try to provide identifiers for the 4 major wikis: gbf.wiki, gbf-wiki.com (JA), Kamigame (JA) and Gamewith ( +JA). Here's how: + +#### gbf.wiki + +This is simply the item's name, as it appears after `https://gbf.wiki/` in the URL. + +``` +https://gbf.wiki/Bahamut -> Bahamut +``` + +#### Gamewith + +This is a 5 to 6 digit string that appears at the end of the URL. + +``` +https://xn--bck3aza1a2if6kra4ee0hf.gamewith.jp/article/show/21612 -> 21612 +``` + +#### Kamigame + +Use a [URL decoder](https://www.urldecoder.org/) to extract the Japanese characters from the URL after the final forward +slash (/) and before `.html`. + +``` +https://kamigame.jp/%E3%82%B0%E3%83%A9%E3%83%96%E3%83%AB/%E3%82%AD%E3%83%A3%E3%83%A9%E3%82%AF%E3%82%BF%E3%83%BC/SSR%E3%83%A4%E3%83%81%E3%83%9E.html +-(decoder)-> +https://kamigame.jp/グラブル/キャラクター/SSRヤチマ.html +-(value)-> +SSRヤチマ +``` + +#### gbf-wiki.com + +Use a [URL decoder](https://www.urldecoder.org/) to extract the Japanese characters from the URL after the question +mark. Replace the `+` with a space. + +``` +https://gbf-wiki.com/?%E3%83%A4%E3%83%81%E3%83%9E+(SSR)%E3%83%AA%E3%83%9F%E3%83%86%E3%83%83%E3%83%89%E3%83%90%E3%83%BC%E3%82%B8%E3%83%A7%E3%83%B3 +-(decoder)-> +https://gbf-wiki.com/?ヤチマ+(SSR)リミテッドバージョン +-(value)-> +ヤチマ (SSR)リミテッドバージョン +``` diff --git a/db/seed/updates/20250101-characters-000.csv.example b/db/seed/updates/20250101-characters-000.csv.example new file mode 100644 index 0000000..153b244 --- /dev/null +++ b/db/seed/updates/20250101-characters-000.csv.example @@ -0,0 +1,2 @@ +name_en,name_jp,granblue_id,rarity,element,proficiency1,proficiency2,gender,race1,race2,flb,min_hp,max_hp,max_hp_flb,min_atk,max_atk,max_atk_flb,base_da,base_ta,ougi_ratio,ougi_ratio_flb,special,ulb,max_hp_ulb,max_atk_ulb,character_id,wiki_en,release_date,flb_date,ulb_date,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp +Gran,グラン,3040001000,3,1,1,2,1,1,0,true,1200,6800,8160,6400,8000,9600,6,3,4.5,5.0,false,false,0,0,{1},/wiki/Gran,2014-03-10,2016-04-01,,/wiki/グラン,/gran,/3040001000,{MC},"{主人公,グラン様}" diff --git a/db/seed/updates/20250101-summons-000.csv.example b/db/seed/updates/20250101-summons-000.csv.example new file mode 100644 index 0000000..bc594d8 --- /dev/null +++ b/db/seed/updates/20250101-summons-000.csv.example @@ -0,0 +1,2 @@ +name_en,name_jp,granblue_id,rarity,element,series,flb,ulb,max_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,subaura,limit,transcendence,max_atk_xlb,max_hp_xlb,summon_id,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,transcendence_date,nicknames_en,nicknames_jp +Bahamut,バハムート,2040003000,3,0,Divine Beasts,true,true,100,600,2400,2880,3360,1500,2000,2400,2800,true,true,true,3200,3840,1,2014-03-10,2016-04-01,2020-07-15,/wiki/Bahamut,/wiki/バハムート,/bahamut,/2040003000,2023-01-15,{Baha},"{バハ,黒龍}" diff --git a/db/seed/updates/20250101-weapons-000.csv.example b/db/seed/updates/20250101-weapons-000.csv.example new file mode 100644 index 0000000..6312256 --- /dev/null +++ b/db/seed/updates/20250101-weapons-000.csv.example @@ -0,0 +1,2 @@ +name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits_id,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date +Aschallon,アスカロン,1040014000,4,1,1,1,true,true,150,15,220,2200,2640,3080,2450,2940,3528,4116,false,1,true,true,,5,2014-03-10,2016-04-01,2020-07-15,/wiki/Aschallon,/wiki/アスカロン,/aschallon,/1040014000,{Asca},"{アスカ,アスカロン}",true,2023-01-15 diff --git a/lib/granblue/data_importer.rb b/lib/granblue/data_importer.rb new file mode 100644 index 0000000..1be5b54 --- /dev/null +++ b/lib/granblue/data_importer.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Granblue + class DataImporter + def initialize(test_mode: false, verbose: false) + @test_mode = test_mode + @verbose = verbose + @import_logs = [] + end + + def process_all_files(&block) + files = Dir.glob(Rails.root.join('db', 'seed', 'updates', '*.csv')).sort + + files.each do |file| + if (new_records = import_csv(file)) + block.call(new_records) if block_given? + end + end + + print_summary if @test_mode + end + + private + + def import_csv(file_path) + filename = File.basename(file_path) + return if already_imported?(filename) + + importer = create_importer(filename, file_path) + return unless importer + + log_info "Processing #{filename} in #{@test_mode ? 'test' : 'live'} mode..." + result = importer.import + log_import(filename, result) + log_info "Successfully processed #{filename}" + result + end + + def log_import_results(result) + return unless @verbose + + result[:new].each do |type, ids| + log_info "Created #{ids.size} new #{type.pluralize}" if ids.any? + end + result[:updated].each do |type, ids| + log_info "Updated #{ids.size} existing #{type.pluralize}" if ids.any? + end + end + + def create_importer(filename, file_path) + # This pattern matches both singular and plural: character(s), weapon(s), summon(s) + match = filename.match(/\A\d{8}-(character(?:s)?|weapon(?:s)?|summon(?:s)?)-\d+\.csv\z/) + return unless match + + matched_type = match[1] + singular_type = matched_type.sub(/s$/, '') + importer_class = "Granblue::Importers::#{singular_type.capitalize}Importer".constantize + + importer_class.new( + file_path, + test_mode: @test_mode, + verbose: @verbose, + logger: self + ) + rescue NameError + log_info "No importer found for type: #{singular_type}" + nil + end + + def already_imported?(filename) + DataVersion.imported?(filename) + end + + def log_import(filename, result = nil) + return if @test_mode + + DataVersion.mark_as_imported(filename) + + if result && @verbose + result[:new].each do |type, ids| + log_info "Created #{ids.size} new #{type.pluralize}" if ids.any? + end + result[:updated].each do |type, ids| + log_info "Updated #{ids.size} existing #{type.pluralize}" if ids.any? + end + end + end + + def log_operation(operation) + if @test_mode + @import_logs << operation + log_info "[TEST MODE] Would perform: #{operation}" + end + end + + def print_summary + log_info "\nTest Mode Summary:" + log_info "Would perform #{@import_logs.size} operations" + if @import_logs.any? + log_info 'Sample of operations:' + @import_logs.first(3).each { |log| log_info "- #{log}" } + log_info '...' if @import_logs.size > 3 + end + end + + def log_info(message) + puts message if @verbose || @test_mode + end + end +end diff --git a/lib/granblue/download_manager.rb b/lib/granblue/download_manager.rb new file mode 100644 index 0000000..44b0204 --- /dev/null +++ b/lib/granblue/download_manager.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module Granblue + module Downloader + class DownloadManager + class << self + def download_for_object(type, granblue_id, test_mode: false, verbose: false, storage: :both) + @test_mode = test_mode + @verbose = verbose + @storage = storage + + case type + when 'character' + download_character(granblue_id) + when 'weapon' + download_weapon(granblue_id) + when 'summon' + download_summon(granblue_id) + else + log_info "Unknown object type: #{type}" + end + end + + private + + def download_character(id) + character = Character.find_by(granblue_id: id) + return unless character + + downloader_options = { + test_mode: @test_mode, + verbose: @verbose, + storage: @storage + } + + %W[#{id}_01 #{id}_02].each do |variant_id| + CharacterDownloader.new(variant_id, **downloader_options).download + end + + CharacterDownloader.new("#{id}_03", **downloader_options).download if character.flb + CharacterDownloader.new("#{id}_04", **downloader_options).download if character.ulb + end + + def download_weapon(id) + weapon = Weapon.find_by(granblue_id: id) + return unless weapon + + downloader_options = { + test_mode: @test_mode, + verbose: @verbose, + storage: @storage + } + + WeaponDownloader.new(id, **downloader_options).download + + return unless weapon.transcendence + + WeaponDownloader.new("#{id}_02", **downloader_options).download + WeaponDownloader.new("#{id}_03", **downloader_options).download + + end + + def download_summon(id) + summon = Summon.find_by(granblue_id: id) + return unless summon + + downloader_options = { + test_mode: @test_mode, + verbose: @verbose, + storage: @storage + } + + SummonDownloader.new(id, **downloader_options).download + SummonDownloader.new("#{id}_02", **downloader_options).download if summon.ulb + + return unless summon.transcendence + + SummonDownloader.new("#{id}_03", **downloader_options).download + SummonDownloader.new("#{id}_04", **downloader_options).download + + end + + def log_info(message) + puts message if @verbose || @test_mode + end + end + end + end +end diff --git a/lib/granblue/downloaders/base_downloader.rb b/lib/granblue/downloaders/base_downloader.rb new file mode 100644 index 0000000..f65e71e --- /dev/null +++ b/lib/granblue/downloaders/base_downloader.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module Granblue + module Downloader + class BaseDownloader + SIZES = %w[main grid square].freeze + + def initialize(id, test_mode: false, verbose: false, storage: :both) + @id = id + @base_url = base_url + @test_mode = test_mode + @verbose = verbose + @storage = storage + @aws_service = AwsService.new + ensure_directories_exist unless @test_mode + end + + def download + log_info "-> #{@id}" + return if @test_mode + + SIZES.each_with_index do |size, index| + path = download_path(size) + url = build_url(size) + process_download(url, size, path, last: index == SIZES.size - 1) + end + end + + private + + def process_download(url, size, path, last: false) + filename = File.basename(url) + s3_key = build_s3_key(size, filename) + download_uri = "#{path}/#{filename}" + + should_process = should_download?(download_uri, s3_key) + return unless should_process + + if last + log_info "\t└ #{size}: #{url}..." + else + log_info "\t├ #{size}: #{url}..." + end + + case @storage + when :local + download_to_local(url, download_uri) + when :s3 + stream_to_s3(url, s3_key) + when :both + download_to_both(url, download_uri, s3_key) + end + rescue OpenURI::HTTPError + log_info "\t404 returned\t#{url}" + end + + def download_to_local(url, download_uri) + download = URI.parse(url).open + IO.copy_stream(download, download_uri) + end + + def stream_to_s3(url, s3_key) + return if @aws_service.file_exists?(s3_key) + + URI.parse(url).open do |file| + @aws_service.upload_stream(file, s3_key) + end + end + + def download_to_both(url, download_uri, s3_key) + download = URI.parse(url).open + + # Write to local file + IO.copy_stream(download, download_uri) + + # Reset file pointer for S3 upload + download.rewind + + # Upload to S3 if it doesn't exist + unless @aws_service.file_exists?(s3_key) + @aws_service.upload_stream(download, s3_key) + end + end + + def should_download?(local_path, s3_key) + case @storage + when :local + !File.exist?(local_path) + when :s3 + !@aws_service.file_exists?(s3_key) + when :both + !File.exist?(local_path) || !@aws_service.file_exists?(s3_key) + end + end + + def ensure_directories_exist + return unless store_locally? + + SIZES.each do |size| + FileUtils.mkdir_p(download_path(size)) + end + end + + def store_locally? + %i[local both].include?(@storage) + end + + def download_path(size) + "#{Rails.root}/download/#{object_type}-#{size}" + end + + def build_s3_key(size, filename) + "#{object_type}-#{size}/#{filename}" + end + + def log_info(message) + puts message if @verbose + end + + def download_elemental_image(url, size, path, filename) + return if @test_mode + + filepath = "#{path}/#{filename}" + download = URI.parse(url).open + log_info "-> #{size}:\t#{url}..." + IO.copy_stream(download, filepath) + rescue OpenURI::HTTPError + log_info "\t404 returned\t#{url}" + end + + def object_type + raise NotImplementedError, 'Subclasses must define object_type' + end + + def base_url + raise NotImplementedError, 'Subclasses must define base_url' + end + + def directory_for_size(size) + raise NotImplementedError, 'Subclasses must define directory_for_size' + end + + def build_url(size) + directory = directory_for_size(size) + "#{@base_url}/#{directory}/#{@id}.jpg" + end + end + end +end diff --git a/lib/granblue/downloaders/character_downloader.rb b/lib/granblue/downloaders/character_downloader.rb new file mode 100644 index 0000000..34aa8a6 --- /dev/null +++ b/lib/granblue/downloaders/character_downloader.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Granblue + module Downloader + class CharacterDownloader < BaseDownloader + private + + def object_type + 'character' + end + + def base_url + 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/npc' + end + + def directory_for_size(size) + case size.to_s + when 'main' then 'f' + when 'grid' then 'm' + when 'square' then 's' + end + end + end + end +end diff --git a/lib/granblue/downloaders/elemental_weapon_downloader.rb b/lib/granblue/downloaders/elemental_weapon_downloader.rb new file mode 100644 index 0000000..9ad5e79 --- /dev/null +++ b/lib/granblue/downloaders/elemental_weapon_downloader.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require_relative 'weapon_downloader' + +module Granblue + module Downloader + class ElementalWeaponDownloader < WeaponDownloader + SUFFIXES = [2, 3, 4, 1, 6, 5].freeze + + def initialize(id_base) + @id_base = id_base.to_i + end + + def download + (1..6).each do |i| + id = @id_base + (i - 1) * 100 + suffix = SUFFIXES[i - 1] + + puts "Elemental Weapon #{id}_#{suffix}" + SIZES.each do |size| + path = download_path(size) + url = build_url_for_id(id, size) + filename = "#{id}_#{suffix}.jpg" + download_elemental_image(url, size, path, filename) + end + + progress_reporter(count: i, total: 6, result: "Elemental Weapon #{id}_#{suffix}") + end + end + + private + + def build_url_for_id(id, size) + directory = directory_for_size(size) + "#{base_url}/#{directory}/#{id}.jpg" + end + + def progress_reporter(count:, total:, result:, bar_len: 40) + filled_len = (bar_len * count / total).round + status = result + percents = (100.0 * count / total).round(1) + bar = '=' * filled_len + '-' * (bar_len - filled_len) + print("\n[#{bar}] #{percents}% ...#{' ' * 14}#{status}\n") + end + end + end +end diff --git a/lib/granblue/downloaders/summon_downloader.rb b/lib/granblue/downloaders/summon_downloader.rb new file mode 100644 index 0000000..2d9ba8d --- /dev/null +++ b/lib/granblue/downloaders/summon_downloader.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Granblue + module Downloader + class SummonDownloader < BaseDownloader + private + + def object_type + 'summon' + end + + def base_url + 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/summon' + end + + def directory_for_size(size) + case size.to_s + when 'main' then 'party_main' + when 'grid' then 'party_sub' + when 'square' then 's' + end + end + end + end +end diff --git a/lib/granblue/downloaders/weapon_downloader.rb b/lib/granblue/downloaders/weapon_downloader.rb new file mode 100644 index 0000000..38f158f --- /dev/null +++ b/lib/granblue/downloaders/weapon_downloader.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Granblue + module Downloader + class WeaponDownloader < BaseDownloader + private + + def object_type + 'weapon' + end + + def base_url + 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/weapon' + end + + def directory_for_size(size) + case size.to_s + when 'main' then 'ls' + when 'grid' then 'm' + when 'square' then 's' + end + end + end + end +end diff --git a/lib/granblue/importers/base_importer.rb b/lib/granblue/importers/base_importer.rb new file mode 100644 index 0000000..cb97f22 --- /dev/null +++ b/lib/granblue/importers/base_importer.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +module Granblue + module Importers + class BaseImporter + attr_reader :new_records, :updated_records + + def initialize(file_path, test_mode: false, verbose: false, logger: nil) + @file_path = file_path + @test_mode = test_mode + @verbose = verbose + @logger = logger + @new_records = Hash.new { |h, k| h[k] = [] } + @updated_records = Hash.new { |h, k| h[k] = [] } + end + + def import + CSV.foreach(@file_path, headers: true) do |row| + import_row(row) + end + { new: @new_records, updated: @updated_records } + end + + private + + def import_row(row) + attributes = build_attributes(row) + # Remove nil values from attributes hash for updates + # Keep them for new records to ensure proper defaults + record = find_or_create_record(attributes) + track_record(record) if record + end + + def find_or_create_record(attributes) + existing_record = model_class.find_by(granblue_id: attributes[:granblue_id]) + + if existing_record + if @test_mode + log_test_update(existing_record, attributes) + nil + else + # For updates, only include non-nil attributes + update_attributes = attributes.compact + was_updated = update_attributes.any? { |key, value| existing_record[key] != value } + existing_record.update!(update_attributes) if was_updated + [existing_record, was_updated] + end + else + if @test_mode + log_test_creation(attributes) + nil + else + # For new records, use all attributes including nil values + [model_class.create!(attributes), false] + end + end + end + + def track_record(result) + record, was_updated = result + type = model_class.name.demodulize.downcase + + if was_updated + @updated_records[type] << record.granblue_id + log_updated_record(record) if @verbose + else + @new_records[type] << record.granblue_id + log_new_record(record) if @verbose + end + end + + def log_test_update(record, attributes) + # For test mode, show only the attributes that would be updated + update_attributes = attributes.compact + @logger&.send(:log_operation, "Update #{model_class.name} #{record.granblue_id}: #{update_attributes.inspect}") + end + + def log_test_creation(attributes) + @logger&.send(:log_operation, "Create #{model_class.name}: #{attributes.inspect}") + end + + def log_new_record(record) + puts "Created #{model_class.name} with ID: #{record.granblue_id}" + end + + def log_updated_record(record) + puts "Updated #{model_class.name} with ID: #{record.granblue_id}" + end + + def parse_value(value) + return nil if value.nil? || value.strip.empty? + + value + end + + def parse_integer(value) + return nil if value.nil? || value.strip.empty? + + value.to_i + end + + def parse_float(value) + return nil if value.nil? || value.strip.empty? + + value.to_f + end + + def parse_boolean(value) + return nil if value.nil? || value.strip.empty? + + value == 'true' + end + + def parse_date(date_str) + return nil if date_str.nil? || date_str.strip.empty? + + Date.parse(date_str) rescue nil + end + + def parse_array(array_str) + return [] if array_str.nil? || array_str.strip.empty? + + array_str.tr('{}', '').split(',') + end + + def parse_integer_array(array_str) + parse_array(array_str).map(&:to_i) + end + + def model_class + raise NotImplementedError, 'Subclasses must define model_class' + end + + def build_attributes(row) + raise NotImplementedError, 'Subclasses must define build_attributes' + end + end + end +end diff --git a/lib/granblue/importers/character_importer.rb b/lib/granblue/importers/character_importer.rb new file mode 100644 index 0000000..2343fa0 --- /dev/null +++ b/lib/granblue/importers/character_importer.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Granblue + module Importers + class CharacterImporter < BaseImporter + private + + def model_class + Character + end + + def build_attributes(row) + { + name_en: parse_value(row['name_en']), + name_jp: parse_value(row['name_jp']), + granblue_id: parse_value(row['granblue_id']), + rarity: parse_integer(row['rarity']), + element: parse_integer(row['element']), + proficiency1: parse_integer(row['proficiency1']), + proficiency2: parse_integer(row['proficiency2']), + gender: parse_integer(row['gender']), + race1: parse_integer(row['race1']), + race2: parse_integer(row['race2']), + flb: parse_boolean(row['flb']), + min_hp: parse_integer(row['min_hp']), + max_hp: parse_integer(row['max_hp']), + max_hp_flb: parse_integer(row['max_hp_flb']), + min_atk: parse_integer(row['min_atk']), + max_atk: parse_integer(row['max_atk']), + max_atk_flb: parse_integer(row['max_atk_flb']), + base_da: parse_integer(row['base_da']), + base_ta: parse_integer(row['base_ta']), + ougi_ratio: parse_float(row['ougi_ratio']), + ougi_ratio_flb: parse_float(row['ougi_ratio_flb']), + special: parse_boolean(row['special']), + ulb: parse_boolean(row['ulb']), + max_hp_ulb: parse_integer(row['max_hp_ulb']), + max_atk_ulb: parse_integer(row['max_atk_ulb']), + character_id: parse_integer_array(row['character_id']), + wiki_en: parse_value(row['wiki_en']), + release_date: parse_value(row['release_date']), + flb_date: parse_value(row['flb_date']), + ulb_date: parse_value(row['ulb_date']), + wiki_ja: parse_value(row['wiki_ja']), + gamewith: parse_value(row['gamewith']), + kamigame: parse_value(row['kamigame']), + nicknames_en: parse_array(row['nicknames_en']), + nicknames_jp: parse_array(row['nicknames_jp']) + } + end + end + end +end diff --git a/lib/granblue/importers/summon_importer.rb b/lib/granblue/importers/summon_importer.rb new file mode 100644 index 0000000..c68ab67 --- /dev/null +++ b/lib/granblue/importers/summon_importer.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Granblue + module Importers + class SummonImporter < BaseImporter + private + + def model_class + Summon + end + + def build_attributes(row) + { + name_en: parse_value(row['name_en']), + name_jp: parse_value(row['name_jp']), + granblue_id: parse_value(row['granblue_id']), + rarity: parse_integer(row['rarity']), + element: parse_integer(row['element']), + series: parse_value(row['series']), + flb: parse_boolean(row['flb']), + ulb: parse_boolean(row['ulb']), + max_level: parse_integer(row['max_level']), + min_hp: parse_integer(row['min_hp']), + max_hp: parse_integer(row['max_hp']), + max_hp_flb: parse_integer(row['max_hp_flb']), + max_hp_ulb: parse_integer(row['max_hp_ulb']), + min_atk: parse_integer(row['min_atk']), + max_atk: parse_integer(row['max_atk']), + max_atk_flb: parse_integer(row['max_atk_flb']), + max_atk_ulb: parse_integer(row['max_atk_ulb']), + subaura: parse_boolean(row['subaura']), + limit: parse_boolean(row['limit']), + transcendence: parse_boolean(row['transcendence']), + max_atk_xlb: parse_integer(row['max_atk_xlb']), + max_hp_xlb: parse_integer(row['max_hp_xlb']), + summon_id: parse_integer(row['summon_id']), + release_date: parse_value(row['release_date']), + flb_date: parse_value(row['flb_date']), + ulb_date: parse_value(row['ulb_date']), + wiki_en: parse_value(row['wiki_en']), + wiki_ja: parse_value(row['wiki_ja']), + gamewith: parse_value(row['gamewith']), + kamigame: parse_value(row['kamigame']), + transcendence_date: parse_value(row['transcendence_date']), + nicknames_en: parse_array(row['nicknames_en']), + nicknames_jp: parse_array(row['nicknames_jp']) + } + end + end + end +end diff --git a/lib/granblue/importers/weapon_importer.rb b/lib/granblue/importers/weapon_importer.rb new file mode 100644 index 0000000..a6fbcae --- /dev/null +++ b/lib/granblue/importers/weapon_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Granblue + module Importers + class WeaponImporter < BaseImporter + private + + def model_class + Weapon + end + + def build_attributes(row) + { + name_en: parse_value(row['name_en']), + name_jp: parse_value(row['name_jp']), + granblue_id: parse_value(row['granblue_id']), + rarity: parse_integer(row['rarity']), + element: parse_integer(row['element']), + proficiency: parse_integer(row['proficiency']), + series: parse_integer(row['series']), + flb: parse_boolean(row['flb']), + ulb: parse_boolean(row['ulb']), + max_level: parse_integer(row['max_level']), + max_skill_level: parse_integer(row['max_skill_level']), + min_hp: parse_integer(row['min_hp']), + max_hp: parse_integer(row['max_hp']), + max_hp_flb: parse_integer(row['max_hp_flb']), + max_hp_ulb: parse_integer(row['max_hp_ulb']), + min_atk: parse_integer(row['min_atk']), + max_atk: parse_integer(row['max_atk']), + max_atk_flb: parse_integer(row['max_atk_flb']), + max_atk_ulb: parse_integer(row['max_atk_ulb']), + extra: parse_boolean(row['extra']), + ax_type: parse_integer(row['ax_type']), + limit: parse_boolean(row['limit']), + ax: parse_boolean(row['ax']), + recruits_id: parse_value(row['recruits_id']), + max_awakening_level: parse_integer(row['max_awakening_level']), + release_date: parse_value(row['release_date']), + flb_date: parse_value(row['flb_date']), + ulb_date: parse_value(row['ulb_date']), + wiki_en: parse_value(row['wiki_en']), + wiki_ja: parse_value(row['wiki_ja']), + gamewith: parse_value(row['gamewith']), + kamigame: parse_value(row['kamigame']), + nicknames_en: parse_array(row['nicknames_en']), + nicknames_jp: parse_array(row['nicknames_jp']), + transcendence: parse_boolean(row['transcendence']), + transcendence_date: parse_value(row['transcendence_date']) + } + end + end + end +end diff --git a/lib/granblue/post_deployment_manager.rb b/lib/granblue/post_deployment_manager.rb new file mode 100644 index 0000000..928d618 --- /dev/null +++ b/lib/granblue/post_deployment_manager.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require_relative '../logging_helper' + +class PostDeploymentManager + include LoggingHelper + + STORAGE_DESCRIPTIONS = { + local: 'to local disk', + s3: 'to S3', + both: 'to local disk and S3' + }.freeze + + def initialize(options = {}) + @test_mode = options.fetch(:test_mode, false) + @verbose = options.fetch(:verbose, false) + @storage = options.fetch(:storage, :both) + @new_records = Hash.new { |h, k| h[k] = [] } + @updated_records = Hash.new { |h, k| h[k] = [] } + end + + def run + import_new_data + display_import_summary + download_images + rebuild_search_indices + display_completion_message + end + + private + + def import_new_data + log_header 'Importing new data...' + puts "\n" + importer = Granblue::DataImporter.new( + test_mode: @test_mode, + verbose: @verbose + ) + + process_imports(importer) + end + + def process_imports(importer) + importer.process_all_files do |result| + result[:new].each do |type, ids| + @new_records[type].concat(ids) + end + result[:updated].each do |type, ids| + @updated_records[type].concat(ids) + end + end + end + + def rebuild_search_indices + log_header 'Rebuilding search indices...', '-' + puts "\n" + [Character, Summon, Weapon, Job].each do |model| + log_verbose "• #{model.name}... " + PgSearch::Multisearch.rebuild(model) + log_verbose "✅ done!\n" + end + end + + def display_import_summary + if @new_records.size > 0 || @updated_records.size > 0 + log_header 'Import Summary', '-' + puts "\n" + display_record_summary('New', @new_records) + display_record_summary('Updated', @updated_records) + else + log_step "\nNo new records imported." + end + end + + def display_record_summary(label, records) + records.each do |type, ids| + next if ids.empty? + puts "#{type.capitalize}: #{ids.size} #{label.downcase} records" + puts "IDs: #{ids.inspect}" if @verbose + end + end + + def download_images + return if all_records_empty? + + if @test_mode + log_step "\nTEST MODE: Would download images for new and updated records..." + else + log_header 'Downloading images...', '+' + end + + [@new_records, @updated_records].each do |records| + records.each do |type, ids| + next if ids.empty? + download_type_images(type, ids) + end + end + end + + def download_type_images(type, ids) + log_step "\nProcessing new #{type.pluralize} (#{ids.size} records)..." + download_options = { + test_mode: @test_mode, + verbose: @verbose, + storage: @storage + } + + ids.each do |id| + download_single_image(type, id, download_options) + end + end + + def download_single_image(type, id, options) + action_text = @test_mode ? 'Would download' : 'Downloading' + storage_text = STORAGE_DESCRIPTIONS[options[:storage]] + log_verbose "\n#{action_text} images #{storage_text} for #{type} #{id}...\n" + + Granblue::Downloader::DownloadManager.download_for_object( + type, + id, + **options + ) + rescue => e + error_message = "Error #{@test_mode ? 'would occur' : 'occurred'} downloading images for #{type} #{id}: #{e.message}" + puts error_message + puts e.backtrace.take(5) if @verbose + end + + def display_completion_message + if @test_mode + log_step "\n✓ Test run completed successfully!" + else + log_step "\n✓ Post-deployment tasks completed successfully!" + end + end + + def all_records_empty? + @new_records.values.all?(&:empty?) && @updated_records.values.all?(&:empty?) + end +end diff --git a/lib/logging_helper.rb b/lib/logging_helper.rb new file mode 100644 index 0000000..2e4fb10 --- /dev/null +++ b/lib/logging_helper.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module LoggingHelper + def log_step(message) + puts message + end + + def log_verbose(message) + print message if @verbose + end + + def log_divider(character = '+', leading_newline = true, trailing_newlines = 1) + output = "" + output += "\n" if leading_newline + output += character * 35 + output += "\n" * trailing_newlines + log_step output + end + + def log_header(title, character = '+', leading_newline = true) + log_divider(character, leading_newline, 0) + log_step title + log_divider(character, false) + end +end diff --git a/lib/tasks/deploy.rake b/lib/tasks/deploy.rake new file mode 100644 index 0000000..ab7416b --- /dev/null +++ b/lib/tasks/deploy.rake @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require_relative '../granblue/downloaders/base_downloader' +require_relative '../logging_helper' + +namespace :deploy do + desc 'Post-deployment tasks: Import new data and download related images. Options: TEST=true for test mode, VERBOSE=true for verbose output, STORAGE=local|s3|both' + task post_deployment: :environment do + include LoggingHelper + + Dir[Rails.root.join('lib', 'granblue', '**', '*.rb')].each { |file| require file } + + # Ensure Rails environment is loaded + Rails.application.eager_load! + + log_header('Starting post-deploy script...', '-', false) + print "\n" + + # Parse and validate storage option + storage = (ENV['STORAGE'] || 'both').to_sym + unless [:local, :s3, :both].include?(storage) + puts 'Invalid STORAGE option. Must be one of: local, s3, both' + exit 1 + end + + options = { + test_mode: ENV['TEST'] == 'true', + verbose: ENV['VERBOSE'] == 'true', + storage: storage + } + + print "Test mode:\t" + if options[:test_mode] + print "✅ Enabled\n" + else + print "❌ Disabled\n" + end + + print "Verbose output:\t" + if options[:verbose] + print "✅ Enabled\n" + else + print "❌ Disabled\n" + end + + puts "Storage mode:\t#{storage}" + + # Execute the task + manager = PostDeploymentManager.new(options) + manager.run + end +end diff --git a/lib/tasks/download_images.rake b/lib/tasks/download_images.rake index 8bf0982..5012fb8 100644 --- a/lib/tasks/download_images.rake +++ b/lib/tasks/download_images.rake @@ -1,188 +1,22 @@ namespace :granblue do - def _progress_reporter(count:, total:, result:, bar_len: 40, multi: true) - filled_len = (bar_len * count / total).round - status = File.basename(result) - percents = (100.0 * count / total).round(1) - bar = '=' * filled_len + '-' * (bar_len - filled_len) + desc 'Downloads images for the given Granblue_IDs' + task :download_images, %i[object] => :environment do |_t, args| + require_relative '../granblue/downloaders/base_downloader' + Dir[Rails.root.join('lib', 'granblue', 'image_downloader', '*.rb')].each { |file| require file } - if !multi - print("[#{bar}] #{percents}% ...#{' ' * 14}#{status}\n") - else - print "\n" - end - end + object = args[:object] + list = args.extras - def build_weapon_url(id, size) - # Set up URL - base_url = 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/weapon' - extension = '.jpg' - - directory = 'ls' if size.to_s == 'main' - directory = 'm' if size.to_s == 'grid' - directory = 's' if size.to_s == 'square' - - "#{base_url}/#{directory}/#{id}#{extension}" - end - - def build_summon_url(id, size) - # Set up URL - base_url = 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/summon' - extension = '.jpg' - - directory = 'party_main' if size.to_s == 'main' - directory = 'party_sub' if size.to_s == 'grid' - directory = 's' if size.to_s == 'square' - - "#{base_url}/#{directory}/#{id}#{extension}" - end - - def build_chara_url(id, size) - # Set up URL - base_url = 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/npc' - extension = '.jpg' - - directory = 'f' if size.to_s == 'main' - directory = 'm' if size.to_s == 'grid' - directory = 's' if size.to_s == 'square' - - "#{base_url}/#{directory}/#{id}#{extension}" - end - - def download_images(url, size, path) - download = URI.parse(url).open - download_URI = "#{path}/#{download.base_uri.to_s.split('/')[-1]}" - if File.exist?(download_URI) - puts "\tSkipping #{size}\t#{url}" - else - puts "\tDownloading #{size}\t#{url}..." - IO.copy_stream(download, "#{path}/#{download.base_uri.to_s.split('/')[-1]}") - end - rescue OpenURI::HTTPError - puts "\t404 returned\t#{url}" - end - - def download_elemental_images(url, size, path, filename) - filepath = "#{path}/#{filename}" - download = URI.parse(url).open - puts "\tDownloading #{size}\t#{url}..." - IO.copy_stream(download, filepath) - rescue OpenURI::HTTPError - puts "\t404 returned\t#{url}" - end - - def download_chara_images(id) - sizes = %w[main grid square] - - url = { - 'main': build_chara_url(id, 'main'), - 'grid': build_chara_url(id, 'grid'), - 'square': build_chara_url(id, 'square') - } - - puts "Character #{id}" - sizes.each do |size| - path = "#{Rails.root}/download/character-#{size}" - download_images(url[size.to_sym], size, path) - end - end - - def download_weapon_images(id) - sizes = %w[main grid square] - - url = { - 'main': build_weapon_url(id, 'main'), - 'grid': build_weapon_url(id, 'grid'), - 'square': build_weapon_url(id, 'square') - } - - puts "Weapon #{id}" - sizes.each do |size| - path = "#{Rails.root}/download/weapon-#{size}" - download_images(url[size.to_sym], size, path) - end - end - - def download_summon_images(id) - sizes = %w[main grid square] - - url = { - 'main': build_summon_url(id, 'main'), - 'grid': build_summon_url(id, 'grid'), - 'square': build_summon_url(id, 'square') - } - - puts "Summon #{id}" - sizes.each do |size| - path = "#{Rails.root}/download/summon-#{size}" - download_images(url[size.to_sym], size, path) + list.each do |id| + Granblue::Downloader::DownloadManager.download_for_object(object, id) end end desc 'Downloads elemental weapon images' task :download_elemental_images, [:id_base] => :environment do |_t, args| - id_base = args[:id_base].to_i - suffixes = [2, 3, 4, 1, 6, 5] + require_relative '../granblue/downloaders/base_downloader' + Dir[Rails.root.join('lib', 'granblue', 'image_downloader', '*.rb')].each { |file| require file } - (1..6).each do |i| - id = id_base + (i - 1) * 100 - - sizes = %w[main grid square] - - url = { - 'main': build_weapon_url(id, 'main'), - 'grid': build_weapon_url(id, 'grid'), - 'square': build_weapon_url(id, 'square') - } - - puts "Elemental Weapon #{id}_#{suffixes[i - 1]}" - sizes.each do |size| - path = "#{Rails.root}/download/weapon-#{size}" - filename = "#{id}_#{suffixes[i - 1]}.jpg" - download_elemental_images(url[size.to_sym], size, path, filename) - end - - _progress_reporter(count: i, total: 6, result: "Elemental Weapon #{id}_#{suffixes[i - 1]}", bar_len: 40, - multi: true) - end - end - - desc 'Downloads images for the given Granblue_IDs' - task :download_images, %i[object] => :environment do |_t, args| - object = args[:object] - list = args.extras - - list.each do |id| - case object - when 'character' - character = Character.find_by(granblue_id: id) - next unless character - - download_chara_images("#{id}_01") - download_chara_images("#{id}_02") - download_chara_images("#{id}_03") if character.flb - download_chara_images("#{id}_04") if character.ulb - when 'weapon' - weapon = Weapon.find_by(granblue_id: id) - next unless weapon - - download_weapon_images(id) - - if weapon.transcendence - download_weapon_images("#{id}_02") - download_weapon_images("#{id}_03") - end - when 'summon' - summon = Summon.find_by(granblue_id: id) - next unless summon - - download_summon_images("#{id}") - download_summon_images("#{id}_02") if summon.ulb - - if summon.transcendence - download_summon_images("#{id}_03") - download_summon_images("#{id}_04") - end - end - end + Granblue::Downloader::ElementalWeaponDownloader.new(args[:id_base]).download end end diff --git a/lib/tasks/import_data.rake b/lib/tasks/import_data.rake new file mode 100644 index 0000000..6e15a05 --- /dev/null +++ b/lib/tasks/import_data.rake @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +namespace :granblue do + desc "Import weapon, summon and character data from db/seed/updates. Use TEST=true for test mode." + task import_data: :environment do + require 'csv' + Dir[Rails.root.join('lib', 'granblue', '**', '*.rb')].each { |file| require file } + + test_mode = ENV['TEST'] == 'true' + importer = Granblue::DataImporter.new(test_mode: test_mode) + importer.process_all_files + end +end