From 0d5d4d5f59836f876e6704ccc7836a4fb2ef54fe Mon Sep 17 00:00:00 2001 From: Justin Edmund Date: Fri, 17 Jan 2025 12:02:12 -0800 Subject: [PATCH] Jedmund/import (#167) * Move app/helpers/granblue_wiki to lib/parsers/wiki This clears up the namespace beginning with "Granblue" * Removed some top-level Granblue libs DataImporter and DownloadManager exist inside of the PostDeployment namespace now so these files are redundant * Fix Downloaders namespace Our namespace was singular Downloader, now it is plural Downloaders to match the folder name * Fix import paths * DownloadManager was moved to downloaders/ * import_data task now uses the PostDeployment version of DataImporter * Update application.rb Eager-Load/Autoload the lib/ folder * Update cors.rb Add Granblue website and Extension ID to CORS * Add transformers Transformers take raw data from Granblue Fantasy and transforms them into hensei-compatible JSON. Transformers heavily borrow from vazkii/hensei-transfer. * Add ImportController and route This adds the controller that handles creating a full party from transformed Granblue Fantasy data --- app/controllers/api/v1/import_controller.rb | 276 ++++++++++++++++ app/helpers/character_parser.rb | 278 ---------------- app/helpers/granblue_wiki.rb | 118 ------- app/helpers/summon_parser.rb | 251 --------------- app/helpers/validation_error_serializer.rb | 35 -- app/helpers/validation_errors_serializer.rb | 17 - app/helpers/weapon_parser.rb | 296 ----------------- config/application.rb | 3 + config/initializers/cors.rb | 4 +- config/routes.rb | 2 + lib/granblue/data_importer.rb | 110 ------- lib/granblue/downloaders/base_downloader.rb | 2 +- .../downloaders/character_downloader.rb | 2 +- .../{ => downloaders}/download_manager.rb | 2 +- .../elemental_weapon_downloader.rb | 2 +- lib/granblue/downloaders/summon_downloader.rb | 2 +- lib/granblue/downloaders/weapon_downloader.rb | 2 +- lib/granblue/parsers/character_parser.rb | 283 +++++++++++++++++ lib/granblue/parsers/summon_parser.rb | 255 +++++++++++++++ .../parsers/validation_error_serializer.rb | 39 +++ .../parsers/validation_errors_serializer.rb | 22 ++ lib/granblue/parsers/weapon_parser.rb | 300 ++++++++++++++++++ lib/granblue/parsers/wiki.rb | 122 +++++++ .../transformers/base_deck_transformer.rb | 91 ++++++ lib/granblue/transformers/base_transformer.rb | 74 +++++ .../transformers/character_transformer.rb | 51 +++ .../transformers/summon_transformer.rb | 73 +++++ .../transformers/transformer_error.rb | 12 + .../transformers/weapon_transformer.rb | 136 ++++++++ lib/post_deployment/image_downloader.rb | 8 +- lib/tasks/import_data.rake | 2 +- 31 files changed, 1752 insertions(+), 1118 deletions(-) create mode 100644 app/controllers/api/v1/import_controller.rb delete mode 100644 app/helpers/character_parser.rb delete mode 100644 app/helpers/granblue_wiki.rb delete mode 100644 app/helpers/summon_parser.rb delete mode 100644 app/helpers/validation_error_serializer.rb delete mode 100644 app/helpers/validation_errors_serializer.rb delete mode 100644 app/helpers/weapon_parser.rb delete mode 100644 lib/granblue/data_importer.rb rename lib/granblue/{ => downloaders}/download_manager.rb (97%) create mode 100644 lib/granblue/parsers/character_parser.rb create mode 100644 lib/granblue/parsers/summon_parser.rb create mode 100644 lib/granblue/parsers/validation_error_serializer.rb create mode 100644 lib/granblue/parsers/validation_errors_serializer.rb create mode 100644 lib/granblue/parsers/weapon_parser.rb create mode 100644 lib/granblue/parsers/wiki.rb create mode 100644 lib/granblue/transformers/base_deck_transformer.rb create mode 100644 lib/granblue/transformers/base_transformer.rb create mode 100644 lib/granblue/transformers/character_transformer.rb create mode 100644 lib/granblue/transformers/summon_transformer.rb create mode 100644 lib/granblue/transformers/transformer_error.rb create mode 100644 lib/granblue/transformers/weapon_transformer.rb diff --git a/app/controllers/api/v1/import_controller.rb b/app/controllers/api/v1/import_controller.rb new file mode 100644 index 0000000..7d64775 --- /dev/null +++ b/app/controllers/api/v1/import_controller.rb @@ -0,0 +1,276 @@ +# frozen_string_literal: true + +module Api + module V1 + class ImportController < Api::V1::ApiController + ELEMENT_MAPPING = { + 0 => nil, + 1 => 4, + 2 => 2, + 3 => 3, + 4 => 1, + 5 => 6, + 6 => 5 + }.freeze + + def create + Rails.logger.info "[IMPORT] Starting import..." + + # Parse JSON request body + raw_body = request.raw_post + begin + raw_params = JSON.parse(raw_body) if raw_body.present? + Rails.logger.info "[IMPORT] Raw game data: #{raw_params.inspect}" + rescue JSON::ParserError => e + Rails.logger.error "[IMPORT] Invalid JSON in request body: #{e.message}" + render json: { error: 'Invalid JSON data' }, status: :bad_request + return + end + + if raw_params.nil? || !raw_params.is_a?(Hash) + Rails.logger.error "[IMPORT] Missing or invalid game data" + render json: { error: 'Missing or invalid game data' }, status: :bad_request + return + end + + # Transform game data + transformer = ::Granblue::Transformers::BaseDeckTransformer.new(raw_params) + transformed_data = transformer.transform + Rails.logger.info "[IMPORT] Transformed data: #{transformed_data.inspect}" + + # Validate transformed data + unless transformed_data[:name].present? && transformed_data[:lang].present? + Rails.logger.error "[IMPORT] Missing required fields in transformed data" + render json: { error: 'Missing required fields name or lang' }, status: :unprocessable_entity + return + end + + # Create party + party = Party.new(user: current_user) + + ActiveRecord::Base.transaction do + # Basic party data + party.name = transformed_data[:name] + party.extra = transformed_data[:extra] + party.save! + + # Process job and skills + if transformed_data[:class].present? + process_job(party, transformed_data[:class], transformed_data[:subskills]) + end + + # Process characters + if transformed_data[:characters].present? + process_characters(party, transformed_data[:characters]) + end + + # Process weapons + if transformed_data[:weapons].present? + process_weapons(party, transformed_data[:weapons]) + end + + # Process summons + if transformed_data[:summons].present? + process_summons(party, transformed_data[:summons], transformed_data[:friend_summon]) + end + + # Process sub summons + if transformed_data[:sub_summons].present? + process_sub_summons(party, transformed_data[:sub_summons]) + end + end + + # Return shortcode for redirection + render json: { shortcode: party.shortcode }, status: :created + rescue StandardError => e + Rails.logger.error "[IMPORT] Error processing import: #{e.message}" + Rails.logger.error "[IMPORT] Backtrace: #{e.backtrace.join("\n")}" + render json: { error: 'Error processing import' }, status: :unprocessable_entity + end + + private + + def process_job(party, job_name, subskills) + return unless job_name + job = Job.find_by("name_en = ? OR name_jp = ?", job_name, job_name) + unless job + Rails.logger.warn "[IMPORT] Could not find job: #{job_name}" + return + end + + party.job = job + party.save! + Rails.logger.info "[IMPORT] Assigned job=#{job_name} to party_id=#{party.id}" + + return unless subskills&.any? + subskills.each_with_index do |skill_name, idx| + next if skill_name.blank? + skill = JobSkill.find_by("(name_en = ? OR name_jp = ?) AND job_id = ?", skill_name, skill_name, job.id) + unless skill + Rails.logger.warn "[IMPORT] Could not find skill=#{skill_name} for job_id=#{job.id}" + next + end + party["skill#{idx + 1}_id"] = skill.id + Rails.logger.info "[IMPORT] Assigned skill=#{skill_name} at position #{idx + 1}" + end + end + + def process_characters(party, characters) + return unless characters&.any? + Rails.logger.info "[IMPORT] Processing #{characters.length} characters" + + characters.each_with_index do |char_data, idx| + character = Character.find_by(granblue_id: char_data[:id]) + unless character + Rails.logger.warn "[IMPORT] Character not found: #{char_data[:id]}" + next + end + + GridCharacter.create!( + party: party, + character_id: character.id, + position: idx, + uncap_level: char_data[:uncap], + perpetuity: char_data[:ringed] || false, + transcendence_step: char_data[:transcend] || 0 + ) + Rails.logger.info "[IMPORT] Added character: #{character.name_en} at position #{idx}" + end + end + + def process_weapons(party, weapons) + return unless weapons&.any? + Rails.logger.info "[IMPORT] Processing #{weapons.length} weapons" + + weapons.each_with_index do |weapon_data, idx| + weapon = Weapon.find_by(granblue_id: weapon_data[:id]) + unless weapon + Rails.logger.warn "[IMPORT] Weapon not found: #{weapon_data[:id]}" + next + end + + grid_weapon = GridWeapon.create!( + party: party, + weapon_id: weapon.id, + position: idx - 1, + mainhand: idx.zero?, + uncap_level: weapon_data[:uncap], + transcendence_step: weapon_data[:transcend] || 0, + element: weapon_data[:attr] ? ELEMENT_MAPPING[weapon_data[:attr]] : nil + ) + + process_weapon_keys(grid_weapon, weapon_data[:keys]) if weapon_data[:keys] + process_weapon_ax(grid_weapon, weapon_data[:ax]) if weapon_data[:ax] + + Rails.logger.info "[IMPORT] Added weapon: #{weapon.name_en} at position #{idx - 1}" + end + end + + def process_weapon_keys(grid_weapon, keys) + keys.each_with_index do |key_id, idx| + key = WeaponKey.find_by(granblue_id: key_id) + unless key + Rails.logger.warn "[IMPORT] WeaponKey not found: #{key_id}" + next + end + grid_weapon["weapon_key#{idx + 1}_id"] = key.id + grid_weapon.save! + end + end + + def process_weapon_ax(grid_weapon, ax_skills) + ax_skills.each_with_index do |ax, idx| + grid_weapon["ax_modifier#{idx + 1}"] = ax[:id].to_i + grid_weapon["ax_strength#{idx + 1}"] = ax[:val].to_s.gsub(/[+%]/, '').to_i + end + grid_weapon.save! + end + + def process_summons(party, summons, friend_summon = nil) + return unless summons&.any? + Rails.logger.info "[IMPORT] Processing #{summons.length} summons" + + # Main and sub summons + summons.each_with_index do |summon_data, idx| + summon = Summon.find_by(granblue_id: summon_data[:id]) + unless summon + Rails.logger.warn "[IMPORT] Summon not found: #{summon_data[:id]}" + next + end + + grid_summon = GridSummon.new( + party: party, + summon_id: summon.id, + position: idx, + main: idx.zero?, + friend: false, + uncap_level: summon_data[:uncap], + transcendence_step: summon_data[:transcend] || 0, + quick_summon: summon_data[:qs] || false + ) + + if grid_summon.save + Rails.logger.info "[IMPORT] Added summon: #{summon.name_en} at position #{idx}" + else + Rails.logger.error "[IMPORT] Failed to save summon: #{grid_summon.errors.full_messages}" + end + end + + # Friend summon if provided + process_friend_summon(party, friend_summon) if friend_summon.present? + end + + def process_friend_summon(party, friend_summon) + friend = Summon.find_by("name_en = ? OR name_jp = ?", friend_summon, friend_summon) + unless friend + Rails.logger.warn "[IMPORT] Friend summon not found: #{friend_summon}" + return + end + + grid_summon = GridSummon.new( + party: party, + summon_id: friend.id, + position: 6, + main: false, + friend: true, + uncap_level: friend.ulb ? 5 : (friend.flb ? 4 : 3) + ) + + if grid_summon.save + Rails.logger.info "[IMPORT] Added friend summon: #{friend.name_en}" + else + Rails.logger.error "[IMPORT] Failed to save friend summon: #{grid_summon.errors.full_messages}" + end + end + + def process_sub_summons(party, sub_summons) + return unless sub_summons&.any? + Rails.logger.info "[IMPORT] Processing #{sub_summons.length} sub summons" + + sub_summons.each_with_index do |summon_data, idx| + summon = Summon.find_by(granblue_id: summon_data[:id]) + unless summon + Rails.logger.warn "[IMPORT] Sub summon not found: #{summon_data[:id]}" + next + end + + grid_summon = GridSummon.new( + party: party, + summon_id: summon.id, + position: idx + 5, + main: false, + friend: false, + uncap_level: summon_data[:uncap], + transcendence_step: summon_data[:transcend] || 0 + ) + + if grid_summon.save + Rails.logger.info "[IMPORT] Added sub summon: #{summon.name_en} at position #{idx + 5}" + else + Rails.logger.error "[IMPORT] Failed to save sub summon: #{grid_summon.errors.full_messages}" + end + end + end + end + end +end diff --git a/app/helpers/character_parser.rb b/app/helpers/character_parser.rb deleted file mode 100644 index 4c78a3f..0000000 --- a/app/helpers/character_parser.rb +++ /dev/null @@ -1,278 +0,0 @@ -# frozen_string_literal: true - -require 'pry' - -# CharacterParser parses character data from gbf.wiki -class CharacterParser - attr_reader :granblue_id - - def initialize(granblue_id: String, debug: false) - @character = Character.find_by(granblue_id: granblue_id) - @wiki = GranblueWiki.new - @debug = debug || false - end - - # Fetches using @wiki and then processes the response - # Returns true if successful, false if not - # Raises an exception if something went wrong - def fetch(save: false) - response = fetch_wiki_info - return false if response.nil? - - redirect = handle_redirected_string(response) - return fetch(save: save) unless redirect.nil? - - handle_fetch_success(response, save) - end - - private - - # Determines whether or not the response is a redirect - # If it is, it will update the character's wiki_en value - def handle_redirected_string(response) - redirect = extract_redirected_string(response) - return unless redirect - - @character.wiki_en = redirect - if @character.save! - ap "Saved new wiki_en value for #{@character.granblue_id}: #{redirect}" if @debug - redirect - else - ap "Unable to save new wiki_en value for #{@character.granblue_id}: #{redirect}" if @debug - nil - end - end - - # Handle the response from the wiki if the response is successful - # If the save flag is set, it will persist the data to the database - def handle_fetch_success(response, save) - ap "#{@character.granblue_id}: Successfully fetched info for #{@character.wiki_en}" if @debug - extracted = parse_string(response) - info = parse(extracted) - persist(info) if save - true - end - - # Determines whether the response string - # should be treated as a redirect - def extract_redirected_string(string) - string.match(/#REDIRECT \[\[(.*?)\]\]/)&.captures&.first - end - - # Parses the response string into a hash - def parse_string(string) - lines = string.split("\n") - data = {} - stop_loop = false - - lines.each do |line| - next if stop_loop - - if line.include?('Gameplay Notes') - stop_loop = true - next - end - - next unless line[0] == '|' && line.size > 2 - - key, value = line[1..].split('=', 2).map(&:strip) - data[key] = value if value - end - - data - end - - # Fetches data from the GranblueWiki object - def fetch_wiki_info - @wiki.fetch(@character.wiki_en) - rescue WikiError => e - ap "There was an error fetching #{e.page}: #{e.message}" if @debug - nil - end - - # Iterates over all characters in the database and fetches their data - # If the save flag is set, data is saved to the database - # If the overwrite flag is set, data is fetched even if it already exists - # If the debug flag is set, additional information is printed to the console - def self.fetch_all(save: false, overwrite: false, debug: false) - errors = [] - - count = Character.count - Character.all.each_with_index do |c, i| - percentage = ((i + 1) / count.to_f * 100).round(2) - ap "#{percentage}%: Fetching #{c.name_en}... (#{i + 1}/#{count})" if debug - next unless c.release_date.nil? || overwrite - - begin - CharacterParser.new(granblue_id: c.granblue_id, - debug: debug).fetch(save: save) - rescue WikiError => e - errors.push(e.page) - end - end - - ap 'The following pages were unable to be fetched:' - ap errors - end - - def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) - errors = [] - - start_index = start.nil? ? 0 : list.index { |id| id == start } - count = list.drop(start_index).count - - # ap "Start index: #{start_index}" - - list.drop(start_index).each_with_index do |id, i| - chara = Character.find_by(granblue_id: id) - percentage = ((i + 1) / count.to_f * 100).round(2) - ap "#{percentage}%: Fetching #{chara.wiki_en}... (#{i + 1}/#{count})" if debug - next unless chara.release_date.nil? || overwrite - - begin - WeaponParser.new(granblue_id: chara.granblue_id, - debug: debug).fetch(save: save) - rescue WikiError => e - errors.push(e.page) - end - end - - ap 'The following pages were unable to be fetched:' - ap errors - end - - # Parses the hash into a format that can be saved to the database - def parse(hash) - info = {} - - info[:name] = { en: hash['name'], ja: hash['jpname'] } - info[:id] = hash['id'] - info[:charid] = hash['charid'].scan(/\b\d{4}\b/) - - info[:flb] = GranblueWiki.boolean.fetch(hash['5star'], false) - info[:ulb] = hash['max_evo'].to_i == 6 - - info[:rarity] = GranblueWiki.rarities.fetch(hash['rarity'], 0) - info[:element] = GranblueWiki.elements.fetch(hash['element'], 0) - info[:gender] = GranblueWiki.genders.fetch(hash['gender'], 0) - - info[:proficiencies] = proficiencies_from_hash(hash['weapon']) - info[:races] = races_from_hash(hash['race']) - - info[:hp] = { - min_hp: hash['min_hp'].to_i, - max_hp: hash['max_hp'].to_i, - max_hp_flb: hash['flb_hp'].to_i - } - - info[:atk] = { - min_atk: hash['min_atk'].to_i, - max_atk: hash['max_atk'].to_i, - max_atk_flb: hash['flb_atk'].to_i - } - - info[:dates] = { - release_date: parse_date(hash['release_date']), - flb_date: parse_date(hash['5star_date']), - ulb_date: parse_date(hash['6star_date']) - } - - info[:links] = { - wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, - gamewith: hash['link_gamewith'], - kamigame: hash['link_kamigame'] - } - - info.compact - end - - # Saves select fields to the database - def persist(hash) - @character.release_date = hash[:dates][:release_date] - @character.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) - @character.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) - - @character.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) - @character.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) - @character.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) - - if @character.save - ap "#{@character.granblue_id}: Successfully saved info for #{@character.name_en}" if @debug - puts - true - end - - false - end - - # Converts proficiencies from a string to a hash - def proficiencies_from_hash(character) - character.to_s.split(',').map.with_index do |prof, i| - { "proficiency#{i + 1}" => GranblueWiki.proficiencies[prof] } - end.reduce({}, :merge) - end - - # Converts races from a string to a hash - def races_from_hash(race) - race.to_s.split(',').map.with_index do |r, i| - { "race#{i + 1}" => GranblueWiki.races[r] } - end.reduce({}, :merge) - end - - # Parses a date string into a Date object - def parse_date(date_str) - Date.parse(date_str) unless date_str.blank? - end - - # Unused methods for now - def extract_abilities(hash) - abilities = [] - hash.each do |key, value| - next unless key =~ /^a(\d+)_/ - - ability_number = Regexp.last_match(1).to_i - abilities[ability_number] ||= {} - - case key.gsub(/^a\d+_/, '') - when 'cd' - cooldown = parse_substring(value) - abilities[ability_number]['cooldown'] = cooldown - when 'dur' - duration = parse_substring(value) - abilities[ability_number]['duration'] = duration - when 'oblevel' - obtained = parse_substring(value) - abilities[ability_number]['obtained'] = obtained - else - abilities[ability_number][key.gsub(/^a\d+_/, '')] = value - end - end - - { 'abilities' => abilities.compact } - end - - def parse_substring(string) - hash = {} - - string.scan(/\|([^|=]+?)=([^|]+)/) do |key, value| - value.gsub!(/\}\}$/, '') if value.include?('}}') - hash[key] = value - end - - hash - end - - def extract_ougis(hash) - ougi = [] - hash.each do |key, value| - next unless key =~ /^ougi(\d*)_(.*)/ - - ougi_number = Regexp.last_match(1) - ougi_key = Regexp.last_match(2) - ougi[ougi_number.to_i] ||= {} - ougi[ougi_number.to_i][ougi_key] = value - end - - { 'ougis' => ougi.compact } - end -end diff --git a/app/helpers/granblue_wiki.rb b/app/helpers/granblue_wiki.rb deleted file mode 100644 index 3e006c2..0000000 --- a/app/helpers/granblue_wiki.rb +++ /dev/null @@ -1,118 +0,0 @@ -# frozen_string_literal: true - -require 'httparty' - -# GranblueWiki fetches and parses data from gbf.wiki -class GranblueWiki - class_attribute :base_uri - - class_attribute :proficiencies - class_attribute :elements - class_attribute :rarities - class_attribute :genders - class_attribute :races - class_attribute :bullets - class_attribute :boolean - - self.base_uri = 'https://gbf.wiki/api.php' - - self.proficiencies = { - 'Sabre' => 1, - 'Dagger' => 2, - 'Axe' => 3, - 'Spear' => 4, - 'Bow' => 5, - 'Staff' => 6, - 'Melee' => 7, - 'Harp' => 8, - 'Gun' => 9, - 'Katana' => 10 - }.freeze - - self.elements = { - 'Wind' => 1, - 'Fire' => 2, - 'Water' => 3, - 'Earth' => 4, - 'Dark' => 5, - 'Light' => 6 - }.freeze - - self.rarities = { - 'R' => 1, - 'SR' => 2, - 'SSR' => 3 - }.freeze - - self.races = { - 'Other' => 0, - 'Human' => 1, - 'Erune' => 2, - 'Draph' => 3, - 'Harvin' => 4, - 'Primal' => 5 - }.freeze - - self.genders = { - 'o' => 0, - 'm' => 1, - 'f' => 2, - 'mf' => 3 - }.freeze - - self.bullets = { - 'cartridge' => 1, - 'rifle' => 2, - 'parabellum' => 3, - 'aetherial' => 4 - }.freeze - - self.boolean = { - 'yes' => true, - 'no' => false - }.freeze - - def initialize(props: ['wikitext'], debug: false) - @debug = debug - @props = props.join('|') - end - - def fetch(page) - query_params = params(page).map do |key, value| - "#{key}=#{value}" - end.join('&') - - destination = "#{base_uri}?#{query_params}" - ap "--> Fetching #{destination}" if @debug - - response = HTTParty.get(destination) - - handle_response(response, page) - end - - private - - def handle_response(response, page) - case response.code - when 200 - if response.key?('error') - raise WikiError.new(code: response['error']['code'], - message: response['error']['info'], - page: page) - end - - response['parse']['wikitext']['*'] - when 404 then puts "Page #{page} not found" - when 500...600 then puts "Server error: #{response.code}" - end - end - - def params(page) - { - action: 'parse', - format: 'json', - page: page, - prop: @props - } - end -end diff --git a/app/helpers/summon_parser.rb b/app/helpers/summon_parser.rb deleted file mode 100644 index 045d43f..0000000 --- a/app/helpers/summon_parser.rb +++ /dev/null @@ -1,251 +0,0 @@ -# frozen_string_literal: true - -require 'pry' - -# SummonParser parses summon data from gbf.wiki -class SummonParser - attr_reader :granblue_id - - def initialize(granblue_id: String, debug: false) - @summon = Summon.find_by(granblue_id: granblue_id) - @wiki = GranblueWiki.new(debug: debug) - @debug = debug || false - end - - # Fetches using @wiki and then processes the response - # Returns true if successful, false if not - # Raises an exception if something went wrong - def fetch(name = nil, save: false) - response = fetch_wiki_info(name) - return false if response.nil? - - if response.starts_with?('#REDIRECT') - # Fetch the string inside of [[]] - redirect = response[/\[\[(.*?)\]\]/m, 1] - fetch(redirect, save: save) - else - # return response if response[:error] - handle_fetch_success(response, save) - end - end - - private - - # Handle the response from the wiki if the response is successful - # If the save flag is set, it will persist the data to the database - def handle_fetch_success(response, save) - ap "#{@summon.granblue_id}: Successfully fetched info for #{@summon.wiki_en}" if @debug - - extracted = parse_string(response) - - unless extracted[:template].nil? - template = @wiki.fetch("Template:#{extracted[:template]}") - extracted.merge!(parse_string(template)) - end - - info, skills = parse(extracted) - - # ap info - # ap skills - - persist(info[:info]) if save - true - end - - # Fetches the wiki info from the wiki - # Returns the response body - # Raises an exception if something went wrong - def fetch_wiki_info(name = nil) - @wiki.fetch(name || @summon.wiki_en) - rescue WikiError => e - ap e - # ap "There was an error fetching #{e.page}: #{e.message}" if @debug - { - error: { - name: @summon.wiki_en, - granblue_id: @summon.granblue_id - } - } - end - - # Iterates over all summons in the database and fetches their data - # If the save flag is set, data is saved to the database - # If the overwrite flag is set, data is fetched even if it already exists - # If the debug flag is set, additional information is printed to the console - def self.fetch_all(save: false, overwrite: false, debug: false, start: nil) - errors = [] - - summons = Summon.all.order(:granblue_id) - - start_index = start.nil? ? 0 : summons.index { |w| w.granblue_id == start } - count = summons.drop(start_index).count - - # ap "Start index: #{start_index}" - - summons.drop(start_index).each_with_index do |w, i| - percentage = ((i + 1) / count.to_f * 100).round(2) - ap "#{percentage}%: Fetching #{w.wiki_en}... (#{i + 1}/#{count})" if debug - next unless w.release_date.nil? || overwrite - - begin - SummonParser.new(granblue_id: w.granblue_id, - debug: debug).fetch(save: save) - rescue WikiError => e - errors.push(e.page) - end - end - - ap 'The following pages were unable to be fetched:' - ap errors - end - - def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) - errors = [] - - start_index = start.nil? ? 0 : list.index { |id| id == start } - count = list.drop(start_index).count - - # ap "Start index: #{start_index}" - - list.drop(start_index).each_with_index do |id, i| - summon = Summon.find_by(granblue_id: id) - percentage = ((i + 1) / count.to_f * 100).round(2) - ap "#{percentage}%: Fetching #{summon.wiki_en}... (#{i + 1}/#{count})" if debug - next unless summon.release_date.nil? || overwrite - - begin - SummonParser.new(granblue_id: summon.granblue_id, - debug: debug).fetch(save: save) - rescue WikiError => e - errors.push(e.page) - end - end - - ap 'The following pages were unable to be fetched:' - ap errors - end - - # Parses the response string into a hash - def parse_string(string) - data = {} - lines = string.split("\n") - stop_loop = false - - lines.each do |line| - next if stop_loop - - if line.include?('Gameplay Notes') - stop_loop = true - next - end - - if line.starts_with?('{{') - substr = line[2..].strip! || line[2..] - - # All template tags start with {{ so we can skip the first two characters - disallowed = %w[#vardefine #lsth About] - next if substr.start_with?(*disallowed) - - if substr.start_with?('Summon') - ap "--> Found template: #{substr}" if @debug - - substr = substr.split('|').first - data[:template] = substr if substr != 'Summon' - next - end - end - - next unless line[0] == '|' && line.size > 2 - - key, value = line[1..].split('=', 2).map(&:strip) - - regex = /\A\{\{\{.*\|\}\}\}\z/ - next if value =~ regex - - data[key] = value if value - end - - data - end - - # Parses the hash into a format that can be saved to the database - def parse(hash) - info = {} - skills = {} - - info[:name] = { en: hash['name'], ja: hash['jpname'] } - info[:flavor] = { en: hash['flavor'], ja: hash['jpflavor'] } - info[:id] = hash['id'] - - info[:flb] = hash['evo_max'].to_i >= 4 - info[:ulb] = hash['evo_max'].to_i >= 5 - info[:transcendence] = hash['evo_max'].to_i == 6 - - info[:rarity] = rarity_from_hash(hash['rarity']) - info[:series] = hash['series'] - info[:obtain] = hash['obtain'] - - info[:hp] = { - min_hp: hash['hp1'].to_i, - max_hp: hash['hp2'].to_i, - max_hp_flb: hash['hp3'].to_i, - max_hp_ulb: hash['hp4'].to_i.zero? ? nil : hash['hp4'].to_i, - max_hp_xlb: hash['hp5'].to_i.zero? ? nil : hash['hp5'].to_i - } - - info[:atk] = { - min_atk: hash['atk1'].to_i, - max_atk: hash['atk2'].to_i, - max_atk_flb: hash['atk3'].to_i, - max_atk_ulb: hash['atk4'].to_i.zero? ? nil : hash['atk4'].to_i, - max_atk_xlb: hash['atk5'].to_i.zero? ? nil : hash['atk5'].to_i - } - - info[:dates] = { - release_date: parse_date(hash['release_date']), - flb_date: parse_date(hash['4star_date']), - ulb_date: parse_date(hash['5star_date']), - transcendence_date: parse_date(hash['6star_date']) - } - - info[:links] = { - wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, - gamewith: hash['link_gamewith'], - kamigame: hash['link_kamigame'] - } - - { - info: info.compact - # skills: skills.compact - } - end - - # Saves select fields to the database - def persist(hash) - @summon.release_date = hash[:dates][:release_date] - @summon.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) - @summon.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) - - @summon.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) - @summon.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) - @summon.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) - - if @summon.save - ap "#{@summon.granblue_id}: Successfully saved info for #{@summon.wiki_en}" if @debug - puts - true - end - - false - end - - # Converts rarities from a string to a hash - def rarity_from_hash(string) - string ? GranblueWiki.rarities[string.upcase] : nil - end - - # Parses a date string into a Date object - def parse_date(date_str) - Date.parse(date_str) unless date_str.blank? - end -end diff --git a/app/helpers/validation_error_serializer.rb b/app/helpers/validation_error_serializer.rb deleted file mode 100644 index ad22738..0000000 --- a/app/helpers/validation_error_serializer.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -class ValidationErrorSerializer - def initialize(record, field, details) - @record = record - @field = field - @details = details - end - - def serialize - { - resource: resource, - field: field, - code: code - } - end - - private - - def resource - @record.class.to_s - end - - def field - @field.to_s - end - - def code - @details[:error].to_s - end - - def underscored_resource_name - @record.class.to_s.gsub('::', '').underscore - end -end diff --git a/app/helpers/validation_errors_serializer.rb b/app/helpers/validation_errors_serializer.rb deleted file mode 100644 index 820ebd2..0000000 --- a/app/helpers/validation_errors_serializer.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class ValidationErrorsSerializer - attr_reader :record - - def initialize(record) - @record = record - end - - def serialize - record.errors.details.map do |field, details| - details.map do |error_details| - ValidationErrorSerializer.new(record, field, error_details).serialize - end - end.flatten - end -end diff --git a/app/helpers/weapon_parser.rb b/app/helpers/weapon_parser.rb deleted file mode 100644 index 81abdb8..0000000 --- a/app/helpers/weapon_parser.rb +++ /dev/null @@ -1,296 +0,0 @@ -# frozen_string_literal: true - -require 'pry' - -# WeaponParser parses weapon data from gbf.wiki -class WeaponParser - attr_reader :granblue_id - - def initialize(granblue_id: String, debug: false) - @weapon = Weapon.find_by(granblue_id: granblue_id) - @wiki = GranblueWiki.new(debug: debug) - @debug = debug || false - end - - # Fetches using @wiki and then processes the response - # Returns true if successful, false if not - # Raises an exception if something went wrong - def fetch(save: false) - response = fetch_wiki_info - return false if response.nil? - - # return response if response[:error] - - handle_fetch_success(response, save) - end - - private - - # Handle the response from the wiki if the response is successful - # If the save flag is set, it will persist the data to the database - def handle_fetch_success(response, save) - ap "#{@weapon.granblue_id}: Successfully fetched info for #{@weapon.wiki_en}" if @debug - extracted = parse_string(response) - - unless extracted[:template].nil? - template = @wiki.fetch("Template:#{extracted[:template]}") - extracted.merge!(parse_string(template)) - end - - info, skills = parse(extracted) - - # ap info - # ap skills - - persist(info[:info]) if save - true - end - - # Fetches the wiki info from the wiki - # Returns the response body - # Raises an exception if something went wrong - def fetch_wiki_info - @wiki.fetch(@weapon.wiki_en) - rescue WikiError => e - ap e - # ap "There was an error fetching #{e.page}: #{e.message}" if @debug - { - error: { - name: @weapon.wiki_en, - granblue_id: @weapon.granblue_id - } - } - end - - # Iterates over all weapons in the database and fetches their data - # If the save flag is set, data is saved to the database - # If the overwrite flag is set, data is fetched even if it already exists - # If the debug flag is set, additional information is printed to the console - def self.fetch_all(save: false, overwrite: false, debug: false, start: nil) - errors = [] - - weapons = Weapon.all.order(:granblue_id) - - start_index = start.nil? ? 0 : weapons.index { |w| w.granblue_id == start } - count = weapons.drop(start_index).count - - # ap "Start index: #{start_index}" - - weapons.drop(start_index).each_with_index do |w, i| - percentage = ((i + 1) / count.to_f * 100).round(2) - ap "#{percentage}%: Fetching #{w.wiki_en}... (#{i + 1}/#{count})" if debug - next if w.wiki_en.include?('Element Changed') || w.wiki_en.include?('Awakened') - next unless w.release_date.nil? || overwrite - - begin - WeaponParser.new(granblue_id: w.granblue_id, - debug: debug).fetch(save: save) - rescue WikiError => e - errors.push(e.page) - end - end - - ap 'The following pages were unable to be fetched:' - ap errors - end - - def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) - errors = [] - - start_index = start.nil? ? 0 : list.index { |id| id == start } - count = list.drop(start_index).count - - # ap "Start index: #{start_index}" - - list.drop(start_index).each_with_index do |id, i| - weapon = Weapon.find_by(granblue_id: id) - percentage = ((i + 1) / count.to_f * 100).round(2) - ap "#{percentage}%: Fetching #{weapon.wiki_en}... (#{i + 1}/#{count})" if debug - next unless weapon.release_date.nil? || overwrite - - begin - WeaponParser.new(granblue_id: weapon.granblue_id, - debug: debug).fetch(save: save) - rescue WikiError => e - errors.push(e.page) - end - end - - ap 'The following pages were unable to be fetched:' - ap errors - end - - # Parses the response string into a hash - def parse_string(string) - data = {} - lines = string.split("\n") - stop_loop = false - - lines.each do |line| - next if stop_loop - - if line.include?('Gameplay Notes') - stop_loop = true - next - end - - if line.starts_with?('{{') - substr = line[2..].strip! || line[2..] - - # All template tags start with {{ so we can skip the first two characters - disallowed = %w[#vardefine #lsth About] - next if substr.start_with?(*disallowed) - - if substr.start_with?('Weapon') - ap "--> Found template: #{substr}" if @debug - - substr = substr.split('|').first - data[:template] = substr if substr != 'Weapon' - next - end - end - - next unless line[0] == '|' && line.size > 2 - - key, value = line[1..].split('=', 2).map(&:strip) - - regex = /\A\{\{\{.*\|\}\}\}\z/ - next if value =~ regex - - data[key] = value if value - end - - data - end - - # Parses the hash into a format that can be saved to the database - def parse(hash) - info = {} - skills = {} - - info[:name] = { en: hash['name'], ja: hash['jpname'] } - info[:flavor] = { en: hash['flavor'], ja: hash['jpflavor'] } - info[:id] = hash['id'] - - info[:flb] = hash['evo_max'].to_i >= 4 - info[:ulb] = hash['evo_max'].to_i == 5 - - info[:rarity] = rarity_from_hash(hash['rarity']) - info[:proficiency] = proficiency_from_hash(hash['weapon']) - info[:series] = hash['series'] - info[:obtain] = hash['obtain'] - - if hash.key?('bullets') - info[:bullets] = { - count: hash['bullets'].to_i, - loadout: [ - bullet_from_hash(hash['bullet1']), - bullet_from_hash(hash['bullet2']), - bullet_from_hash(hash['bullet3']), - bullet_from_hash(hash['bullet4']), - bullet_from_hash(hash['bullet5']), - bullet_from_hash(hash['bullet6']) - ] - } - end - - info[:hp] = { - min_hp: hash['hp1'].to_i, - max_hp: hash['hp2'].to_i, - max_hp_flb: hash['hp3'].to_i, - max_hp_ulb: hash['hp4'].to_i.zero? ? nil : hash['hp4'].to_i - } - - info[:atk] = { - min_atk: hash['atk1'].to_i, - max_atk: hash['atk2'].to_i, - max_atk_flb: hash['atk3'].to_i, - max_atk_ulb: hash['atk4'].to_i.zero? ? nil : hash['atk4'].to_i - } - - info[:dates] = { - release_date: parse_date(hash['release_date']), - flb_date: parse_date(hash['4star_date']), - ulb_date: parse_date(hash['5star_date']) - } - - info[:links] = { - wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, - gamewith: hash['link_gamewith'], - kamigame: hash['link_kamigame'] - } - - skills[:charge_attack] = { - name: { en: hash['ougi_name'], ja: hash['jpougi_name'] }, - description: { - mlb: { - en: hash['enougi'], - ja: hash['jpougi'] - }, - flb: { - en: hash['enougi_4s'], - ja: hash['jpougi_4s'] - } - } - } - - skills[:skills] = [ - { - name: { en: hash['s1_name'], ja: nil }, - description: { en: hash['ens1_desc'] || hash['s1_desc'], ja: nil } - }, - { - name: { en: hash['s2_name'], ja: nil }, - description: { en: hash['ens2_desc'] || hash['s2_desc'], ja: nil } - }, - { - name: { en: hash['s3_name'], ja: nil }, - description: { en: hash['ens3_desc'] || hash['s3_desc'], ja: nil } - } - ] - - { - info: info.compact, - skills: skills.compact - } - end - - # Saves select fields to the database - def persist(hash) - @weapon.release_date = hash[:dates][:release_date] - @weapon.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) - @weapon.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) - - @weapon.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) - @weapon.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) - @weapon.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) - - if @weapon.save - ap "#{@weapon.granblue_id}: Successfully saved info for #{@weapon.wiki_en}" if @debug - puts - true - end - - false - end - - # Converts rarities from a string to a hash - def rarity_from_hash(string) - string ? GranblueWiki.rarities[string.upcase] : nil - end - - # Converts proficiencies from a string to a hash - def proficiency_from_hash(string) - GranblueWiki.proficiencies[string] - end - - # Converts a bullet type from a string to a hash - def bullet_from_hash(string) - string ? GranblueWiki.bullets[string] : nil - end - - # Parses a date string into a Date object - def parse_date(date_str) - Date.parse(date_str) unless date_str.blank? - end -end diff --git a/config/application.rb b/config/application.rb index d5011bc..399fabe 100644 --- a/config/application.rb +++ b/config/application.rb @@ -30,6 +30,9 @@ module HenseiApi # # config.time_zone = "Central Time (US & Canada)" # config.eager_load_paths << Rails.root.join("extras") + + config.autoload_paths << Rails.root.join("lib") + config.eager_load_paths << Rails.root.join("lib") # Only loads a smaller set of middleware suitable for API only apps. # Middleware like session, flash, cookies can be added back manually. diff --git a/config/initializers/cors.rb b/config/initializers/cors.rb index 339a4e9..14c0860 100644 --- a/config/initializers/cors.rb +++ b/config/initializers/cors.rb @@ -8,9 +8,9 @@ Rails.application.config.middleware.insert_before 0, Rack::Cors do allow do if Rails.env.production? - origins %w[granblue.team app.granblue.team hensei-web-production.up.railway.app] + origins %w[granblue.team app.granblue.team hensei-web-production.up.railway.app game.granbluefantasy.jp chrome-extension://ahacbogimbikgiodaahmacboojcpdfpf] else - origins %w[staging.granblue.team 127.0.0.1:1234] + origins %w[staging.granblue.team 127.0.0.1:1234 game.granbluefantasy.jp chrome-extension://ahacbogimbikgiodaahmacboojcpdfpf] end resource '*', diff --git a/config/routes.rb b/config/routes.rb index bc8811d..4a7e1dc 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -18,6 +18,8 @@ Rails.application.routes.draw do get 'version', to: 'api#version' + post 'import', to: 'import#create' + get 'users/info/:id', to: 'users#info' get 'parties/favorites', to: 'parties#favorites' diff --git a/lib/granblue/data_importer.rb b/lib/granblue/data_importer.rb deleted file mode 100644 index 1be5b54..0000000 --- a/lib/granblue/data_importer.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true - -module Granblue - class DataImporter - def initialize(test_mode: false, verbose: false) - @test_mode = test_mode - @verbose = verbose - @import_logs = [] - end - - def process_all_files(&block) - files = Dir.glob(Rails.root.join('db', 'seed', 'updates', '*.csv')).sort - - files.each do |file| - if (new_records = import_csv(file)) - block.call(new_records) if block_given? - end - end - - print_summary if @test_mode - end - - private - - def import_csv(file_path) - filename = File.basename(file_path) - return if already_imported?(filename) - - importer = create_importer(filename, file_path) - return unless importer - - log_info "Processing #{filename} in #{@test_mode ? 'test' : 'live'} mode..." - result = importer.import - log_import(filename, result) - log_info "Successfully processed #{filename}" - result - end - - def log_import_results(result) - return unless @verbose - - result[:new].each do |type, ids| - log_info "Created #{ids.size} new #{type.pluralize}" if ids.any? - end - result[:updated].each do |type, ids| - log_info "Updated #{ids.size} existing #{type.pluralize}" if ids.any? - end - end - - def create_importer(filename, file_path) - # This pattern matches both singular and plural: character(s), weapon(s), summon(s) - match = filename.match(/\A\d{8}-(character(?:s)?|weapon(?:s)?|summon(?:s)?)-\d+\.csv\z/) - return unless match - - matched_type = match[1] - singular_type = matched_type.sub(/s$/, '') - importer_class = "Granblue::Importers::#{singular_type.capitalize}Importer".constantize - - importer_class.new( - file_path, - test_mode: @test_mode, - verbose: @verbose, - logger: self - ) - rescue NameError - log_info "No importer found for type: #{singular_type}" - nil - end - - def already_imported?(filename) - DataVersion.imported?(filename) - end - - def log_import(filename, result = nil) - return if @test_mode - - DataVersion.mark_as_imported(filename) - - if result && @verbose - result[:new].each do |type, ids| - log_info "Created #{ids.size} new #{type.pluralize}" if ids.any? - end - result[:updated].each do |type, ids| - log_info "Updated #{ids.size} existing #{type.pluralize}" if ids.any? - end - end - end - - def log_operation(operation) - if @test_mode - @import_logs << operation - log_info "[TEST MODE] Would perform: #{operation}" - end - end - - def print_summary - log_info "\nTest Mode Summary:" - log_info "Would perform #{@import_logs.size} operations" - if @import_logs.any? - log_info 'Sample of operations:' - @import_logs.first(3).each { |log| log_info "- #{log}" } - log_info '...' if @import_logs.size > 3 - end - end - - def log_info(message) - puts message if @verbose || @test_mode - end - end -end diff --git a/lib/granblue/downloaders/base_downloader.rb b/lib/granblue/downloaders/base_downloader.rb index f65e71e..09aaac8 100644 --- a/lib/granblue/downloaders/base_downloader.rb +++ b/lib/granblue/downloaders/base_downloader.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Granblue - module Downloader + module Downloaders class BaseDownloader SIZES = %w[main grid square].freeze diff --git a/lib/granblue/downloaders/character_downloader.rb b/lib/granblue/downloaders/character_downloader.rb index 2ad492b..825b1d8 100644 --- a/lib/granblue/downloaders/character_downloader.rb +++ b/lib/granblue/downloaders/character_downloader.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Granblue - module Downloader + module Downloaders class CharacterDownloader < BaseDownloader def download character = Character.find_by(granblue_id: @id) diff --git a/lib/granblue/download_manager.rb b/lib/granblue/downloaders/download_manager.rb similarity index 97% rename from lib/granblue/download_manager.rb rename to lib/granblue/downloaders/download_manager.rb index 9c2b20b..db2cfe8 100644 --- a/lib/granblue/download_manager.rb +++ b/lib/granblue/downloaders/download_manager.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Granblue - module Downloader + module Downloaders class DownloadManager class << self def download_for_object(type, granblue_id, test_mode: false, verbose: false, storage: :both) diff --git a/lib/granblue/downloaders/elemental_weapon_downloader.rb b/lib/granblue/downloaders/elemental_weapon_downloader.rb index 9ad5e79..2525c3d 100644 --- a/lib/granblue/downloaders/elemental_weapon_downloader.rb +++ b/lib/granblue/downloaders/elemental_weapon_downloader.rb @@ -3,7 +3,7 @@ require_relative 'weapon_downloader' module Granblue - module Downloader + module Downloaders class ElementalWeaponDownloader < WeaponDownloader SUFFIXES = [2, 3, 4, 1, 6, 5].freeze diff --git a/lib/granblue/downloaders/summon_downloader.rb b/lib/granblue/downloaders/summon_downloader.rb index 5874f78..20fcfc3 100644 --- a/lib/granblue/downloaders/summon_downloader.rb +++ b/lib/granblue/downloaders/summon_downloader.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Granblue - module Downloader + module Downloaders class SummonDownloader < BaseDownloader def download summon = Summon.find_by(granblue_id: @id) diff --git a/lib/granblue/downloaders/weapon_downloader.rb b/lib/granblue/downloaders/weapon_downloader.rb index 5879ff0..66aeae2 100644 --- a/lib/granblue/downloaders/weapon_downloader.rb +++ b/lib/granblue/downloaders/weapon_downloader.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Granblue - module Downloader + module Downloaders class WeaponDownloader < BaseDownloader def download weapon = Weapon.find_by(granblue_id: @id) diff --git a/lib/granblue/parsers/character_parser.rb b/lib/granblue/parsers/character_parser.rb new file mode 100644 index 0000000..b98a3e8 --- /dev/null +++ b/lib/granblue/parsers/character_parser.rb @@ -0,0 +1,283 @@ +# frozen_string_literal: true + +require 'pry' + +module Granblue + module Parsers + + # CharacterParser parses character data from gbf.wiki + class CharacterParser + attr_reader :granblue_id + + def initialize(granblue_id: String, debug: false) + @character = Character.find_by(granblue_id: granblue_id) + @wiki = GranblueWiki.new + @debug = debug || false + end + + # Fetches using @wiki and then processes the response + # Returns true if successful, false if not + # Raises an exception if something went wrong + def fetch(save: false) + response = fetch_wiki_info + return false if response.nil? + + redirect = handle_redirected_string(response) + return fetch(save: save) unless redirect.nil? + + handle_fetch_success(response, save) + end + + private + + # Determines whether or not the response is a redirect + # If it is, it will update the character's wiki_en value + def handle_redirected_string(response) + redirect = extract_redirected_string(response) + return unless redirect + + @character.wiki_en = redirect + if @character.save! + ap "Saved new wiki_en value for #{@character.granblue_id}: #{redirect}" if @debug + redirect + else + ap "Unable to save new wiki_en value for #{@character.granblue_id}: #{redirect}" if @debug + nil + end + end + + # Handle the response from the wiki if the response is successful + # If the save flag is set, it will persist the data to the database + def handle_fetch_success(response, save) + ap "#{@character.granblue_id}: Successfully fetched info for #{@character.wiki_en}" if @debug + extracted = parse_string(response) + info = parse(extracted) + persist(info) if save + true + end + + # Determines whether the response string + # should be treated as a redirect + def extract_redirected_string(string) + string.match(/#REDIRECT \[\[(.*?)\]\]/)&.captures&.first + end + + # Parses the response string into a hash + def parse_string(string) + lines = string.split("\n") + data = {} + stop_loop = false + + lines.each do |line| + next if stop_loop + + if line.include?('Gameplay Notes') + stop_loop = true + next + end + + next unless line[0] == '|' && line.size > 2 + + key, value = line[1..].split('=', 2).map(&:strip) + data[key] = value if value + end + + data + end + + # Fetches data from the GranblueWiki object + def fetch_wiki_info + @wiki.fetch(@character.wiki_en) + rescue WikiError => e + ap "There was an error fetching #{e.page}: #{e.message}" if @debug + nil + end + + # Iterates over all characters in the database and fetches their data + # If the save flag is set, data is saved to the database + # If the overwrite flag is set, data is fetched even if it already exists + # If the debug flag is set, additional information is printed to the console + def self.fetch_all(save: false, overwrite: false, debug: false) + errors = [] + + count = Character.count + Character.all.each_with_index do |c, i| + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{c.name_en}... (#{i + 1}/#{count})" if debug + next unless c.release_date.nil? || overwrite + + begin + CharacterParser.new(granblue_id: c.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) + errors = [] + + start_index = start.nil? ? 0 : list.index { |id| id == start } + count = list.drop(start_index).count + + # ap "Start index: #{start_index}" + + list.drop(start_index).each_with_index do |id, i| + chara = Character.find_by(granblue_id: id) + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{chara.wiki_en}... (#{i + 1}/#{count})" if debug + next unless chara.release_date.nil? || overwrite + + begin + WeaponParser.new(granblue_id: chara.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + # Parses the hash into a format that can be saved to the database + def parse(hash) + info = {} + + info[:name] = { en: hash['name'], ja: hash['jpname'] } + info[:id] = hash['id'] + info[:charid] = hash['charid'].scan(/\b\d{4}\b/) + + info[:flb] = GranblueWiki.boolean.fetch(hash['5star'], false) + info[:ulb] = hash['max_evo'].to_i == 6 + + info[:rarity] = GranblueWiki.rarities.fetch(hash['rarity'], 0) + info[:element] = GranblueWiki.elements.fetch(hash['element'], 0) + info[:gender] = GranblueWiki.genders.fetch(hash['gender'], 0) + + info[:proficiencies] = proficiencies_from_hash(hash['weapon']) + info[:races] = races_from_hash(hash['race']) + + info[:hp] = { + min_hp: hash['min_hp'].to_i, + max_hp: hash['max_hp'].to_i, + max_hp_flb: hash['flb_hp'].to_i + } + + info[:atk] = { + min_atk: hash['min_atk'].to_i, + max_atk: hash['max_atk'].to_i, + max_atk_flb: hash['flb_atk'].to_i + } + + info[:dates] = { + release_date: parse_date(hash['release_date']), + flb_date: parse_date(hash['5star_date']), + ulb_date: parse_date(hash['6star_date']) + } + + info[:links] = { + wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, + gamewith: hash['link_gamewith'], + kamigame: hash['link_kamigame'] + } + + info.compact + end + + # Saves select fields to the database + def persist(hash) + @character.release_date = hash[:dates][:release_date] + @character.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) + @character.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) + + @character.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) + @character.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) + @character.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) + + if @character.save + ap "#{@character.granblue_id}: Successfully saved info for #{@character.name_en}" if @debug + puts + true + end + + false + end + + # Converts proficiencies from a string to a hash + def proficiencies_from_hash(character) + character.to_s.split(',').map.with_index do |prof, i| + { "proficiency#{i + 1}" => GranblueWiki.proficiencies[prof] } + end.reduce({}, :merge) + end + + # Converts races from a string to a hash + def races_from_hash(race) + race.to_s.split(',').map.with_index do |r, i| + { "race#{i + 1}" => GranblueWiki.races[r] } + end.reduce({}, :merge) + end + + # Parses a date string into a Date object + def parse_date(date_str) + Date.parse(date_str) unless date_str.blank? + end + + # Unused methods for now + def extract_abilities(hash) + abilities = [] + hash.each do |key, value| + next unless key =~ /^a(\d+)_/ + + ability_number = Regexp.last_match(1).to_i + abilities[ability_number] ||= {} + + case key.gsub(/^a\d+_/, '') + when 'cd' + cooldown = parse_substring(value) + abilities[ability_number]['cooldown'] = cooldown + when 'dur' + duration = parse_substring(value) + abilities[ability_number]['duration'] = duration + when 'oblevel' + obtained = parse_substring(value) + abilities[ability_number]['obtained'] = obtained + else + abilities[ability_number][key.gsub(/^a\d+_/, '')] = value + end + end + + { 'abilities' => abilities.compact } + end + + def parse_substring(string) + hash = {} + + string.scan(/\|([^|=]+?)=([^|]+)/) do |key, value| + value.gsub!(/\}\}$/, '') if value.include?('}}') + hash[key] = value + end + + hash + end + + def extract_ougis(hash) + ougi = [] + hash.each do |key, value| + next unless key =~ /^ougi(\d*)_(.*)/ + + ougi_number = Regexp.last_match(1) + ougi_key = Regexp.last_match(2) + ougi[ougi_number.to_i] ||= {} + ougi[ougi_number.to_i][ougi_key] = value + end + + { 'ougis' => ougi.compact } + end + end + end +end diff --git a/lib/granblue/parsers/summon_parser.rb b/lib/granblue/parsers/summon_parser.rb new file mode 100644 index 0000000..5150525 --- /dev/null +++ b/lib/granblue/parsers/summon_parser.rb @@ -0,0 +1,255 @@ +# frozen_string_literal: true + +require 'pry' + +module Granblue + module Parsers + # SummonParser parses summon data from gbf.wiki + class SummonParser + attr_reader :granblue_id + + def initialize(granblue_id: String, debug: false) + @summon = Summon.find_by(granblue_id: granblue_id) + @wiki = GranblueWiki.new(debug: debug) + @debug = debug || false + end + + # Fetches using @wiki and then processes the response + # Returns true if successful, false if not + # Raises an exception if something went wrong + def fetch(name = nil, save: false) + response = fetch_wiki_info(name) + return false if response.nil? + + if response.starts_with?('#REDIRECT') + # Fetch the string inside of [[]] + redirect = response[/\[\[(.*?)\]\]/m, 1] + fetch(redirect, save: save) + else + # return response if response[:error] + handle_fetch_success(response, save) + end + end + + private + + # Handle the response from the wiki if the response is successful + # If the save flag is set, it will persist the data to the database + def handle_fetch_success(response, save) + ap "#{@summon.granblue_id}: Successfully fetched info for #{@summon.wiki_en}" if @debug + + extracted = parse_string(response) + + unless extracted[:template].nil? + template = @wiki.fetch("Template:#{extracted[:template]}") + extracted.merge!(parse_string(template)) + end + + info, skills = parse(extracted) + + # ap info + # ap skills + + persist(info[:info]) if save + true + end + + # Fetches the wiki info from the wiki + # Returns the response body + # Raises an exception if something went wrong + def fetch_wiki_info(name = nil) + @wiki.fetch(name || @summon.wiki_en) + rescue WikiError => e + ap e + # ap "There was an error fetching #{e.page}: #{e.message}" if @debug + { + error: { + name: @summon.wiki_en, + granblue_id: @summon.granblue_id + } + } + end + + # Iterates over all summons in the database and fetches their data + # If the save flag is set, data is saved to the database + # If the overwrite flag is set, data is fetched even if it already exists + # If the debug flag is set, additional information is printed to the console + def self.fetch_all(save: false, overwrite: false, debug: false, start: nil) + errors = [] + + summons = Summon.all.order(:granblue_id) + + start_index = start.nil? ? 0 : summons.index { |w| w.granblue_id == start } + count = summons.drop(start_index).count + + # ap "Start index: #{start_index}" + + summons.drop(start_index).each_with_index do |w, i| + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{w.wiki_en}... (#{i + 1}/#{count})" if debug + next unless w.release_date.nil? || overwrite + + begin + SummonParser.new(granblue_id: w.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) + errors = [] + + start_index = start.nil? ? 0 : list.index { |id| id == start } + count = list.drop(start_index).count + + # ap "Start index: #{start_index}" + + list.drop(start_index).each_with_index do |id, i| + summon = Summon.find_by(granblue_id: id) + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{summon.wiki_en}... (#{i + 1}/#{count})" if debug + next unless summon.release_date.nil? || overwrite + + begin + SummonParser.new(granblue_id: summon.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + # Parses the response string into a hash + def parse_string(string) + data = {} + lines = string.split("\n") + stop_loop = false + + lines.each do |line| + next if stop_loop + + if line.include?('Gameplay Notes') + stop_loop = true + next + end + + if line.starts_with?('{{') + substr = line[2..].strip! || line[2..] + + # All template tags start with {{ so we can skip the first two characters + disallowed = %w[#vardefine #lsth About] + next if substr.start_with?(*disallowed) + + if substr.start_with?('Summon') + ap "--> Found template: #{substr}" if @debug + + substr = substr.split('|').first + data[:template] = substr if substr != 'Summon' + next + end + end + + next unless line[0] == '|' && line.size > 2 + + key, value = line[1..].split('=', 2).map(&:strip) + + regex = /\A\{\{\{.*\|\}\}\}\z/ + next if value =~ regex + + data[key] = value if value + end + + data + end + + # Parses the hash into a format that can be saved to the database + def parse(hash) + info = {} + skills = {} + + info[:name] = { en: hash['name'], ja: hash['jpname'] } + info[:flavor] = { en: hash['flavor'], ja: hash['jpflavor'] } + info[:id] = hash['id'] + + info[:flb] = hash['evo_max'].to_i >= 4 + info[:ulb] = hash['evo_max'].to_i >= 5 + info[:transcendence] = hash['evo_max'].to_i == 6 + + info[:rarity] = rarity_from_hash(hash['rarity']) + info[:series] = hash['series'] + info[:obtain] = hash['obtain'] + + info[:hp] = { + min_hp: hash['hp1'].to_i, + max_hp: hash['hp2'].to_i, + max_hp_flb: hash['hp3'].to_i, + max_hp_ulb: hash['hp4'].to_i.zero? ? nil : hash['hp4'].to_i, + max_hp_xlb: hash['hp5'].to_i.zero? ? nil : hash['hp5'].to_i + } + + info[:atk] = { + min_atk: hash['atk1'].to_i, + max_atk: hash['atk2'].to_i, + max_atk_flb: hash['atk3'].to_i, + max_atk_ulb: hash['atk4'].to_i.zero? ? nil : hash['atk4'].to_i, + max_atk_xlb: hash['atk5'].to_i.zero? ? nil : hash['atk5'].to_i + } + + info[:dates] = { + release_date: parse_date(hash['release_date']), + flb_date: parse_date(hash['4star_date']), + ulb_date: parse_date(hash['5star_date']), + transcendence_date: parse_date(hash['6star_date']) + } + + info[:links] = { + wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, + gamewith: hash['link_gamewith'], + kamigame: hash['link_kamigame'] + } + + { + info: info.compact + # skills: skills.compact + } + end + + # Saves select fields to the database + def persist(hash) + @summon.release_date = hash[:dates][:release_date] + @summon.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) + @summon.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) + + @summon.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) + @summon.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) + @summon.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) + + if @summon.save + ap "#{@summon.granblue_id}: Successfully saved info for #{@summon.wiki_en}" if @debug + puts + true + end + + false + end + + # Converts rarities from a string to a hash + def rarity_from_hash(string) + string ? GranblueWiki.rarities[string.upcase] : nil + end + + # Parses a date string into a Date object + def parse_date(date_str) + Date.parse(date_str) unless date_str.blank? + end + end + end +end diff --git a/lib/granblue/parsers/validation_error_serializer.rb b/lib/granblue/parsers/validation_error_serializer.rb new file mode 100644 index 0000000..34a3be7 --- /dev/null +++ b/lib/granblue/parsers/validation_error_serializer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Granblue + module Parsers + class ValidationErrorSerializer + def initialize(record, field, details) + @record = record + @field = field + @details = details + end + + def serialize + { + resource: resource, + field: field, + code: code + } + end + + private + + def resource + @record.class.to_s + end + + def field + @field.to_s + end + + def code + @details[:error].to_s + end + + def underscored_resource_name + @record.class.to_s.gsub('::', '').underscore + end + end + end +end diff --git a/lib/granblue/parsers/validation_errors_serializer.rb b/lib/granblue/parsers/validation_errors_serializer.rb new file mode 100644 index 0000000..60898d7 --- /dev/null +++ b/lib/granblue/parsers/validation_errors_serializer.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Granblue + module Parsers + + class ValidationErrorsSerializer + attr_reader :record + + def initialize(record) + @record = record + end + + def serialize + record.errors.details.map do |field, details| + details.map do |error_details| + ValidationErrorSerializer.new(record, field, error_details).serialize + end + end.flatten + end + end + end +end diff --git a/lib/granblue/parsers/weapon_parser.rb b/lib/granblue/parsers/weapon_parser.rb new file mode 100644 index 0000000..30d44dd --- /dev/null +++ b/lib/granblue/parsers/weapon_parser.rb @@ -0,0 +1,300 @@ +# frozen_string_literal: true + +require 'pry' + +module Granblue + module Parsers + # WeaponParser parses weapon data from gbf.wiki + class WeaponParser + attr_reader :granblue_id + + def initialize(granblue_id: String, debug: false) + @weapon = Weapon.find_by(granblue_id: granblue_id) + @wiki = GranblueWiki.new(debug: debug) + @debug = debug || false + end + + # Fetches using @wiki and then processes the response + # Returns true if successful, false if not + # Raises an exception if something went wrong + def fetch(save: false) + response = fetch_wiki_info + return false if response.nil? + + # return response if response[:error] + + handle_fetch_success(response, save) + end + + private + + # Handle the response from the wiki if the response is successful + # If the save flag is set, it will persist the data to the database + def handle_fetch_success(response, save) + ap "#{@weapon.granblue_id}: Successfully fetched info for #{@weapon.wiki_en}" if @debug + extracted = parse_string(response) + + unless extracted[:template].nil? + template = @wiki.fetch("Template:#{extracted[:template]}") + extracted.merge!(parse_string(template)) + end + + info, skills = parse(extracted) + + # ap info + # ap skills + + persist(info[:info]) if save + true + end + + # Fetches the wiki info from the wiki + # Returns the response body + # Raises an exception if something went wrong + def fetch_wiki_info + @wiki.fetch(@weapon.wiki_en) + rescue WikiError => e + ap e + # ap "There was an error fetching #{e.page}: #{e.message}" if @debug + { + error: { + name: @weapon.wiki_en, + granblue_id: @weapon.granblue_id + } + } + end + + # Iterates over all weapons in the database and fetches their data + # If the save flag is set, data is saved to the database + # If the overwrite flag is set, data is fetched even if it already exists + # If the debug flag is set, additional information is printed to the console + def self.fetch_all(save: false, overwrite: false, debug: false, start: nil) + errors = [] + + weapons = Weapon.all.order(:granblue_id) + + start_index = start.nil? ? 0 : weapons.index { |w| w.granblue_id == start } + count = weapons.drop(start_index).count + + # ap "Start index: #{start_index}" + + weapons.drop(start_index).each_with_index do |w, i| + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{w.wiki_en}... (#{i + 1}/#{count})" if debug + next if w.wiki_en.include?('Element Changed') || w.wiki_en.include?('Awakened') + next unless w.release_date.nil? || overwrite + + begin + WeaponParser.new(granblue_id: w.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) + errors = [] + + start_index = start.nil? ? 0 : list.index { |id| id == start } + count = list.drop(start_index).count + + # ap "Start index: #{start_index}" + + list.drop(start_index).each_with_index do |id, i| + weapon = Weapon.find_by(granblue_id: id) + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{weapon.wiki_en}... (#{i + 1}/#{count})" if debug + next unless weapon.release_date.nil? || overwrite + + begin + WeaponParser.new(granblue_id: weapon.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + # Parses the response string into a hash + def parse_string(string) + data = {} + lines = string.split("\n") + stop_loop = false + + lines.each do |line| + next if stop_loop + + if line.include?('Gameplay Notes') + stop_loop = true + next + end + + if line.starts_with?('{{') + substr = line[2..].strip! || line[2..] + + # All template tags start with {{ so we can skip the first two characters + disallowed = %w[#vardefine #lsth About] + next if substr.start_with?(*disallowed) + + if substr.start_with?('Weapon') + ap "--> Found template: #{substr}" if @debug + + substr = substr.split('|').first + data[:template] = substr if substr != 'Weapon' + next + end + end + + next unless line[0] == '|' && line.size > 2 + + key, value = line[1..].split('=', 2).map(&:strip) + + regex = /\A\{\{\{.*\|\}\}\}\z/ + next if value =~ regex + + data[key] = value if value + end + + data + end + + # Parses the hash into a format that can be saved to the database + def parse(hash) + info = {} + skills = {} + + info[:name] = { en: hash['name'], ja: hash['jpname'] } + info[:flavor] = { en: hash['flavor'], ja: hash['jpflavor'] } + info[:id] = hash['id'] + + info[:flb] = hash['evo_max'].to_i >= 4 + info[:ulb] = hash['evo_max'].to_i == 5 + + info[:rarity] = rarity_from_hash(hash['rarity']) + info[:proficiency] = proficiency_from_hash(hash['weapon']) + info[:series] = hash['series'] + info[:obtain] = hash['obtain'] + + if hash.key?('bullets') + info[:bullets] = { + count: hash['bullets'].to_i, + loadout: [ + bullet_from_hash(hash['bullet1']), + bullet_from_hash(hash['bullet2']), + bullet_from_hash(hash['bullet3']), + bullet_from_hash(hash['bullet4']), + bullet_from_hash(hash['bullet5']), + bullet_from_hash(hash['bullet6']) + ] + } + end + + info[:hp] = { + min_hp: hash['hp1'].to_i, + max_hp: hash['hp2'].to_i, + max_hp_flb: hash['hp3'].to_i, + max_hp_ulb: hash['hp4'].to_i.zero? ? nil : hash['hp4'].to_i + } + + info[:atk] = { + min_atk: hash['atk1'].to_i, + max_atk: hash['atk2'].to_i, + max_atk_flb: hash['atk3'].to_i, + max_atk_ulb: hash['atk4'].to_i.zero? ? nil : hash['atk4'].to_i + } + + info[:dates] = { + release_date: parse_date(hash['release_date']), + flb_date: parse_date(hash['4star_date']), + ulb_date: parse_date(hash['5star_date']) + } + + info[:links] = { + wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, + gamewith: hash['link_gamewith'], + kamigame: hash['link_kamigame'] + } + + skills[:charge_attack] = { + name: { en: hash['ougi_name'], ja: hash['jpougi_name'] }, + description: { + mlb: { + en: hash['enougi'], + ja: hash['jpougi'] + }, + flb: { + en: hash['enougi_4s'], + ja: hash['jpougi_4s'] + } + } + } + + skills[:skills] = [ + { + name: { en: hash['s1_name'], ja: nil }, + description: { en: hash['ens1_desc'] || hash['s1_desc'], ja: nil } + }, + { + name: { en: hash['s2_name'], ja: nil }, + description: { en: hash['ens2_desc'] || hash['s2_desc'], ja: nil } + }, + { + name: { en: hash['s3_name'], ja: nil }, + description: { en: hash['ens3_desc'] || hash['s3_desc'], ja: nil } + } + ] + + { + info: info.compact, + skills: skills.compact + } + end + + # Saves select fields to the database + def persist(hash) + @weapon.release_date = hash[:dates][:release_date] + @weapon.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) + @weapon.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) + + @weapon.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) + @weapon.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) + @weapon.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) + + if @weapon.save + ap "#{@weapon.granblue_id}: Successfully saved info for #{@weapon.wiki_en}" if @debug + puts + true + end + + false + end + + # Converts rarities from a string to a hash + def rarity_from_hash(string) + string ? GranblueWiki.rarities[string.upcase] : nil + end + + # Converts proficiencies from a string to a hash + def proficiency_from_hash(string) + GranblueWiki.proficiencies[string] + end + + # Converts a bullet type from a string to a hash + def bullet_from_hash(string) + string ? GranblueWiki.bullets[string] : nil + end + + # Parses a date string into a Date object + def parse_date(date_str) + Date.parse(date_str) unless date_str.blank? + end + end + end +end diff --git a/lib/granblue/parsers/wiki.rb b/lib/granblue/parsers/wiki.rb new file mode 100644 index 0000000..eca1b02 --- /dev/null +++ b/lib/granblue/parsers/wiki.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require 'httparty' + +# GranblueWiki fetches and parses data from gbf.wiki +module Granblue + module Parsers + class Wiki + class_attribute :base_uri + + class_attribute :proficiencies + class_attribute :elements + class_attribute :rarities + class_attribute :genders + class_attribute :races + class_attribute :bullets + class_attribute :boolean + + self.base_uri = 'https://gbf.wiki/api.php' + + self.proficiencies = { + 'Sabre' => 1, + 'Dagger' => 2, + 'Axe' => 3, + 'Spear' => 4, + 'Bow' => 5, + 'Staff' => 6, + 'Melee' => 7, + 'Harp' => 8, + 'Gun' => 9, + 'Katana' => 10 + }.freeze + + self.elements = { + 'Wind' => 1, + 'Fire' => 2, + 'Water' => 3, + 'Earth' => 4, + 'Dark' => 5, + 'Light' => 6 + }.freeze + + self.rarities = { + 'R' => 1, + 'SR' => 2, + 'SSR' => 3 + }.freeze + + self.races = { + 'Other' => 0, + 'Human' => 1, + 'Erune' => 2, + 'Draph' => 3, + 'Harvin' => 4, + 'Primal' => 5 + }.freeze + + self.genders = { + 'o' => 0, + 'm' => 1, + 'f' => 2, + 'mf' => 3 + }.freeze + + self.bullets = { + 'cartridge' => 1, + 'rifle' => 2, + 'parabellum' => 3, + 'aetherial' => 4 + }.freeze + + self.boolean = { + 'yes' => true, + 'no' => false + }.freeze + + def initialize(props: ['wikitext'], debug: false) + @debug = debug + @props = props.join('|') + end + + def fetch(page) + query_params = params(page).map do |key, value| + "#{key}=#{value}" + end.join('&') + + destination = "#{base_uri}?#{query_params}" + ap "--> Fetching #{destination}" if @debug + + response = HTTParty.get(destination) + + handle_response(response, page) + end + + private + + def handle_response(response, page) + case response.code + when 200 + if response.key?('error') + raise WikiError.new(code: response['error']['code'], + message: response['error']['info'], + page: page) + end + + response['parse']['wikitext']['*'] + when 404 then puts "Page #{page} not found" + when 500...600 then puts "Server error: #{response.code}" + end + end + + def params(page) + { + action: 'parse', + format: 'json', + page: page, + prop: @props + } + end + end + end +end diff --git a/lib/granblue/transformers/base_deck_transformer.rb b/lib/granblue/transformers/base_deck_transformer.rb new file mode 100644 index 0000000..b6a7faa --- /dev/null +++ b/lib/granblue/transformers/base_deck_transformer.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Granblue + module Transformers + class BaseDeckTransformer < BaseTransformer + def transform + Rails.logger.info "[TRANSFORM] Starting BaseDeckTransformer#transform" + Rails.logger.info "[TRANSFORM] Data class: #{data.class}" + + # Handle already transformed parameters + if data.is_a?(ActionController::Parameters) && data.key?(:name) + Rails.logger.info "[TRANSFORM] Found existing parameters, returning as is" + return data.to_h.symbolize_keys + end + + # Handle raw game data + Rails.logger.info "[TRANSFORM] Processing raw game data" + input_data = data['import'] if data.is_a?(Hash) + unless input_data + Rails.logger.error "[TRANSFORM] No import data found" + return {} + end + + Rails.logger.info "[TRANSFORM] Found import data" + deck = input_data['deck'] + pc = deck['pc'] if deck + + unless deck && pc + Rails.logger.error "[TRANSFORM] Missing deck or pc data" + Rails.logger.error "[TRANSFORM] deck present: #{!!deck}" + Rails.logger.error "[TRANSFORM] pc present: #{!!pc}" + return {} + end + + Rails.logger.info "[TRANSFORM] Building deck data structure" + result = { + lang: language, + name: deck['name'] || 'Untitled', + class: pc.dig('job', 'master', 'name'), + extra: pc['isExtraDeck'] || false, + subskills: transform_subskills(pc['set_action']), + characters: transform_characters(deck['npc']), + weapons: transform_weapons(pc['weapons']), + summons: transform_summons(pc['summons'], pc['quick_user_summon_id']), + sub_summons: transform_summons(pc['sub_summons']), + friend_summon: pc.dig('damage_info', 'summon_name') + } + + Rails.logger.info "[TRANSFORM] Completed transformation" + Rails.logger.debug "[TRANSFORM] Result: #{result}" + + result + end + + private + + def transform_subskills(set_action) + Rails.logger.info "[TRANSFORM] Processing subskills" + unless set_action.is_a?(Array) && !set_action.empty? + Rails.logger.info "[TRANSFORM] No valid set_action data" + return [] + end + + skills = set_action[0] + unless skills.is_a?(Array) + Rails.logger.info "[TRANSFORM] Invalid skills array" + return [] + end + + results = skills.map { |skill| skill['name'] if skill.is_a?(Hash) }.compact + Rails.logger.info "[TRANSFORM] Found #{results.length} subskills" + results + end + + def transform_characters(npc_data) + Rails.logger.info "[TRANSFORM] Processing characters" + CharacterTransformer.new(npc_data, options).transform + end + + def transform_weapons(weapons_data) + Rails.logger.info "[TRANSFORM] Processing weapons" + WeaponTransformer.new(weapons_data, options).transform + end + + def transform_summons(summons_data, quick_summon_id = nil) + Rails.logger.info "[TRANSFORM] Processing summons" + SummonTransformer.new(summons_data, quick_summon_id, options).transform + end + end + end +end diff --git a/lib/granblue/transformers/base_transformer.rb b/lib/granblue/transformers/base_transformer.rb new file mode 100644 index 0000000..e49a3df --- /dev/null +++ b/lib/granblue/transformers/base_transformer.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Granblue + module Transformers + class TransformerError < StandardError + attr_reader :details + + def initialize(message, details = nil) + @details = details + super(message) + end + end + + class BaseTransformer + ELEMENT_MAPPING = { + 0 => nil, + 1 => 4, # Wind -> Earth + 2 => 2, # Fire -> Fire + 3 => 3, # Water -> Water + 4 => 1, # Earth -> Wind + 5 => 6, # Dark -> Light + 6 => 5 # Light -> Dark + }.freeze + + def initialize(data, options = {}) + @data = data + @options = options + @language = options[:language] || 'en' + Rails.logger.info "[TRANSFORM] Initializing #{self.class.name} with data: #{data.class}" + validate_data + end + + def transform + raise NotImplementedError, "#{self.class} must implement #transform" + end + + protected + + attr_reader :data, :options, :language + + def validate_data + Rails.logger.info "[TRANSFORM] Validating data: #{data.inspect[0..100]}..." + + if data.nil? + Rails.logger.info "[TRANSFORM] Data is nil" + return true + end + + if data.empty? + Rails.logger.info "[TRANSFORM] Data is empty" + return true + end + + # Data validation successful + true + end + + def get_master_param(obj) + return [nil, nil] unless obj.is_a?(Hash) + + master = obj['master'] + param = obj['param'] + Rails.logger.debug "[TRANSFORM] Extracted master: #{!!master}, param: #{!!param}" + + [master, param] + end + + def log_debug(message) + return unless options[:debug] + Rails.logger.debug "[TRANSFORM-DEBUG] #{self.class.name}: #{message}" + end + end + end +end diff --git a/lib/granblue/transformers/character_transformer.rb b/lib/granblue/transformers/character_transformer.rb new file mode 100644 index 0000000..210228a --- /dev/null +++ b/lib/granblue/transformers/character_transformer.rb @@ -0,0 +1,51 @@ +module Granblue + module Transformers + class CharacterTransformer < BaseTransformer + def transform + Rails.logger.info "[TRANSFORM] Starting CharacterTransformer#transform" + + unless data.is_a?(Hash) + Rails.logger.error "[TRANSFORM] Invalid character data structure" + return [] + end + + characters = [] + data.each_value do |char_data| + next unless char_data['master'] && char_data['param'] + + master = char_data['master'] + param = char_data['param'] + + Rails.logger.debug "[TRANSFORM] Processing character: #{master['name']}" + + character = { + name: master['name'], + id: master['id'], + uncap: param['evolution'].to_i + } + + Rails.logger.debug "[TRANSFORM] Base character data: #{character}" + + # Add perpetuity (rings) if present + if param['has_npcaugment_constant'] + character[:ringed] = true + Rails.logger.debug "[TRANSFORM] Character is ringed" + end + + # Add transcendence if present + phase = param['phase'].to_i + if phase && phase.positive? + character[:transcend] = phase + Rails.logger.debug "[TRANSFORM] Character has transcendence: #{phase}" + end + + characters << character unless master['id'].nil? + Rails.logger.info "[TRANSFORM] Successfully processed character #{character[:name]}" + end + + Rails.logger.info "[TRANSFORM] Completed processing #{characters.length} characters" + characters + end + end + end +end diff --git a/lib/granblue/transformers/summon_transformer.rb b/lib/granblue/transformers/summon_transformer.rb new file mode 100644 index 0000000..f149c87 --- /dev/null +++ b/lib/granblue/transformers/summon_transformer.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Granblue + module Transformers + class SummonTransformer < BaseTransformer + TRANSCENDENCE_LEVELS = [210, 220, 230, 240].freeze + + def initialize(data, quick_summon_id = nil, options = {}) + super(data, options) + @quick_summon_id = quick_summon_id + Rails.logger.info "[TRANSFORM] Initializing SummonTransformer with quick_summon_id: #{quick_summon_id}" + end + + def transform + Rails.logger.info "[TRANSFORM] Starting SummonTransformer#transform" + + unless data.is_a?(Hash) + Rails.logger.error "[TRANSFORM] Invalid summon data structure" + Rails.logger.error "[TRANSFORM] Data class: #{data.class}" + return [] + end + + summons = [] + data.each_value do |summon_data| + Rails.logger.debug "[TRANSFORM] Processing summon: #{summon_data['master']['name'] if summon_data['master']}" + + master, param = get_master_param(summon_data) + unless master && param + Rails.logger.debug "[TRANSFORM] Skipping summon - missing master or param data" + next + end + + summon = { + name: master['name'], + id: master['id'], + uncap: param['evolution'].to_i + } + + Rails.logger.debug "[TRANSFORM] Base summon data: #{summon}" + + # Add transcendence if applicable + if summon[:uncap] > 5 + level = param['level'].to_i + trans = calculate_transcendence_level(level) + summon[:transcend] = trans + Rails.logger.debug "[TRANSFORM] Added transcendence level: #{trans}" + end + + # Mark quick summon if applicable + if @quick_summon_id && param['id'].to_s == @quick_summon_id.to_s + summon[:qs] = true + Rails.logger.debug "[TRANSFORM] Marked as quick summon" + end + + summons << summon + Rails.logger.info "[TRANSFORM] Successfully processed summon #{summon[:name]}" + end + + Rails.logger.info "[TRANSFORM] Completed processing #{summons.length} summons" + summons + end + + private + + def calculate_transcendence_level(level) + return 1 unless level + level = 1 + TRANSCENDENCE_LEVELS.count { |cutoff| level > cutoff } + Rails.logger.debug "[TRANSFORM] Calculated transcendence level: #{level}" + level + end + end + end +end diff --git a/lib/granblue/transformers/transformer_error.rb b/lib/granblue/transformers/transformer_error.rb new file mode 100644 index 0000000..be8b13c --- /dev/null +++ b/lib/granblue/transformers/transformer_error.rb @@ -0,0 +1,12 @@ +module Granblue + module Transformers + class TransformerError < StandardError + attr_reader :details + + def initialize(message, details = nil) + @details = details + super(message) + end + end + end +end diff --git a/lib/granblue/transformers/weapon_transformer.rb b/lib/granblue/transformers/weapon_transformer.rb new file mode 100644 index 0000000..db5e964 --- /dev/null +++ b/lib/granblue/transformers/weapon_transformer.rb @@ -0,0 +1,136 @@ +module Granblue + module Transformers + class WeaponTransformer < BaseTransformer + UNCAP_LEVELS = [40, 60, 80, 100, 150, 200].freeze + TRANSCENDENCE_LEVELS = [210, 220, 230, 240].freeze + MULTIELEMENT_SERIES = [13, 17, 19].freeze + + def transform + Rails.logger.info "[TRANSFORM] Starting WeaponTransformer#transform" + + unless data.is_a?(Hash) + Rails.logger.error "[TRANSFORM] Invalid weapon data structure" + return [] + end + + weapons = [] + data.each_value do |weapon_data| + next unless weapon_data['master'] && weapon_data['param'] + + master = weapon_data['master'] + param = weapon_data['param'] + + Rails.logger.debug "[TRANSFORM] Processing weapon: #{master['name']}" + + weapon = transform_base_attributes(master, param) + Rails.logger.debug "[TRANSFORM] Base weapon attributes: #{weapon}" + + weapon.merge!(transform_awakening(param)) + Rails.logger.debug "[TRANSFORM] After awakening: #{weapon[:awakening] if weapon[:awakening]}" + + weapon.merge!(transform_ax_skills(param)) + Rails.logger.debug "[TRANSFORM] After AX skills: #{weapon[:ax] if weapon[:ax]}" + + weapon.merge!(transform_weapon_keys(weapon_data)) + Rails.logger.debug "[TRANSFORM] After weapon keys: #{weapon[:keys] if weapon[:keys]}" + + weapons << weapon unless master['id'].nil? + Rails.logger.info "[TRANSFORM] Successfully processed weapon #{weapon[:name]}" + end + + Rails.logger.info "[TRANSFORM] Completed processing #{weapons.length} weapons" + weapons + end + + private + + def transform_base_attributes(master, param) + Rails.logger.debug "[TRANSFORM] Processing base attributes for weapon" + + series = master['series_id'].to_i + weapon = { + name: master['name'], + id: master['id'] + } + + # Handle multi-element weapons + if MULTIELEMENT_SERIES.include?(series) + element = master['attribute'].to_i - 1 + weapon[:attr] = element + weapon[:id] = (master['id'].to_i - (element * 100)).to_s + Rails.logger.debug "[TRANSFORM] Multi-element weapon adjustments made" + end + + # Calculate uncap level + level = param['level'].to_i + uncap = calculate_uncap_level(level) + weapon[:uncap] = uncap + Rails.logger.debug "[TRANSFORM] Calculated uncap level: #{uncap}" + + # Add transcendence if applicable + if uncap > 5 + trans = calculate_transcendence_level(level) + weapon[:transcend] = trans + Rails.logger.debug "[TRANSFORM] Added transcendence level: #{trans}" + end + + weapon + end + + def transform_awakening(param) + return {} unless param['arousal']&.[]('is_arousal_weapon') + + Rails.logger.debug "[TRANSFORM] Processing weapon awakening" + { + awakening: { + type: param['arousal']['form_name'], + lvl: param['arousal']['level'] + } + } + end + + def transform_ax_skills(param) + augments = param['augment_skill_info'] + return {} unless augments&.first&.any? + + Rails.logger.debug "[TRANSFORM] Processing AX skills" + ax = [] + augments.first.each_value do |augment| + ax_skill = { + id: augment['skill_id'].to_s, + val: augment['show_value'] + } + ax << ax_skill + Rails.logger.debug "[TRANSFORM] Added AX skill: #{ax_skill}" + end + + { ax: ax } + end + + def transform_weapon_keys(weapon_data) + Rails.logger.debug "[TRANSFORM] Processing weapon keys" + keys = [] + + # Add weapon keys if they exist + ['skill1', 'skill2', 'skill3'].each do |skill_key| + if weapon_data[skill_key]&.[]('id') + keys << weapon_data[skill_key]['id'] + Rails.logger.debug "[TRANSFORM] Added weapon key: #{weapon_data[skill_key]['id']}" + end + end + + keys.any? ? { keys: keys } : {} + end + + def calculate_uncap_level(level) + return 0 unless level + UNCAP_LEVELS.count { |cutoff| level.to_i > cutoff } + end + + def calculate_transcendence_level(level) + return 1 unless level + 1 + TRANSCENDENCE_LEVELS.count { |cutoff| level.to_i > cutoff } + end + end + end +end diff --git a/lib/post_deployment/image_downloader.rb b/lib/post_deployment/image_downloader.rb index 7027bf3..02a1862 100644 --- a/lib/post_deployment/image_downloader.rb +++ b/lib/post_deployment/image_downloader.rb @@ -6,7 +6,7 @@ require_relative '../granblue/downloaders/character_downloader' require_relative '../granblue/downloaders/weapon_downloader' require_relative '../granblue/downloaders/summon_downloader' require_relative '../granblue/downloaders/elemental_weapon_downloader' -require_relative '../granblue/download_manager' +require_relative '../granblue/downloaders/download_manager' module PostDeployment class ImageDownloader @@ -19,9 +19,9 @@ module PostDeployment }.freeze SUPPORTED_TYPES = { - 'character' => Granblue::Downloader::CharacterDownloader, - 'summon' => Granblue::Downloader::SummonDownloader, - 'weapon' => Granblue::Downloader::WeaponDownloader + 'character' => Granblue::Downloaders::CharacterDownloader, + 'summon' => Granblue::Downloaders::SummonDownloader, + 'weapon' => Granblue::Downloaders::WeaponDownloader }.freeze def initialize(test_mode:, verbose:, storage:, new_records:, updated_records:) diff --git a/lib/tasks/import_data.rake b/lib/tasks/import_data.rake index 6e15a05..0dae8c4 100644 --- a/lib/tasks/import_data.rake +++ b/lib/tasks/import_data.rake @@ -7,7 +7,7 @@ namespace :granblue do Dir[Rails.root.join('lib', 'granblue', '**', '*.rb')].each { |file| require file } test_mode = ENV['TEST'] == 'true' - importer = Granblue::DataImporter.new(test_mode: test_mode) + importer = Granblue::PostDeployment::DataImporter.new(test_mode: test_mode) importer.process_all_files end end