diff --git a/db/migrate/20250301143956_add_raw_data_columns.rb b/db/migrate/20250301143956_add_raw_data_columns.rb new file mode 100644 index 0000000..06615c5 --- /dev/null +++ b/db/migrate/20250301143956_add_raw_data_columns.rb @@ -0,0 +1,15 @@ +class AddRawDataColumns < ActiveRecord::Migration[8.0] + def change + add_column :characters, :wiki_raw, :text + add_column :characters, :game_raw_en, :text + add_column :characters, :game_raw_jp, :text + + add_column :summons, :wiki_raw, :text + add_column :summons, :game_raw_en, :text + add_column :summons, :game_raw_jp, :text + + add_column :weapons, :wiki_raw, :text + add_column :weapons, :game_raw_en, :text + add_column :weapons, :game_raw_jp, :text + end +end diff --git a/db/migrate/20250301143956_add_wiki_raw_to_characters.rb b/db/migrate/20250301143956_add_wiki_raw_to_characters.rb new file mode 100644 index 0000000..58d069b --- /dev/null +++ b/db/migrate/20250301143956_add_wiki_raw_to_characters.rb @@ -0,0 +1,7 @@ +class AddWikiRawToCharacters < ActiveRecord::Migration[8.0] + def change + add_column :characters, :wiki_raw, :text + add_column :characters, :game_raw_en, :text + add_column :characters, :game_raw_jp, :text + end +end diff --git a/lib/granblue/parsers/character_parser.rb b/lib/granblue/parsers/character_parser.rb index b98a3e8..ec44cc3 100644 --- a/lib/granblue/parsers/character_parser.rb +++ b/lib/granblue/parsers/character_parser.rb @@ -4,21 +4,26 @@ require 'pry' module Granblue module Parsers - # CharacterParser parses character data from gbf.wiki class CharacterParser attr_reader :granblue_id - def initialize(granblue_id: String, debug: false) + def initialize(granblue_id: String, debug: false, use_local: false) @character = Character.find_by(granblue_id: granblue_id) - @wiki = GranblueWiki.new + @wiki = Granblue::Parsers::Wiki.new @debug = debug || false + @use_local = use_local end # Fetches using @wiki and then processes the response # Returns true if successful, false if not # Raises an exception if something went wrong def fetch(save: false) + if @use_local && @character.wiki_raw.present? + wikitext = @character.wiki_raw + return handle_fetch_success(wikitext, save) + end + response = fetch_wiki_info return false if response.nil? @@ -49,6 +54,9 @@ module Granblue # Handle the response from the wiki if the response is successful # If the save flag is set, it will persist the data to the database def handle_fetch_success(response, save) + @character.wiki_raw = response + @character.save! + ap "#{@character.granblue_id}: Successfully fetched info for #{@character.wiki_en}" if @debug extracted = parse_string(response) info = parse(extracted) @@ -152,12 +160,12 @@ module Granblue info[:id] = hash['id'] info[:charid] = hash['charid'].scan(/\b\d{4}\b/) - info[:flb] = GranblueWiki.boolean.fetch(hash['5star'], false) + info[:flb] = Granblue::Parsers::Wiki.boolean.fetch(hash['5star'], false) info[:ulb] = hash['max_evo'].to_i == 6 - info[:rarity] = GranblueWiki.rarities.fetch(hash['rarity'], 0) - info[:element] = GranblueWiki.elements.fetch(hash['element'], 0) - info[:gender] = GranblueWiki.genders.fetch(hash['gender'], 0) + info[:rarity] = Granblue::Parsers::Wiki.rarities.fetch(hash['rarity'], 0) + info[:element] = Granblue::Parsers::Wiki.elements.fetch(hash['element'], 0) + info[:gender] = Granblue::Parsers::Wiki.genders.fetch(hash['gender'], 0) info[:proficiencies] = proficiencies_from_hash(hash['weapon']) info[:races] = races_from_hash(hash['race']) @@ -211,14 +219,14 @@ module Granblue # Converts proficiencies from a string to a hash def proficiencies_from_hash(character) character.to_s.split(',').map.with_index do |prof, i| - { "proficiency#{i + 1}" => GranblueWiki.proficiencies[prof] } + { "proficiency#{i + 1}" => Granblue::Parsers::Wiki.proficiencies[prof] } end.reduce({}, :merge) end # Converts races from a string to a hash def races_from_hash(race) race.to_s.split(',').map.with_index do |r, i| - { "race#{i + 1}" => GranblueWiki.races[r] } + { "race#{i + 1}" => Granblue::Parsers::Wiki.races[r] } end.reduce({}, :merge) end diff --git a/lib/granblue/parsers/summon_parser.rb b/lib/granblue/parsers/summon_parser.rb index 5150525..1807996 100644 --- a/lib/granblue/parsers/summon_parser.rb +++ b/lib/granblue/parsers/summon_parser.rb @@ -10,7 +10,7 @@ module Granblue def initialize(granblue_id: String, debug: false) @summon = Summon.find_by(granblue_id: granblue_id) - @wiki = GranblueWiki.new(debug: debug) + @wiki = Granblue::Parsers::Wiki.new @debug = debug || false end diff --git a/lib/granblue/parsers/weapon_parser.rb b/lib/granblue/parsers/weapon_parser.rb index 30d44dd..19d5ed5 100644 --- a/lib/granblue/parsers/weapon_parser.rb +++ b/lib/granblue/parsers/weapon_parser.rb @@ -10,7 +10,7 @@ module Granblue def initialize(granblue_id: String, debug: false) @weapon = Weapon.find_by(granblue_id: granblue_id) - @wiki = GranblueWiki.new(debug: debug) + @wiki = Granblue::Parsers::Wiki.new @debug = debug || false end @@ -278,17 +278,17 @@ module Granblue # Converts rarities from a string to a hash def rarity_from_hash(string) - string ? GranblueWiki.rarities[string.upcase] : nil + string ? Granblue::Parsers::Wiki.rarities[string.upcase] : nil end # Converts proficiencies from a string to a hash def proficiency_from_hash(string) - GranblueWiki.proficiencies[string] + Granblue::Parsers::Wiki.proficiencies[string] end # Converts a bullet type from a string to a hash def bullet_from_hash(string) - string ? GranblueWiki.bullets[string] : nil + string ? Granblue::Parsers::Wiki.bullets[string] : nil end # Parses a date string into a Date object diff --git a/lib/tasks/fetch_wiki.rake b/lib/tasks/fetch_wiki.rake new file mode 100644 index 0000000..03cc7aa --- /dev/null +++ b/lib/tasks/fetch_wiki.rake @@ -0,0 +1,79 @@ +namespace :granblue do + desc <<~DESC + Fetch and store raw wiki data for objects (Character, Weapon, Summon). + + Usage: + rake granblue:fetch_wiki_data # Fetch all Characters (default) + rake granblue:fetch_wiki_data type=Weapon # Fetch all Weapons + rake granblue:fetch_wiki_data type=Summon # Fetch all Summons + rake granblue:fetch_wiki_data type=Character id=5 # Fetch specific Character by ID + rake granblue:fetch_wiki_data force=true # Force re-download even if data exists + DESC + task fetch_wiki_data: :environment do + # Get parameters from environment + type = (ENV['type'] || 'Character').classify + id = ENV['id'] + force = ENV['force'] == 'true' + + # Validate object type + valid_types = %w[Character Weapon Summon] + unless valid_types.include?(type) + puts "Error: Invalid type '#{type}'. Must be one of: #{valid_types.join(', ')}" + exit 1 + end + + # Get the class from the type string + klass = type.constantize + + # Setup query - either all objects or specific one + query = id.present? ? klass.where(granblue_id: id) : klass.all + + errors = [] + count = 0 + + query.find_each do |object| + # Skip objects that already have wiki_raw if force is not set + if object.wiki_raw.present? && !force + puts "Skipping #{object.name_en} (already has wiki_raw)." + next + end + + # If the object doesn't have a wiki page specified, skip + if object.wiki_en.blank? + puts "Skipping #{object.name_en} (no wiki_en set)." + next + end + + begin + # 1) Fetch raw wikitext from the wiki + wiki_text = Granblue::Parsers::Wiki.new.fetch(object.wiki_en) + + # 2) Check if the page is a redirect + redirect_match = wiki_text.match(/#REDIRECT \[\[(.*?)\]\]/) + if redirect_match + redirect_target = redirect_match[1] + # Update object to new wiki_en so we don't keep fetching the old page + object.update!(wiki_en: redirect_target) + # Fetch again with the new page name + wiki_text = Granblue::Parsers::Wiki.new.fetch(redirect_target) + end + puts wiki_text + + # 3) Save raw wiki text in the object record + object.update!(wiki_raw: wiki_text) + puts "Saved wiki data for #{object.name_en} (#{object.id})" + count += 1 + rescue StandardError => e + errors << { object_id: object.id, type: type, error: e.message } + puts "Error fetching data for #{object.name_en}: #{e.message}" + end + end + + if errors.any? + puts "#{errors.size} #{type.pluralize} had errors:" + errors.each { |err| puts " - #{err[:type]} ##{err[:object_id]} => #{err[:error]}" } + else + puts "Wiki data fetch complete for #{count} #{type.pluralize} with no errors!" + end + end +end