Merge pull request #193 from jedmund/jedmund/wiki-fields

Implement columns for raw wiki/game data

db/migrate/20250301143956_add_raw_data_columns.rb

@@ -0,0 +1,15 @@
+class AddRawDataColumns < ActiveRecord::Migration[8.0]
+  def change
+    add_column :characters, :wiki_raw, :text
+    add_column :characters, :game_raw_en, :text
+    add_column :characters, :game_raw_jp, :text
+
+    add_column :summons, :wiki_raw, :text
+    add_column :summons, :game_raw_en, :text
+    add_column :summons, :game_raw_jp, :text
+
+    add_column :weapons, :wiki_raw, :text
+    add_column :weapons, :game_raw_en, :text
+    add_column :weapons, :game_raw_jp, :text
+  end
+end

db/migrate/20250301143956_add_wiki_raw_to_characters.rb

@@ -0,0 +1,7 @@
+class AddWikiRawToCharacters < ActiveRecord::Migration[8.0]
+  def change
+    add_column :characters, :wiki_raw, :text
+    add_column :characters, :game_raw_en, :text
+    add_column :characters, :game_raw_jp, :text
+  end
+end

lib/granblue/parsers/character_parser.rb

@@ -4,21 +4,26 @@ require 'pry'
 
 module Granblue
   module Parsers
-
     # CharacterParser parses character data from gbf.wiki
     class CharacterParser
       attr_reader :granblue_id
 
-      def initialize(granblue_id: String, debug: false)
+      def initialize(granblue_id: String, debug: false, use_local: false)
         @character = Character.find_by(granblue_id: granblue_id)
-        @wiki = GranblueWiki.new
+        @wiki = Granblue::Parsers::Wiki.new
         @debug = debug || false
+        @use_local = use_local
       end
 
       # Fetches using @wiki and then processes the response
       # Returns true if successful, false if not
       # Raises an exception if something went wrong
       def fetch(save: false)
+        if @use_local && @character.wiki_raw.present?
+          wikitext = @character.wiki_raw
+          return handle_fetch_success(wikitext, save)
+        end
+
         response = fetch_wiki_info
         return false if response.nil?
 
@@ -49,6 +54,9 @@ module Granblue
       # Handle the response from the wiki if the response is successful
       # If the save flag is set, it will persist the data to the database
       def handle_fetch_success(response, save)
+        @character.wiki_raw = response
+        @character.save!
+
         ap "#{@character.granblue_id}: Successfully fetched info for #{@character.wiki_en}" if @debug
         extracted = parse_string(response)
         info = parse(extracted)
@@ -152,12 +160,12 @@ module Granblue
         info[:id] = hash['id']
         info[:charid] = hash['charid'].scan(/\b\d{4}\b/)
 
-        info[:flb] = GranblueWiki.boolean.fetch(hash['5star'], false)
+        info[:flb] = Granblue::Parsers::Wiki.boolean.fetch(hash['5star'], false)
         info[:ulb] = hash['max_evo'].to_i == 6
 
-        info[:rarity] = GranblueWiki.rarities.fetch(hash['rarity'], 0)
+        info[:rarity] = Granblue::Parsers::Wiki.rarities.fetch(hash['rarity'], 0)
-        info[:element] = GranblueWiki.elements.fetch(hash['element'], 0)
+        info[:element] = Granblue::Parsers::Wiki.elements.fetch(hash['element'], 0)
-        info[:gender] = GranblueWiki.genders.fetch(hash['gender'], 0)
+        info[:gender] = Granblue::Parsers::Wiki.genders.fetch(hash['gender'], 0)
 
         info[:proficiencies] = proficiencies_from_hash(hash['weapon'])
         info[:races] = races_from_hash(hash['race'])
@@ -211,14 +219,14 @@ module Granblue
       # Converts proficiencies from a string to a hash
       def proficiencies_from_hash(character)
         character.to_s.split(',').map.with_index do |prof, i|
-          { "proficiency#{i + 1}" => GranblueWiki.proficiencies[prof] }
+          { "proficiency#{i + 1}" => Granblue::Parsers::Wiki.proficiencies[prof] }
         end.reduce({}, :merge)
       end
 
       # Converts races from a string to a hash
       def races_from_hash(race)
         race.to_s.split(',').map.with_index do |r, i|
-          { "race#{i + 1}" => GranblueWiki.races[r] }
+          { "race#{i + 1}" => Granblue::Parsers::Wiki.races[r] }
         end.reduce({}, :merge)
       end
 

lib/granblue/parsers/summon_parser.rb

@@ -10,7 +10,7 @@ module Granblue
 
       def initialize(granblue_id: String, debug: false)
         @summon = Summon.find_by(granblue_id: granblue_id)
-        @wiki = GranblueWiki.new(debug: debug)
+        @wiki = Granblue::Parsers::Wiki.new
         @debug = debug || false
       end
 

lib/granblue/parsers/weapon_parser.rb

@@ -10,7 +10,7 @@ module Granblue
 
       def initialize(granblue_id: String, debug: false)
         @weapon = Weapon.find_by(granblue_id: granblue_id)
-        @wiki = GranblueWiki.new(debug: debug)
+        @wiki = Granblue::Parsers::Wiki.new
         @debug = debug || false
       end
 
@@ -278,17 +278,17 @@ module Granblue
 
       # Converts rarities from a string to a hash
       def rarity_from_hash(string)
-        string ? GranblueWiki.rarities[string.upcase] : nil
+        string ? Granblue::Parsers::Wiki.rarities[string.upcase] : nil
       end
 
       # Converts proficiencies from a string to a hash
       def proficiency_from_hash(string)
-        GranblueWiki.proficiencies[string]
+        Granblue::Parsers::Wiki.proficiencies[string]
       end
 
       # Converts a bullet type from a string to a hash
       def bullet_from_hash(string)
-        string ? GranblueWiki.bullets[string] : nil
+        string ? Granblue::Parsers::Wiki.bullets[string] : nil
       end
 
       # Parses a date string into a Date object

lib/tasks/fetch_wiki.rake

@@ -0,0 +1,79 @@
+namespace :granblue do
+  desc <<~DESC
+    Fetch and store raw wiki data for objects (Character, Weapon, Summon).
+
+    Usage:
+      rake granblue:fetch_wiki_data                     # Fetch all Characters (default)
+      rake granblue:fetch_wiki_data type=Weapon         # Fetch all Weapons
+      rake granblue:fetch_wiki_data type=Summon         # Fetch all Summons
+      rake granblue:fetch_wiki_data type=Character id=5 # Fetch specific Character by ID
+      rake granblue:fetch_wiki_data force=true          # Force re-download even if data exists
+  DESC
+  task fetch_wiki_data: :environment do
+    # Get parameters from environment
+    type = (ENV['type'] || 'Character').classify
+    id = ENV['id']
+    force = ENV['force'] == 'true'
+
+    # Validate object type
+    valid_types = %w[Character Weapon Summon]
+    unless valid_types.include?(type)
+      puts "Error: Invalid type '#{type}'. Must be one of: #{valid_types.join(', ')}"
+      exit 1
+    end
+
+    # Get the class from the type string
+    klass = type.constantize
+
+    # Setup query - either all objects or specific one
+    query = id.present? ? klass.where(granblue_id: id) : klass.all
+
+    errors = []
+    count = 0
+
+    query.find_each do |object|
+      # Skip objects that already have wiki_raw if force is not set
+      if object.wiki_raw.present? && !force
+        puts "Skipping #{object.name_en} (already has wiki_raw)."
+        next
+      end
+
+      # If the object doesn't have a wiki page specified, skip
+      if object.wiki_en.blank?
+        puts "Skipping #{object.name_en} (no wiki_en set)."
+        next
+      end
+
+      begin
+        # 1) Fetch raw wikitext from the wiki
+        wiki_text = Granblue::Parsers::Wiki.new.fetch(object.wiki_en)
+
+        # 2) Check if the page is a redirect
+        redirect_match = wiki_text.match(/#REDIRECT \[\[(.*?)\]\]/)
+        if redirect_match
+          redirect_target = redirect_match[1]
+          # Update object to new wiki_en so we don't keep fetching the old page
+          object.update!(wiki_en: redirect_target)
+          # Fetch again with the new page name
+          wiki_text = Granblue::Parsers::Wiki.new.fetch(redirect_target)
+        end
+        puts wiki_text
+
+        # 3) Save raw wiki text in the object record
+        object.update!(wiki_raw: wiki_text)
+        puts "Saved wiki data for #{object.name_en} (#{object.id})"
+        count += 1
+      rescue StandardError => e
+        errors << { object_id: object.id, type: type, error: e.message }
+        puts "Error fetching data for #{object.name_en}: #{e.message}"
+      end
+    end
+
+    if errors.any?
+      puts "#{errors.size} #{type.pluralize} had errors:"
+      errors.each { |err| puts "  - #{err[:type]} ##{err[:object_id]} => #{err[:error]}" }
+    else
+      puts "Wiki data fetch complete for #{count} #{type.pluralize} with no errors!"
+    end
+  end
+end