Fix error in weapons (#199)

Adds items from the March 2025 Legend Festival/Event

app/controllers/api/v1/import_controller.rb

@@ -27,6 +27,8 @@ module Api
         6 => 5
       }.freeze
 
+      before_action :ensure_admin_role, only: %i[weapons summons characters]
+
       ##
       # Processes an import request.
       #
@@ -51,7 +53,7 @@ module Api
         unless raw_params['deck'].is_a?(Hash) &&
                raw_params['deck'].key?('pc') &&
                raw_params['deck'].key?('npc')
-          Rails.logger.error "[IMPORT] Deck data incomplete or missing."
+          Rails.logger.error '[IMPORT] Deck data incomplete or missing.'
           return render json: { error: 'Invalid deck data' }, status: :unprocessable_content
         end
 
@@ -68,8 +70,111 @@ module Api
         render json: { error: e.message }, status: :unprocessable_content
       end
 
+      def weapons
+        Rails.logger.info '[IMPORT] Checking weapon gamedata input...'
+
+        body = parse_request_body
+        return unless body
+
+        weapon = Weapon.find_by(granblue_id: body['id'])
+        unless weapon
+          Rails.logger.error "[IMPORT] Weapon not found with ID: #{body['id']}"
+          return render json: { error: 'Weapon not found' }, status: :not_found
+        end
+
+        lang = params[:lang]
+        unless %w[en jp].include?(lang)
+          Rails.logger.error "[IMPORT] Invalid language: #{lang}"
+          return render json: { error: 'Invalid language' }, status: :unprocessable_content
+        end
+
+        begin
+          weapon.update!(
+            "game_raw_#{lang}" => body.to_json
+          )
+          render json: { message: 'Weapon gamedata updated successfully' }, status: :ok
+        rescue StandardError => e
+          Rails.logger.error "[IMPORT] Failed to update weapon gamedata: #{e.message}"
+          render json: { error: e.message }, status: :unprocessable_content
+        end
+      end
+
+      def summons
+        Rails.logger.info '[IMPORT] Checking summon gamedata input...'
+
+        body = parse_request_body
+        return unless body
+
+        summon = Summon.find_by(granblue_id: body['id'])
+        unless summon
+          Rails.logger.error "[IMPORT] Summon not found with ID: #{body['id']}"
+          return render json: { error: 'Summon not found' }, status: :not_found
+        end
+
+        lang = params[:lang]
+        unless %w[en jp].include?(lang)
+          Rails.logger.error "[IMPORT] Invalid language: #{lang}"
+          return render json: { error: 'Invalid language' }, status: :unprocessable_content
+        end
+
+        begin
+          summon.update!(
+            "game_raw_#{lang}" => body.to_json
+          )
+          render json: { message: 'Summon gamedata updated successfully' }, status: :ok
+        rescue StandardError => e
+          Rails.logger.error "[IMPORT] Failed to update summon gamedata: #{e.message}"
+          render json: { error: e.message }, status: :unprocessable_content
+        end
+      end
+
+      ##
+      # Updates character gamedata from JSON blob.
+      #
+      # @return [void] Renders JSON response with success or error message.
+      def characters
+        Rails.logger.info '[IMPORT] Checking character gamedata input...'
+
+        body = parse_request_body
+        return unless body
+
+        character = Character.find_by(granblue_id: body['id'])
+        unless character
+          Rails.logger.error "[IMPORT] Character not found with ID: #{body['id']}"
+          return render json: { error: 'Character not found' }, status: :not_found
+        end
+
+        lang = params[:lang]
+        unless %w[en jp].include?(lang)
+          Rails.logger.error "[IMPORT] Invalid language: #{lang}"
+          return render json: { error: 'Invalid language' }, status: :unprocessable_content
+        end
+
+        begin
+          character.update!(
+            "game_raw_#{lang}" => body.to_json
+          )
+          render json: { message: 'Character gamedata updated successfully' }, status: :ok
+        rescue StandardError => e
+          Rails.logger.error "[IMPORT] Failed to update character gamedata: #{e.message}"
+          render json: { error: e.message }, status: :unprocessable_content
+        end
+      end
+
       private
 
+      ##
+      # Ensures the current user has admin role (role 9).
+      # Renders an error if the user is not an admin.
+      #
+      # @return [void]
+      def ensure_admin_role
+        return if current_user&.role == 9
+
+        Rails.logger.error "[IMPORT] Unauthorized access attempt by user #{current_user&.id}"
+        render json: { error: 'Unauthorized' }, status: :unauthorized
+      end
+
       ##
       # Reads and parses the raw JSON request body.
       #

app/services/dataminer.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+class Dataminer
+  include HTTParty
+
+  BOT_UID = '39094985'
+  GAME_VERSION = '1741068713'
+
+  base_uri 'https://game.granbluefantasy.jp'
+  format :json
+
+  HEADERS = {
+    'Accept' => 'application/json, text/javascript, */*; q=0.01',
+    'Accept-Language' => 'en-US,en;q=0.9',
+    'Accept-Encoding' => 'gzip, deflate, br, zstd',
+    'Content-Type' => 'application/json',
+    'DNT' => '1',
+    'Origin' => 'https://game.granbluefantasy.jp',
+    'Referer' => 'https://game.granbluefantasy.jp/',
+    'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
+    'X-Requested-With' => 'XMLHttpRequest'
+  }.freeze
+
+  attr_reader :page, :cookies, :logger, :debug
+
+  def initialize(page:, access_token:, wing:, midship:, t: 'dummy', debug: false)
+    @page = page
+    @cookies = {
+      access_gbtk: access_token,
+      wing: wing,
+      t: t,
+      midship: midship
+    }
+    @debug = debug
+    setup_logger
+  end
+
+  def fetch
+    timestamp = Time.now.to_i * 1000
+    response = self.class.post(
+      "/#{page}?_=#{timestamp}&t=#{timestamp}&uid=#{BOT_UID}",
+      headers: HEADERS.merge(
+        'Cookie' => format_cookies,
+        'X-VERSION' => GAME_VERSION
+      )
+    )
+
+    raise AuthenticationError if auth_failed?(response)
+
+    response
+  end
+
+  def fetch_character(granblue_id)
+    timestamp = Time.now.to_i * 1000
+    url = "/archive/npc_detail?_=#{timestamp}&t=#{timestamp}&uid=#{BOT_UID}"
+    body = {
+      special_token: nil,
+      user_id: BOT_UID,
+      kind_name: '0',
+      attribute: '0',
+      event_id: nil,
+      story_id: nil,
+      style: 1,
+      character_id: granblue_id
+    }
+
+    response = fetch_detail(url, body)
+    update_game_data('Character', granblue_id, response) if response
+    response
+  end
+
+  def fetch_weapon(granblue_id)
+    timestamp = Time.now.to_i * 1000
+    url = "/archive/weapon_detail?_=#{timestamp}&t=#{timestamp}&uid=#{BOT_UID}"
+    body = {
+      special_token: nil,
+      user_id: BOT_UID,
+      kind_name: '0',
+      attribute: '0',
+      event_id: nil,
+      story_id: nil,
+      weapon_id: granblue_id
+    }
+
+    response = fetch_detail(url, body)
+    update_game_data('Weapon', granblue_id, response) if response
+    response
+  end
+
+  def fetch_summon(granblue_id)
+    timestamp = Time.now.to_i * 1000
+    url = "/archive/summon_detail?_=#{timestamp}&t=#{timestamp}&uid=#{BOT_UID}"
+    body = {
+      special_token: nil,
+      user_id: BOT_UID,
+      kind_name: '0',
+      attribute: '0',
+      event_id: nil,
+      story_id: nil,
+      summon_id: granblue_id
+    }
+
+    response = fetch_detail(url, body)
+    update_game_data('Summon', granblue_id, response) if response
+    response
+  end
+
+  # Public batch processing methods
+  def fetch_all_characters(only_missing: false)
+    process_all_records('Character', only_missing: only_missing)
+  end
+
+  def fetch_all_weapons(only_missing: false)
+    process_all_records('Weapon', only_missing: only_missing)
+  end
+
+  def fetch_all_summons(only_missing: false)
+    process_all_records('Summon', only_missing: only_missing)
+  end
+
+  private
+
+  def format_cookies
+    cookies.map { |k, v| "#{k}=#{v}" }.join('; ')
+  end
+
+  def auth_failed?(response)
+    return true if response.code != 200
+
+    begin
+      parsed = JSON.parse(response.body)
+      parsed.is_a?(Hash) && parsed['auth_status'] == 'require_auth'
+    rescue JSON::ParserError
+      true
+    end
+  end
+
+  def setup_logger
+    @logger = ::Logger.new($stdout)
+    @logger.level = debug ? ::Logger::DEBUG : ::Logger::INFO
+    @logger.formatter = proc do |severity, _datetime, _progname, msg|
+      case severity
+      when 'DEBUG'
+        debug ? "#{msg}\n" : ''
+      else
+        "#{msg}\n"
+      end
+    end
+
+    # Suppress SQL logs in non-debug mode
+    return if debug
+
+    ActiveRecord::Base.logger.level = ::Logger::INFO if defined?(ActiveRecord::Base)
+  end
+
+  def fetch_detail(url, body)
+    logger.debug "\n=== Request Details ==="
+    logger.debug "URL: #{url}"
+    logger.debug 'Headers:'
+    logger.debug HEADERS.merge(
+      'Cookie' => format_cookies,
+      'X-VERSION' => GAME_VERSION
+    ).inspect
+    logger.debug 'Body:'
+    logger.debug body.to_json
+    logger.debug '===================='
+
+    response = self.class.post(
+      url,
+      headers: HEADERS.merge(
+        'Cookie' => format_cookies,
+        'X-VERSION' => GAME_VERSION
+      ),
+      body: body.to_json
+    )
+
+    logger.debug "\n=== Response Details ==="
+    logger.debug "Response code: #{response.code}"
+    logger.debug 'Response headers:'
+    logger.debug response.headers.inspect
+    logger.debug 'Raw response body:'
+    logger.debug response.body.inspect
+    begin
+      logger.debug 'Parsed response body (if JSON):'
+      logger.debug JSON.parse(response.body).inspect
+    rescue JSON::ParserError => e
+      logger.debug "Could not parse as JSON: #{e.message}"
+    end
+    logger.debug '======================'
+
+    raise AuthenticationError if auth_failed?(response)
+
+    JSON.parse(response.body)
+  end
+
+  def update_game_data(model_name, granblue_id, response_data)
+    return unless response_data.is_a?(Hash)
+
+    model = Object.const_get(model_name)
+    record = model.find_by(granblue_id: granblue_id)
+
+    if record
+      record.update(game_raw_en: response_data)
+      logger.debug "Updated #{model_name} #{granblue_id}"
+    else
+      logger.warn "#{model_name} with granblue_id #{granblue_id} not found in database"
+    end
+  rescue StandardError => e
+    logger.error "Error updating #{model_name} #{granblue_id}: #{e.message}"
+  end
+
+  def process_all_records(model_name, only_missing: false)
+    model = Object.const_get(model_name)
+    scope = model
+    scope = scope.where(game_raw_en: nil) if only_missing
+
+    total = scope.count
+    success_count = 0
+    error_count = 0
+
+    logger.info "Starting to fetch #{total} #{model_name.downcase}s#{' (missing data only)' if only_missing}..."
+
+    scope.find_each do |record|
+      logger.info "\nProcessing #{model_name} #{record.granblue_id} (#{success_count + error_count + 1}/#{total})"
+
+      response = case model_name
+                 when 'Character'
+                   fetch_character(record.granblue_id)
+                 when 'Weapon'
+                   fetch_weapon(record.granblue_id)
+                 when 'Summon'
+                   fetch_summon(record.granblue_id)
+                 end
+
+      success_count += 1
+      logger.debug "Successfully processed #{model_name} #{record.granblue_id}"
+
+      sleep(1)
+    rescue StandardError => e
+      error_count += 1
+      logger.error "Error processing #{model_name} #{record.granblue_id}: #{e.message}"
+    end
+
+    logger.info "\nProcessing complete!"
+    logger.info "Total: #{total}"
+    logger.info "Successful: #{success_count}"
+    logger.info "Failed: #{error_count}"
+  end
+
+  class AuthenticationError < StandardError; end
+end

config/routes.rb

@@ -20,6 +20,9 @@ Rails.application.routes.draw do
     get 'version', to: 'api#version'
 
     post 'import', to: 'import#create'
+    post 'import/weapons', to: 'import#weapons'
+    post 'import/summons', to: 'import#summons'
+    post 'import/characters', to: 'import#characters'
 
     get 'users/info/:id', to: 'users#info'
 

db/migrate/20250301143956_add_raw_data_columns.rb

@@ -0,0 +1,15 @@
+class AddRawDataColumns < ActiveRecord::Migration[8.0]
+  def change
+    add_column :characters, :wiki_raw, :text
+    add_column :characters, :game_raw_en, :text
+    add_column :characters, :game_raw_jp, :text
+
+    add_column :summons, :wiki_raw, :text
+    add_column :summons, :game_raw_en, :text
+    add_column :summons, :game_raw_jp, :text
+
+    add_column :weapons, :wiki_raw, :text
+    add_column :weapons, :game_raw_en, :text
+    add_column :weapons, :game_raw_jp, :text
+  end
+end

db/migrate/20250327044028_add_classic_ii_and_collab_to_gacha.rb

@@ -0,0 +1,6 @@
+class AddClassicIiAndCollabToGacha < ActiveRecord::Migration[8.0]
+  def change
+    add_column :gacha, :classic_ii, :boolean, default: false
+    add_column :gacha, :collab, :boolean, default: false
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2025_02_18_025315) do
+ActiveRecord::Schema[8.0].define(version: 2025_03_01_143956) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "btree_gin"
   enable_extension "pg_catalog.plpgsql"
@@ -67,6 +67,9 @@ ActiveRecord::Schema[8.0].define(version: 2025_02_18_025315) do
     t.string "kamigame", default: ""
     t.string "nicknames_en", default: [], null: false, array: true
     t.string "nicknames_jp", default: [], null: false, array: true
+    t.text "wiki_raw"
+    t.text "game_raw_en"
+    t.text "game_raw_jp"
     t.index ["granblue_id"], name: "index_characters_on_granblue_id"
     t.index ["name_en"], name: "index_characters_on_name_en", opclass: :gin_trgm_ops, using: :gin
   end
@@ -420,6 +423,9 @@ ActiveRecord::Schema[8.0].define(version: 2025_02_18_025315) do
     t.date "transcendence_date"
     t.string "nicknames_en", default: [], null: false, array: true
     t.string "nicknames_jp", default: [], null: false, array: true
+    t.text "wiki_raw"
+    t.text "game_raw_en"
+    t.text "game_raw_jp"
     t.index ["granblue_id"], name: "index_summons_on_granblue_id"
     t.index ["name_en"], name: "index_summons_on_name_en", opclass: :gin_trgm_ops, using: :gin
   end
@@ -495,6 +501,10 @@ ActiveRecord::Schema[8.0].define(version: 2025_02_18_025315) do
     t.date "transcendence_date"
     t.string "recruits"
     t.integer "series"
+    t.integer "new_series"
+    t.text "wiki_raw"
+    t.text "game_raw_en"
+    t.text "game_raw_jp"
     t.index ["granblue_id"], name: "index_weapons_on_granblue_id"
     t.index ["name_en"], name: "index_weapons_on_name_en", opclass: :gin_trgm_ops, using: :gin
     t.index ["recruits"], name: "index_weapons_on_recruits"

db/seed/updates/20250301-characters-024.csv

@@ -0,0 +1,10 @@
+name_en,name_jp,granblue_id,rarity,element,proficiency1,proficiency2,gender,race1,race2,flb,min_hp,max_hp,max_hp_flb,min_atk,max_atk,max_atk_flb,base_da,base_ta,ougi_ratio,ougi_ratio_flb,special,ulb,max_hp_ulb,max_atk_ulb,character_id,wiki_en,release_date,flb_date,ulb_date,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp
+Kaguya (Grand),,3040486000,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Kaguya,
+,,3040519000,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Fenie,
+,,3040501000,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Uriel,
+,,3040254000,,,,,,,,,,,,,,,,,,,,,,,{2030},,,,,,,,,
+,,3040158000,,,,,,,,,,,,,,,,,,,,,,,{3098},,,,,,,,,
+,,3040103000,,,,,,,,,,,,,,,,,,,,,,,{3070},,,,,,,,,
+,,3040377000,,,,,,,,,,,,,,,,,,,,,,,{3192},,,,,,,,,
+,,3040058000,,,,,,,,,,,,,,,,,,,,,,,{3045},,,,,,,,,
+,,3040500000,,,,,,,,,,,,,,,,,,,,,,,{1027},,,,,,,,,

db/seed/updates/20250310-summons-007.csv

@@ -0,0 +1,7 @@
+name_en,name_jp,granblue_id,rarity,element,series,flb,ulb,max_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,subaura,limit,transcendence,max_atk_xlb,max_hp_xlb,summon_id,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,transcendence_date,nicknames_en,nicknames_jp
+,,2040361000,,,,true,,,,,,,,,,,,,,,,,,2025-03-10,,,,,,,,
+,,2040363000,,,,true,,,,,,,,,,,,,,,,,,2025-03-10,,,,,,,,
+,,2040368000,,,,true,,,,,,,,,,,,,,,,,,2025-03-10,,,,,,,,
+,,2040366000,,,,true,,,,,,,,,,,,,,,,,,2025-03-10,,,,,,,,
+,,2040381000,,,,true,,,,,,,,,,,,,,,,,,2025-03-10,,,,,,,,
+,,2040385000,,,,true,,,,,,,,,,,,,,,,,,2025-03-10,,,,,,,,

db/seed/updates/20250318-characters-025.csv

@@ -0,0 +1,5 @@
+name_en,name_jp,granblue_id,rarity,element,proficiency1,proficiency2,gender,race1,race2,flb,min_hp,max_hp,max_hp_flb,min_atk,max_atk,max_atk_flb,base_da,base_ta,ougi_ratio,ougi_ratio_flb,special,ulb,max_hp_ulb,max_atk_ulb,character_id,wiki_en,release_date,flb_date,ulb_date,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp
+Basara (Grand),バサラ (リミテッドver),3040582000,3,6,10,7,1,2,,false,192,1128,,1740,8760,,,,,,false,false,,,{3271},Basara,2025-03-17,,,バサラ (SSR)リミテッドバージョン,489323,SSRバサラ,,
+Mahira (Summer),マキラ(水着ver),3040584000,3,1,8,7,2,3,,false,281,1217,,1408,8915,,,,,,false,false,,,{3073},Mahira (Summer),2025-03-17,,,マキラ (SSR)水着バージョン,489328,SSR水着マキラ,,
+Lu Woh (Summer),ル・オー(水着ver),3040583000,3,4,7,6,0,2,,false,236,1172,,1385,7820,,,,,,false,false,,,{3221},Lu Woh (Summer),2025-03-17,,,ル・オー (SSR)水着バージョン,490119,SSR水着ルオー,,
+Joy (Event SSR),ジョイ(イベントSSR),3040588000,3,1,7,5,0,0,,false,108,1080,,1711,9011,,,,,,false,false,,,{2146},Joy (Event SSR),2025-03-11,,,ジョイ (SSR),489593,SSRジョイ,,

db/seed/updates/20250327-weapons-024.csv

@@ -0,0 +1,8 @@
+name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date, , , , , , , , , , , , , 
+Canifortis,天干地支刀・戌之威,1040917200,3,6,10,2,true,false,150,15,38,215,259,,474,2895,3500,,false,,false,false,3040582000,,2025-03-17,2025-03-17,,Canifortis,天干地支刀・戌之威 (SSR),490121,天干地支刀・戌之威,,,false,,,,,,,,,,,,,,
+Tenth Crow of the Clutch,第十酉行筒,1040517200,3,1,9,99,false,false,100,10,33,196,,,434,2606,,,false,,false,false,3040584000,,2025-03-17,,,Tenth Crow of the Clutch,第十酉行筒 (SSR),490124,第十酉行筒,,,false,,,,,,,,,,,,,,
+Lu Woh Float,ル・オー・フロート,1040817100,3,4,8,99,false,false,100,10,51,303,,,344,2073,,,false,,false,false,3040583000,,2025-03-17,,,Lu Woh Float,ル・オー・フロート (SSR),490125,ル・オー・フロート,,,false,,,,,,,,,,,,,,
+Exo Heliocentrum,神杖エクス・ヘリオセント,1040424300,3,6,6,39,true,false,150,15,41,260,315,,333,2029,2453,,false,,false,false,,,2025-03-22,2025-03-22,,Exo Heliocentrum,神杖エクス・ヘリオセント (SSR),490430,神杖エクス・ヘリオセント,,,false,,,,,,,,,,,,,,
+Onmyoji's Reito,陰陽之霊刀,1040917300,3,0,10,19,true,true,200,20,35,,,281,430,,,3856,false,,false,false,,,2025-03-25,2025-03-25,2025-03-25,Onmyoji%27s_Reito,陰陽之霊刀 (SSR),490757,陰陽之霊刀,,,false,,,,,,,,,,,,,,
+Ouranosphaira Ravdos,ウラニアスフェラ・ラヴドス,1040424400,3,0,6,19,true,true,200,20,44,,,371,379,,,3390,false,,false,false,,,2025-03-25,2025-03-25,2025-03-25,Ouranosphaira Ravdos,ウラニアスフェラ・ラヴドス (SSR),490756,ウラニアスフェラ・ラヴドス,,,false,,,,,,,,,,,,,,
+Cometa Sica,コメーテス・シーカ,1040121300,3,0,2,19,true,true,200,20,43,,,397,384,,,3251,false,,false,false,,,2025-03-25,2025-03-25,2025-03-25,Cometa Sica,コメーテス・シーカ (SSR),490753,コメーテス・シーカ,,,false,,,,,,,,,,,,,,

db/seed/updates/20250327-weapons-025.csv

@@ -0,0 +1,16 @@
+name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date, , , , , , , , , , , , , 
+,,1040022600,,,,,true,,150,15,,,221,,,,2603,,,,,,,,,2025-03-14,,,,,,,,,,,,,,,,,,,,,,
+,,1040216800,,,,,true,,150,15,,,264,,,,2380,,,,,,,,,2025-03-14,,,,,,,,,,,,,,,,,,,,,,
+,,1040618900,,,,,true,,150,15,,,297,,,,2854,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040713700,,,,,true,,150,15,,,270,,,,3048,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040423400,,,,,true,,150,15,,,354,,,,2568,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040916600,,,,,true,,150,15,,,247,,,,3125,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040119100,,,,,true,,150,15,,,278,,,,2943,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040518000,,,,,true,,150,15,,,213,,,,3267,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040617200,,,,,true,,150,15,,,302,,,,2828,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040916400,,,,,true,,150,15,,,214,,,,3288,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040119600,,,,,true,,150,15,,,291,,,,2879,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040816900,,,,,true,,150,15,,,335,,,,2659,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040026000,,,,,true,,150,15,,,302,,,,2821,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+,,1040816800,,,,,true,,150,15,,,366,,,,2507,,,,,,,,,2025-03-10,,,,,,,,,,,,,,,,,,,,,,
+Decorus Sicarius,,1040121100,,,,,,,,,,,,,,,,,,,,,,,,,,Decorus Sicarius,,,,,,,,,,,,,,,,,,,,

db/seed/updates/20250330-characters-026.csv

@@ -0,0 +1,4 @@
+name_en,name_jp,granblue_id,rarity,element,proficiency1,proficiency2,gender,race1,race2,flb,min_hp,max_hp,max_hp_flb,min_atk,max_atk,max_atk_flb,base_da,base_ta,ougi_ratio,ougi_ratio_flb,special,ulb,max_hp_ulb,max_atk_ulb,character_id,wiki_en,release_date,flb_date,ulb_date,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp
+Seofon (Yukata),シエテ(浴衣ver),3040586000,3,5,1,10,1,1,,false,277,1277,,1777,9777,,,,,,false,false,,,{4007},Seofon (Yukata),2025-03-30,,,シエテ (SSR)浴衣バージョン,489325,SSR浴衣シエテ,,
+Vikala (Yukata),ビカラ(浴衣ver),3040585000,3,6,3,7,2,1,,false,280,1550,,1600,8250,,,,,,false,false,,,{3150},Vikala (Yukata),2025-03-30,,,ビカラ (SSR)浴衣バージョン,491855,SSR浴衣ビカラ,,
+,,3040087000,,,,,,,,true,300,1600,1900,1500,8000,9500,,,,,,,,,,,2016-06-30,2025-03-29,,ロザミア (SSR),33985,SSRロザミア,,

db/seed/updates/20250330-weapons-026.csv

@@ -0,0 +1,5 @@
+name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date, , , , , , , , , , , , , 
+First Fling of the Mischief,第一子行弾弓,1040714000,3,6,5,99,true,false,150,15,46,236,284,,377,2460,2981,,false,,false,false,,,2025-03-30,2025-03-30,,First Fling of the Mischief,第一子行弾弓 (SSR),,第一子行弾弓,,,false,,,,,,,,,,,,,,
+Prismatic Trientalis,七彩華刀,1040917900,3,5,10,99,false,false,100,10,18,174,,,515,2737,,,false,,false,false,,,2025-03-30,,,Prismatic Trientalis,七彩華刀 (SSR),,七彩華刀,,,false,,,,,,,,,,,,,,
+Scarface,スカーフェイス,1040517300,3,5,9,98,false,false,100,10,11,132,,,408,2221,,,false,,false,false,,,2025-03-29,,,Scarface,スカーフェイス (SSR),491802,スカーフェイス,,,false,,,,,,,,,,,,,,
+Henchman,ヘンチマン,1030109100,2,5,2,98,false,false,75,10,8,117,,,271,1368,,,false,,false,false,,,2025-03-29,,,Henchman,ヘンチマン (SR),,ヘンチマン,,,false,,,,,,,,,,,,,,

lib/granblue/parsers/character_parser.rb

@@ -4,21 +4,26 @@ require 'pry'
 
 module Granblue
   module Parsers
-
     # CharacterParser parses character data from gbf.wiki
     class CharacterParser
       attr_reader :granblue_id
 
-      def initialize(granblue_id: String, debug: false)
+      def initialize(granblue_id: String, debug: false, use_local: false)
         @character = Character.find_by(granblue_id: granblue_id)
-        @wiki = GranblueWiki.new
+        @wiki = Granblue::Parsers::Wiki.new
         @debug = debug || false
+        @use_local = use_local
       end
 
       # Fetches using @wiki and then processes the response
       # Returns true if successful, false if not
       # Raises an exception if something went wrong
       def fetch(save: false)
+        if @use_local && @character.wiki_raw.present?
+          wikitext = @character.wiki_raw
+          return handle_fetch_success(wikitext, save)
+        end
+
         response = fetch_wiki_info
         return false if response.nil?
 
@@ -49,6 +54,9 @@ module Granblue
       # Handle the response from the wiki if the response is successful
       # If the save flag is set, it will persist the data to the database
       def handle_fetch_success(response, save)
+        @character.wiki_raw = response
+        @character.save!
+
         ap "#{@character.granblue_id}: Successfully fetched info for #{@character.wiki_en}" if @debug
         extracted = parse_string(response)
         info = parse(extracted)
@@ -152,12 +160,12 @@ module Granblue
         info[:id] = hash['id']
         info[:charid] = hash['charid'].scan(/\b\d{4}\b/)
 
-        info[:flb] = GranblueWiki.boolean.fetch(hash['5star'], false)
+        info[:flb] = Granblue::Parsers::Wiki.boolean.fetch(hash['5star'], false)
         info[:ulb] = hash['max_evo'].to_i == 6
 
-        info[:rarity] = GranblueWiki.rarities.fetch(hash['rarity'], 0)
+        info[:rarity] = Granblue::Parsers::Wiki.rarities.fetch(hash['rarity'], 0)
-        info[:element] = GranblueWiki.elements.fetch(hash['element'], 0)
+        info[:element] = Granblue::Parsers::Wiki.elements.fetch(hash['element'], 0)
-        info[:gender] = GranblueWiki.genders.fetch(hash['gender'], 0)
+        info[:gender] = Granblue::Parsers::Wiki.genders.fetch(hash['gender'], 0)
 
         info[:proficiencies] = proficiencies_from_hash(hash['weapon'])
         info[:races] = races_from_hash(hash['race'])
@@ -211,14 +219,14 @@ module Granblue
       # Converts proficiencies from a string to a hash
       def proficiencies_from_hash(character)
         character.to_s.split(',').map.with_index do |prof, i|
-          { "proficiency#{i + 1}" => GranblueWiki.proficiencies[prof] }
+          { "proficiency#{i + 1}" => Granblue::Parsers::Wiki.proficiencies[prof] }
         end.reduce({}, :merge)
       end
 
       # Converts races from a string to a hash
       def races_from_hash(race)
         race.to_s.split(',').map.with_index do |r, i|
-          { "race#{i + 1}" => GranblueWiki.races[r] }
+          { "race#{i + 1}" => Granblue::Parsers::Wiki.races[r] }
         end.reduce({}, :merge)
       end
 

lib/granblue/parsers/summon_parser.rb

@@ -10,7 +10,7 @@ module Granblue
 
       def initialize(granblue_id: String, debug: false)
         @summon = Summon.find_by(granblue_id: granblue_id)
-        @wiki = GranblueWiki.new(debug: debug)
+        @wiki = Granblue::Parsers::Wiki.new
         @debug = debug || false
       end
 

lib/granblue/parsers/weapon_parser.rb

@@ -10,7 +10,7 @@ module Granblue
 
       def initialize(granblue_id: String, debug: false)
         @weapon = Weapon.find_by(granblue_id: granblue_id)
-        @wiki = GranblueWiki.new(debug: debug)
+        @wiki = Granblue::Parsers::Wiki.new
         @debug = debug || false
       end
 
@@ -278,17 +278,17 @@ module Granblue
 
       # Converts rarities from a string to a hash
       def rarity_from_hash(string)
-        string ? GranblueWiki.rarities[string.upcase] : nil
+        string ? Granblue::Parsers::Wiki.rarities[string.upcase] : nil
       end
 
       # Converts proficiencies from a string to a hash
       def proficiency_from_hash(string)
-        GranblueWiki.proficiencies[string]
+        Granblue::Parsers::Wiki.proficiencies[string]
       end
 
       # Converts a bullet type from a string to a hash
       def bullet_from_hash(string)
-        string ? GranblueWiki.bullets[string] : nil
+        string ? Granblue::Parsers::Wiki.bullets[string] : nil
       end
 
       # Parses a date string into a Date object

lib/tasks/fetch_wiki.rake

@@ -0,0 +1,79 @@
+namespace :granblue do
+  desc <<~DESC
+    Fetch and store raw wiki data for objects (Character, Weapon, Summon).
+
+    Usage:
+      rake granblue:fetch_wiki_data                     # Fetch all Characters (default)
+      rake granblue:fetch_wiki_data type=Weapon         # Fetch all Weapons
+      rake granblue:fetch_wiki_data type=Summon         # Fetch all Summons
+      rake granblue:fetch_wiki_data type=Character id=5 # Fetch specific Character by ID
+      rake granblue:fetch_wiki_data force=true          # Force re-download even if data exists
+  DESC
+  task fetch_wiki_data: :environment do
+    # Get parameters from environment
+    type = (ENV['type'] || 'Character').classify
+    id = ENV['id']
+    force = ENV['force'] == 'true'
+
+    # Validate object type
+    valid_types = %w[Character Weapon Summon]
+    unless valid_types.include?(type)
+      puts "Error: Invalid type '#{type}'. Must be one of: #{valid_types.join(', ')}"
+      exit 1
+    end
+
+    # Get the class from the type string
+    klass = type.constantize
+
+    # Setup query - either all objects or specific one
+    query = id.present? ? klass.where(granblue_id: id) : klass.all
+
+    errors = []
+    count = 0
+
+    query.find_each do |object|
+      # Skip objects that already have wiki_raw if force is not set
+      if object.wiki_raw.present? && !force
+        puts "Skipping #{object.name_en} (already has wiki_raw)."
+        next
+      end
+
+      # If the object doesn't have a wiki page specified, skip
+      if object.wiki_en.blank?
+        puts "Skipping #{object.name_en} (no wiki_en set)."
+        next
+      end
+
+      begin
+        # 1) Fetch raw wikitext from the wiki
+        wiki_text = Granblue::Parsers::Wiki.new.fetch(object.wiki_en)
+
+        # 2) Check if the page is a redirect
+        redirect_match = wiki_text.match(/#REDIRECT \[\[(.*?)\]\]/)
+        if redirect_match
+          redirect_target = redirect_match[1]
+          # Update object to new wiki_en so we don't keep fetching the old page
+          object.update!(wiki_en: redirect_target)
+          # Fetch again with the new page name
+          wiki_text = Granblue::Parsers::Wiki.new.fetch(redirect_target)
+        end
+        puts wiki_text
+
+        # 3) Save raw wiki text in the object record
+        object.update!(wiki_raw: wiki_text)
+        puts "Saved wiki data for #{object.name_en} (#{object.id})"
+        count += 1
+      rescue StandardError => e
+        errors << { object_id: object.id, type: type, error: e.message }
+        puts "Error fetching data for #{object.name_en}: #{e.message}"
+      end
+    end
+
+    if errors.any?
+      puts "#{errors.size} #{type.pluralize} had errors:"
+      errors.each { |err| puts "  - #{err[:type]} ##{err[:object_id]} => #{err[:error]}" }
+    else
+      puts "Wiki data fetch complete for #{count} #{type.pluralize} with no errors!"
+    end
+  end
+end