Bring up-to-date with main

Gemfile

@@ -50,6 +50,12 @@ gem 'data_migrate'
 # A ruby gem to allow the copying of ActiveRecord objects and their associated children, configurable with a DSL on the model
 gem 'amoeba'
 
+# Makes http fun again!
+gem 'httparty'
+
+# StringScanner provides for lexical scanning operations on a String.
+gem 'strscan'
+
 group :doc do
   gem 'apipie-rails'
   gem 'sdoc'
@@ -60,6 +66,7 @@ group :development, :test do
   gem 'dotenv-rails'
   gem 'factory_bot_rails'
   gem 'faker'
+  gem 'pry'
   gem 'rspec_junit_formatter'
   gem 'rspec-rails'
 end
@@ -72,8 +79,8 @@ group :development do
 end
 
 group :tools do
-  gem 'squasher', '>= 0.6.0'
   gem 'rubocop'
+  gem 'squasher', '>= 0.6.0'
 end
 
 group :test do

Gemfile.lock

@@ -87,6 +87,7 @@ GEM
       msgpack (~> 1.2)
     builder (3.2.4)
     byebug (11.1.3)
+    coderay (1.1.3)
     concurrent-ruby (1.1.10)
     crass (1.0.6)
     data_migrate (8.5.0)
@@ -126,6 +127,9 @@ GEM
       gemoji (>= 2.1.0)
     globalid (1.0.1)
       activesupport (>= 5.0)
+    httparty (0.20.0)
+      mime-types (~> 3.0)
+      multi_xml (>= 0.5.2)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
@@ -147,10 +151,14 @@ GEM
       net-smtp
     marcel (1.0.2)
     method_source (1.0.0)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
     mini_portile2 (2.8.1)
     minitest (5.17.0)
     msgpack (1.6.0)
+    multi_xml (0.6.0)
     net-imap (0.3.4)
       date
       net-protocol
@@ -172,6 +180,9 @@ GEM
     pg_search (2.3.6)
       activerecord (>= 5.2)
       activesupport (>= 5.2)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     psych (5.0.2)
       stringio
     puma (6.0.2)
@@ -295,6 +306,7 @@ GEM
       sprockets (>= 3.0.0)
     squasher (0.7.2)
     stringio (3.0.4)
+    strscan (3.0.0)
     thor (1.2.1)
     tilt (2.0.11)
     timeout (0.3.1)
@@ -332,10 +344,12 @@ DEPENDENCIES
   faker
   figaro
   gemoji-parser
+  httparty
   listen
   oj
   pg
   pg_search
+  pry
   puma
   rack-cors
   rails
@@ -351,10 +365,11 @@ DEPENDENCIES
   spring-commands-rspec
   sprockets-rails
   squasher (>= 0.6.0)
+  strscan
   will_paginate (~> 3.3)
 
 RUBY VERSION
    ruby 3.0.0p0
 
 BUNDLED WITH
-   2.3.26
+   2.4.2

app/controllers/api/v1/search_controller.rb

@@ -72,7 +72,7 @@ module Api
                          Character.en_search(search_params[:query]).where(conditions)
                        end
                      else
-                       Character.where(conditions)
+                       Character.where(conditions).order(Arel.sql('greatest(release_date, flb_date, ulb_date) desc'))
                      end
 
         count = characters.length
@@ -110,7 +110,7 @@ module Api
                       Weapon.en_search(search_params[:query]).where(conditions)
                     end
                   else
-                    Weapon.where(conditions)
+                    Weapon.where(conditions).order(Arel.sql('greatest(release_date, flb_date, ulb_date) desc'))
                   end
 
         count = weapons.length
@@ -143,7 +143,7 @@ module Api
                       Summon.en_search(search_params[:query]).where(conditions)
                     end
                   else
-                    Summon.where(conditions)
+                    Summon.where(conditions).order(release_date: :desc).order(Arel.sql('greatest(release_date, flb_date, ulb_date, xlb_date) desc'))
                   end
 
         count = summons.length

app/errors/WikiError.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class WikiError < StandardError
+  def initialize(code: nil, page: nil, message: nil)
+    super
+    @code = code
+    @page = page
+    @message = message
+  end
+
+  def to_hash
+    {
+      message: @message,
+      code: @code,
+      page: @page
+    }
+  end
+end

app/helpers/character_parser.rb

@@ -0,0 +1,278 @@
+# frozen_string_literal: true
+
+require 'pry'
+
+# CharacterParser parses character data from gbf.wiki
+class CharacterParser
+  attr_reader :granblue_id
+
+  def initialize(granblue_id: String, debug: false)
+    @character = Character.find_by(granblue_id: granblue_id)
+    @wiki = GranblueWiki.new
+    @debug = debug || false
+  end
+
+  # Fetches using @wiki and then processes the response
+  # Returns true if successful, false if not
+  # Raises an exception if something went wrong
+  def fetch(save: false)
+    response = fetch_wiki_info
+    return false if response.nil?
+
+    redirect = handle_redirected_string(response)
+    return fetch(save: save) unless redirect.nil?
+
+    handle_fetch_success(response, save)
+  end
+
+  private
+
+  # Determines whether or not the response is a redirect
+  # If it is, it will update the character's wiki_en value
+  def handle_redirected_string(response)
+    redirect = extract_redirected_string(response)
+    return unless redirect
+
+    @character.wiki_en = redirect
+    if @character.save!
+      ap "Saved new wiki_en value for #{@character.granblue_id}: #{redirect}" if @debug
+      redirect
+    else
+      ap "Unable to save new wiki_en value for #{@character.granblue_id}: #{redirect}" if @debug
+      nil
+    end
+  end
+
+  # Handle the response from the wiki if the response is successful
+  # If the save flag is set, it will persist the data to the database
+  def handle_fetch_success(response, save)
+    ap "#{@character.granblue_id}: Successfully fetched info for #{@character.wiki_en}" if @debug
+    extracted = parse_string(response)
+    info = parse(extracted)
+    persist(info) if save
+    true
+  end
+
+  # Determines whether the response string
+  # should be treated as a redirect
+  def extract_redirected_string(string)
+    string.match(/#REDIRECT \[\[(.*?)\]\]/)&.captures&.first
+  end
+
+  # Parses the response string into a hash
+  def parse_string(string)
+    lines = string.split("\n")
+    data = {}
+    stop_loop = false
+
+    lines.each do |line|
+      next if stop_loop
+
+      if line.include?('Gameplay Notes')
+        stop_loop = true
+        next
+      end
+
+      next unless line[0] == '|' && line.size > 2
+
+      key, value = line[1..].split('=', 2).map(&:strip)
+      data[key] = value if value
+    end
+
+    data
+  end
+
+  # Fetches data from the GranblueWiki object
+  def fetch_wiki_info
+    @wiki.fetch(@character.wiki_en)
+  rescue WikiError => e
+    ap "There was an error fetching #{e.page}: #{e.message}" if @debug
+    nil
+  end
+
+  # Iterates over all characters in the database and fetches their data
+  # If the save flag is set, data is saved to the database
+  # If the overwrite flag is set, data is fetched even if it already exists
+  # If the debug flag is set, additional information is printed to the console
+  def self.fetch_all(save: false, overwrite: false, debug: false)
+    errors = []
+
+    count = Character.count
+    Character.all.each_with_index do |c, i|
+      percentage = ((i + 1) / count.to_f * 100).round(2)
+      ap "#{percentage}%: Fetching #{c.name_en}... (#{i + 1}/#{count})" if debug
+      next unless c.release_date.nil? || overwrite
+
+      begin
+        CharacterParser.new(granblue_id: c.granblue_id,
+                            debug: debug).fetch(save: save)
+      rescue WikiError => e
+        errors.push(e.page)
+      end
+    end
+
+    ap 'The following pages were unable to be fetched:'
+    ap errors
+  end
+
+  def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil)
+    errors = []
+
+    start_index = start.nil? ? 0 : list.index { |id| id == start }
+    count = list.drop(start_index).count
+
+    # ap "Start index: #{start_index}"
+
+    list.drop(start_index).each_with_index do |id, i|
+      chara = Character.find_by(granblue_id: id)
+      percentage = ((i + 1) / count.to_f * 100).round(2)
+      ap "#{percentage}%: Fetching #{chara.wiki_en}... (#{i + 1}/#{count})" if debug
+      next unless chara.release_date.nil? || overwrite
+
+      begin
+        WeaponParser.new(granblue_id: chara.granblue_id,
+                         debug: debug).fetch(save: save)
+      rescue WikiError => e
+        errors.push(e.page)
+      end
+    end
+
+    ap 'The following pages were unable to be fetched:'
+    ap errors
+  end
+
+  # Parses the hash into a format that can be saved to the database
+  def parse(hash)
+    info = {}
+
+    info[:name] = { en: hash['name'], ja: hash['jpname'] }
+    info[:id] = hash['id']
+    info[:charid] = hash['charid'].scan(/\b\d{4}\b/)
+
+    info[:flb] = GranblueWiki.boolean.fetch(hash['5star'], false)
+    info[:ulb] = hash['max_evo'].to_i == 6
+
+    info[:rarity] = GranblueWiki.rarities.fetch(hash['rarity'], 0)
+    info[:element] = GranblueWiki.elements.fetch(hash['element'], 0)
+    info[:gender] = GranblueWiki.genders.fetch(hash['gender'], 0)
+
+    info[:proficiencies] = proficiencies_from_hash(hash['weapon'])
+    info[:races] = races_from_hash(hash['race'])
+
+    info[:hp] = {
+      min_hp: hash['min_hp'].to_i,
+      max_hp: hash['max_hp'].to_i,
+      max_hp_flb: hash['flb_hp'].to_i
+    }
+
+    info[:atk] = {
+      min_atk: hash['min_atk'].to_i,
+      max_atk: hash['max_atk'].to_i,
+      max_atk_flb: hash['flb_atk'].to_i
+    }
+
+    info[:dates] = {
+      release_date: parse_date(hash['release_date']),
+      flb_date: parse_date(hash['5star_date']),
+      ulb_date: parse_date(hash['6star_date'])
+    }
+
+    info[:links] = {
+      wiki: { en: hash['name'], ja: hash['link_jpwiki'] },
+      gamewith: hash['link_gamewith'],
+      kamigame: hash['link_kamigame']
+    }
+
+    info.compact
+  end
+
+  # Saves select fields to the database
+  def persist(hash)
+    @character.release_date = hash[:dates][:release_date]
+    @character.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date)
+    @character.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date)
+
+    @character.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja)
+    @character.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith)
+    @character.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame)
+
+    if @character.save
+      ap "#{@character.granblue_id}: Successfully saved info for #{@character.name_en}" if @debug
+      puts
+      true
+    end
+
+    false
+  end
+
+  # Converts proficiencies from a string to a hash
+  def proficiencies_from_hash(character)
+    character.to_s.split(',').map.with_index do |prof, i|
+      { "proficiency#{i + 1}" => GranblueWiki.proficiencies[prof] }
+    end.reduce({}, :merge)
+  end
+
+  # Converts races from a string to a hash
+  def races_from_hash(race)
+    race.to_s.split(',').map.with_index do |r, i|
+      { "race#{i + 1}" => GranblueWiki.races[r] }
+    end.reduce({}, :merge)
+  end
+
+  # Parses a date string into a Date object
+  def parse_date(date_str)
+    Date.parse(date_str) unless date_str.blank?
+  end
+
+  # Unused methods for now
+  def extract_abilities(hash)
+    abilities = []
+    hash.each do |key, value|
+      next unless key =~ /^a(\d+)_/
+
+      ability_number = Regexp.last_match(1).to_i
+      abilities[ability_number] ||= {}
+
+      case key.gsub(/^a\d+_/, '')
+      when 'cd'
+        cooldown = parse_substring(value)
+        abilities[ability_number]['cooldown'] = cooldown
+      when 'dur'
+        duration = parse_substring(value)
+        abilities[ability_number]['duration'] = duration
+      when 'oblevel'
+        obtained = parse_substring(value)
+        abilities[ability_number]['obtained'] = obtained
+      else
+        abilities[ability_number][key.gsub(/^a\d+_/, '')] = value
+      end
+    end
+
+    { 'abilities' => abilities.compact }
+  end
+
+  def parse_substring(string)
+    hash = {}
+
+    string.scan(/\|([^|=]+?)=([^|]+)/) do |key, value|
+      value.gsub!(/\}\}$/, '') if value.include?('}}')
+      hash[key] = value
+    end
+
+    hash
+  end
+
+  def extract_ougis(hash)
+    ougi = []
+    hash.each do |key, value|
+      next unless key =~ /^ougi(\d*)_(.*)/
+
+      ougi_number = Regexp.last_match(1)
+      ougi_key = Regexp.last_match(2)
+      ougi[ougi_number.to_i] ||= {}
+      ougi[ougi_number.to_i][ougi_key] = value
+    end
+
+    { 'ougis' => ougi.compact }
+  end
+end

app/helpers/granblue_wiki.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'httparty'
+
+# GranblueWiki fetches and parses data from gbf.wiki
+class GranblueWiki
+  class_attribute :base_uri
+
+  class_attribute :proficiencies
+  class_attribute :elements
+  class_attribute :rarities
+  class_attribute :genders
+  class_attribute :races
+  class_attribute :bullets
+  class_attribute :boolean
+
+  self.base_uri = 'https://gbf.wiki/api.php'
+
+  self.proficiencies = {
+    'Sabre' => 1,
+    'Dagger' => 2,
+    'Axe' => 3,
+    'Spear' => 4,
+    'Bow' => 5,
+    'Staff' => 6,
+    'Melee' => 7,
+    'Harp' => 8,
+    'Gun' => 9,
+    'Katana' => 10
+  }.freeze
+
+  self.elements = {
+    'Wind' => 1,
+    'Fire' => 2,
+    'Water' => 3,
+    'Earth' => 4,
+    'Dark' => 5,
+    'Light' => 6
+  }.freeze
+
+  self.rarities = {
+    'R' => 1,
+    'SR' => 2,
+    'SSR' => 3
+  }.freeze
+
+  self.races = {
+    'Other' => 0,
+    'Human' => 1,
+    'Erune' => 2,
+    'Draph' => 3,
+    'Harvin' => 4,
+    'Primal' => 5
+  }.freeze
+
+  self.genders = {
+    'o' => 0,
+    'm' => 1,
+    'f' => 2,
+    'mf' => 3
+  }.freeze
+
+  self.bullets = {
+    'cartridge' => 1,
+    'rifle' => 2,
+    'parabellum' => 3,
+    'aetherial' => 4
+  }.freeze
+
+  self.boolean = {
+    'yes' => true,
+    'no' => false
+  }.freeze
+
+  def initialize(props: ['wikitext'], debug: false)
+    @debug = debug
+    @props = props.join('|')
+  end
+
+  def fetch(page)
+    query_params = params(page).map do |key, value|
+      "#{key}=#{value}"
+    end.join('&')
+
+    destination = "#{base_uri}?#{query_params}"
+    ap "--> Fetching #{destination}" if @debug
+
+    response = HTTParty.get(destination)
+
+    handle_response(response, page)
+  end
+
+  private
+
+  def handle_response(response, page)
+    case response.code
+    when 200
+      if response.key?('error')
+        raise WikiError.new(code: response['error']['code'],
+                            message: response['error']['info'],
+                            page: page)
+      end
+
+      response['parse']['wikitext']['*']
+    when 404 then puts "Page #{page} not found"
+    when 500...600 then puts "Server error: #{response.code}"
+    end
+  end
+
+  def params(page)
+    {
+      action: 'parse',
+      format: 'json',
+      page: page,
+      prop: @props
+    }
+  end
+end

app/helpers/summon_parser.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+require 'pry'
+
+# SummonParser parses summon data from gbf.wiki
+class SummonParser
+  attr_reader :granblue_id
+
+  def initialize(granblue_id: String, debug: false)
+    @summon = Summon.find_by(granblue_id: granblue_id)
+    @wiki = GranblueWiki.new(debug: debug)
+    @debug = debug || false
+  end
+
+  # Fetches using @wiki and then processes the response
+  # Returns true if successful, false if not
+  # Raises an exception if something went wrong
+  def fetch(name = nil, save: false)
+    response = fetch_wiki_info(name)
+    return false if response.nil?
+
+    if response.starts_with?('#REDIRECT')
+      # Fetch the string inside of [[]]
+      redirect = response[/\[\[(.*?)\]\]/m, 1]
+      fetch(redirect, save: save)
+    else
+      # return response if response[:error]
+      handle_fetch_success(response, save)
+    end
+  end
+
+  private
+
+  # Handle the response from the wiki if the response is successful
+  # If the save flag is set, it will persist the data to the database
+  def handle_fetch_success(response, save)
+    ap "#{@summon.granblue_id}: Successfully fetched info for #{@summon.wiki_en}" if @debug
+
+    extracted = parse_string(response)
+
+    unless extracted[:template].nil?
+      template = @wiki.fetch("Template:#{extracted[:template]}")
+      extracted.merge!(parse_string(template))
+    end
+
+    info, skills = parse(extracted)
+
+    # ap info
+    # ap skills
+
+    persist(info[:info]) if save
+    true
+  end
+
+  # Fetches the wiki info from the wiki
+  # Returns the response body
+  # Raises an exception if something went wrong
+  def fetch_wiki_info(name = nil)
+    @wiki.fetch(name || @summon.wiki_en)
+  rescue WikiError => e
+    ap e
+    # ap "There was an error fetching #{e.page}: #{e.message}" if @debug
+    {
+      error: {
+        name: @summon.wiki_en,
+        granblue_id: @summon.granblue_id
+      }
+    }
+  end
+
+  # Iterates over all summons in the database and fetches their data
+  # If the save flag is set, data is saved to the database
+  # If the overwrite flag is set, data is fetched even if it already exists
+  # If the debug flag is set, additional information is printed to the console
+  def self.fetch_all(save: false, overwrite: false, debug: false, start: nil)
+    errors = []
+
+    summons = Summon.all.order(:granblue_id)
+
+    start_index = start.nil? ? 0 : summons.index { |w| w.granblue_id == start }
+    count = summons.drop(start_index).count
+
+    # ap "Start index: #{start_index}"
+
+    summons.drop(start_index).each_with_index do |w, i|
+      percentage = ((i + 1) / count.to_f * 100).round(2)
+      ap "#{percentage}%: Fetching #{w.wiki_en}... (#{i + 1}/#{count})" if debug
+      next unless w.release_date.nil? || overwrite
+
+      begin
+        SummonParser.new(granblue_id: w.granblue_id,
+                         debug: debug).fetch(save: save)
+      rescue WikiError => e
+        errors.push(e.page)
+      end
+    end
+
+    ap 'The following pages were unable to be fetched:'
+    ap errors
+  end
+
+  def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil)
+    errors = []
+
+    start_index = start.nil? ? 0 : list.index { |id| id == start }
+    count = list.drop(start_index).count
+
+    # ap "Start index: #{start_index}"
+
+    list.drop(start_index).each_with_index do |id, i|
+      summon = Summon.find_by(granblue_id: id)
+      percentage = ((i + 1) / count.to_f * 100).round(2)
+      ap "#{percentage}%: Fetching #{summon.wiki_en}... (#{i + 1}/#{count})" if debug
+      next unless summon.release_date.nil? || overwrite
+
+      begin
+        SummonParser.new(granblue_id: summon.granblue_id,
+                         debug: debug).fetch(save: save)
+      rescue WikiError => e
+        errors.push(e.page)
+      end
+    end
+
+    ap 'The following pages were unable to be fetched:'
+    ap errors
+  end
+
+  # Parses the response string into a hash
+  def parse_string(string)
+    data = {}
+    lines = string.split("\n")
+    stop_loop = false
+
+    lines.each do |line|
+      next if stop_loop
+
+      if line.include?('Gameplay Notes')
+        stop_loop = true
+        next
+      end
+
+      if line.starts_with?('{{')
+        substr = line[2..].strip! || line[2..]
+
+        # All template tags start with {{ so we can skip the first two characters
+        disallowed = %w[#vardefine #lsth About]
+        next if substr.start_with?(*disallowed)
+
+        if substr.start_with?('Summon')
+          ap "--> Found template: #{substr}" if @debug
+
+          substr = substr.split('|').first
+          data[:template] = substr if substr != 'Summon'
+          next
+        end
+      end
+
+      next unless line[0] == '|' && line.size > 2
+
+      key, value = line[1..].split('=', 2).map(&:strip)
+
+      regex = /\A\{\{\{.*\|\}\}\}\z/
+      next if value =~ regex
+
+      data[key] = value if value
+    end
+
+    data
+  end
+
+  # Parses the hash into a format that can be saved to the database
+  def parse(hash)
+    info = {}
+    skills = {}
+
+    info[:name] = { en: hash['name'], ja: hash['jpname'] }
+    info[:flavor] = { en: hash['flavor'], ja: hash['jpflavor'] }
+    info[:id] = hash['id']
+
+    info[:flb] = hash['evo_max'].to_i >= 4
+    info[:ulb] = hash['evo_max'].to_i >= 5
+    info[:xlb] = hash['evo_max'].to_i == 6
+
+    info[:rarity] = rarity_from_hash(hash['rarity'])
+    info[:series] = hash['series']
+    info[:obtain] = hash['obtain']
+
+    info[:hp] = {
+      min_hp: hash['hp1'].to_i,
+      max_hp: hash['hp2'].to_i,
+      max_hp_flb: hash['hp3'].to_i,
+      max_hp_ulb: hash['hp4'].to_i.zero? ? nil : hash['hp4'].to_i,
+      max_hp_xlb: hash['hp5'].to_i.zero? ? nil : hash['hp5'].to_i
+    }
+
+    info[:atk] = {
+      min_atk: hash['atk1'].to_i,
+      max_atk: hash['atk2'].to_i,
+      max_atk_flb: hash['atk3'].to_i,
+      max_atk_ulb: hash['atk4'].to_i.zero? ? nil : hash['atk4'].to_i,
+      max_atk_xlb: hash['atk5'].to_i.zero? ? nil : hash['atk5'].to_i
+    }
+
+    info[:dates] = {
+      release_date: parse_date(hash['release_date']),
+      flb_date: parse_date(hash['4star_date']),
+      ulb_date: parse_date(hash['5star_date']),
+      xlb_date: parse_date(hash['6star_date'])
+    }
+
+    info[:links] = {
+      wiki: { en: hash['name'], ja: hash['link_jpwiki'] },
+      gamewith: hash['link_gamewith'],
+      kamigame: hash['link_kamigame']
+    }
+
+    {
+      info: info.compact
+      # skills: skills.compact
+    }
+  end
+
+  # Saves select fields to the database
+  def persist(hash)
+    @summon.release_date = hash[:dates][:release_date]
+    @summon.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date)
+    @summon.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date)
+
+    @summon.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja)
+    @summon.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith)
+    @summon.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame)
+
+    if @summon.save
+      ap "#{@summon.granblue_id}: Successfully saved info for #{@summon.wiki_en}" if @debug
+      puts
+      true
+    end
+
+    false
+  end
+
+  # Converts rarities from a string to a hash
+  def rarity_from_hash(string)
+    string ? GranblueWiki.rarities[string.upcase] : nil
+  end
+
+  # Parses a date string into a Date object
+  def parse_date(date_str)
+    Date.parse(date_str) unless date_str.blank?
+  end
+end

app/helpers/weapon_parser.rb

@@ -0,0 +1,296 @@
+# frozen_string_literal: true
+
+require 'pry'
+
+# WeaponParser parses weapon data from gbf.wiki
+class WeaponParser
+  attr_reader :granblue_id
+
+  def initialize(granblue_id: String, debug: false)
+    @weapon = Weapon.find_by(granblue_id: granblue_id)
+    @wiki = GranblueWiki.new(debug: debug)
+    @debug = debug || false
+  end
+
+  # Fetches using @wiki and then processes the response
+  # Returns true if successful, false if not
+  # Raises an exception if something went wrong
+  def fetch(save: false)
+    response = fetch_wiki_info
+    return false if response.nil?
+
+    # return response if response[:error]
+
+    handle_fetch_success(response, save)
+  end
+
+  private
+
+  # Handle the response from the wiki if the response is successful
+  # If the save flag is set, it will persist the data to the database
+  def handle_fetch_success(response, save)
+    ap "#{@weapon.granblue_id}: Successfully fetched info for #{@weapon.wiki_en}" if @debug
+    extracted = parse_string(response)
+
+    unless extracted[:template].nil?
+      template = @wiki.fetch("Template:#{extracted[:template]}")
+      extracted.merge!(parse_string(template))
+    end
+
+    info, skills = parse(extracted)
+
+    # ap info
+    # ap skills
+
+    persist(info[:info]) if save
+    true
+  end
+
+  # Fetches the wiki info from the wiki
+  # Returns the response body
+  # Raises an exception if something went wrong
+  def fetch_wiki_info
+    @wiki.fetch(@weapon.wiki_en)
+  rescue WikiError => e
+    ap e
+    # ap "There was an error fetching #{e.page}: #{e.message}" if @debug
+    {
+      error: {
+        name: @weapon.wiki_en,
+        granblue_id: @weapon.granblue_id
+      }
+    }
+  end
+
+  # Iterates over all weapons in the database and fetches their data
+  # If the save flag is set, data is saved to the database
+  # If the overwrite flag is set, data is fetched even if it already exists
+  # If the debug flag is set, additional information is printed to the console
+  def self.fetch_all(save: false, overwrite: false, debug: false, start: nil)
+    errors = []
+
+    weapons = Weapon.all.order(:granblue_id)
+
+    start_index = start.nil? ? 0 : weapons.index { |w| w.granblue_id == start }
+    count = weapons.drop(start_index).count
+
+    # ap "Start index: #{start_index}"
+
+    weapons.drop(start_index).each_with_index do |w, i|
+      percentage = ((i + 1) / count.to_f * 100).round(2)
+      ap "#{percentage}%: Fetching #{w.wiki_en}... (#{i + 1}/#{count})" if debug
+      next if w.wiki_en.include?('Element Changed') || w.wiki_en.include?('Awakened')
+      next unless w.release_date.nil? || overwrite
+
+      begin
+        WeaponParser.new(granblue_id: w.granblue_id,
+                         debug: debug).fetch(save: save)
+      rescue WikiError => e
+        errors.push(e.page)
+      end
+    end
+
+    ap 'The following pages were unable to be fetched:'
+    ap errors
+  end
+
+  def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil)
+    errors = []
+
+    start_index = start.nil? ? 0 : list.index { |id| id == start }
+    count = list.drop(start_index).count
+
+    # ap "Start index: #{start_index}"
+
+    list.drop(start_index).each_with_index do |id, i|
+      weapon = Weapon.find_by(granblue_id: id)
+      percentage = ((i + 1) / count.to_f * 100).round(2)
+      ap "#{percentage}%: Fetching #{weapon.wiki_en}... (#{i + 1}/#{count})" if debug
+      next unless weapon.release_date.nil? || overwrite
+
+      begin
+        WeaponParser.new(granblue_id: weapon.granblue_id,
+                         debug: debug).fetch(save: save)
+      rescue WikiError => e
+        errors.push(e.page)
+      end
+    end
+
+    ap 'The following pages were unable to be fetched:'
+    ap errors
+  end
+
+  # Parses the response string into a hash
+  def parse_string(string)
+    data = {}
+    lines = string.split("\n")
+    stop_loop = false
+
+    lines.each do |line|
+      next if stop_loop
+
+      if line.include?('Gameplay Notes')
+        stop_loop = true
+        next
+      end
+
+      if line.starts_with?('{{')
+        substr = line[2..].strip! || line[2..]
+
+        # All template tags start with {{ so we can skip the first two characters
+        disallowed = %w[#vardefine #lsth About]
+        next if substr.start_with?(*disallowed)
+
+        if substr.start_with?('Weapon')
+          ap "--> Found template: #{substr}" if @debug
+
+          substr = substr.split('|').first
+          data[:template] = substr if substr != 'Weapon'
+          next
+        end
+      end
+
+      next unless line[0] == '|' && line.size > 2
+
+      key, value = line[1..].split('=', 2).map(&:strip)
+
+      regex = /\A\{\{\{.*\|\}\}\}\z/
+      next if value =~ regex
+
+      data[key] = value if value
+    end
+
+    data
+  end
+
+  # Parses the hash into a format that can be saved to the database
+  def parse(hash)
+    info = {}
+    skills = {}
+
+    info[:name] = { en: hash['name'], ja: hash['jpname'] }
+    info[:flavor] = { en: hash['flavor'], ja: hash['jpflavor'] }
+    info[:id] = hash['id']
+
+    info[:flb] = hash['evo_max'].to_i >= 4
+    info[:ulb] = hash['evo_max'].to_i == 5
+
+    info[:rarity] = rarity_from_hash(hash['rarity'])
+    info[:proficiency] = proficiency_from_hash(hash['weapon'])
+    info[:series] = hash['series']
+    info[:obtain] = hash['obtain']
+
+    if hash.key?('bullets')
+      info[:bullets] = {
+        count: hash['bullets'].to_i,
+        loadout: [
+          bullet_from_hash(hash['bullet1']),
+          bullet_from_hash(hash['bullet2']),
+          bullet_from_hash(hash['bullet3']),
+          bullet_from_hash(hash['bullet4']),
+          bullet_from_hash(hash['bullet5']),
+          bullet_from_hash(hash['bullet6'])
+        ]
+      }
+    end
+
+    info[:hp] = {
+      min_hp: hash['hp1'].to_i,
+      max_hp: hash['hp2'].to_i,
+      max_hp_flb: hash['hp3'].to_i,
+      max_hp_ulb: hash['hp4'].to_i.zero? ? nil : hash['hp4'].to_i
+    }
+
+    info[:atk] = {
+      min_atk: hash['atk1'].to_i,
+      max_atk: hash['atk2'].to_i,
+      max_atk_flb: hash['atk3'].to_i,
+      max_atk_ulb: hash['atk4'].to_i.zero? ? nil : hash['atk4'].to_i
+    }
+
+    info[:dates] = {
+      release_date: parse_date(hash['release_date']),
+      flb_date: parse_date(hash['4star_date']),
+      ulb_date: parse_date(hash['5star_date'])
+    }
+
+    info[:links] = {
+      wiki: { en: hash['name'], ja: hash['link_jpwiki'] },
+      gamewith: hash['link_gamewith'],
+      kamigame: hash['link_kamigame']
+    }
+
+    skills[:charge_attack] = {
+      name: { en: hash['ougi_name'], ja: hash['jpougi_name'] },
+      description: {
+        mlb: {
+          en: hash['enougi'],
+          ja: hash['jpougi']
+        },
+        flb: {
+          en: hash['enougi_4s'],
+          ja: hash['jpougi_4s']
+        }
+      }
+    }
+
+    skills[:skills] = [
+      {
+        name: { en: hash['s1_name'], ja: nil },
+        description: { en: hash['ens1_desc'] || hash['s1_desc'], ja: nil }
+      },
+      {
+        name: { en: hash['s2_name'], ja: nil },
+        description: { en: hash['ens2_desc'] || hash['s2_desc'], ja: nil }
+      },
+      {
+        name: { en: hash['s3_name'], ja: nil },
+        description: { en: hash['ens3_desc'] || hash['s3_desc'], ja: nil }
+      }
+    ]
+
+    {
+      info: info.compact,
+      skills: skills.compact
+    }
+  end
+
+  # Saves select fields to the database
+  def persist(hash)
+    @weapon.release_date = hash[:dates][:release_date]
+    @weapon.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date)
+    @weapon.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date)
+
+    @weapon.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja)
+    @weapon.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith)
+    @weapon.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame)
+
+    if @weapon.save
+      ap "#{@weapon.granblue_id}: Successfully saved info for #{@weapon.wiki_en}" if @debug
+      puts
+      true
+    end
+
+    false
+  end
+
+  # Converts rarities from a string to a hash
+  def rarity_from_hash(string)
+    string ? GranblueWiki.rarities[string.upcase] : nil
+  end
+
+  # Converts proficiencies from a string to a hash
+  def proficiency_from_hash(string)
+    GranblueWiki.proficiencies[string]
+  end
+
+  # Converts a bullet type from a string to a hash
+  def bullet_from_hash(string)
+    string ? GranblueWiki.bullets[string] : nil
+  end
+
+  # Parses a date string into a Date object
+  def parse_date(date_str)
+    Date.parse(date_str) unless date_str.blank?
+  end
+end

app/models/character.rb

@@ -14,15 +14,19 @@ class Character < ApplicationRecord
                   }
 
   pg_search_scope :en_search,
-                  against: :name_en,
+                  against: %i[name_en nicknames_en],
                   using: {
+                    tsearch: {
+                      prefix: true,
+                      dictionary: 'simple'
+                    },
                     trigram: {
                       threshold: 0.18
                     }
                   }
 
   pg_search_scope :ja_search,
-                  against: :name_jp,
+                  against: %i[name_jp nicknames_jp],
                   using: {
                     tsearch: {
                       prefix: true,

app/models/summon.rb

@@ -14,15 +14,19 @@ class Summon < ApplicationRecord
                   }
 
   pg_search_scope :en_search,
-                  against: :name_en,
+                  against: %i[name_en nicknames_en],
                   using: {
+                    tsearch: {
+                      prefix: true,
+                      dictionary: 'simple'
+                    },
                     trigram: {
                       threshold: 0.18
                     }
                   }
 
   pg_search_scope :ja_search,
-                  against: :name_jp,
+                  against: %i[name_jp nicknames_jp],
                   using: {
                     tsearch: {
                       prefix: true,

app/models/weapon.rb

@@ -14,15 +14,19 @@ class Weapon < ApplicationRecord
                   }
 
   pg_search_scope :en_search,
-                  against: :name_en,
+                  against: %i[name_en nicknames_en],
                   using: {
+                    tsearch: {
+                      prefix: true,
+                      dictionary: 'simple'
+                    },
                     trigram: {
                       threshold: 0.18
                     }
                   }
 
   pg_search_scope :ja_search,
-                  against: :name_jp,
+                  against: %i[name_jp nicknames_jp],
                   using: {
                     tsearch: {
                       prefix: true,

db/data/20230716132721_populate_wiki_en_column.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class PopulateWikiEnColumn < ActiveRecord::Migration[7.0]
+  def up
+    Character.all.each do |c|
+      c.wiki_en = c.name_en
+      c.save
+    end
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/data/20230816061005_populate_wiki_columns.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class PopulateWikiColumns < ActiveRecord::Migration[7.0]
+  def up
+    Weapon.all.each do |c|
+      c.wiki_en = c.name_en
+      c.save
+    end
+
+    Summon.all.each do |c|
+      c.wiki_en = c.name_en
+      c.save
+    end
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/data_schema.rb

@@ -1 +1 @@
-DataMigrate::Data.define(version: 20230702035600)
+DataMigrate::Data.define(version: 20230816061005)

db/migrate/20230716132629_add_gbf_wiki_to_character.rb

@@ -0,0 +1,5 @@
+class AddGbfWikiToCharacter < ActiveRecord::Migration[7.0]
+  def change
+    add_column :characters, :wiki_en, :string, null: false, default: ''
+  end
+end

db/migrate/20230717011150_add_columns_to_characters.rb

@@ -0,0 +1,11 @@
+class AddColumnsToCharacters < ActiveRecord::Migration[7.0]
+  def change
+    add_column :characters, :release_date, :date
+    add_column :characters, :flb_date, :date
+    add_column :characters, :ulb_date, :date
+
+    add_column :characters, :wiki_ja, :string, null: false, default: ''
+    add_column :characters, :gamewith, :string, null: false, default: ''
+    add_column :characters, :kamigame, :string, null: false, default: ''
+  end
+end

db/migrate/20230816015828_add_columns_to_weapons.rb

@@ -0,0 +1,12 @@
+class AddColumnsToWeapons < ActiveRecord::Migration[7.0]
+  def change
+    add_column :weapons, :release_date, :date
+    add_column :weapons, :flb_date, :date
+    add_column :weapons, :ulb_date, :date
+
+    add_column :weapons, :wiki_en, :string, default: ''
+    add_column :weapons, :wiki_ja, :string, default: ''
+    add_column :weapons, :gamewith, :string, default: ''
+    add_column :weapons, :kamigame, :string, default: ''
+  end
+end

db/migrate/20230816015904_add_columns_to_summons.rb

@@ -0,0 +1,13 @@
+class AddColumnsToSummons < ActiveRecord::Migration[7.0]
+  def change
+    add_column :summons, :summon_id, :integer
+    add_column :summons, :release_date, :date
+    add_column :summons, :flb_date, :date
+    add_column :summons, :ulb_date, :date
+
+    add_column :summons, :wiki_en, :string, default: ''
+    add_column :summons, :wiki_ja, :string, default: ''
+    add_column :summons, :gamewith, :string, default: ''
+    add_column :summons, :kamigame, :string, default: ''
+  end
+end

db/migrate/20230820045019_add_xlb_date_to_summons.rb

@@ -0,0 +1,5 @@
+class AddXlbDateToSummons < ActiveRecord::Migration[7.0]
+  def change
+    add_column :summons, :xlb_date, :date
+  end
+end

db/migrate/20230820113800_add_nicknames_to_characters.rb

@@ -0,0 +1,6 @@
+class AddNicknamesToCharacters < ActiveRecord::Migration[7.0]
+  def change
+    add_column :characters, :nicknames_en, :string, array: true, default: [], null: false, if_not_exists: true
+    add_column :characters, :nicknames_jp, :string, array: true, default: [], null: false, if_not_exists: true
+  end
+end

db/migrate/20230820113810_add_nicknames_to_summons.rb

@@ -0,0 +1,6 @@
+class AddNicknamesToSummons < ActiveRecord::Migration[7.0]
+  def change
+    add_column :summons, :nicknames_en, :string, array: true, default: [], null: false, if_not_exists: true
+    add_column :summons, :nicknames_jp, :string, array: true, default: [], null: false, if_not_exists: true
+  end
+end

db/migrate/20230820113900_add_nicknames_to_weapons.rb

@@ -0,0 +1,6 @@
+class AddNicknamesToWeapons < ActiveRecord::Migration[7.0]
+  def change
+    add_column :weapons, :nicknames_en, :string, array: true, default: [], null: false, if_not_exists: true
+    add_column :weapons, :nicknames_jp, :string, array: true, default: [], null: false, if_not_exists: true
+  end
+end