diff --git a/Gemfile b/Gemfile index a138342..84a05ea 100644 --- a/Gemfile +++ b/Gemfile @@ -50,6 +50,12 @@ gem 'data_migrate' # A ruby gem to allow the copying of ActiveRecord objects and their associated children, configurable with a DSL on the model gem 'amoeba' +# Makes http fun again! +gem 'httparty' + +# StringScanner provides for lexical scanning operations on a String. +gem 'strscan' + group :doc do gem 'apipie-rails' gem 'sdoc' @@ -60,6 +66,7 @@ group :development, :test do gem 'dotenv-rails' gem 'factory_bot_rails' gem 'faker' + gem 'pry' gem 'rspec_junit_formatter' gem 'rspec-rails' end @@ -72,8 +79,8 @@ group :development do end group :tools do - gem 'squasher', '>= 0.6.0' gem 'rubocop' + gem 'squasher', '>= 0.6.0' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index fef5a6c..f02220c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -87,6 +87,7 @@ GEM msgpack (~> 1.2) builder (3.2.4) byebug (11.1.3) + coderay (1.1.3) concurrent-ruby (1.1.10) crass (1.0.6) data_migrate (8.5.0) @@ -126,6 +127,9 @@ GEM gemoji (>= 2.1.0) globalid (1.0.1) activesupport (>= 5.0) + httparty (0.20.0) + mime-types (~> 3.0) + multi_xml (>= 0.5.2) i18n (1.12.0) concurrent-ruby (~> 1.0) jaro_winkler (1.5.4) @@ -147,10 +151,14 @@ GEM net-smtp marcel (1.0.2) method_source (1.0.0) + mime-types (3.4.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2022.0105) mini_mime (1.1.2) mini_portile2 (2.8.1) minitest (5.17.0) msgpack (1.6.0) + multi_xml (0.6.0) net-imap (0.3.4) date net-protocol @@ -172,6 +180,9 @@ GEM pg_search (2.3.6) activerecord (>= 5.2) activesupport (>= 5.2) + pry (0.14.1) + coderay (~> 1.1) + method_source (~> 1.0) psych (5.0.2) stringio puma (6.0.2) @@ -295,6 +306,7 @@ GEM sprockets (>= 3.0.0) squasher (0.7.2) stringio (3.0.4) + strscan (3.0.0) thor (1.2.1) tilt (2.0.11) timeout (0.3.1) @@ -332,10 +344,12 @@ DEPENDENCIES faker figaro gemoji-parser + httparty listen oj pg pg_search + pry puma rack-cors rails @@ -351,10 +365,11 @@ DEPENDENCIES spring-commands-rspec sprockets-rails squasher (>= 0.6.0) + strscan will_paginate (~> 3.3) RUBY VERSION ruby 3.0.0p0 BUNDLED WITH - 2.3.26 + 2.4.2 diff --git a/app/errors/WikiError.rb b/app/errors/WikiError.rb new file mode 100644 index 0000000..82089b5 --- /dev/null +++ b/app/errors/WikiError.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class WikiError < StandardError + def initialize(code: nil, page: nil, message: nil) + super + @code = code + @page = page + @message = message + end + + def to_hash + { + message: @message, + code: @code, + page: @page + } + end +end diff --git a/app/helpers/character_parser.rb b/app/helpers/character_parser.rb new file mode 100644 index 0000000..4c78a3f --- /dev/null +++ b/app/helpers/character_parser.rb @@ -0,0 +1,278 @@ +# frozen_string_literal: true + +require 'pry' + +# CharacterParser parses character data from gbf.wiki +class CharacterParser + attr_reader :granblue_id + + def initialize(granblue_id: String, debug: false) + @character = Character.find_by(granblue_id: granblue_id) + @wiki = GranblueWiki.new + @debug = debug || false + end + + # Fetches using @wiki and then processes the response + # Returns true if successful, false if not + # Raises an exception if something went wrong + def fetch(save: false) + response = fetch_wiki_info + return false if response.nil? + + redirect = handle_redirected_string(response) + return fetch(save: save) unless redirect.nil? + + handle_fetch_success(response, save) + end + + private + + # Determines whether or not the response is a redirect + # If it is, it will update the character's wiki_en value + def handle_redirected_string(response) + redirect = extract_redirected_string(response) + return unless redirect + + @character.wiki_en = redirect + if @character.save! + ap "Saved new wiki_en value for #{@character.granblue_id}: #{redirect}" if @debug + redirect + else + ap "Unable to save new wiki_en value for #{@character.granblue_id}: #{redirect}" if @debug + nil + end + end + + # Handle the response from the wiki if the response is successful + # If the save flag is set, it will persist the data to the database + def handle_fetch_success(response, save) + ap "#{@character.granblue_id}: Successfully fetched info for #{@character.wiki_en}" if @debug + extracted = parse_string(response) + info = parse(extracted) + persist(info) if save + true + end + + # Determines whether the response string + # should be treated as a redirect + def extract_redirected_string(string) + string.match(/#REDIRECT \[\[(.*?)\]\]/)&.captures&.first + end + + # Parses the response string into a hash + def parse_string(string) + lines = string.split("\n") + data = {} + stop_loop = false + + lines.each do |line| + next if stop_loop + + if line.include?('Gameplay Notes') + stop_loop = true + next + end + + next unless line[0] == '|' && line.size > 2 + + key, value = line[1..].split('=', 2).map(&:strip) + data[key] = value if value + end + + data + end + + # Fetches data from the GranblueWiki object + def fetch_wiki_info + @wiki.fetch(@character.wiki_en) + rescue WikiError => e + ap "There was an error fetching #{e.page}: #{e.message}" if @debug + nil + end + + # Iterates over all characters in the database and fetches their data + # If the save flag is set, data is saved to the database + # If the overwrite flag is set, data is fetched even if it already exists + # If the debug flag is set, additional information is printed to the console + def self.fetch_all(save: false, overwrite: false, debug: false) + errors = [] + + count = Character.count + Character.all.each_with_index do |c, i| + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{c.name_en}... (#{i + 1}/#{count})" if debug + next unless c.release_date.nil? || overwrite + + begin + CharacterParser.new(granblue_id: c.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) + errors = [] + + start_index = start.nil? ? 0 : list.index { |id| id == start } + count = list.drop(start_index).count + + # ap "Start index: #{start_index}" + + list.drop(start_index).each_with_index do |id, i| + chara = Character.find_by(granblue_id: id) + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{chara.wiki_en}... (#{i + 1}/#{count})" if debug + next unless chara.release_date.nil? || overwrite + + begin + WeaponParser.new(granblue_id: chara.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + # Parses the hash into a format that can be saved to the database + def parse(hash) + info = {} + + info[:name] = { en: hash['name'], ja: hash['jpname'] } + info[:id] = hash['id'] + info[:charid] = hash['charid'].scan(/\b\d{4}\b/) + + info[:flb] = GranblueWiki.boolean.fetch(hash['5star'], false) + info[:ulb] = hash['max_evo'].to_i == 6 + + info[:rarity] = GranblueWiki.rarities.fetch(hash['rarity'], 0) + info[:element] = GranblueWiki.elements.fetch(hash['element'], 0) + info[:gender] = GranblueWiki.genders.fetch(hash['gender'], 0) + + info[:proficiencies] = proficiencies_from_hash(hash['weapon']) + info[:races] = races_from_hash(hash['race']) + + info[:hp] = { + min_hp: hash['min_hp'].to_i, + max_hp: hash['max_hp'].to_i, + max_hp_flb: hash['flb_hp'].to_i + } + + info[:atk] = { + min_atk: hash['min_atk'].to_i, + max_atk: hash['max_atk'].to_i, + max_atk_flb: hash['flb_atk'].to_i + } + + info[:dates] = { + release_date: parse_date(hash['release_date']), + flb_date: parse_date(hash['5star_date']), + ulb_date: parse_date(hash['6star_date']) + } + + info[:links] = { + wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, + gamewith: hash['link_gamewith'], + kamigame: hash['link_kamigame'] + } + + info.compact + end + + # Saves select fields to the database + def persist(hash) + @character.release_date = hash[:dates][:release_date] + @character.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) + @character.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) + + @character.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) + @character.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) + @character.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) + + if @character.save + ap "#{@character.granblue_id}: Successfully saved info for #{@character.name_en}" if @debug + puts + true + end + + false + end + + # Converts proficiencies from a string to a hash + def proficiencies_from_hash(character) + character.to_s.split(',').map.with_index do |prof, i| + { "proficiency#{i + 1}" => GranblueWiki.proficiencies[prof] } + end.reduce({}, :merge) + end + + # Converts races from a string to a hash + def races_from_hash(race) + race.to_s.split(',').map.with_index do |r, i| + { "race#{i + 1}" => GranblueWiki.races[r] } + end.reduce({}, :merge) + end + + # Parses a date string into a Date object + def parse_date(date_str) + Date.parse(date_str) unless date_str.blank? + end + + # Unused methods for now + def extract_abilities(hash) + abilities = [] + hash.each do |key, value| + next unless key =~ /^a(\d+)_/ + + ability_number = Regexp.last_match(1).to_i + abilities[ability_number] ||= {} + + case key.gsub(/^a\d+_/, '') + when 'cd' + cooldown = parse_substring(value) + abilities[ability_number]['cooldown'] = cooldown + when 'dur' + duration = parse_substring(value) + abilities[ability_number]['duration'] = duration + when 'oblevel' + obtained = parse_substring(value) + abilities[ability_number]['obtained'] = obtained + else + abilities[ability_number][key.gsub(/^a\d+_/, '')] = value + end + end + + { 'abilities' => abilities.compact } + end + + def parse_substring(string) + hash = {} + + string.scan(/\|([^|=]+?)=([^|]+)/) do |key, value| + value.gsub!(/\}\}$/, '') if value.include?('}}') + hash[key] = value + end + + hash + end + + def extract_ougis(hash) + ougi = [] + hash.each do |key, value| + next unless key =~ /^ougi(\d*)_(.*)/ + + ougi_number = Regexp.last_match(1) + ougi_key = Regexp.last_match(2) + ougi[ougi_number.to_i] ||= {} + ougi[ougi_number.to_i][ougi_key] = value + end + + { 'ougis' => ougi.compact } + end +end diff --git a/app/helpers/granblue_wiki.rb b/app/helpers/granblue_wiki.rb new file mode 100644 index 0000000..3e006c2 --- /dev/null +++ b/app/helpers/granblue_wiki.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require 'httparty' + +# GranblueWiki fetches and parses data from gbf.wiki +class GranblueWiki + class_attribute :base_uri + + class_attribute :proficiencies + class_attribute :elements + class_attribute :rarities + class_attribute :genders + class_attribute :races + class_attribute :bullets + class_attribute :boolean + + self.base_uri = 'https://gbf.wiki/api.php' + + self.proficiencies = { + 'Sabre' => 1, + 'Dagger' => 2, + 'Axe' => 3, + 'Spear' => 4, + 'Bow' => 5, + 'Staff' => 6, + 'Melee' => 7, + 'Harp' => 8, + 'Gun' => 9, + 'Katana' => 10 + }.freeze + + self.elements = { + 'Wind' => 1, + 'Fire' => 2, + 'Water' => 3, + 'Earth' => 4, + 'Dark' => 5, + 'Light' => 6 + }.freeze + + self.rarities = { + 'R' => 1, + 'SR' => 2, + 'SSR' => 3 + }.freeze + + self.races = { + 'Other' => 0, + 'Human' => 1, + 'Erune' => 2, + 'Draph' => 3, + 'Harvin' => 4, + 'Primal' => 5 + }.freeze + + self.genders = { + 'o' => 0, + 'm' => 1, + 'f' => 2, + 'mf' => 3 + }.freeze + + self.bullets = { + 'cartridge' => 1, + 'rifle' => 2, + 'parabellum' => 3, + 'aetherial' => 4 + }.freeze + + self.boolean = { + 'yes' => true, + 'no' => false + }.freeze + + def initialize(props: ['wikitext'], debug: false) + @debug = debug + @props = props.join('|') + end + + def fetch(page) + query_params = params(page).map do |key, value| + "#{key}=#{value}" + end.join('&') + + destination = "#{base_uri}?#{query_params}" + ap "--> Fetching #{destination}" if @debug + + response = HTTParty.get(destination) + + handle_response(response, page) + end + + private + + def handle_response(response, page) + case response.code + when 200 + if response.key?('error') + raise WikiError.new(code: response['error']['code'], + message: response['error']['info'], + page: page) + end + + response['parse']['wikitext']['*'] + when 404 then puts "Page #{page} not found" + when 500...600 then puts "Server error: #{response.code}" + end + end + + def params(page) + { + action: 'parse', + format: 'json', + page: page, + prop: @props + } + end +end diff --git a/app/helpers/summon_parser.rb b/app/helpers/summon_parser.rb new file mode 100644 index 0000000..eff61a8 --- /dev/null +++ b/app/helpers/summon_parser.rb @@ -0,0 +1,251 @@ +# frozen_string_literal: true + +require 'pry' + +# SummonParser parses summon data from gbf.wiki +class SummonParser + attr_reader :granblue_id + + def initialize(granblue_id: String, debug: false) + @summon = Summon.find_by(granblue_id: granblue_id) + @wiki = GranblueWiki.new(debug: debug) + @debug = debug || false + end + + # Fetches using @wiki and then processes the response + # Returns true if successful, false if not + # Raises an exception if something went wrong + def fetch(name = nil, save: false) + response = fetch_wiki_info(name) + return false if response.nil? + + if response.starts_with?('#REDIRECT') + # Fetch the string inside of [[]] + redirect = response[/\[\[(.*?)\]\]/m, 1] + fetch(redirect, save: save) + else + # return response if response[:error] + handle_fetch_success(response, save) + end + end + + private + + # Handle the response from the wiki if the response is successful + # If the save flag is set, it will persist the data to the database + def handle_fetch_success(response, save) + ap "#{@summon.granblue_id}: Successfully fetched info for #{@summon.wiki_en}" if @debug + + extracted = parse_string(response) + + unless extracted[:template].nil? + template = @wiki.fetch("Template:#{extracted[:template]}") + extracted.merge!(parse_string(template)) + end + + info, skills = parse(extracted) + + # ap info + # ap skills + + persist(info[:info]) if save + true + end + + # Fetches the wiki info from the wiki + # Returns the response body + # Raises an exception if something went wrong + def fetch_wiki_info(name = nil) + @wiki.fetch(name || @summon.wiki_en) + rescue WikiError => e + ap e + # ap "There was an error fetching #{e.page}: #{e.message}" if @debug + { + error: { + name: @summon.wiki_en, + granblue_id: @summon.granblue_id + } + } + end + + # Iterates over all summons in the database and fetches their data + # If the save flag is set, data is saved to the database + # If the overwrite flag is set, data is fetched even if it already exists + # If the debug flag is set, additional information is printed to the console + def self.fetch_all(save: false, overwrite: false, debug: false, start: nil) + errors = [] + + summons = Summon.all.order(:granblue_id) + + start_index = start.nil? ? 0 : summons.index { |w| w.granblue_id == start } + count = summons.drop(start_index).count + + # ap "Start index: #{start_index}" + + summons.drop(start_index).each_with_index do |w, i| + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{w.wiki_en}... (#{i + 1}/#{count})" if debug + next unless w.release_date.nil? || overwrite + + begin + SummonParser.new(granblue_id: w.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) + errors = [] + + start_index = start.nil? ? 0 : list.index { |id| id == start } + count = list.drop(start_index).count + + # ap "Start index: #{start_index}" + + list.drop(start_index).each_with_index do |id, i| + summon = Summon.find_by(granblue_id: id) + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{summon.wiki_en}... (#{i + 1}/#{count})" if debug + next unless summon.release_date.nil? || overwrite + + begin + SummonParser.new(granblue_id: summon.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + # Parses the response string into a hash + def parse_string(string) + data = {} + lines = string.split("\n") + stop_loop = false + + lines.each do |line| + next if stop_loop + + if line.include?('Gameplay Notes') + stop_loop = true + next + end + + if line.starts_with?('{{') + substr = line[2..].strip! || line[2..] + + # All template tags start with {{ so we can skip the first two characters + disallowed = %w[#vardefine #lsth About] + next if substr.start_with?(*disallowed) + + if substr.start_with?('Summon') + ap "--> Found template: #{substr}" if @debug + + substr = substr.split('|').first + data[:template] = substr if substr != 'Summon' + next + end + end + + next unless line[0] == '|' && line.size > 2 + + key, value = line[1..].split('=', 2).map(&:strip) + + regex = /\A\{\{\{.*\|\}\}\}\z/ + next if value =~ regex + + data[key] = value if value + end + + data + end + + # Parses the hash into a format that can be saved to the database + def parse(hash) + info = {} + skills = {} + + info[:name] = { en: hash['name'], ja: hash['jpname'] } + info[:flavor] = { en: hash['flavor'], ja: hash['jpflavor'] } + info[:id] = hash['id'] + + info[:flb] = hash['evo_max'].to_i >= 4 + info[:ulb] = hash['evo_max'].to_i >= 5 + info[:xlb] = hash['evo_max'].to_i == 6 + + info[:rarity] = rarity_from_hash(hash['rarity']) + info[:series] = hash['series'] + info[:obtain] = hash['obtain'] + + info[:hp] = { + min_hp: hash['hp1'].to_i, + max_hp: hash['hp2'].to_i, + max_hp_flb: hash['hp3'].to_i, + max_hp_ulb: hash['hp4'].to_i.zero? ? nil : hash['hp4'].to_i, + max_hp_xlb: hash['hp5'].to_i.zero? ? nil : hash['hp5'].to_i + } + + info[:atk] = { + min_atk: hash['atk1'].to_i, + max_atk: hash['atk2'].to_i, + max_atk_flb: hash['atk3'].to_i, + max_atk_ulb: hash['atk4'].to_i.zero? ? nil : hash['atk4'].to_i, + max_atk_xlb: hash['atk5'].to_i.zero? ? nil : hash['atk5'].to_i + } + + info[:dates] = { + release_date: parse_date(hash['release_date']), + flb_date: parse_date(hash['4star_date']), + ulb_date: parse_date(hash['5star_date']), + xlb_date: parse_date(hash['6star_date']) + } + + info[:links] = { + wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, + gamewith: hash['link_gamewith'], + kamigame: hash['link_kamigame'] + } + + { + info: info.compact + # skills: skills.compact + } + end + + # Saves select fields to the database + def persist(hash) + @summon.release_date = hash[:dates][:release_date] + @summon.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) + @summon.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) + + @summon.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) + @summon.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) + @summon.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) + + if @summon.save + ap "#{@summon.granblue_id}: Successfully saved info for #{@summon.wiki_en}" if @debug + puts + true + end + + false + end + + # Converts rarities from a string to a hash + def rarity_from_hash(string) + string ? GranblueWiki.rarities[string.upcase] : nil + end + + # Parses a date string into a Date object + def parse_date(date_str) + Date.parse(date_str) unless date_str.blank? + end +end diff --git a/app/helpers/weapon_parser.rb b/app/helpers/weapon_parser.rb new file mode 100644 index 0000000..81abdb8 --- /dev/null +++ b/app/helpers/weapon_parser.rb @@ -0,0 +1,296 @@ +# frozen_string_literal: true + +require 'pry' + +# WeaponParser parses weapon data from gbf.wiki +class WeaponParser + attr_reader :granblue_id + + def initialize(granblue_id: String, debug: false) + @weapon = Weapon.find_by(granblue_id: granblue_id) + @wiki = GranblueWiki.new(debug: debug) + @debug = debug || false + end + + # Fetches using @wiki and then processes the response + # Returns true if successful, false if not + # Raises an exception if something went wrong + def fetch(save: false) + response = fetch_wiki_info + return false if response.nil? + + # return response if response[:error] + + handle_fetch_success(response, save) + end + + private + + # Handle the response from the wiki if the response is successful + # If the save flag is set, it will persist the data to the database + def handle_fetch_success(response, save) + ap "#{@weapon.granblue_id}: Successfully fetched info for #{@weapon.wiki_en}" if @debug + extracted = parse_string(response) + + unless extracted[:template].nil? + template = @wiki.fetch("Template:#{extracted[:template]}") + extracted.merge!(parse_string(template)) + end + + info, skills = parse(extracted) + + # ap info + # ap skills + + persist(info[:info]) if save + true + end + + # Fetches the wiki info from the wiki + # Returns the response body + # Raises an exception if something went wrong + def fetch_wiki_info + @wiki.fetch(@weapon.wiki_en) + rescue WikiError => e + ap e + # ap "There was an error fetching #{e.page}: #{e.message}" if @debug + { + error: { + name: @weapon.wiki_en, + granblue_id: @weapon.granblue_id + } + } + end + + # Iterates over all weapons in the database and fetches their data + # If the save flag is set, data is saved to the database + # If the overwrite flag is set, data is fetched even if it already exists + # If the debug flag is set, additional information is printed to the console + def self.fetch_all(save: false, overwrite: false, debug: false, start: nil) + errors = [] + + weapons = Weapon.all.order(:granblue_id) + + start_index = start.nil? ? 0 : weapons.index { |w| w.granblue_id == start } + count = weapons.drop(start_index).count + + # ap "Start index: #{start_index}" + + weapons.drop(start_index).each_with_index do |w, i| + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{w.wiki_en}... (#{i + 1}/#{count})" if debug + next if w.wiki_en.include?('Element Changed') || w.wiki_en.include?('Awakened') + next unless w.release_date.nil? || overwrite + + begin + WeaponParser.new(granblue_id: w.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil) + errors = [] + + start_index = start.nil? ? 0 : list.index { |id| id == start } + count = list.drop(start_index).count + + # ap "Start index: #{start_index}" + + list.drop(start_index).each_with_index do |id, i| + weapon = Weapon.find_by(granblue_id: id) + percentage = ((i + 1) / count.to_f * 100).round(2) + ap "#{percentage}%: Fetching #{weapon.wiki_en}... (#{i + 1}/#{count})" if debug + next unless weapon.release_date.nil? || overwrite + + begin + WeaponParser.new(granblue_id: weapon.granblue_id, + debug: debug).fetch(save: save) + rescue WikiError => e + errors.push(e.page) + end + end + + ap 'The following pages were unable to be fetched:' + ap errors + end + + # Parses the response string into a hash + def parse_string(string) + data = {} + lines = string.split("\n") + stop_loop = false + + lines.each do |line| + next if stop_loop + + if line.include?('Gameplay Notes') + stop_loop = true + next + end + + if line.starts_with?('{{') + substr = line[2..].strip! || line[2..] + + # All template tags start with {{ so we can skip the first two characters + disallowed = %w[#vardefine #lsth About] + next if substr.start_with?(*disallowed) + + if substr.start_with?('Weapon') + ap "--> Found template: #{substr}" if @debug + + substr = substr.split('|').first + data[:template] = substr if substr != 'Weapon' + next + end + end + + next unless line[0] == '|' && line.size > 2 + + key, value = line[1..].split('=', 2).map(&:strip) + + regex = /\A\{\{\{.*\|\}\}\}\z/ + next if value =~ regex + + data[key] = value if value + end + + data + end + + # Parses the hash into a format that can be saved to the database + def parse(hash) + info = {} + skills = {} + + info[:name] = { en: hash['name'], ja: hash['jpname'] } + info[:flavor] = { en: hash['flavor'], ja: hash['jpflavor'] } + info[:id] = hash['id'] + + info[:flb] = hash['evo_max'].to_i >= 4 + info[:ulb] = hash['evo_max'].to_i == 5 + + info[:rarity] = rarity_from_hash(hash['rarity']) + info[:proficiency] = proficiency_from_hash(hash['weapon']) + info[:series] = hash['series'] + info[:obtain] = hash['obtain'] + + if hash.key?('bullets') + info[:bullets] = { + count: hash['bullets'].to_i, + loadout: [ + bullet_from_hash(hash['bullet1']), + bullet_from_hash(hash['bullet2']), + bullet_from_hash(hash['bullet3']), + bullet_from_hash(hash['bullet4']), + bullet_from_hash(hash['bullet5']), + bullet_from_hash(hash['bullet6']) + ] + } + end + + info[:hp] = { + min_hp: hash['hp1'].to_i, + max_hp: hash['hp2'].to_i, + max_hp_flb: hash['hp3'].to_i, + max_hp_ulb: hash['hp4'].to_i.zero? ? nil : hash['hp4'].to_i + } + + info[:atk] = { + min_atk: hash['atk1'].to_i, + max_atk: hash['atk2'].to_i, + max_atk_flb: hash['atk3'].to_i, + max_atk_ulb: hash['atk4'].to_i.zero? ? nil : hash['atk4'].to_i + } + + info[:dates] = { + release_date: parse_date(hash['release_date']), + flb_date: parse_date(hash['4star_date']), + ulb_date: parse_date(hash['5star_date']) + } + + info[:links] = { + wiki: { en: hash['name'], ja: hash['link_jpwiki'] }, + gamewith: hash['link_gamewith'], + kamigame: hash['link_kamigame'] + } + + skills[:charge_attack] = { + name: { en: hash['ougi_name'], ja: hash['jpougi_name'] }, + description: { + mlb: { + en: hash['enougi'], + ja: hash['jpougi'] + }, + flb: { + en: hash['enougi_4s'], + ja: hash['jpougi_4s'] + } + } + } + + skills[:skills] = [ + { + name: { en: hash['s1_name'], ja: nil }, + description: { en: hash['ens1_desc'] || hash['s1_desc'], ja: nil } + }, + { + name: { en: hash['s2_name'], ja: nil }, + description: { en: hash['ens2_desc'] || hash['s2_desc'], ja: nil } + }, + { + name: { en: hash['s3_name'], ja: nil }, + description: { en: hash['ens3_desc'] || hash['s3_desc'], ja: nil } + } + ] + + { + info: info.compact, + skills: skills.compact + } + end + + # Saves select fields to the database + def persist(hash) + @weapon.release_date = hash[:dates][:release_date] + @weapon.flb_date = hash[:dates][:flb_date] if hash[:dates].key?(:flb_date) + @weapon.ulb_date = hash[:dates][:ulb_date] if hash[:dates].key?(:ulb_date) + + @weapon.wiki_ja = hash[:links][:wiki][:ja] if hash[:links].key?(:wiki) && hash[:links][:wiki].key?(:ja) + @weapon.gamewith = hash[:links][:gamewith] if hash[:links].key?(:gamewith) + @weapon.kamigame = hash[:links][:kamigame] if hash[:links].key?(:kamigame) + + if @weapon.save + ap "#{@weapon.granblue_id}: Successfully saved info for #{@weapon.wiki_en}" if @debug + puts + true + end + + false + end + + # Converts rarities from a string to a hash + def rarity_from_hash(string) + string ? GranblueWiki.rarities[string.upcase] : nil + end + + # Converts proficiencies from a string to a hash + def proficiency_from_hash(string) + GranblueWiki.proficiencies[string] + end + + # Converts a bullet type from a string to a hash + def bullet_from_hash(string) + string ? GranblueWiki.bullets[string] : nil + end + + # Parses a date string into a Date object + def parse_date(date_str) + Date.parse(date_str) unless date_str.blank? + end +end diff --git a/db/data/20230716132721_populate_wiki_en_column.rb b/db/data/20230716132721_populate_wiki_en_column.rb new file mode 100644 index 0000000..9b546ac --- /dev/null +++ b/db/data/20230716132721_populate_wiki_en_column.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class PopulateWikiEnColumn < ActiveRecord::Migration[7.0] + def up + Character.all.each do |c| + c.wiki_en = c.name_en + c.save + end + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/data/20230816061005_populate_wiki_columns.rb b/db/data/20230816061005_populate_wiki_columns.rb new file mode 100644 index 0000000..2f6bfff --- /dev/null +++ b/db/data/20230816061005_populate_wiki_columns.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class PopulateWikiColumns < ActiveRecord::Migration[7.0] + def up + Weapon.all.each do |c| + c.wiki_en = c.name_en + c.save + end + + Summon.all.each do |c| + c.wiki_en = c.name_en + c.save + end + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/data_schema.rb b/db/data_schema.rb index 439fbc9..d620ac0 100644 --- a/db/data_schema.rb +++ b/db/data_schema.rb @@ -1 +1 @@ -DataMigrate::Data.define(version: 20230702035600) +DataMigrate::Data.define(version: 20230816061005) diff --git a/db/migrate/20230716132629_add_gbf_wiki_to_character.rb b/db/migrate/20230716132629_add_gbf_wiki_to_character.rb new file mode 100644 index 0000000..c4ddd72 --- /dev/null +++ b/db/migrate/20230716132629_add_gbf_wiki_to_character.rb @@ -0,0 +1,5 @@ +class AddGbfWikiToCharacter < ActiveRecord::Migration[7.0] + def change + add_column :characters, :wiki_en, :string, null: false, default: '' + end +end diff --git a/db/migrate/20230717011150_add_columns_to_characters.rb b/db/migrate/20230717011150_add_columns_to_characters.rb new file mode 100644 index 0000000..f6c0378 --- /dev/null +++ b/db/migrate/20230717011150_add_columns_to_characters.rb @@ -0,0 +1,11 @@ +class AddColumnsToCharacters < ActiveRecord::Migration[7.0] + def change + add_column :characters, :release_date, :date + add_column :characters, :flb_date, :date + add_column :characters, :ulb_date, :date + + add_column :characters, :wiki_ja, :string, null: false, default: '' + add_column :characters, :gamewith, :string, null: false, default: '' + add_column :characters, :kamigame, :string, null: false, default: '' + end +end diff --git a/db/migrate/20230816015828_add_columns_to_weapons.rb b/db/migrate/20230816015828_add_columns_to_weapons.rb new file mode 100644 index 0000000..5fdb6e2 --- /dev/null +++ b/db/migrate/20230816015828_add_columns_to_weapons.rb @@ -0,0 +1,12 @@ +class AddColumnsToWeapons < ActiveRecord::Migration[7.0] + def change + add_column :weapons, :release_date, :date + add_column :weapons, :flb_date, :date + add_column :weapons, :ulb_date, :date + + add_column :weapons, :wiki_en, :string, default: '' + add_column :weapons, :wiki_ja, :string, default: '' + add_column :weapons, :gamewith, :string, default: '' + add_column :weapons, :kamigame, :string, default: '' + end +end diff --git a/db/migrate/20230816015904_add_columns_to_summons.rb b/db/migrate/20230816015904_add_columns_to_summons.rb new file mode 100644 index 0000000..cb9b651 --- /dev/null +++ b/db/migrate/20230816015904_add_columns_to_summons.rb @@ -0,0 +1,13 @@ +class AddColumnsToSummons < ActiveRecord::Migration[7.0] + def change + add_column :summons, :summon_id, :integer + add_column :summons, :release_date, :date + add_column :summons, :flb_date, :date + add_column :summons, :ulb_date, :date + + add_column :summons, :wiki_en, :string, default: '' + add_column :summons, :wiki_ja, :string, default: '' + add_column :summons, :gamewith, :string, default: '' + add_column :summons, :kamigame, :string, default: '' + end +end diff --git a/db/migrate/20230820045019_add_xlb_date_to_summons.rb b/db/migrate/20230820045019_add_xlb_date_to_summons.rb new file mode 100644 index 0000000..7f739bb --- /dev/null +++ b/db/migrate/20230820045019_add_xlb_date_to_summons.rb @@ -0,0 +1,5 @@ +class AddXlbDateToSummons < ActiveRecord::Migration[7.0] + def change + add_column :summons, :xlb_date, :date + end +end diff --git a/db/schema.rb b/db/schema.rb index 6abca66..4a2715e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_07_05_065015) do +ActiveRecord::Schema[7.0].define(version: 2023_08_20_045019) do # These are extensions that must be enabled in order to support this database enable_extension "btree_gin" enable_extension "pg_trgm" @@ -103,6 +103,13 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_05_065015) do t.integer "character_id", default: [], null: false, array: true t.string "nicknames_en", default: [], null: false, array: true t.string "nicknames_jp", default: [], null: false, array: true + t.string "wiki_en", default: "", null: false + t.date "release_date" + t.date "flb_date" + t.date "ulb_date" + t.string "wiki_ja", default: "", null: false + t.string "gamewith", default: "", null: false + t.string "kamigame", default: "", null: false t.index ["name_en"], name: "index_characters_on_name_en", opclass: :gin_trgm_ops, using: :gin end @@ -434,6 +441,15 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_05_065015) do t.integer "max_hp_xlb" t.string "nicknames_en", default: [], null: false, array: true t.string "nicknames_jp", default: [], null: false, array: true + t.integer "summon_id" + t.date "release_date" + t.date "flb_date" + t.date "ulb_date" + t.string "wiki_en", default: "" + t.string "wiki_ja", default: "" + t.string "gamewith", default: "" + t.string "kamigame", default: "" + t.date "xlb_date" t.index ["name_en"], name: "index_summons_on_name_en", opclass: :gin_trgm_ops, using: :gin end @@ -498,6 +514,13 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_05_065015) do t.string "nicknames_jp", default: [], null: false, array: true t.uuid "recruits_id" t.integer "max_awakening_level" + t.date "release_date" + t.date "flb_date" + t.date "ulb_date" + t.string "wiki_en", default: "" + t.string "wiki_ja", default: "" + t.string "gamewith", default: "" + t.string "kamigame", default: "" t.index ["name_en"], name: "index_weapons_on_name_en", opclass: :gin_trgm_ops, using: :gin t.index ["recruits_id"], name: "index_weapons_on_recruits_id" end