Add fetch_from_list static method

This backports the `fetch_from_list` static method to CharacterParser and WeaponParser. This will let us fetch data for a specific set of items instead of having to fetch one-by-one or fetch the entire dataset again.
This commit is contained in:
Justin Edmund 2023-08-19 23:01:27 -07:00
parent f21ca5e659
commit 2a9d1ce766
2 changed files with 52 additions and 0 deletions

View file

@ -115,6 +115,32 @@ class CharacterParser
ap errors
end
def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil)
errors = []
start_index = start.nil? ? 0 : list.index { |id| id == start }
count = list.drop(start_index).count
# ap "Start index: #{start_index}"
list.drop(start_index).each_with_index do |id, i|
chara = Character.find_by(granblue_id: id)
percentage = ((i + 1) / count.to_f * 100).round(2)
ap "#{percentage}%: Fetching #{chara.wiki_en}... (#{i + 1}/#{count})" if debug
next unless chara.release_date.nil? || overwrite
begin
WeaponParser.new(granblue_id: chara.granblue_id,
debug: debug).fetch(save: save)
rescue WikiError => e
errors.push(e.page)
end
end
ap 'The following pages were unable to be fetched:'
ap errors
end
# Parses the hash into a format that can be saved to the database
def parse(hash)
info = {}

View file

@ -94,6 +94,32 @@ class WeaponParser
ap errors
end
def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil)
errors = []
start_index = start.nil? ? 0 : list.index { |id| id == start }
count = list.drop(start_index).count
# ap "Start index: #{start_index}"
list.drop(start_index).each_with_index do |id, i|
weapon = Weapon.find_by(granblue_id: id)
percentage = ((i + 1) / count.to_f * 100).round(2)
ap "#{percentage}%: Fetching #{weapon.wiki_en}... (#{i + 1}/#{count})" if debug
next unless weapon.release_date.nil? || overwrite
begin
WeaponParser.new(granblue_id: weapon.granblue_id,
debug: debug).fetch(save: save)
rescue WikiError => e
errors.push(e.page)
end
end
ap 'The following pages were unable to be fetched:'
ap errors
end
# Parses the response string into a hash
def parse_string(string)
data = {}