Add fetch_from_list static method
This backports the `fetch_from_list` static method to CharacterParser and WeaponParser. This will let us fetch data for a specific set of items instead of having to fetch one-by-one or fetch the entire dataset again.
This commit is contained in:
parent
f21ca5e659
commit
2a9d1ce766
2 changed files with 52 additions and 0 deletions
|
|
@ -115,6 +115,32 @@ class CharacterParser
|
|||
ap errors
|
||||
end
|
||||
|
||||
def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil)
|
||||
errors = []
|
||||
|
||||
start_index = start.nil? ? 0 : list.index { |id| id == start }
|
||||
count = list.drop(start_index).count
|
||||
|
||||
# ap "Start index: #{start_index}"
|
||||
|
||||
list.drop(start_index).each_with_index do |id, i|
|
||||
chara = Character.find_by(granblue_id: id)
|
||||
percentage = ((i + 1) / count.to_f * 100).round(2)
|
||||
ap "#{percentage}%: Fetching #{chara.wiki_en}... (#{i + 1}/#{count})" if debug
|
||||
next unless chara.release_date.nil? || overwrite
|
||||
|
||||
begin
|
||||
WeaponParser.new(granblue_id: chara.granblue_id,
|
||||
debug: debug).fetch(save: save)
|
||||
rescue WikiError => e
|
||||
errors.push(e.page)
|
||||
end
|
||||
end
|
||||
|
||||
ap 'The following pages were unable to be fetched:'
|
||||
ap errors
|
||||
end
|
||||
|
||||
# Parses the hash into a format that can be saved to the database
|
||||
def parse(hash)
|
||||
info = {}
|
||||
|
|
|
|||
|
|
@ -94,6 +94,32 @@ class WeaponParser
|
|||
ap errors
|
||||
end
|
||||
|
||||
def self.fetch_list(list: [], save: false, overwrite: false, debug: false, start: nil)
|
||||
errors = []
|
||||
|
||||
start_index = start.nil? ? 0 : list.index { |id| id == start }
|
||||
count = list.drop(start_index).count
|
||||
|
||||
# ap "Start index: #{start_index}"
|
||||
|
||||
list.drop(start_index).each_with_index do |id, i|
|
||||
weapon = Weapon.find_by(granblue_id: id)
|
||||
percentage = ((i + 1) / count.to_f * 100).round(2)
|
||||
ap "#{percentage}%: Fetching #{weapon.wiki_en}... (#{i + 1}/#{count})" if debug
|
||||
next unless weapon.release_date.nil? || overwrite
|
||||
|
||||
begin
|
||||
WeaponParser.new(granblue_id: weapon.granblue_id,
|
||||
debug: debug).fetch(save: save)
|
||||
rescue WikiError => e
|
||||
errors.push(e.page)
|
||||
end
|
||||
end
|
||||
|
||||
ap 'The following pages were unable to be fetched:'
|
||||
ap errors
|
||||
end
|
||||
|
||||
# Parses the response string into a hash
|
||||
def parse_string(string)
|
||||
data = {}
|
||||
|
|
|
|||
Loading…
Reference in a new issue