add raid image downloader
This commit is contained in:
parent
31045324f7
commit
66a7edfcb5
3 changed files with 298 additions and 0 deletions
87
app/jobs/download_raid_images_job.rb
Normal file
87
app/jobs/download_raid_images_job.rb
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Background job for downloading raid images from Granblue servers to S3.
|
||||||
|
# Stores progress in Redis for status polling.
|
||||||
|
#
|
||||||
|
# @example Enqueue a download job
|
||||||
|
# job = DownloadRaidImagesJob.perform_later(raid.id)
|
||||||
|
# # Poll status with: DownloadRaidImagesJob.status(raid.id)
|
||||||
|
class DownloadRaidImagesJob < ApplicationJob
|
||||||
|
queue_as :downloads
|
||||||
|
|
||||||
|
retry_on StandardError, wait: :exponentially_longer, attempts: 3
|
||||||
|
|
||||||
|
discard_on ActiveRecord::RecordNotFound do |job, _error|
|
||||||
|
raid_id = job.arguments.first
|
||||||
|
Rails.logger.error "[DownloadRaidImages] Raid #{raid_id} not found"
|
||||||
|
update_status(raid_id, 'failed', error: 'Raid not found')
|
||||||
|
end
|
||||||
|
|
||||||
|
# Status keys for Redis storage
|
||||||
|
REDIS_KEY_PREFIX = 'raid_image_download'
|
||||||
|
STATUS_TTL = 1.hour.to_i
|
||||||
|
|
||||||
|
class << self
|
||||||
|
# Get the current status of a download job for a raid
|
||||||
|
#
|
||||||
|
# @param raid_id [String] UUID of the raid
|
||||||
|
# @return [Hash] Status hash with :status, :progress, :images_downloaded, :images_total, :error
|
||||||
|
def status(raid_id)
|
||||||
|
data = redis.get(redis_key(raid_id))
|
||||||
|
return { status: 'not_found' } unless data
|
||||||
|
|
||||||
|
JSON.parse(data, symbolize_names: true)
|
||||||
|
end
|
||||||
|
|
||||||
|
def redis_key(raid_id)
|
||||||
|
"#{REDIS_KEY_PREFIX}:#{raid_id}"
|
||||||
|
end
|
||||||
|
|
||||||
|
def redis
|
||||||
|
@redis ||= Redis.new(url: ENV.fetch('REDIS_URL', 'redis://localhost:6379/0'))
|
||||||
|
end
|
||||||
|
|
||||||
|
def update_status(raid_id, status, **attrs)
|
||||||
|
data = { status: status, updated_at: Time.current.iso8601 }.merge(attrs)
|
||||||
|
redis.setex(redis_key(raid_id), STATUS_TTL, data.to_json)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def perform(raid_id, force: false, size: 'all')
|
||||||
|
Rails.logger.info "[DownloadRaidImages] Starting download for raid #{raid_id}"
|
||||||
|
|
||||||
|
raid = Raid.find(raid_id)
|
||||||
|
update_status(raid_id, 'processing', progress: 0, images_downloaded: 0)
|
||||||
|
|
||||||
|
service = RaidImageDownloadService.new(
|
||||||
|
raid,
|
||||||
|
force: force,
|
||||||
|
size: size,
|
||||||
|
storage: :s3
|
||||||
|
)
|
||||||
|
|
||||||
|
result = service.download
|
||||||
|
|
||||||
|
if result.success?
|
||||||
|
Rails.logger.info "[DownloadRaidImages] Completed for raid #{raid_id}"
|
||||||
|
update_status(
|
||||||
|
raid_id,
|
||||||
|
'completed',
|
||||||
|
progress: 100,
|
||||||
|
images_downloaded: result.total,
|
||||||
|
images_total: result.total,
|
||||||
|
images: result.images
|
||||||
|
)
|
||||||
|
else
|
||||||
|
Rails.logger.error "[DownloadRaidImages] Failed for raid #{raid_id}: #{result.error}"
|
||||||
|
update_status(raid_id, 'failed', error: result.error)
|
||||||
|
raise StandardError, result.error # Trigger retry
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def update_status(raid_id, status, **attrs)
|
||||||
|
self.class.update_status(raid_id, status, **attrs)
|
||||||
|
end
|
||||||
|
end
|
||||||
72
app/services/raid_image_download_service.rb
Normal file
72
app/services/raid_image_download_service.rb
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Service wrapper for downloading raid images from Granblue servers to S3.
|
||||||
|
# Uses the RaidDownloader but provides a cleaner interface for controllers.
|
||||||
|
#
|
||||||
|
# @example Download images for a raid
|
||||||
|
# service = RaidImageDownloadService.new(raid)
|
||||||
|
# result = service.download
|
||||||
|
# if result.success?
|
||||||
|
# puts result.images
|
||||||
|
# else
|
||||||
|
# puts result.error
|
||||||
|
# end
|
||||||
|
class RaidImageDownloadService
|
||||||
|
Result = Struct.new(:success?, :images, :error, :total, keyword_init: true)
|
||||||
|
|
||||||
|
def initialize(raid, options = {})
|
||||||
|
@raid = raid
|
||||||
|
@force = options[:force] || false
|
||||||
|
@size = options[:size] || 'all'
|
||||||
|
@storage = options[:storage] || :s3
|
||||||
|
end
|
||||||
|
|
||||||
|
# Downloads images for the raid
|
||||||
|
#
|
||||||
|
# @return [Result] Struct with success status, images manifest, and any errors
|
||||||
|
def download
|
||||||
|
downloader = Granblue::Downloaders::RaidDownloader.new(
|
||||||
|
@raid,
|
||||||
|
storage: @storage,
|
||||||
|
force: @force,
|
||||||
|
verbose: Rails.env.development?
|
||||||
|
)
|
||||||
|
|
||||||
|
selected_size = @size == 'all' ? nil : @size
|
||||||
|
downloader.download(selected_size)
|
||||||
|
|
||||||
|
manifest = build_image_manifest
|
||||||
|
|
||||||
|
Result.new(
|
||||||
|
success?: true,
|
||||||
|
images: manifest,
|
||||||
|
total: count_total_images(manifest)
|
||||||
|
)
|
||||||
|
rescue StandardError => e
|
||||||
|
Rails.logger.error "[RaidImageDownload] Failed for #{@raid.slug}: #{e.message}"
|
||||||
|
Result.new(
|
||||||
|
success?: false,
|
||||||
|
error: e.message
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def build_image_manifest
|
||||||
|
manifest = {}
|
||||||
|
|
||||||
|
if @raid.enemy_id
|
||||||
|
manifest['icon'] = ["#{@raid.enemy_id}.png"]
|
||||||
|
end
|
||||||
|
|
||||||
|
if @raid.summon_id
|
||||||
|
manifest['thumbnail'] = ["#{@raid.summon_id}_high.png"]
|
||||||
|
end
|
||||||
|
|
||||||
|
manifest
|
||||||
|
end
|
||||||
|
|
||||||
|
def count_total_images(manifest)
|
||||||
|
manifest.values.sum(&:size)
|
||||||
|
end
|
||||||
|
end
|
||||||
139
lib/granblue/downloaders/raid_downloader.rb
Normal file
139
lib/granblue/downloaders/raid_downloader.rb
Normal file
|
|
@ -0,0 +1,139 @@
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
module Granblue
|
||||||
|
module Downloaders
|
||||||
|
# Downloads raid image assets from the game server.
|
||||||
|
# Raids have two different image types from different sources:
|
||||||
|
# - Icon: from enemy directory using enemy_id
|
||||||
|
# - Thumbnail: from summon directory using summon_id
|
||||||
|
#
|
||||||
|
# @example Download images for a specific raid
|
||||||
|
# downloader = RaidDownloader.new(raid, storage: :both)
|
||||||
|
# downloader.download
|
||||||
|
#
|
||||||
|
# @note Unlike other downloaders, RaidDownloader takes a Raid model instance
|
||||||
|
# since it needs both enemy_id and summon_id
|
||||||
|
class RaidDownloader < BaseDownloader
|
||||||
|
SIZES = %w[icon thumbnail].freeze
|
||||||
|
|
||||||
|
ICON_BASE_URL = 'https://prd-game-a-granbluefantasy.akamaized.net/assets_en/img/sp/assets/enemy'
|
||||||
|
THUMBNAIL_BASE_URL = 'https://prd-game-a1-granbluefantasy.akamaized.net/assets_en/img/sp/assets/summon'
|
||||||
|
|
||||||
|
# Initialize with a Raid model instead of just an ID
|
||||||
|
# @param raid [Raid] Raid model instance
|
||||||
|
# @param test_mode [Boolean] When true, only logs actions without downloading
|
||||||
|
# @param verbose [Boolean] When true, enables detailed logging
|
||||||
|
# @param storage [Symbol] Storage mode (:local, :s3, or :both)
|
||||||
|
# @param force [Boolean] When true, re-downloads even if file exists
|
||||||
|
def initialize(raid, test_mode: false, verbose: false, storage: :both, force: false, logger: nil)
|
||||||
|
@raid = raid
|
||||||
|
@id = raid.slug # Use slug for logging
|
||||||
|
@test_mode = test_mode
|
||||||
|
@verbose = verbose
|
||||||
|
@storage = storage
|
||||||
|
@force = force
|
||||||
|
@logger = logger || Logger.new($stdout)
|
||||||
|
@aws_service = self.class.aws_service if store_in_s3?
|
||||||
|
ensure_directories_exist unless @test_mode
|
||||||
|
end
|
||||||
|
|
||||||
|
# Download images for all available sizes
|
||||||
|
# @param selected_size [String] The size to download ('icon', 'thumbnail', or nil for all)
|
||||||
|
# @return [void]
|
||||||
|
def download(selected_size = nil)
|
||||||
|
log_info("-> #{@raid.slug}")
|
||||||
|
return if @test_mode
|
||||||
|
|
||||||
|
sizes = selected_size ? [selected_size] : SIZES
|
||||||
|
|
||||||
|
sizes.each_with_index do |size, index|
|
||||||
|
case size
|
||||||
|
when 'icon'
|
||||||
|
download_icon(last: index == sizes.size - 1)
|
||||||
|
when 'thumbnail'
|
||||||
|
download_thumbnail(last: index == sizes.size - 1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
# Download the icon image (from enemy directory)
|
||||||
|
def download_icon(last: false)
|
||||||
|
return unless @raid.enemy_id
|
||||||
|
|
||||||
|
path = download_path('icon')
|
||||||
|
url = build_icon_url
|
||||||
|
filename = "#{@raid.enemy_id}.png"
|
||||||
|
s3_key = build_s3_key('icon', filename)
|
||||||
|
download_uri = "#{path}/#{filename}"
|
||||||
|
|
||||||
|
return unless should_download?(download_uri, s3_key)
|
||||||
|
|
||||||
|
log_download('icon', url, last: last)
|
||||||
|
process_image_download(url, download_uri, s3_key)
|
||||||
|
rescue OpenURI::HTTPError
|
||||||
|
log_info "\t404 returned\t#{url}"
|
||||||
|
end
|
||||||
|
|
||||||
|
# Download the thumbnail image (from summon directory)
|
||||||
|
def download_thumbnail(last: false)
|
||||||
|
return unless @raid.summon_id
|
||||||
|
|
||||||
|
path = download_path('thumbnail')
|
||||||
|
url = build_thumbnail_url
|
||||||
|
filename = "#{@raid.summon_id}_high.png"
|
||||||
|
s3_key = build_s3_key('thumbnail', filename)
|
||||||
|
download_uri = "#{path}/#{filename}"
|
||||||
|
|
||||||
|
return unless should_download?(download_uri, s3_key)
|
||||||
|
|
||||||
|
log_download('thumbnail', url, last: last)
|
||||||
|
process_image_download(url, download_uri, s3_key)
|
||||||
|
rescue OpenURI::HTTPError
|
||||||
|
log_info "\t404 returned\t#{url}"
|
||||||
|
end
|
||||||
|
|
||||||
|
def log_download(size, url, last: false)
|
||||||
|
if last
|
||||||
|
log_info "\t└ #{size}: #{url}..."
|
||||||
|
else
|
||||||
|
log_info "\t├ #{size}: #{url}..."
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def process_image_download(url, download_uri, s3_key)
|
||||||
|
case @storage
|
||||||
|
when :local
|
||||||
|
download_to_local(url, download_uri)
|
||||||
|
when :s3
|
||||||
|
stream_to_s3(url, s3_key)
|
||||||
|
when :both
|
||||||
|
download_to_both(url, download_uri, s3_key)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def build_icon_url
|
||||||
|
"#{ICON_BASE_URL}/m/#{@raid.enemy_id}.png"
|
||||||
|
end
|
||||||
|
|
||||||
|
def build_thumbnail_url
|
||||||
|
"#{THUMBNAIL_BASE_URL}/qm/#{@raid.summon_id}_high.png"
|
||||||
|
end
|
||||||
|
|
||||||
|
def object_type
|
||||||
|
'raid'
|
||||||
|
end
|
||||||
|
|
||||||
|
# Not used for raids since we have custom URLs
|
||||||
|
def base_url
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
|
||||||
|
# Not used for raids since we have custom URL building
|
||||||
|
def directory_for_size(_size)
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
Loading…
Reference in a new issue