From 5668c5c6867da281bd823314b10d2f420302220c Mon Sep 17 00:00:00 2001 From: Justin Edmund Date: Sun, 12 Jan 2025 16:01:06 -0800 Subject: [PATCH] Add aws-sdk-s3 and create aws_service.rb AwsService handles streaming game image files from the Granblue Fantasy server to our S3 instance. --- Gemfile | 3 + Gemfile.lock | 20 ++++- app/services/aws_service.rb | 67 +++++++++++++++ lib/granblue/post_deployment_manager.rb | 107 ++++++++++++++++++++++++ 4 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 app/services/aws_service.rb create mode 100644 lib/granblue/post_deployment_manager.rb diff --git a/Gemfile b/Gemfile index 84a05ea..de7767b 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,9 @@ gem 'gemoji-parser' # An awesome replacement for acts_as_nested_set and better_nested_set. gem 'awesome_nested_set' +# Official AWS Ruby gem for Amazon Simple Storage Service (Amazon S3) +gem 'aws-sdk-s3' + # An email validator for Rails gem 'email_validator' diff --git a/Gemfile.lock b/Gemfile.lock index f02220c..abb4111 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,6 +79,22 @@ GEM ast (2.4.2) awesome_nested_set (3.5.0) activerecord (>= 4.0.0, < 7.1) + aws-eventstream (1.3.0) + aws-partitions (1.1035.0) + aws-sdk-core (3.215.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.96.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-s3 (1.177.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.11.0) + aws-eventstream (~> 1, >= 1.0.2) backport (1.2.0) bcrypt (3.1.18) benchmark (0.2.1) @@ -133,6 +149,7 @@ GEM i18n (1.12.0) concurrent-ruby (~> 1.0) jaro_winkler (1.5.4) + jmespath (1.6.2) json (2.6.3) kramdown (2.4.0) rexml @@ -331,6 +348,7 @@ DEPENDENCIES api_matchers apipie-rails awesome_nested_set + aws-sdk-s3 bcrypt blueprinter bootsnap @@ -372,4 +390,4 @@ RUBY VERSION ruby 3.0.0p0 BUNDLED WITH - 2.4.2 + 2.5.1 diff --git a/app/services/aws_service.rb b/app/services/aws_service.rb new file mode 100644 index 0000000..332b3ec --- /dev/null +++ b/app/services/aws_service.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'aws-sdk-s3' + +class AwsService + class ConfigurationError < StandardError; end + + def initialize + validate_credentials! + + @s3_client = Aws::S3::Client.new( + region: Rails.application.credentials.dig(:aws, :region), + access_key_id: Rails.application.credentials.dig(:aws, :access_key_id), + secret_access_key: Rails.application.credentials.dig(:aws, :secret_access_key) + ) + @bucket = Rails.application.credentials.dig(:aws, :bucket_name) + rescue KeyError => e + raise ConfigurationError, "Missing AWS credential: #{e.message}" + end + + def upload_stream(io, key) + @s3_client.put_object( + bucket: @bucket, + key: key, + body: io + ) + end + + def file_exists?(key) + @s3_client.head_object( + bucket: @bucket, + key: key + ) + true + rescue Aws::S3::Errors::NotFound + false + end + + private + + def credentials + @credentials ||= begin + creds = Rails.application.credentials[:aws] + raise ConfigurationError, 'AWS credentials not found' unless creds + + { + region: creds[:region], + access_key_id: creds[:access_key_id], + secret_access_key: creds[:secret_access_key], + bucket_name: creds[:bucket_name] + } + end + end + + def validate_credentials! + missing = [] + creds = Rails.application.credentials[:aws] + + %i[region access_key_id secret_access_key bucket_name].each do |key| + missing << key unless creds&.dig(key) + end + + return unless missing.any? + + raise ConfigurationError, "Missing AWS credentials: #{missing.join(', ')}" + end +end diff --git a/lib/granblue/post_deployment_manager.rb b/lib/granblue/post_deployment_manager.rb new file mode 100644 index 0000000..d7244e8 --- /dev/null +++ b/lib/granblue/post_deployment_manager.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +class PostDeploymentManager + STORAGE_DESCRIPTIONS = { + local: 'to local disk', + s3: 'to S3', + both: 'to local disk and S3' + }.freeze + + def initialize(options = {}) + @test_mode = options.fetch(:test_mode, false) + @verbose = options.fetch(:verbose, false) + @storage = options.fetch(:storage, :both) + @new_records = Hash.new { |h, k| h[k] = [] } + end + + def run + import_new_data + display_import_summary + download_images + end + + private + + def import_new_data + log_step 'Importing new data...' + importer = Granblue::DataImporter.new( + test_mode: @test_mode, + verbose: @verbose + ) + + process_imports(importer) + end + + def process_imports(importer) + importer.process_all_files do |file_records| + file_records.each do |type, ids| + @new_records[type].concat(ids) + end + end + end + + def display_import_summary + log_step "\nImport Summary:" + @new_records.each do |type, ids| + puts "#{type.capitalize}: #{ids.size} new records" + puts "IDs: #{ids.inspect}" if @verbose + end + end + + def download_images + return if all_records_empty? + + if @test_mode + log_step "\nTEST MODE: Would download images for new records..." + else + log_step "\nDownloading images for new records..." + end + + @new_records.each do |type, ids| + next if ids.empty? + + download_type_images(type, ids) + end + end + + def download_type_images(type, ids) + log_step "\nProcessing new #{type.pluralize} (#{ids.size} records)..." + download_options = { + test_mode: @test_mode, + verbose: @verbose, + storage: @storage + } + + ids.each do |id| + download_single_image(type, id, download_options) + end + end + + def download_single_image(type, id, options) + action_text = @test_mode ? 'Would download' : 'Downloading' + storage_text = STORAGE_DESCRIPTIONS[options[:storage]] + log_verbose "#{action_text} images #{storage_text} for #{type} #{id}..." + + Granblue::Downloader::DownloadManager.download_for_object( + type, + id, + **options + ) + rescue => e + error_message = "Error #{@test_mode ? 'would occur' : 'occurred'} downloading images for #{type} #{id}: #{e.message}" + puts error_message + puts e.backtrace.take(5) if @verbose + end + + def all_records_empty? + @new_records.values.all?(&:empty?) + end + + def log_step(message) + puts message + end + + def log_verbose(message) + puts message if @verbose + end +end