Create pipeline for importing data via PRs (#148)
* Add table for data version and migrate * Modify migration and re-migrate * Create data_version.rb Adds a model for DataVersion * Add aws-sdk-s3 and create aws_service.rb AwsService handles streaming game image files from the Granblue Fantasy server to our S3 instance. * Add importers The Importer libraries take CSV data and import them into the database for each type. We currently support characters, summons and weapons. * Add downloaders Downloaders take Granblue IDs and download images for those items from the Granblue Fantasy server in all relevant sizes. Downloaders can download to disk or stream the file directly to S3. * Create data_importer.rb * Fetches a list of all CSV files present in the updates folder * Checks which have already been imported * Sends unimported data to the appropriate Importer to handle * Create download_manager.rb Creates an appropriate downloader for each Granblue ID it receives * Update download_images.rake Most of this task has been extracted into the Downloader libraries * Update import_data.rake * Create deploy.rake This task is to be run as a post-deploy script. It checks for new unimported data, imports it, then downloads the relevant images to S3 or local disk depending on the parameters provided. * Update credentials.yml.enc * Began working on a README and added example CSVs * Modify importer to handle updates This way we can also add FLBs and other uncaps easier. * Updates only require values that will change When updating a row, fields that don't have a provided value will not be changed * Rebuild search indices in post deploy * Clean up logs with LoggingHelper * More logging adjustments Trying to get a nice-looking output * Change some ASCII characters * Final ASCII changes * Fix issues with Summon and Weapon importers * Finish README for contributing
This commit is contained in:
GitHub
parent
0e490df113
commit
c0922203a7
27 changed files with 1315 additions and 180 deletions
3
Gemfile
3
Gemfile
|
|
@ -35,6 +35,9 @@ gem 'gemoji-parser'
|
|||
# An awesome replacement for acts_as_nested_set and better_nested_set.
|
||||
gem 'awesome_nested_set'
|
||||
|
||||
# Official AWS Ruby gem for Amazon Simple Storage Service (Amazon S3)
|
||||
gem 'aws-sdk-s3'
|
||||
|
||||
# An email validator for Rails
|
||||
gem 'email_validator'
|
||||
|
||||
|
|
|
|||
20
Gemfile.lock
20
Gemfile.lock
|
|
@ -79,6 +79,22 @@ GEM
|
|||
ast (2.4.2)
|
||||
awesome_nested_set (3.5.0)
|
||||
activerecord (>= 4.0.0, < 7.1)
|
||||
aws-eventstream (1.3.0)
|
||||
aws-partitions (1.1035.0)
|
||||
aws-sdk-core (3.215.0)
|
||||
aws-eventstream (~> 1, >= 1.3.0)
|
||||
aws-partitions (~> 1, >= 1.992.0)
|
||||
aws-sigv4 (~> 1.9)
|
||||
jmespath (~> 1, >= 1.6.1)
|
||||
aws-sdk-kms (1.96.0)
|
||||
aws-sdk-core (~> 3, >= 3.210.0)
|
||||
aws-sigv4 (~> 1.5)
|
||||
aws-sdk-s3 (1.177.0)
|
||||
aws-sdk-core (~> 3, >= 3.210.0)
|
||||
aws-sdk-kms (~> 1)
|
||||
aws-sigv4 (~> 1.5)
|
||||
aws-sigv4 (1.11.0)
|
||||
aws-eventstream (~> 1, >= 1.0.2)
|
||||
backport (1.2.0)
|
||||
bcrypt (3.1.18)
|
||||
benchmark (0.2.1)
|
||||
|
|
@ -133,6 +149,7 @@ GEM
|
|||
i18n (1.12.0)
|
||||
concurrent-ruby (~> 1.0)
|
||||
jaro_winkler (1.5.4)
|
||||
jmespath (1.6.2)
|
||||
json (2.6.3)
|
||||
kramdown (2.4.0)
|
||||
rexml
|
||||
|
|
@ -331,6 +348,7 @@ DEPENDENCIES
|
|||
api_matchers
|
||||
apipie-rails
|
||||
awesome_nested_set
|
||||
aws-sdk-s3
|
||||
bcrypt
|
||||
blueprinter
|
||||
bootsnap
|
||||
|
|
@ -372,4 +390,4 @@ RUBY VERSION
|
|||
ruby 3.0.0p0
|
||||
|
||||
BUNDLED WITH
|
||||
2.4.2
|
||||
2.5.1
|
||||
|
|
|
|||
13
app/models/data_version.rb
Normal file
13
app/models/data_version.rb
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
# frozen_string_literal: true
|
||||
class DataVersion < ActiveRecord::Base
|
||||
validates :filename, presence: true, uniqueness: true
|
||||
validates :imported_at, presence: true
|
||||
|
||||
def self.mark_as_imported(filename)
|
||||
create!(filename: filename, imported_at: Time.current)
|
||||
end
|
||||
|
||||
def self.imported?(filename)
|
||||
exists?(filename: filename)
|
||||
end
|
||||
end
|
||||
67
app/services/aws_service.rb
Normal file
67
app/services/aws_service.rb
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'aws-sdk-s3'
|
||||
|
||||
class AwsService
|
||||
class ConfigurationError < StandardError; end
|
||||
|
||||
def initialize
|
||||
validate_credentials!
|
||||
|
||||
@s3_client = Aws::S3::Client.new(
|
||||
region: Rails.application.credentials.dig(:aws, :region),
|
||||
access_key_id: Rails.application.credentials.dig(:aws, :access_key_id),
|
||||
secret_access_key: Rails.application.credentials.dig(:aws, :secret_access_key)
|
||||
)
|
||||
@bucket = Rails.application.credentials.dig(:aws, :bucket_name)
|
||||
rescue KeyError => e
|
||||
raise ConfigurationError, "Missing AWS credential: #{e.message}"
|
||||
end
|
||||
|
||||
def upload_stream(io, key)
|
||||
@s3_client.put_object(
|
||||
bucket: @bucket,
|
||||
key: key,
|
||||
body: io
|
||||
)
|
||||
end
|
||||
|
||||
def file_exists?(key)
|
||||
@s3_client.head_object(
|
||||
bucket: @bucket,
|
||||
key: key
|
||||
)
|
||||
true
|
||||
rescue Aws::S3::Errors::NotFound
|
||||
false
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def credentials
|
||||
@credentials ||= begin
|
||||
creds = Rails.application.credentials[:aws]
|
||||
raise ConfigurationError, 'AWS credentials not found' unless creds
|
||||
|
||||
{
|
||||
region: creds[:region],
|
||||
access_key_id: creds[:access_key_id],
|
||||
secret_access_key: creds[:secret_access_key],
|
||||
bucket_name: creds[:bucket_name]
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def validate_credentials!
|
||||
missing = []
|
||||
creds = Rails.application.credentials[:aws]
|
||||
|
||||
%i[region access_key_id secret_access_key bucket_name].each do |key|
|
||||
missing << key unless creds&.dig(key)
|
||||
end
|
||||
|
||||
return unless missing.any?
|
||||
|
||||
raise ConfigurationError, "Missing AWS credentials: #{missing.join(', ')}"
|
||||
end
|
||||
end
|
||||
|
|
@ -1 +1 @@
|
|||
Fxc8acnxWOFdt+zwWoACR/fskFH2+ZY5izq5cHf8pnGDKRSoI7QYm0h8RwevJtRUvUJQsJ+ja/xzbTYxNC4ABRSBe06lXwHJuCnt5YtR+4l+NiFnS76kGzSfhlfhmvPLtSdfTVRfhRib1vrz7E38jM1pcc2QBkzCxyaoZRu3X65U+gc7EqTjOsg8wpTjJwvfTXW9gkFNwFSen3nOSytewYDcivwUjr/3NUAONKHn4rNhBN3UJiNgOSCGj77Xx60E0Q95CidbkgExcyKAIMMsQgLKGhQRr9yUGxdshMuhA3JhVQSyvtd+jX8PmNX3FQusQIg7YUCh/WpiKo3aimZLQYY2n7lbfeSLpwuishjn138GAxe59Wgm1JhKN4xAkcAq54Q9d4AGFnu/IphMhv1TO03CqnwX1BbfY142--n8Fil7/q3W/SrENe--J31ORG+51iIo29fjiZU3Uw==
|
||||
DNg3u1sscYjhg9KuSpOmx/E5ysJ89hktGTi2aslpe0R5DFzs/MAulFMJwZJPzjNKxXoyxJb4CCASGTUXUQFATasO/aUwws9ZWN/dVQS2CTd4guRICRqR4Kzip43XHgE9ctnOP9E2NMXfGnRIXmQohu+mjMJp0fQnJzr7L70o3cjtB1iQ/KwF3BithKQF/xPi+HCd4OZUxOhyixsG0OhNbCNsb7/tSqAs9JBrslRbN+XiRibzbWGD8rtNapU+IuBMWNK5B++8KpyUNWvUhTJup84L5FNHHysRlP0kAd8XM119EnMOs0rb0QwQsbZk2WfIGXgnKDzqr02XXsUjWtNZrbTM2zqiaLioYIvLxE6EMnFFEMNU+2Bpgj9xUu/x+WIw57xI9/6Iyr8Ck3PmFe5r0gpNLs2xXHkweCrXWDZjyNNzwNhSt3HTb5K+3QsU0JkB2wqGZZnez2CwfrvBfMFjKfAxAVGygeKFZsRY3XCVhs7r5NSHg6Wp/X+/jyYz8MCjlyw/yppyA4c/sAs1bJ1fmzo5K5reOzmpv1K7uqvX57o4--9yICsk5RvHZzyqdC--g3xaeflXn1y3Z/H5v6/oWw==
|
||||
11
db/migrate/20250110070255_create_data_versions.rb
Normal file
11
db/migrate/20250110070255_create_data_versions.rb
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class CreateDataVersions < ActiveRecord::Migration[7.0]
|
||||
def change
|
||||
create_table :data_versions, id: :uuid, default: -> { 'gen_random_uuid()' } do |t|
|
||||
t.string :filename, null: false
|
||||
t.datetime :imported_at, null: false
|
||||
t.index :filename, unique: true
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -10,12 +10,13 @@
|
|||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[7.0].define(version: 2024_01_26_032358) do
|
||||
ActiveRecord::Schema[7.0].define(version: 2025_01_10_070255) do
|
||||
# These are extensions that must be enabled in order to support this database
|
||||
enable_extension "btree_gin"
|
||||
enable_extension "pg_trgm"
|
||||
enable_extension "pgcrypto"
|
||||
enable_extension "plpgsql"
|
||||
enable_extension "uuid-ossp"
|
||||
|
||||
create_table "app_updates", primary_key: "updated_at", id: :datetime, force: :cascade do |t|
|
||||
t.string "update_type", null: false
|
||||
|
|
@ -72,6 +73,12 @@ ActiveRecord::Schema[7.0].define(version: 2024_01_26_032358) do
|
|||
create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t|
|
||||
end
|
||||
|
||||
create_table "data_versions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
|
||||
t.string "filename", null: false
|
||||
t.datetime "imported_at", null: false
|
||||
t.index ["filename"], name: "index_data_versions_on_filename", unique: true
|
||||
end
|
||||
|
||||
create_table "favorites", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
|
||||
t.uuid "user_id"
|
||||
t.uuid "party_id"
|
||||
|
|
|
|||
179
db/seed/README.md
Normal file
179
db/seed/README.md
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
# Adding items to the database
|
||||
|
||||
Anyone can add data to the database via a Pull Request on Github. Fork the repository, then create a new CSV file in the
|
||||
`updates` folder with the date, type of item you're updating, and the next number in the sequence. The most important
|
||||
part is the type of item, which should be plural for convention. This is what tells the service how to process your
|
||||
data.
|
||||
|
||||
```
|
||||
20240618-characters-001.csv
|
||||
20241231-weapons-010.csv
|
||||
20250115-summons-025.csv
|
||||
```
|
||||
|
||||
It's recommended to use a CSV editor to edit this data, but something like Microsoft Excel or Numbers should work fine
|
||||
too.
|
||||
|
||||
## Required values
|
||||
|
||||
You are required to set values for the following fields:
|
||||
TBA
|
||||
|
||||
## Arrays
|
||||
|
||||
When adding the `character_id`, `nicknames_en`, or `nicknames_jp`, make sure to wrap your values in brackets ({}). If
|
||||
there are multiple values, you can separate them with a comma (,) with no spaces on either side.
|
||||
|
||||
```
|
||||
# character_id, Zeta and Vaseraga (Halloween):
|
||||
{3024,3025}
|
||||
|
||||
# nicknames_en, Threo
|
||||
{sarasa,cake,thalatha}
|
||||
```
|
||||
|
||||
## Value tables
|
||||
|
||||
Values for properties like `element` are pre-defined, so you only need to input the corresponding digit.
|
||||
|
||||
#### Rarity
|
||||
|
||||
| Rarity | Value |
|
||||
|--------|-------|
|
||||
| SSR | 3 |
|
||||
| SR | 2 |
|
||||
| R | 1 |
|
||||
|
||||
#### Element
|
||||
|
||||
| Element | Value |
|
||||
|---------|-------|
|
||||
| Wind | 1 |
|
||||
| Fire | 2 |
|
||||
| Water | 3 |
|
||||
| Earth | 4 |
|
||||
| Dark | 5 |
|
||||
| Light | 6 |
|
||||
|
||||
#### Proficiency
|
||||
|
||||
| Proficiency | Value |
|
||||
|-------------|-------|
|
||||
| Sabre | 1 |
|
||||
| Dagger | 2 |
|
||||
| Axe | 3 |
|
||||
| Spear | 4 |
|
||||
| Bow | 5 |
|
||||
| Staff | 6 |
|
||||
| Melee | 7 |
|
||||
| Harp | 8 |
|
||||
| Gun | 9 |
|
||||
| Katana | 10 |
|
||||
|
||||
#### Race
|
||||
|
||||
| Race | Value |
|
||||
|---------|-------|
|
||||
| Unknown | 0 |
|
||||
| Human | 1 |
|
||||
| Erune | 2 |
|
||||
| Draph | 3 |
|
||||
| Harvin | 4 |
|
||||
| Primal | 5 |
|
||||
|
||||
#### Gender
|
||||
|
||||
| Gender | Value |
|
||||
|-------------|-------|
|
||||
| Other | 0 |
|
||||
| Male | 1 |
|
||||
| Female | 2 |
|
||||
| Male/Female | 3 |
|
||||
|
||||
#### Weapon Series (Needs to be cleaned)
|
||||
|
||||
| Series Name | Value | Series Name | Value |
|
||||
|--------------|-------|----------------------|-------|
|
||||
| Seraphic | 0 | Cosmic | 20 |
|
||||
| Grand | 1 | Draconic | 21 |
|
||||
| Dark Opus | 2 | Superlative | 22 |
|
||||
| Draconic | 3 | Vintage | 23 |
|
||||
| Revenant | 4 | Class Champion | 24 |
|
||||
| Beast | 5 | Proven | 25 |
|
||||
| Primal | 6 | Malice | 26 |
|
||||
| Beast | 7 | Menace | 27 |
|
||||
| Regalia | 8 | Sephira | 28 |
|
||||
| Omega | 9 | New World Foundation | 29 |
|
||||
| Olden Primal | 10 | Revans | 30 |
|
||||
| Militis | 11 | Illustrious | 31 |
|
||||
| Hollowsky | 12 | World | 32 |
|
||||
| Xeno | 13 | Exo | 33 |
|
||||
| Astral | 14 | Event | 35 |
|
||||
| Rose Crystal | 15 | Gacha | 36 |
|
||||
| Bahamut | 16 | Celestial | 37 |
|
||||
| Ultima | 17 | Omega Rebirth | 38 |
|
||||
| Epic | 18 | Assorted | -1 |
|
||||
| Ennead | 19 | | |
|
||||
|
||||
#### Summon Series (Needs to be cleaned)
|
||||
|
||||
| Series Name | Value |
|
||||
|--------------|-------|
|
||||
| Providence | 0 |
|
||||
| Genesis | 1 |
|
||||
| Omega | 2 |
|
||||
| Optimus | 3 |
|
||||
| Demi Optimus | 4 |
|
||||
| Archangel | 5 |
|
||||
| Arcarum | 6 |
|
||||
| Epic | 7 |
|
||||
| Dynamis | 9 |
|
||||
| Cryptid | 11 |
|
||||
| Six Dragons | 12 |
|
||||
|
||||
### Wiki links
|
||||
|
||||
You should try to provide identifiers for the 4 major wikis: gbf.wiki, gbf-wiki.com (JA), Kamigame (JA) and Gamewith (
|
||||
JA). Here's how:
|
||||
|
||||
#### gbf.wiki
|
||||
|
||||
This is simply the item's name, as it appears after `https://gbf.wiki/` in the URL.
|
||||
|
||||
```
|
||||
https://gbf.wiki/Bahamut -> Bahamut
|
||||
```
|
||||
|
||||
#### Gamewith
|
||||
|
||||
This is a 5 to 6 digit string that appears at the end of the URL.
|
||||
|
||||
```
|
||||
https://xn--bck3aza1a2if6kra4ee0hf.gamewith.jp/article/show/21612 -> 21612
|
||||
```
|
||||
|
||||
#### Kamigame
|
||||
|
||||
Use a [URL decoder](https://www.urldecoder.org/) to extract the Japanese characters from the URL after the final forward
|
||||
slash (/) and before `.html`.
|
||||
|
||||
```
|
||||
https://kamigame.jp/%E3%82%B0%E3%83%A9%E3%83%96%E3%83%AB/%E3%82%AD%E3%83%A3%E3%83%A9%E3%82%AF%E3%82%BF%E3%83%BC/SSR%E3%83%A4%E3%83%81%E3%83%9E.html
|
||||
-(decoder)->
|
||||
https://kamigame.jp/グラブル/キャラクター/SSRヤチマ.html
|
||||
-(value)->
|
||||
SSRヤチマ
|
||||
```
|
||||
|
||||
#### gbf-wiki.com
|
||||
|
||||
Use a [URL decoder](https://www.urldecoder.org/) to extract the Japanese characters from the URL after the question
|
||||
mark. Replace the `+` with a space.
|
||||
|
||||
```
|
||||
https://gbf-wiki.com/?%E3%83%A4%E3%83%81%E3%83%9E+(SSR)%E3%83%AA%E3%83%9F%E3%83%86%E3%83%83%E3%83%89%E3%83%90%E3%83%BC%E3%82%B8%E3%83%A7%E3%83%B3
|
||||
-(decoder)->
|
||||
https://gbf-wiki.com/?ヤチマ+(SSR)リミテッドバージョン
|
||||
-(value)->
|
||||
ヤチマ (SSR)リミテッドバージョン
|
||||
```
|
||||
2
db/seed/updates/20250101-characters-000.csv.example
Normal file
2
db/seed/updates/20250101-characters-000.csv.example
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
name_en,name_jp,granblue_id,rarity,element,proficiency1,proficiency2,gender,race1,race2,flb,min_hp,max_hp,max_hp_flb,min_atk,max_atk,max_atk_flb,base_da,base_ta,ougi_ratio,ougi_ratio_flb,special,ulb,max_hp_ulb,max_atk_ulb,character_id,wiki_en,release_date,flb_date,ulb_date,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp
|
||||
Gran,グラン,3040001000,3,1,1,2,1,1,0,true,1200,6800,8160,6400,8000,9600,6,3,4.5,5.0,false,false,0,0,{1},/wiki/Gran,2014-03-10,2016-04-01,,/wiki/グラン,/gran,/3040001000,{MC},"{主人公,グラン様}"
|
||||
2
db/seed/updates/20250101-summons-000.csv.example
Normal file
2
db/seed/updates/20250101-summons-000.csv.example
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
name_en,name_jp,granblue_id,rarity,element,series,flb,ulb,max_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,subaura,limit,transcendence,max_atk_xlb,max_hp_xlb,summon_id,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,transcendence_date,nicknames_en,nicknames_jp
|
||||
Bahamut,バハムート,2040003000,3,0,Divine Beasts,true,true,100,600,2400,2880,3360,1500,2000,2400,2800,true,true,true,3200,3840,1,2014-03-10,2016-04-01,2020-07-15,/wiki/Bahamut,/wiki/バハムート,/bahamut,/2040003000,2023-01-15,{Baha},"{バハ,黒龍}"
|
||||
2
db/seed/updates/20250101-weapons-000.csv.example
Normal file
2
db/seed/updates/20250101-weapons-000.csv.example
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
name_en,name_jp,granblue_id,rarity,element,proficiency,series,flb,ulb,max_level,max_skill_level,min_hp,max_hp,max_hp_flb,max_hp_ulb,min_atk,max_atk,max_atk_flb,max_atk_ulb,extra,ax_type,limit,ax,recruits_id,max_awakening_level,release_date,flb_date,ulb_date,wiki_en,wiki_ja,gamewith,kamigame,nicknames_en,nicknames_jp,transcendence,transcendence_date
|
||||
Aschallon,アスカロン,1040014000,4,1,1,1,true,true,150,15,220,2200,2640,3080,2450,2940,3528,4116,false,1,true,true,,5,2014-03-10,2016-04-01,2020-07-15,/wiki/Aschallon,/wiki/アスカロン,/aschallon,/1040014000,{Asca},"{アスカ,アスカロン}",true,2023-01-15
|
||||
110
lib/granblue/data_importer.rb
Normal file
110
lib/granblue/data_importer.rb
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
class DataImporter
|
||||
def initialize(test_mode: false, verbose: false)
|
||||
@test_mode = test_mode
|
||||
@verbose = verbose
|
||||
@import_logs = []
|
||||
end
|
||||
|
||||
def process_all_files(&block)
|
||||
files = Dir.glob(Rails.root.join('db', 'seed', 'updates', '*.csv')).sort
|
||||
|
||||
files.each do |file|
|
||||
if (new_records = import_csv(file))
|
||||
block.call(new_records) if block_given?
|
||||
end
|
||||
end
|
||||
|
||||
print_summary if @test_mode
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def import_csv(file_path)
|
||||
filename = File.basename(file_path)
|
||||
return if already_imported?(filename)
|
||||
|
||||
importer = create_importer(filename, file_path)
|
||||
return unless importer
|
||||
|
||||
log_info "Processing #{filename} in #{@test_mode ? 'test' : 'live'} mode..."
|
||||
result = importer.import
|
||||
log_import(filename, result)
|
||||
log_info "Successfully processed #{filename}"
|
||||
result
|
||||
end
|
||||
|
||||
def log_import_results(result)
|
||||
return unless @verbose
|
||||
|
||||
result[:new].each do |type, ids|
|
||||
log_info "Created #{ids.size} new #{type.pluralize}" if ids.any?
|
||||
end
|
||||
result[:updated].each do |type, ids|
|
||||
log_info "Updated #{ids.size} existing #{type.pluralize}" if ids.any?
|
||||
end
|
||||
end
|
||||
|
||||
def create_importer(filename, file_path)
|
||||
# This pattern matches both singular and plural: character(s), weapon(s), summon(s)
|
||||
match = filename.match(/\A\d{8}-(character(?:s)?|weapon(?:s)?|summon(?:s)?)-\d+\.csv\z/)
|
||||
return unless match
|
||||
|
||||
matched_type = match[1]
|
||||
singular_type = matched_type.sub(/s$/, '')
|
||||
importer_class = "Granblue::Importers::#{singular_type.capitalize}Importer".constantize
|
||||
|
||||
importer_class.new(
|
||||
file_path,
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
logger: self
|
||||
)
|
||||
rescue NameError
|
||||
log_info "No importer found for type: #{singular_type}"
|
||||
nil
|
||||
end
|
||||
|
||||
def already_imported?(filename)
|
||||
DataVersion.imported?(filename)
|
||||
end
|
||||
|
||||
def log_import(filename, result = nil)
|
||||
return if @test_mode
|
||||
|
||||
DataVersion.mark_as_imported(filename)
|
||||
|
||||
if result && @verbose
|
||||
result[:new].each do |type, ids|
|
||||
log_info "Created #{ids.size} new #{type.pluralize}" if ids.any?
|
||||
end
|
||||
result[:updated].each do |type, ids|
|
||||
log_info "Updated #{ids.size} existing #{type.pluralize}" if ids.any?
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def log_operation(operation)
|
||||
if @test_mode
|
||||
@import_logs << operation
|
||||
log_info "[TEST MODE] Would perform: #{operation}"
|
||||
end
|
||||
end
|
||||
|
||||
def print_summary
|
||||
log_info "\nTest Mode Summary:"
|
||||
log_info "Would perform #{@import_logs.size} operations"
|
||||
if @import_logs.any?
|
||||
log_info 'Sample of operations:'
|
||||
@import_logs.first(3).each { |log| log_info "- #{log}" }
|
||||
log_info '...' if @import_logs.size > 3
|
||||
end
|
||||
end
|
||||
|
||||
def log_info(message)
|
||||
puts message if @verbose || @test_mode
|
||||
end
|
||||
end
|
||||
end
|
||||
89
lib/granblue/download_manager.rb
Normal file
89
lib/granblue/download_manager.rb
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Downloader
|
||||
class DownloadManager
|
||||
class << self
|
||||
def download_for_object(type, granblue_id, test_mode: false, verbose: false, storage: :both)
|
||||
@test_mode = test_mode
|
||||
@verbose = verbose
|
||||
@storage = storage
|
||||
|
||||
case type
|
||||
when 'character'
|
||||
download_character(granblue_id)
|
||||
when 'weapon'
|
||||
download_weapon(granblue_id)
|
||||
when 'summon'
|
||||
download_summon(granblue_id)
|
||||
else
|
||||
log_info "Unknown object type: #{type}"
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def download_character(id)
|
||||
character = Character.find_by(granblue_id: id)
|
||||
return unless character
|
||||
|
||||
downloader_options = {
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
storage: @storage
|
||||
}
|
||||
|
||||
%W[#{id}_01 #{id}_02].each do |variant_id|
|
||||
CharacterDownloader.new(variant_id, **downloader_options).download
|
||||
end
|
||||
|
||||
CharacterDownloader.new("#{id}_03", **downloader_options).download if character.flb
|
||||
CharacterDownloader.new("#{id}_04", **downloader_options).download if character.ulb
|
||||
end
|
||||
|
||||
def download_weapon(id)
|
||||
weapon = Weapon.find_by(granblue_id: id)
|
||||
return unless weapon
|
||||
|
||||
downloader_options = {
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
storage: @storage
|
||||
}
|
||||
|
||||
WeaponDownloader.new(id, **downloader_options).download
|
||||
|
||||
return unless weapon.transcendence
|
||||
|
||||
WeaponDownloader.new("#{id}_02", **downloader_options).download
|
||||
WeaponDownloader.new("#{id}_03", **downloader_options).download
|
||||
|
||||
end
|
||||
|
||||
def download_summon(id)
|
||||
summon = Summon.find_by(granblue_id: id)
|
||||
return unless summon
|
||||
|
||||
downloader_options = {
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
storage: @storage
|
||||
}
|
||||
|
||||
SummonDownloader.new(id, **downloader_options).download
|
||||
SummonDownloader.new("#{id}_02", **downloader_options).download if summon.ulb
|
||||
|
||||
return unless summon.transcendence
|
||||
|
||||
SummonDownloader.new("#{id}_03", **downloader_options).download
|
||||
SummonDownloader.new("#{id}_04", **downloader_options).download
|
||||
|
||||
end
|
||||
|
||||
def log_info(message)
|
||||
puts message if @verbose || @test_mode
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
149
lib/granblue/downloaders/base_downloader.rb
Normal file
149
lib/granblue/downloaders/base_downloader.rb
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Downloader
|
||||
class BaseDownloader
|
||||
SIZES = %w[main grid square].freeze
|
||||
|
||||
def initialize(id, test_mode: false, verbose: false, storage: :both)
|
||||
@id = id
|
||||
@base_url = base_url
|
||||
@test_mode = test_mode
|
||||
@verbose = verbose
|
||||
@storage = storage
|
||||
@aws_service = AwsService.new
|
||||
ensure_directories_exist unless @test_mode
|
||||
end
|
||||
|
||||
def download
|
||||
log_info "-> #{@id}"
|
||||
return if @test_mode
|
||||
|
||||
SIZES.each_with_index do |size, index|
|
||||
path = download_path(size)
|
||||
url = build_url(size)
|
||||
process_download(url, size, path, last: index == SIZES.size - 1)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def process_download(url, size, path, last: false)
|
||||
filename = File.basename(url)
|
||||
s3_key = build_s3_key(size, filename)
|
||||
download_uri = "#{path}/#{filename}"
|
||||
|
||||
should_process = should_download?(download_uri, s3_key)
|
||||
return unless should_process
|
||||
|
||||
if last
|
||||
log_info "\t└ #{size}: #{url}..."
|
||||
else
|
||||
log_info "\t├ #{size}: #{url}..."
|
||||
end
|
||||
|
||||
case @storage
|
||||
when :local
|
||||
download_to_local(url, download_uri)
|
||||
when :s3
|
||||
stream_to_s3(url, s3_key)
|
||||
when :both
|
||||
download_to_both(url, download_uri, s3_key)
|
||||
end
|
||||
rescue OpenURI::HTTPError
|
||||
log_info "\t404 returned\t#{url}"
|
||||
end
|
||||
|
||||
def download_to_local(url, download_uri)
|
||||
download = URI.parse(url).open
|
||||
IO.copy_stream(download, download_uri)
|
||||
end
|
||||
|
||||
def stream_to_s3(url, s3_key)
|
||||
return if @aws_service.file_exists?(s3_key)
|
||||
|
||||
URI.parse(url).open do |file|
|
||||
@aws_service.upload_stream(file, s3_key)
|
||||
end
|
||||
end
|
||||
|
||||
def download_to_both(url, download_uri, s3_key)
|
||||
download = URI.parse(url).open
|
||||
|
||||
# Write to local file
|
||||
IO.copy_stream(download, download_uri)
|
||||
|
||||
# Reset file pointer for S3 upload
|
||||
download.rewind
|
||||
|
||||
# Upload to S3 if it doesn't exist
|
||||
unless @aws_service.file_exists?(s3_key)
|
||||
@aws_service.upload_stream(download, s3_key)
|
||||
end
|
||||
end
|
||||
|
||||
def should_download?(local_path, s3_key)
|
||||
case @storage
|
||||
when :local
|
||||
!File.exist?(local_path)
|
||||
when :s3
|
||||
!@aws_service.file_exists?(s3_key)
|
||||
when :both
|
||||
!File.exist?(local_path) || !@aws_service.file_exists?(s3_key)
|
||||
end
|
||||
end
|
||||
|
||||
def ensure_directories_exist
|
||||
return unless store_locally?
|
||||
|
||||
SIZES.each do |size|
|
||||
FileUtils.mkdir_p(download_path(size))
|
||||
end
|
||||
end
|
||||
|
||||
def store_locally?
|
||||
%i[local both].include?(@storage)
|
||||
end
|
||||
|
||||
def download_path(size)
|
||||
"#{Rails.root}/download/#{object_type}-#{size}"
|
||||
end
|
||||
|
||||
def build_s3_key(size, filename)
|
||||
"#{object_type}-#{size}/#{filename}"
|
||||
end
|
||||
|
||||
def log_info(message)
|
||||
puts message if @verbose
|
||||
end
|
||||
|
||||
def download_elemental_image(url, size, path, filename)
|
||||
return if @test_mode
|
||||
|
||||
filepath = "#{path}/#{filename}"
|
||||
download = URI.parse(url).open
|
||||
log_info "-> #{size}:\t#{url}..."
|
||||
IO.copy_stream(download, filepath)
|
||||
rescue OpenURI::HTTPError
|
||||
log_info "\t404 returned\t#{url}"
|
||||
end
|
||||
|
||||
def object_type
|
||||
raise NotImplementedError, 'Subclasses must define object_type'
|
||||
end
|
||||
|
||||
def base_url
|
||||
raise NotImplementedError, 'Subclasses must define base_url'
|
||||
end
|
||||
|
||||
def directory_for_size(size)
|
||||
raise NotImplementedError, 'Subclasses must define directory_for_size'
|
||||
end
|
||||
|
||||
def build_url(size)
|
||||
directory = directory_for_size(size)
|
||||
"#{@base_url}/#{directory}/#{@id}.jpg"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
25
lib/granblue/downloaders/character_downloader.rb
Normal file
25
lib/granblue/downloaders/character_downloader.rb
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Downloader
|
||||
class CharacterDownloader < BaseDownloader
|
||||
private
|
||||
|
||||
def object_type
|
||||
'character'
|
||||
end
|
||||
|
||||
def base_url
|
||||
'http://gbf.game-a.mbga.jp/assets/img/sp/assets/npc'
|
||||
end
|
||||
|
||||
def directory_for_size(size)
|
||||
case size.to_s
|
||||
when 'main' then 'f'
|
||||
when 'grid' then 'm'
|
||||
when 'square' then 's'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
47
lib/granblue/downloaders/elemental_weapon_downloader.rb
Normal file
47
lib/granblue/downloaders/elemental_weapon_downloader.rb
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'weapon_downloader'
|
||||
|
||||
module Granblue
|
||||
module Downloader
|
||||
class ElementalWeaponDownloader < WeaponDownloader
|
||||
SUFFIXES = [2, 3, 4, 1, 6, 5].freeze
|
||||
|
||||
def initialize(id_base)
|
||||
@id_base = id_base.to_i
|
||||
end
|
||||
|
||||
def download
|
||||
(1..6).each do |i|
|
||||
id = @id_base + (i - 1) * 100
|
||||
suffix = SUFFIXES[i - 1]
|
||||
|
||||
puts "Elemental Weapon #{id}_#{suffix}"
|
||||
SIZES.each do |size|
|
||||
path = download_path(size)
|
||||
url = build_url_for_id(id, size)
|
||||
filename = "#{id}_#{suffix}.jpg"
|
||||
download_elemental_image(url, size, path, filename)
|
||||
end
|
||||
|
||||
progress_reporter(count: i, total: 6, result: "Elemental Weapon #{id}_#{suffix}")
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def build_url_for_id(id, size)
|
||||
directory = directory_for_size(size)
|
||||
"#{base_url}/#{directory}/#{id}.jpg"
|
||||
end
|
||||
|
||||
def progress_reporter(count:, total:, result:, bar_len: 40)
|
||||
filled_len = (bar_len * count / total).round
|
||||
status = result
|
||||
percents = (100.0 * count / total).round(1)
|
||||
bar = '=' * filled_len + '-' * (bar_len - filled_len)
|
||||
print("\n[#{bar}] #{percents}% ...#{' ' * 14}#{status}\n")
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
25
lib/granblue/downloaders/summon_downloader.rb
Normal file
25
lib/granblue/downloaders/summon_downloader.rb
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Downloader
|
||||
class SummonDownloader < BaseDownloader
|
||||
private
|
||||
|
||||
def object_type
|
||||
'summon'
|
||||
end
|
||||
|
||||
def base_url
|
||||
'http://gbf.game-a.mbga.jp/assets/img/sp/assets/summon'
|
||||
end
|
||||
|
||||
def directory_for_size(size)
|
||||
case size.to_s
|
||||
when 'main' then 'party_main'
|
||||
when 'grid' then 'party_sub'
|
||||
when 'square' then 's'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
25
lib/granblue/downloaders/weapon_downloader.rb
Normal file
25
lib/granblue/downloaders/weapon_downloader.rb
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Downloader
|
||||
class WeaponDownloader < BaseDownloader
|
||||
private
|
||||
|
||||
def object_type
|
||||
'weapon'
|
||||
end
|
||||
|
||||
def base_url
|
||||
'http://gbf.game-a.mbga.jp/assets/img/sp/assets/weapon'
|
||||
end
|
||||
|
||||
def directory_for_size(size)
|
||||
case size.to_s
|
||||
when 'main' then 'ls'
|
||||
when 'grid' then 'm'
|
||||
when 'square' then 's'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
139
lib/granblue/importers/base_importer.rb
Normal file
139
lib/granblue/importers/base_importer.rb
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Importers
|
||||
class BaseImporter
|
||||
attr_reader :new_records, :updated_records
|
||||
|
||||
def initialize(file_path, test_mode: false, verbose: false, logger: nil)
|
||||
@file_path = file_path
|
||||
@test_mode = test_mode
|
||||
@verbose = verbose
|
||||
@logger = logger
|
||||
@new_records = Hash.new { |h, k| h[k] = [] }
|
||||
@updated_records = Hash.new { |h, k| h[k] = [] }
|
||||
end
|
||||
|
||||
def import
|
||||
CSV.foreach(@file_path, headers: true) do |row|
|
||||
import_row(row)
|
||||
end
|
||||
{ new: @new_records, updated: @updated_records }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def import_row(row)
|
||||
attributes = build_attributes(row)
|
||||
# Remove nil values from attributes hash for updates
|
||||
# Keep them for new records to ensure proper defaults
|
||||
record = find_or_create_record(attributes)
|
||||
track_record(record) if record
|
||||
end
|
||||
|
||||
def find_or_create_record(attributes)
|
||||
existing_record = model_class.find_by(granblue_id: attributes[:granblue_id])
|
||||
|
||||
if existing_record
|
||||
if @test_mode
|
||||
log_test_update(existing_record, attributes)
|
||||
nil
|
||||
else
|
||||
# For updates, only include non-nil attributes
|
||||
update_attributes = attributes.compact
|
||||
was_updated = update_attributes.any? { |key, value| existing_record[key] != value }
|
||||
existing_record.update!(update_attributes) if was_updated
|
||||
[existing_record, was_updated]
|
||||
end
|
||||
else
|
||||
if @test_mode
|
||||
log_test_creation(attributes)
|
||||
nil
|
||||
else
|
||||
# For new records, use all attributes including nil values
|
||||
[model_class.create!(attributes), false]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def track_record(result)
|
||||
record, was_updated = result
|
||||
type = model_class.name.demodulize.downcase
|
||||
|
||||
if was_updated
|
||||
@updated_records[type] << record.granblue_id
|
||||
log_updated_record(record) if @verbose
|
||||
else
|
||||
@new_records[type] << record.granblue_id
|
||||
log_new_record(record) if @verbose
|
||||
end
|
||||
end
|
||||
|
||||
def log_test_update(record, attributes)
|
||||
# For test mode, show only the attributes that would be updated
|
||||
update_attributes = attributes.compact
|
||||
@logger&.send(:log_operation, "Update #{model_class.name} #{record.granblue_id}: #{update_attributes.inspect}")
|
||||
end
|
||||
|
||||
def log_test_creation(attributes)
|
||||
@logger&.send(:log_operation, "Create #{model_class.name}: #{attributes.inspect}")
|
||||
end
|
||||
|
||||
def log_new_record(record)
|
||||
puts "Created #{model_class.name} with ID: #{record.granblue_id}"
|
||||
end
|
||||
|
||||
def log_updated_record(record)
|
||||
puts "Updated #{model_class.name} with ID: #{record.granblue_id}"
|
||||
end
|
||||
|
||||
def parse_value(value)
|
||||
return nil if value.nil? || value.strip.empty?
|
||||
|
||||
value
|
||||
end
|
||||
|
||||
def parse_integer(value)
|
||||
return nil if value.nil? || value.strip.empty?
|
||||
|
||||
value.to_i
|
||||
end
|
||||
|
||||
def parse_float(value)
|
||||
return nil if value.nil? || value.strip.empty?
|
||||
|
||||
value.to_f
|
||||
end
|
||||
|
||||
def parse_boolean(value)
|
||||
return nil if value.nil? || value.strip.empty?
|
||||
|
||||
value == 'true'
|
||||
end
|
||||
|
||||
def parse_date(date_str)
|
||||
return nil if date_str.nil? || date_str.strip.empty?
|
||||
|
||||
Date.parse(date_str) rescue nil
|
||||
end
|
||||
|
||||
def parse_array(array_str)
|
||||
return [] if array_str.nil? || array_str.strip.empty?
|
||||
|
||||
array_str.tr('{}', '').split(',')
|
||||
end
|
||||
|
||||
def parse_integer_array(array_str)
|
||||
parse_array(array_str).map(&:to_i)
|
||||
end
|
||||
|
||||
def model_class
|
||||
raise NotImplementedError, 'Subclasses must define model_class'
|
||||
end
|
||||
|
||||
def build_attributes(row)
|
||||
raise NotImplementedError, 'Subclasses must define build_attributes'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
53
lib/granblue/importers/character_importer.rb
Normal file
53
lib/granblue/importers/character_importer.rb
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Importers
|
||||
class CharacterImporter < BaseImporter
|
||||
private
|
||||
|
||||
def model_class
|
||||
Character
|
||||
end
|
||||
|
||||
def build_attributes(row)
|
||||
{
|
||||
name_en: parse_value(row['name_en']),
|
||||
name_jp: parse_value(row['name_jp']),
|
||||
granblue_id: parse_value(row['granblue_id']),
|
||||
rarity: parse_integer(row['rarity']),
|
||||
element: parse_integer(row['element']),
|
||||
proficiency1: parse_integer(row['proficiency1']),
|
||||
proficiency2: parse_integer(row['proficiency2']),
|
||||
gender: parse_integer(row['gender']),
|
||||
race1: parse_integer(row['race1']),
|
||||
race2: parse_integer(row['race2']),
|
||||
flb: parse_boolean(row['flb']),
|
||||
min_hp: parse_integer(row['min_hp']),
|
||||
max_hp: parse_integer(row['max_hp']),
|
||||
max_hp_flb: parse_integer(row['max_hp_flb']),
|
||||
min_atk: parse_integer(row['min_atk']),
|
||||
max_atk: parse_integer(row['max_atk']),
|
||||
max_atk_flb: parse_integer(row['max_atk_flb']),
|
||||
base_da: parse_integer(row['base_da']),
|
||||
base_ta: parse_integer(row['base_ta']),
|
||||
ougi_ratio: parse_float(row['ougi_ratio']),
|
||||
ougi_ratio_flb: parse_float(row['ougi_ratio_flb']),
|
||||
special: parse_boolean(row['special']),
|
||||
ulb: parse_boolean(row['ulb']),
|
||||
max_hp_ulb: parse_integer(row['max_hp_ulb']),
|
||||
max_atk_ulb: parse_integer(row['max_atk_ulb']),
|
||||
character_id: parse_integer_array(row['character_id']),
|
||||
wiki_en: parse_value(row['wiki_en']),
|
||||
release_date: parse_value(row['release_date']),
|
||||
flb_date: parse_value(row['flb_date']),
|
||||
ulb_date: parse_value(row['ulb_date']),
|
||||
wiki_ja: parse_value(row['wiki_ja']),
|
||||
gamewith: parse_value(row['gamewith']),
|
||||
kamigame: parse_value(row['kamigame']),
|
||||
nicknames_en: parse_array(row['nicknames_en']),
|
||||
nicknames_jp: parse_array(row['nicknames_jp'])
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
51
lib/granblue/importers/summon_importer.rb
Normal file
51
lib/granblue/importers/summon_importer.rb
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Importers
|
||||
class SummonImporter < BaseImporter
|
||||
private
|
||||
|
||||
def model_class
|
||||
Summon
|
||||
end
|
||||
|
||||
def build_attributes(row)
|
||||
{
|
||||
name_en: parse_value(row['name_en']),
|
||||
name_jp: parse_value(row['name_jp']),
|
||||
granblue_id: parse_value(row['granblue_id']),
|
||||
rarity: parse_integer(row['rarity']),
|
||||
element: parse_integer(row['element']),
|
||||
series: parse_value(row['series']),
|
||||
flb: parse_boolean(row['flb']),
|
||||
ulb: parse_boolean(row['ulb']),
|
||||
max_level: parse_integer(row['max_level']),
|
||||
min_hp: parse_integer(row['min_hp']),
|
||||
max_hp: parse_integer(row['max_hp']),
|
||||
max_hp_flb: parse_integer(row['max_hp_flb']),
|
||||
max_hp_ulb: parse_integer(row['max_hp_ulb']),
|
||||
min_atk: parse_integer(row['min_atk']),
|
||||
max_atk: parse_integer(row['max_atk']),
|
||||
max_atk_flb: parse_integer(row['max_atk_flb']),
|
||||
max_atk_ulb: parse_integer(row['max_atk_ulb']),
|
||||
subaura: parse_boolean(row['subaura']),
|
||||
limit: parse_boolean(row['limit']),
|
||||
transcendence: parse_boolean(row['transcendence']),
|
||||
max_atk_xlb: parse_integer(row['max_atk_xlb']),
|
||||
max_hp_xlb: parse_integer(row['max_hp_xlb']),
|
||||
summon_id: parse_integer(row['summon_id']),
|
||||
release_date: parse_value(row['release_date']),
|
||||
flb_date: parse_value(row['flb_date']),
|
||||
ulb_date: parse_value(row['ulb_date']),
|
||||
wiki_en: parse_value(row['wiki_en']),
|
||||
wiki_ja: parse_value(row['wiki_ja']),
|
||||
gamewith: parse_value(row['gamewith']),
|
||||
kamigame: parse_value(row['kamigame']),
|
||||
transcendence_date: parse_value(row['transcendence_date']),
|
||||
nicknames_en: parse_array(row['nicknames_en']),
|
||||
nicknames_jp: parse_array(row['nicknames_jp'])
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
54
lib/granblue/importers/weapon_importer.rb
Normal file
54
lib/granblue/importers/weapon_importer.rb
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module Granblue
|
||||
module Importers
|
||||
class WeaponImporter < BaseImporter
|
||||
private
|
||||
|
||||
def model_class
|
||||
Weapon
|
||||
end
|
||||
|
||||
def build_attributes(row)
|
||||
{
|
||||
name_en: parse_value(row['name_en']),
|
||||
name_jp: parse_value(row['name_jp']),
|
||||
granblue_id: parse_value(row['granblue_id']),
|
||||
rarity: parse_integer(row['rarity']),
|
||||
element: parse_integer(row['element']),
|
||||
proficiency: parse_integer(row['proficiency']),
|
||||
series: parse_integer(row['series']),
|
||||
flb: parse_boolean(row['flb']),
|
||||
ulb: parse_boolean(row['ulb']),
|
||||
max_level: parse_integer(row['max_level']),
|
||||
max_skill_level: parse_integer(row['max_skill_level']),
|
||||
min_hp: parse_integer(row['min_hp']),
|
||||
max_hp: parse_integer(row['max_hp']),
|
||||
max_hp_flb: parse_integer(row['max_hp_flb']),
|
||||
max_hp_ulb: parse_integer(row['max_hp_ulb']),
|
||||
min_atk: parse_integer(row['min_atk']),
|
||||
max_atk: parse_integer(row['max_atk']),
|
||||
max_atk_flb: parse_integer(row['max_atk_flb']),
|
||||
max_atk_ulb: parse_integer(row['max_atk_ulb']),
|
||||
extra: parse_boolean(row['extra']),
|
||||
ax_type: parse_integer(row['ax_type']),
|
||||
limit: parse_boolean(row['limit']),
|
||||
ax: parse_boolean(row['ax']),
|
||||
recruits_id: parse_value(row['recruits_id']),
|
||||
max_awakening_level: parse_integer(row['max_awakening_level']),
|
||||
release_date: parse_value(row['release_date']),
|
||||
flb_date: parse_value(row['flb_date']),
|
||||
ulb_date: parse_value(row['ulb_date']),
|
||||
wiki_en: parse_value(row['wiki_en']),
|
||||
wiki_ja: parse_value(row['wiki_ja']),
|
||||
gamewith: parse_value(row['gamewith']),
|
||||
kamigame: parse_value(row['kamigame']),
|
||||
nicknames_en: parse_array(row['nicknames_en']),
|
||||
nicknames_jp: parse_array(row['nicknames_jp']),
|
||||
transcendence: parse_boolean(row['transcendence']),
|
||||
transcendence_date: parse_value(row['transcendence_date'])
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
140
lib/granblue/post_deployment_manager.rb
Normal file
140
lib/granblue/post_deployment_manager.rb
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../logging_helper'
|
||||
|
||||
class PostDeploymentManager
|
||||
include LoggingHelper
|
||||
|
||||
STORAGE_DESCRIPTIONS = {
|
||||
local: 'to local disk',
|
||||
s3: 'to S3',
|
||||
both: 'to local disk and S3'
|
||||
}.freeze
|
||||
|
||||
def initialize(options = {})
|
||||
@test_mode = options.fetch(:test_mode, false)
|
||||
@verbose = options.fetch(:verbose, false)
|
||||
@storage = options.fetch(:storage, :both)
|
||||
@new_records = Hash.new { |h, k| h[k] = [] }
|
||||
@updated_records = Hash.new { |h, k| h[k] = [] }
|
||||
end
|
||||
|
||||
def run
|
||||
import_new_data
|
||||
display_import_summary
|
||||
download_images
|
||||
rebuild_search_indices
|
||||
display_completion_message
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def import_new_data
|
||||
log_header 'Importing new data...'
|
||||
puts "\n"
|
||||
importer = Granblue::DataImporter.new(
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose
|
||||
)
|
||||
|
||||
process_imports(importer)
|
||||
end
|
||||
|
||||
def process_imports(importer)
|
||||
importer.process_all_files do |result|
|
||||
result[:new].each do |type, ids|
|
||||
@new_records[type].concat(ids)
|
||||
end
|
||||
result[:updated].each do |type, ids|
|
||||
@updated_records[type].concat(ids)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def rebuild_search_indices
|
||||
log_header 'Rebuilding search indices...', '-'
|
||||
puts "\n"
|
||||
[Character, Summon, Weapon, Job].each do |model|
|
||||
log_verbose "• #{model.name}... "
|
||||
PgSearch::Multisearch.rebuild(model)
|
||||
log_verbose "✅ done!\n"
|
||||
end
|
||||
end
|
||||
|
||||
def display_import_summary
|
||||
if @new_records.size > 0 || @updated_records.size > 0
|
||||
log_header 'Import Summary', '-'
|
||||
puts "\n"
|
||||
display_record_summary('New', @new_records)
|
||||
display_record_summary('Updated', @updated_records)
|
||||
else
|
||||
log_step "\nNo new records imported."
|
||||
end
|
||||
end
|
||||
|
||||
def display_record_summary(label, records)
|
||||
records.each do |type, ids|
|
||||
next if ids.empty?
|
||||
puts "#{type.capitalize}: #{ids.size} #{label.downcase} records"
|
||||
puts "IDs: #{ids.inspect}" if @verbose
|
||||
end
|
||||
end
|
||||
|
||||
def download_images
|
||||
return if all_records_empty?
|
||||
|
||||
if @test_mode
|
||||
log_step "\nTEST MODE: Would download images for new and updated records..."
|
||||
else
|
||||
log_header 'Downloading images...', '+'
|
||||
end
|
||||
|
||||
[@new_records, @updated_records].each do |records|
|
||||
records.each do |type, ids|
|
||||
next if ids.empty?
|
||||
download_type_images(type, ids)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def download_type_images(type, ids)
|
||||
log_step "\nProcessing new #{type.pluralize} (#{ids.size} records)..."
|
||||
download_options = {
|
||||
test_mode: @test_mode,
|
||||
verbose: @verbose,
|
||||
storage: @storage
|
||||
}
|
||||
|
||||
ids.each do |id|
|
||||
download_single_image(type, id, download_options)
|
||||
end
|
||||
end
|
||||
|
||||
def download_single_image(type, id, options)
|
||||
action_text = @test_mode ? 'Would download' : 'Downloading'
|
||||
storage_text = STORAGE_DESCRIPTIONS[options[:storage]]
|
||||
log_verbose "\n#{action_text} images #{storage_text} for #{type} #{id}...\n"
|
||||
|
||||
Granblue::Downloader::DownloadManager.download_for_object(
|
||||
type,
|
||||
id,
|
||||
**options
|
||||
)
|
||||
rescue => e
|
||||
error_message = "Error #{@test_mode ? 'would occur' : 'occurred'} downloading images for #{type} #{id}: #{e.message}"
|
||||
puts error_message
|
||||
puts e.backtrace.take(5) if @verbose
|
||||
end
|
||||
|
||||
def display_completion_message
|
||||
if @test_mode
|
||||
log_step "\n✓ Test run completed successfully!"
|
||||
else
|
||||
log_step "\n✓ Post-deployment tasks completed successfully!"
|
||||
end
|
||||
end
|
||||
|
||||
def all_records_empty?
|
||||
@new_records.values.all?(&:empty?) && @updated_records.values.all?(&:empty?)
|
||||
end
|
||||
end
|
||||
25
lib/logging_helper.rb
Normal file
25
lib/logging_helper.rb
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
module LoggingHelper
|
||||
def log_step(message)
|
||||
puts message
|
||||
end
|
||||
|
||||
def log_verbose(message)
|
||||
print message if @verbose
|
||||
end
|
||||
|
||||
def log_divider(character = '+', leading_newline = true, trailing_newlines = 1)
|
||||
output = ""
|
||||
output += "\n" if leading_newline
|
||||
output += character * 35
|
||||
output += "\n" * trailing_newlines
|
||||
log_step output
|
||||
end
|
||||
|
||||
def log_header(title, character = '+', leading_newline = true)
|
||||
log_divider(character, leading_newline, 0)
|
||||
log_step title
|
||||
log_divider(character, false)
|
||||
end
|
||||
end
|
||||
52
lib/tasks/deploy.rake
Normal file
52
lib/tasks/deploy.rake
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../granblue/downloaders/base_downloader'
|
||||
require_relative '../logging_helper'
|
||||
|
||||
namespace :deploy do
|
||||
desc 'Post-deployment tasks: Import new data and download related images. Options: TEST=true for test mode, VERBOSE=true for verbose output, STORAGE=local|s3|both'
|
||||
task post_deployment: :environment do
|
||||
include LoggingHelper
|
||||
|
||||
Dir[Rails.root.join('lib', 'granblue', '**', '*.rb')].each { |file| require file }
|
||||
|
||||
# Ensure Rails environment is loaded
|
||||
Rails.application.eager_load!
|
||||
|
||||
log_header('Starting post-deploy script...', '-', false)
|
||||
print "\n"
|
||||
|
||||
# Parse and validate storage option
|
||||
storage = (ENV['STORAGE'] || 'both').to_sym
|
||||
unless [:local, :s3, :both].include?(storage)
|
||||
puts 'Invalid STORAGE option. Must be one of: local, s3, both'
|
||||
exit 1
|
||||
end
|
||||
|
||||
options = {
|
||||
test_mode: ENV['TEST'] == 'true',
|
||||
verbose: ENV['VERBOSE'] == 'true',
|
||||
storage: storage
|
||||
}
|
||||
|
||||
print "Test mode:\t"
|
||||
if options[:test_mode]
|
||||
print "✅ Enabled\n"
|
||||
else
|
||||
print "❌ Disabled\n"
|
||||
end
|
||||
|
||||
print "Verbose output:\t"
|
||||
if options[:verbose]
|
||||
print "✅ Enabled\n"
|
||||
else
|
||||
print "❌ Disabled\n"
|
||||
end
|
||||
|
||||
puts "Storage mode:\t#{storage}"
|
||||
|
||||
# Execute the task
|
||||
manager = PostDeploymentManager.new(options)
|
||||
manager.run
|
||||
end
|
||||
end
|
||||
|
|
@ -1,188 +1,22 @@
|
|||
namespace :granblue do
|
||||
def _progress_reporter(count:, total:, result:, bar_len: 40, multi: true)
|
||||
filled_len = (bar_len * count / total).round
|
||||
status = File.basename(result)
|
||||
percents = (100.0 * count / total).round(1)
|
||||
bar = '=' * filled_len + '-' * (bar_len - filled_len)
|
||||
desc 'Downloads images for the given Granblue_IDs'
|
||||
task :download_images, %i[object] => :environment do |_t, args|
|
||||
require_relative '../granblue/downloaders/base_downloader'
|
||||
Dir[Rails.root.join('lib', 'granblue', 'image_downloader', '*.rb')].each { |file| require file }
|
||||
|
||||
if !multi
|
||||
print("[#{bar}] #{percents}% ...#{' ' * 14}#{status}\n")
|
||||
else
|
||||
print "\n"
|
||||
end
|
||||
end
|
||||
object = args[:object]
|
||||
list = args.extras
|
||||
|
||||
def build_weapon_url(id, size)
|
||||
# Set up URL
|
||||
base_url = 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/weapon'
|
||||
extension = '.jpg'
|
||||
|
||||
directory = 'ls' if size.to_s == 'main'
|
||||
directory = 'm' if size.to_s == 'grid'
|
||||
directory = 's' if size.to_s == 'square'
|
||||
|
||||
"#{base_url}/#{directory}/#{id}#{extension}"
|
||||
end
|
||||
|
||||
def build_summon_url(id, size)
|
||||
# Set up URL
|
||||
base_url = 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/summon'
|
||||
extension = '.jpg'
|
||||
|
||||
directory = 'party_main' if size.to_s == 'main'
|
||||
directory = 'party_sub' if size.to_s == 'grid'
|
||||
directory = 's' if size.to_s == 'square'
|
||||
|
||||
"#{base_url}/#{directory}/#{id}#{extension}"
|
||||
end
|
||||
|
||||
def build_chara_url(id, size)
|
||||
# Set up URL
|
||||
base_url = 'http://gbf.game-a.mbga.jp/assets/img/sp/assets/npc'
|
||||
extension = '.jpg'
|
||||
|
||||
directory = 'f' if size.to_s == 'main'
|
||||
directory = 'm' if size.to_s == 'grid'
|
||||
directory = 's' if size.to_s == 'square'
|
||||
|
||||
"#{base_url}/#{directory}/#{id}#{extension}"
|
||||
end
|
||||
|
||||
def download_images(url, size, path)
|
||||
download = URI.parse(url).open
|
||||
download_URI = "#{path}/#{download.base_uri.to_s.split('/')[-1]}"
|
||||
if File.exist?(download_URI)
|
||||
puts "\tSkipping #{size}\t#{url}"
|
||||
else
|
||||
puts "\tDownloading #{size}\t#{url}..."
|
||||
IO.copy_stream(download, "#{path}/#{download.base_uri.to_s.split('/')[-1]}")
|
||||
end
|
||||
rescue OpenURI::HTTPError
|
||||
puts "\t404 returned\t#{url}"
|
||||
end
|
||||
|
||||
def download_elemental_images(url, size, path, filename)
|
||||
filepath = "#{path}/#{filename}"
|
||||
download = URI.parse(url).open
|
||||
puts "\tDownloading #{size}\t#{url}..."
|
||||
IO.copy_stream(download, filepath)
|
||||
rescue OpenURI::HTTPError
|
||||
puts "\t404 returned\t#{url}"
|
||||
end
|
||||
|
||||
def download_chara_images(id)
|
||||
sizes = %w[main grid square]
|
||||
|
||||
url = {
|
||||
'main': build_chara_url(id, 'main'),
|
||||
'grid': build_chara_url(id, 'grid'),
|
||||
'square': build_chara_url(id, 'square')
|
||||
}
|
||||
|
||||
puts "Character #{id}"
|
||||
sizes.each do |size|
|
||||
path = "#{Rails.root}/download/character-#{size}"
|
||||
download_images(url[size.to_sym], size, path)
|
||||
end
|
||||
end
|
||||
|
||||
def download_weapon_images(id)
|
||||
sizes = %w[main grid square]
|
||||
|
||||
url = {
|
||||
'main': build_weapon_url(id, 'main'),
|
||||
'grid': build_weapon_url(id, 'grid'),
|
||||
'square': build_weapon_url(id, 'square')
|
||||
}
|
||||
|
||||
puts "Weapon #{id}"
|
||||
sizes.each do |size|
|
||||
path = "#{Rails.root}/download/weapon-#{size}"
|
||||
download_images(url[size.to_sym], size, path)
|
||||
end
|
||||
end
|
||||
|
||||
def download_summon_images(id)
|
||||
sizes = %w[main grid square]
|
||||
|
||||
url = {
|
||||
'main': build_summon_url(id, 'main'),
|
||||
'grid': build_summon_url(id, 'grid'),
|
||||
'square': build_summon_url(id, 'square')
|
||||
}
|
||||
|
||||
puts "Summon #{id}"
|
||||
sizes.each do |size|
|
||||
path = "#{Rails.root}/download/summon-#{size}"
|
||||
download_images(url[size.to_sym], size, path)
|
||||
list.each do |id|
|
||||
Granblue::Downloader::DownloadManager.download_for_object(object, id)
|
||||
end
|
||||
end
|
||||
|
||||
desc 'Downloads elemental weapon images'
|
||||
task :download_elemental_images, [:id_base] => :environment do |_t, args|
|
||||
id_base = args[:id_base].to_i
|
||||
suffixes = [2, 3, 4, 1, 6, 5]
|
||||
require_relative '../granblue/downloaders/base_downloader'
|
||||
Dir[Rails.root.join('lib', 'granblue', 'image_downloader', '*.rb')].each { |file| require file }
|
||||
|
||||
(1..6).each do |i|
|
||||
id = id_base + (i - 1) * 100
|
||||
|
||||
sizes = %w[main grid square]
|
||||
|
||||
url = {
|
||||
'main': build_weapon_url(id, 'main'),
|
||||
'grid': build_weapon_url(id, 'grid'),
|
||||
'square': build_weapon_url(id, 'square')
|
||||
}
|
||||
|
||||
puts "Elemental Weapon #{id}_#{suffixes[i - 1]}"
|
||||
sizes.each do |size|
|
||||
path = "#{Rails.root}/download/weapon-#{size}"
|
||||
filename = "#{id}_#{suffixes[i - 1]}.jpg"
|
||||
download_elemental_images(url[size.to_sym], size, path, filename)
|
||||
end
|
||||
|
||||
_progress_reporter(count: i, total: 6, result: "Elemental Weapon #{id}_#{suffixes[i - 1]}", bar_len: 40,
|
||||
multi: true)
|
||||
end
|
||||
end
|
||||
|
||||
desc 'Downloads images for the given Granblue_IDs'
|
||||
task :download_images, %i[object] => :environment do |_t, args|
|
||||
object = args[:object]
|
||||
list = args.extras
|
||||
|
||||
list.each do |id|
|
||||
case object
|
||||
when 'character'
|
||||
character = Character.find_by(granblue_id: id)
|
||||
next unless character
|
||||
|
||||
download_chara_images("#{id}_01")
|
||||
download_chara_images("#{id}_02")
|
||||
download_chara_images("#{id}_03") if character.flb
|
||||
download_chara_images("#{id}_04") if character.ulb
|
||||
when 'weapon'
|
||||
weapon = Weapon.find_by(granblue_id: id)
|
||||
next unless weapon
|
||||
|
||||
download_weapon_images(id)
|
||||
|
||||
if weapon.transcendence
|
||||
download_weapon_images("#{id}_02")
|
||||
download_weapon_images("#{id}_03")
|
||||
end
|
||||
when 'summon'
|
||||
summon = Summon.find_by(granblue_id: id)
|
||||
next unless summon
|
||||
|
||||
download_summon_images("#{id}")
|
||||
download_summon_images("#{id}_02") if summon.ulb
|
||||
|
||||
if summon.transcendence
|
||||
download_summon_images("#{id}_03")
|
||||
download_summon_images("#{id}_04")
|
||||
end
|
||||
end
|
||||
end
|
||||
Granblue::Downloader::ElementalWeaponDownloader.new(args[:id_base]).download
|
||||
end
|
||||
end
|
||||
|
|
|
|||
13
lib/tasks/import_data.rake
Normal file
13
lib/tasks/import_data.rake
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
namespace :granblue do
|
||||
desc "Import weapon, summon and character data from db/seed/updates. Use TEST=true for test mode."
|
||||
task import_data: :environment do
|
||||
require 'csv'
|
||||
Dir[Rails.root.join('lib', 'granblue', '**', '*.rb')].each { |file| require file }
|
||||
|
||||
test_mode = ENV['TEST'] == 'true'
|
||||
importer = Granblue::DataImporter.new(test_mode: test_mode)
|
||||
importer.process_all_files
|
||||
end
|
||||
end
|
||||
Loading…
Reference in a new issue