From 28a6b1894e68380602737efc3abafe01380d4549 Mon Sep 17 00:00:00 2001 From: Justin Edmund Date: Sun, 2 Mar 2025 16:29:05 -0800 Subject: [PATCH 1/3] Delete db/migrate/20250301143956_add_wiki_raw_to_characters.rb (#194) --- db/migrate/20250301143956_add_wiki_raw_to_characters.rb | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 db/migrate/20250301143956_add_wiki_raw_to_characters.rb diff --git a/db/migrate/20250301143956_add_wiki_raw_to_characters.rb b/db/migrate/20250301143956_add_wiki_raw_to_characters.rb deleted file mode 100644 index 58d069b..0000000 --- a/db/migrate/20250301143956_add_wiki_raw_to_characters.rb +++ /dev/null @@ -1,7 +0,0 @@ -class AddWikiRawToCharacters < ActiveRecord::Migration[8.0] - def change - add_column :characters, :wiki_raw, :text - add_column :characters, :game_raw_en, :text - add_column :characters, :game_raw_jp, :text - end -end From 7880ac76ccf08c46770ddcfb2940aca6ba18a7b9 Mon Sep 17 00:00:00 2001 From: Justin Edmund Date: Sun, 2 Mar 2025 17:48:35 -0800 Subject: [PATCH 2/3] Add data ingestion endpoints (#195) * Update schema.rb * Add endpoints for importing game data This lets privileged users import canonical data for characters, weapons and summons directly from the game --- app/controllers/api/v1/import_controller.rb | 111 +++++++++++++++++++- config/routes.rb | 3 + db/schema.rb | 12 ++- 3 files changed, 122 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/v1/import_controller.rb b/app/controllers/api/v1/import_controller.rb index 928768f..84f9e07 100644 --- a/app/controllers/api/v1/import_controller.rb +++ b/app/controllers/api/v1/import_controller.rb @@ -27,6 +27,8 @@ module Api 6 => 5 }.freeze + before_action :ensure_admin_role, only: %i[weapons summons characters] + ## # Processes an import request. # @@ -49,9 +51,9 @@ module Api end unless raw_params['deck'].is_a?(Hash) && - raw_params['deck'].key?('pc') && - raw_params['deck'].key?('npc') - Rails.logger.error "[IMPORT] Deck data incomplete or missing." + raw_params['deck'].key?('pc') && + raw_params['deck'].key?('npc') + Rails.logger.error '[IMPORT] Deck data incomplete or missing.' return render json: { error: 'Invalid deck data' }, status: :unprocessable_content end @@ -68,8 +70,111 @@ module Api render json: { error: e.message }, status: :unprocessable_content end + def weapons + Rails.logger.info '[IMPORT] Checking weapon gamedata input...' + + body = parse_request_body + return unless body + + weapon = Weapon.find_by(granblue_id: body['id']) + unless weapon + Rails.logger.error "[IMPORT] Weapon not found with ID: #{body['id']}" + return render json: { error: 'Weapon not found' }, status: :not_found + end + + lang = params[:lang] + unless %w[en jp].include?(lang) + Rails.logger.error "[IMPORT] Invalid language: #{lang}" + return render json: { error: 'Invalid language' }, status: :unprocessable_content + end + + begin + weapon.update!( + "game_raw_#{lang}" => body.to_json + ) + render json: { message: 'Weapon gamedata updated successfully' }, status: :ok + rescue StandardError => e + Rails.logger.error "[IMPORT] Failed to update weapon gamedata: #{e.message}" + render json: { error: e.message }, status: :unprocessable_content + end + end + + def summons + Rails.logger.info '[IMPORT] Checking summon gamedata input...' + + body = parse_request_body + return unless body + + summon = Summon.find_by(granblue_id: body['id']) + unless summon + Rails.logger.error "[IMPORT] Summon not found with ID: #{body['id']}" + return render json: { error: 'Summon not found' }, status: :not_found + end + + lang = params[:lang] + unless %w[en jp].include?(lang) + Rails.logger.error "[IMPORT] Invalid language: #{lang}" + return render json: { error: 'Invalid language' }, status: :unprocessable_content + end + + begin + summon.update!( + "game_raw_#{lang}" => body.to_json + ) + render json: { message: 'Summon gamedata updated successfully' }, status: :ok + rescue StandardError => e + Rails.logger.error "[IMPORT] Failed to update summon gamedata: #{e.message}" + render json: { error: e.message }, status: :unprocessable_content + end + end + + ## + # Updates character gamedata from JSON blob. + # + # @return [void] Renders JSON response with success or error message. + def characters + Rails.logger.info '[IMPORT] Checking character gamedata input...' + + body = parse_request_body + return unless body + + character = Character.find_by(granblue_id: body['id']) + unless character + Rails.logger.error "[IMPORT] Character not found with ID: #{body['id']}" + return render json: { error: 'Character not found' }, status: :not_found + end + + lang = params[:lang] + unless %w[en jp].include?(lang) + Rails.logger.error "[IMPORT] Invalid language: #{lang}" + return render json: { error: 'Invalid language' }, status: :unprocessable_content + end + + begin + character.update!( + "game_raw_#{lang}" => body.to_json + ) + render json: { message: 'Character gamedata updated successfully' }, status: :ok + rescue StandardError => e + Rails.logger.error "[IMPORT] Failed to update character gamedata: #{e.message}" + render json: { error: e.message }, status: :unprocessable_content + end + end + private + ## + # Ensures the current user has admin role (role 9). + # Renders an error if the user is not an admin. + # + # @return [void] + def ensure_admin_role + return if current_user&.role == 9 + + Rails.logger.error "[IMPORT] Unauthorized access attempt by user #{current_user&.id}" + render json: { error: 'Unauthorized' }, status: :unauthorized + end + ## # Reads and parses the raw JSON request body. # diff --git a/config/routes.rb b/config/routes.rb index fafbf95..7523e2e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -20,6 +20,9 @@ Rails.application.routes.draw do get 'version', to: 'api#version' post 'import', to: 'import#create' + post 'import/weapons', to: 'import#weapons' + post 'import/summons', to: 'import#summons' + post 'import/characters', to: 'import#characters' get 'users/info/:id', to: 'users#info' diff --git a/db/schema.rb b/db/schema.rb index 80636d3..a69bf75 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_02_18_025315) do +ActiveRecord::Schema[8.0].define(version: 2025_03_01_143956) do # These are extensions that must be enabled in order to support this database enable_extension "btree_gin" enable_extension "pg_catalog.plpgsql" @@ -67,6 +67,9 @@ ActiveRecord::Schema[8.0].define(version: 2025_02_18_025315) do t.string "kamigame", default: "" t.string "nicknames_en", default: [], null: false, array: true t.string "nicknames_jp", default: [], null: false, array: true + t.text "wiki_raw" + t.text "game_raw_en" + t.text "game_raw_jp" t.index ["granblue_id"], name: "index_characters_on_granblue_id" t.index ["name_en"], name: "index_characters_on_name_en", opclass: :gin_trgm_ops, using: :gin end @@ -420,6 +423,9 @@ ActiveRecord::Schema[8.0].define(version: 2025_02_18_025315) do t.date "transcendence_date" t.string "nicknames_en", default: [], null: false, array: true t.string "nicknames_jp", default: [], null: false, array: true + t.text "wiki_raw" + t.text "game_raw_en" + t.text "game_raw_jp" t.index ["granblue_id"], name: "index_summons_on_granblue_id" t.index ["name_en"], name: "index_summons_on_name_en", opclass: :gin_trgm_ops, using: :gin end @@ -495,6 +501,10 @@ ActiveRecord::Schema[8.0].define(version: 2025_02_18_025315) do t.date "transcendence_date" t.string "recruits" t.integer "series" + t.integer "new_series" + t.text "wiki_raw" + t.text "game_raw_en" + t.text "game_raw_jp" t.index ["granblue_id"], name: "index_weapons_on_granblue_id" t.index ["name_en"], name: "index_weapons_on_name_en", opclass: :gin_trgm_ops, using: :gin t.index ["recruits"], name: "index_weapons_on_recruits" From 2f04a7d3a7a35fbe9c381ddb0b388413389a5f21 Mon Sep 17 00:00:00 2001 From: Justin Edmund Date: Thu, 6 Mar 2025 19:12:44 -0800 Subject: [PATCH 3/3] Add a data miner for downloading data from GBF (#196) * First pass at dataminer service * Got output printing from dataminer * Fetches summons, characters and weapons * Can loop over objects * Finish dataminer Adds logger and continuing downloads --- app/services/dataminer.rb | 251 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 app/services/dataminer.rb diff --git a/app/services/dataminer.rb b/app/services/dataminer.rb new file mode 100644 index 0000000..c63528b --- /dev/null +++ b/app/services/dataminer.rb @@ -0,0 +1,251 @@ +# frozen_string_literal: true + +class Dataminer + include HTTParty + + BOT_UID = '39094985' + GAME_VERSION = '1741068713' + + base_uri 'https://game.granbluefantasy.jp' + format :json + + HEADERS = { + 'Accept' => 'application/json, text/javascript, */*; q=0.01', + 'Accept-Language' => 'en-US,en;q=0.9', + 'Accept-Encoding' => 'gzip, deflate, br, zstd', + 'Content-Type' => 'application/json', + 'DNT' => '1', + 'Origin' => 'https://game.granbluefantasy.jp', + 'Referer' => 'https://game.granbluefantasy.jp/', + 'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36', + 'X-Requested-With' => 'XMLHttpRequest' + }.freeze + + attr_reader :page, :cookies, :logger, :debug + + def initialize(page:, access_token:, wing:, midship:, t: 'dummy', debug: false) + @page = page + @cookies = { + access_gbtk: access_token, + wing: wing, + t: t, + midship: midship + } + @debug = debug + setup_logger + end + + def fetch + timestamp = Time.now.to_i * 1000 + response = self.class.post( + "/#{page}?_=#{timestamp}&t=#{timestamp}&uid=#{BOT_UID}", + headers: HEADERS.merge( + 'Cookie' => format_cookies, + 'X-VERSION' => GAME_VERSION + ) + ) + + raise AuthenticationError if auth_failed?(response) + + response + end + + def fetch_character(granblue_id) + timestamp = Time.now.to_i * 1000 + url = "/archive/npc_detail?_=#{timestamp}&t=#{timestamp}&uid=#{BOT_UID}" + body = { + special_token: nil, + user_id: BOT_UID, + kind_name: '0', + attribute: '0', + event_id: nil, + story_id: nil, + style: 1, + character_id: granblue_id + } + + response = fetch_detail(url, body) + update_game_data('Character', granblue_id, response) if response + response + end + + def fetch_weapon(granblue_id) + timestamp = Time.now.to_i * 1000 + url = "/archive/weapon_detail?_=#{timestamp}&t=#{timestamp}&uid=#{BOT_UID}" + body = { + special_token: nil, + user_id: BOT_UID, + kind_name: '0', + attribute: '0', + event_id: nil, + story_id: nil, + weapon_id: granblue_id + } + + response = fetch_detail(url, body) + update_game_data('Weapon', granblue_id, response) if response + response + end + + def fetch_summon(granblue_id) + timestamp = Time.now.to_i * 1000 + url = "/archive/summon_detail?_=#{timestamp}&t=#{timestamp}&uid=#{BOT_UID}" + body = { + special_token: nil, + user_id: BOT_UID, + kind_name: '0', + attribute: '0', + event_id: nil, + story_id: nil, + summon_id: granblue_id + } + + response = fetch_detail(url, body) + update_game_data('Summon', granblue_id, response) if response + response + end + + # Public batch processing methods + def fetch_all_characters(only_missing: false) + process_all_records('Character', only_missing: only_missing) + end + + def fetch_all_weapons(only_missing: false) + process_all_records('Weapon', only_missing: only_missing) + end + + def fetch_all_summons(only_missing: false) + process_all_records('Summon', only_missing: only_missing) + end + + private + + def format_cookies + cookies.map { |k, v| "#{k}=#{v}" }.join('; ') + end + + def auth_failed?(response) + return true if response.code != 200 + + begin + parsed = JSON.parse(response.body) + parsed.is_a?(Hash) && parsed['auth_status'] == 'require_auth' + rescue JSON::ParserError + true + end + end + + def setup_logger + @logger = ::Logger.new($stdout) + @logger.level = debug ? ::Logger::DEBUG : ::Logger::INFO + @logger.formatter = proc do |severity, _datetime, _progname, msg| + case severity + when 'DEBUG' + debug ? "#{msg}\n" : '' + else + "#{msg}\n" + end + end + + # Suppress SQL logs in non-debug mode + return if debug + + ActiveRecord::Base.logger.level = ::Logger::INFO if defined?(ActiveRecord::Base) + end + + def fetch_detail(url, body) + logger.debug "\n=== Request Details ===" + logger.debug "URL: #{url}" + logger.debug 'Headers:' + logger.debug HEADERS.merge( + 'Cookie' => format_cookies, + 'X-VERSION' => GAME_VERSION + ).inspect + logger.debug 'Body:' + logger.debug body.to_json + logger.debug '====================' + + response = self.class.post( + url, + headers: HEADERS.merge( + 'Cookie' => format_cookies, + 'X-VERSION' => GAME_VERSION + ), + body: body.to_json + ) + + logger.debug "\n=== Response Details ===" + logger.debug "Response code: #{response.code}" + logger.debug 'Response headers:' + logger.debug response.headers.inspect + logger.debug 'Raw response body:' + logger.debug response.body.inspect + begin + logger.debug 'Parsed response body (if JSON):' + logger.debug JSON.parse(response.body).inspect + rescue JSON::ParserError => e + logger.debug "Could not parse as JSON: #{e.message}" + end + logger.debug '======================' + + raise AuthenticationError if auth_failed?(response) + + JSON.parse(response.body) + end + + def update_game_data(model_name, granblue_id, response_data) + return unless response_data.is_a?(Hash) + + model = Object.const_get(model_name) + record = model.find_by(granblue_id: granblue_id) + + if record + record.update(game_raw_en: response_data) + logger.debug "Updated #{model_name} #{granblue_id}" + else + logger.warn "#{model_name} with granblue_id #{granblue_id} not found in database" + end + rescue StandardError => e + logger.error "Error updating #{model_name} #{granblue_id}: #{e.message}" + end + + def process_all_records(model_name, only_missing: false) + model = Object.const_get(model_name) + scope = model + scope = scope.where(game_raw_en: nil) if only_missing + + total = scope.count + success_count = 0 + error_count = 0 + + logger.info "Starting to fetch #{total} #{model_name.downcase}s#{' (missing data only)' if only_missing}..." + + scope.find_each do |record| + logger.info "\nProcessing #{model_name} #{record.granblue_id} (#{success_count + error_count + 1}/#{total})" + + response = case model_name + when 'Character' + fetch_character(record.granblue_id) + when 'Weapon' + fetch_weapon(record.granblue_id) + when 'Summon' + fetch_summon(record.granblue_id) + end + + success_count += 1 + logger.debug "Successfully processed #{model_name} #{record.granblue_id}" + + sleep(1) + rescue StandardError => e + error_count += 1 + logger.error "Error processing #{model_name} #{record.granblue_id}: #{e.message}" + end + + logger.info "\nProcessing complete!" + logger.info "Total: #{total}" + logger.info "Successful: #{success_count}" + logger.info "Failed: #{error_count}" + end + + class AuthenticationError < StandardError; end +end