Fix Cloudinary media audit

This commit is contained in:
Justin Edmund 2025-06-17 08:13:43 +01:00
parent fdf1ce5e21
commit 5da6f4c736
6 changed files with 191 additions and 91 deletions

View file

@ -42,6 +42,9 @@ Required environment variables:
- `LASTFM_API_KEY` - Last.fm API key for music data
- `REDIS_URL` - Redis connection URL for caching
Optional environment variables:
- `DEBUG` - Enable debug logging for specific categories (e.g., `DEBUG=music` for music-related logs)
## Commands
- `npm run dev` - Start development server

View file

@ -7,6 +7,7 @@ import type {
} from '$lib/types/apple-music'
import { isAppleMusicError } from '$lib/types/apple-music'
import { ApiRateLimiter } from './rate-limiter'
import { logger } from './logger'
const APPLE_MUSIC_API_BASE = 'https://api.music.apple.com/v1'
const DEFAULT_STOREFRONT = 'us' // Default to US storefront
@ -37,7 +38,7 @@ async function makeAppleMusicRequest<T>(endpoint: string, identifier?: string):
const url = `${APPLE_MUSIC_API_BASE}${endpoint}`
const headers = getAppleMusicHeaders()
console.log('Making Apple Music API request:', {
logger.music('debug', 'Making Apple Music API request:', {
url,
headers: {
...headers,
@ -50,11 +51,11 @@ async function makeAppleMusicRequest<T>(endpoint: string, identifier?: string):
if (!response.ok) {
const errorText = await response.text()
console.error('Apple Music API error response:', {
logger.error('Apple Music API error response:', undefined, {
status: response.status,
statusText: response.statusText,
body: errorText
})
}, 'music')
// Record failure and handle rate limiting
if (identifier) {
@ -82,7 +83,7 @@ async function makeAppleMusicRequest<T>(endpoint: string, identifier?: string):
return await response.json()
} catch (error) {
console.error('Apple Music API request failed:', error)
logger.error('Apple Music API request failed:', error as Error, undefined, 'music')
throw error
}
}
@ -127,7 +128,7 @@ export async function getAlbumDetails(id: string): Promise<AppleMusicAlbum | nul
included?: AppleMusicTrack[]
}>(endpoint, `album:${id}`)
console.log(`Album details for ${id}:`, {
logger.music('debug', `Album details for ${id}:`, {
hasData: !!response.data?.[0],
hasRelationships: !!response.data?.[0]?.relationships,
hasTracks: !!response.data?.[0]?.relationships?.tracks,
@ -137,12 +138,12 @@ export async function getAlbumDetails(id: string): Promise<AppleMusicAlbum | nul
// Check if tracks are in the included array
if (response.included?.length) {
console.log('First included track:', JSON.stringify(response.included[0], null, 2))
logger.music('debug', 'First included track:', { track: response.included[0] })
}
return response.data?.[0] || null
} catch (error) {
console.error(`Failed to get album details for ID ${id}:`, error)
logger.error(`Failed to get album details for ID ${id}:`, error as Error, undefined, 'music')
return null
}
}
@ -158,7 +159,7 @@ export async function findAlbum(artist: string, album: string): Promise<AppleMus
// Check if this album was already marked as not found
if (await rateLimiter.isNotFoundCached(identifier)) {
console.log(`Album "${album}" by "${artist}" is cached as not found`)
logger.music('debug', `Album "${album}" by "${artist}" is cached as not found`)
return null
}
@ -176,19 +177,18 @@ export async function findAlbum(artist: string, album: string): Promise<AppleMus
const searchQuery = `${artist} ${searchAlbum}`
const response = await searchAlbums(searchQuery, 5, storefront)
console.log(
`Search results for "${searchQuery}" in ${storefront} storefront:`,
JSON.stringify(response, null, 2)
)
logger.music('debug', `Search results for "${searchQuery}" in ${storefront} storefront:`, {
response
})
if (!response.results?.albums?.data?.length) {
console.log(`No albums found in ${storefront} storefront`)
logger.music('debug', `No albums found in ${storefront} storefront`)
return null
}
// Try to find the best match
const albums = response.results.albums.data
console.log(`Found ${albums.length} albums`)
logger.music('debug', `Found ${albums.length} albums`)
// First try exact match with original album name
let match = albums.find(
@ -224,7 +224,7 @@ export async function findAlbum(artist: string, album: string): Promise<AppleMus
// If no match, try Japanese storefront
if (!result) {
console.log(`No match found in US storefront, trying Japanese storefront`)
logger.music('debug', `No match found in US storefront, trying Japanese storefront`)
result = await searchAndMatch(album, JAPANESE_STOREFRONT)
}
@ -232,14 +232,14 @@ export async function findAlbum(artist: string, album: string): Promise<AppleMus
if (!result) {
const cleanedAlbum = removeLeadingPunctuation(album)
if (cleanedAlbum !== album && cleanedAlbum.length > 0) {
console.log(
logger.music('debug',
`No match found for "${album}", trying without leading punctuation: "${cleanedAlbum}"`
)
result = await searchAndMatch(cleanedAlbum)
// Also try Japanese storefront with cleaned album name
if (!result) {
console.log(`Still no match, trying Japanese storefront with cleaned name`)
logger.music('debug', `Still no match, trying Japanese storefront with cleaned name`)
result = await searchAndMatch(cleanedAlbum, JAPANESE_STOREFRONT)
}
}
@ -258,7 +258,7 @@ export async function findAlbum(artist: string, album: string): Promise<AppleMus
// Return the match
return result.album
} catch (error) {
console.error(`Failed to find album "${album}" by "${artist}":`, error)
logger.error(`Failed to find album "${album}" by "${artist}":`, error as Error, undefined, 'music')
// Don't cache as not found on error - might be temporary
return null
}
@ -285,7 +285,7 @@ export async function transformAlbumData(appleMusicAlbum: AppleMusicAlbum) {
included?: AppleMusicTrack[]
}>(endpoint, `album:${appleMusicAlbum.id}`)
console.log(`Album details response structure:`, {
logger.music('debug', `Album details response structure:`, {
hasData: !!response.data,
dataLength: response.data?.length,
hasIncluded: !!response.included,
@ -300,7 +300,7 @@ export async function transformAlbumData(appleMusicAlbum: AppleMusicAlbum) {
const tracksData = albumData?.relationships?.tracks?.data
if (tracksData?.length) {
console.log(`Found ${tracksData.length} tracks for album "${attributes.name}"`)
logger.music('debug', `Found ${tracksData.length} tracks for album "${attributes.name}"`)
// Process all tracks
tracks = tracksData
@ -313,7 +313,7 @@ export async function transformAlbumData(appleMusicAlbum: AppleMusicAlbum) {
// Log track details
tracks.forEach((track, index) => {
console.log(
logger.music('debug',
`Track ${index + 1}: ${track.name} - Preview: ${track.previewUrl ? 'Yes' : 'No'} - Duration: ${track.durationMs}ms`
)
})
@ -323,16 +323,16 @@ export async function transformAlbumData(appleMusicAlbum: AppleMusicAlbum) {
for (const track of tracksData) {
if (track.type === 'songs' && track.attributes?.previews?.[0]?.url) {
previewUrl = track.attributes.previews[0].url
console.log(`Using preview URL from track "${track.attributes.name}"`)
logger.music('debug', `Using preview URL from track "${track.attributes.name}"`)
break
}
}
}
} else {
console.log('No tracks found in album response')
logger.music('debug', 'No tracks found in album response')
}
} catch (error) {
console.error('Failed to fetch album tracks:', error)
logger.error('Failed to fetch album tracks:', error as Error, undefined, 'music')
}
}

View file

@ -1,6 +1,7 @@
import { dev } from '$app/environment'
export type LogLevel = 'debug' | 'info' | 'warn' | 'error'
export type LogCategory = 'music' | 'api' | 'db' | 'media' | 'general'
interface LogEntry {
level: LogLevel
@ -8,19 +9,46 @@ interface LogEntry {
timestamp: string
context?: Record<string, any>
error?: Error
category?: LogCategory
}
class Logger {
private shouldLog(level: LogLevel): boolean {
// In development, log everything
if (dev) return true
private debugCategories: Set<LogCategory> = new Set()
constructor() {
// Parse DEBUG environment variable to enable specific categories
const debugEnv = process.env.DEBUG || ''
if (debugEnv) {
const categories = debugEnv.split(',').map(c => c.trim()) as LogCategory[]
categories.forEach(cat => this.debugCategories.add(cat))
}
}
private shouldLog(level: LogLevel, category?: LogCategory): boolean {
// Always log warnings and errors
if (level === 'warn' || level === 'error') return true
// In development, check if category debugging is enabled
if (dev && category && this.debugCategories.size > 0) {
return this.debugCategories.has(category)
}
// In development without category debugging, log everything except music logs
if (dev && !category) return true
if (dev && category === 'music') return this.debugCategories.has('music')
// In production, only log warnings and errors
return level === 'warn' || level === 'error'
return false
}
private formatLog(entry: LogEntry): string {
const parts = [`[${entry.timestamp}]`, `[${entry.level.toUpperCase()}]`, entry.message]
const parts = [`[${entry.timestamp}]`, `[${entry.level.toUpperCase()}]`]
if (entry.category) {
parts.push(`[${entry.category.toUpperCase()}]`)
}
parts.push(entry.message)
if (entry.context) {
parts.push(JSON.stringify(entry.context, null, 2))
@ -36,15 +64,16 @@ class Logger {
return parts.join(' ')
}
private log(level: LogLevel, message: string, context?: Record<string, any>, error?: Error) {
if (!this.shouldLog(level)) return
private log(level: LogLevel, message: string, context?: Record<string, any>, error?: Error, category?: LogCategory) {
if (!this.shouldLog(level, category)) return
const entry: LogEntry = {
level,
message,
timestamp: new Date().toISOString(),
context,
error
error,
category
}
const formatted = this.formatLog(entry)
@ -63,20 +92,25 @@ class Logger {
}
}
debug(message: string, context?: Record<string, any>) {
this.log('debug', message, context)
debug(message: string, context?: Record<string, any>, category?: LogCategory) {
this.log('debug', message, context, undefined, category)
}
info(message: string, context?: Record<string, any>) {
this.log('info', message, context)
info(message: string, context?: Record<string, any>, category?: LogCategory) {
this.log('info', message, context, undefined, category)
}
warn(message: string, context?: Record<string, any>) {
this.log('warn', message, context)
warn(message: string, context?: Record<string, any>, category?: LogCategory) {
this.log('warn', message, context, undefined, category)
}
error(message: string, error?: Error, context?: Record<string, any>) {
this.log('error', message, context, error)
error(message: string, error?: Error, context?: Record<string, any>, category?: LogCategory) {
this.log('error', message, context, error, category)
}
// Convenience method for music-related logs
music(level: LogLevel, message: string, context?: Record<string, any>) {
this.log(level, message, context, undefined, 'music')
}
// Log API requests

View file

@ -51,6 +51,8 @@
.filter((f) => selectedFiles.has(f.publicId))
.reduce((sum, f) => sum + f.size, 0) || 0
$: console.log('Reactive state:', { hasSelection, selectedFilesSize: selectedFiles.size, deleting, showDeleteModal, showCleanupModal })
onMount(() => {
runAudit()
})
@ -93,12 +95,14 @@
}
function toggleFile(publicId: string) {
console.log('toggleFile called', publicId)
if (selectedFiles.has(publicId)) {
selectedFiles.delete(publicId)
} else {
selectedFiles.add(publicId)
}
selectedFiles = selectedFiles // Trigger reactivity
console.log('selectedFiles after toggle:', Array.from(selectedFiles))
}
async function deleteSelected(dryRun = true) {
@ -286,13 +290,16 @@
{/if}
</div>
<div class="actions">
<Button variant="text" size="small" onclick={toggleSelectAll}>
<Button variant="text" buttonSize="small" onclick={toggleSelectAll}>
{allSelected ? 'Deselect All' : 'Select All'}
</Button>
<Button
variant="danger"
size="small"
onclick={() => (showDeleteModal = true)}
buttonSize="small"
onclick={() => {
console.log('Delete Selected clicked', { hasSelection, deleting, selectedFiles: Array.from(selectedFiles) })
showDeleteModal = true
}}
disabled={!hasSelection || deleting}
icon={Trash2}
iconPosition="left"
@ -379,8 +386,11 @@
</p>
<Button
variant="secondary"
size="small"
onclick={() => (showCleanupModal = true)}
buttonSize="small"
onclick={() => {
console.log('Clean Up Broken References clicked', { cleaningUp, missingReferencesCount: auditData?.missingReferences.length })
showCleanupModal = true
}}
disabled={cleaningUp}
icon={AlertCircle}
iconPosition="left"
@ -405,33 +415,55 @@
</AdminPage>
<!-- Delete Confirmation Modal -->
<Modal bind:open={showDeleteModal} title="Delete Orphaned Files">
<Modal bind:isOpen={showDeleteModal}>
<div class="audit-modal-content">
<div class="modal-header">
<h2>Delete Orphaned Files</h2>
</div>
<div class="delete-confirmation">
<p>Are you sure you want to delete {selectedFiles.size} orphaned files?</p>
<p class="size-info">This will free up {formatBytes(selectedSize)} of storage.</p>
<p class="warning">⚠️ This action cannot be undone.</p>
</div>
<div slot="actions">
<Button variant="secondary" onclick={() => (showDeleteModal = false)}>Cancel</Button>
<Button variant="danger" onclick={() => deleteSelected(false)} disabled={deleting}>
<div class="modal-actions">
<Button variant="secondary" onclick={() => {
console.log('Cancel clicked')
showDeleteModal = false
}}>Cancel</Button>
<Button variant="danger" onclick={() => {
console.log('Delete Files clicked')
deleteSelected(false)
}} disabled={deleting}>
{deleting ? 'Deleting...' : 'Delete Files'}
</Button>
</div>
</div>
</Modal>
<!-- Cleanup Confirmation Modal -->
<Modal bind:open={showCleanupModal} title="Clean Up Broken References">
<Modal bind:isOpen={showCleanupModal}>
<div class="audit-modal-content">
<div class="modal-header">
<h2>Clean Up Broken References</h2>
</div>
<div class="cleanup-confirmation">
<p>Are you sure you want to clean up {auditData?.missingReferences.length || 0} broken references?</p>
<p class="warning">⚠️ This will remove Cloudinary URLs from database records where the files no longer exist.</p>
<p>This action cannot be undone.</p>
</div>
<div slot="actions">
<Button variant="secondary" onclick={() => (showCleanupModal = false)}>Cancel</Button>
<Button variant="danger" onclick={cleanupBrokenReferences} disabled={cleaningUp}>
<div class="modal-actions">
<Button variant="secondary" onclick={() => {
console.log('Cancel cleanup clicked')
showCleanupModal = false
}}>Cancel</Button>
<Button variant="danger" onclick={() => {
console.log('Clean Up References clicked')
cleanupBrokenReferences()
}} disabled={cleaningUp}>
{cleaningUp ? 'Cleaning Up...' : 'Clean Up References'}
</Button>
</div>
</div>
</Modal>
<style lang="scss">
@ -815,6 +847,33 @@
}
}
.modal-header {
margin-bottom: 1rem;
h2 {
margin: 0;
font-size: 1.25rem;
font-weight: 600;
color: $grey-10;
}
}
.audit-modal-content {
display: flex;
flex-direction: column;
padding: 1.5rem;
min-width: 400px;
}
.modal-actions {
display: flex;
justify-content: flex-end;
gap: 0.75rem;
margin-top: 1.5rem;
padding-top: 1.5rem;
border-top: 1px solid $grey-90;
}
@keyframes spin {
to {
transform: rotate(360deg);

View file

@ -5,6 +5,7 @@ import type { Album, AlbumImages } from '$lib/types/lastfm'
import type { LastfmImage } from '@musicorum/lastfm/dist/types/packages/common'
import { findAlbum, transformAlbumData } from '$lib/server/apple-music-client'
import redis from '../redis-client'
import { logger } from '$lib/server/logger'
const LASTFM_API_KEY = process.env.LASTFM_API_KEY
const USERNAME = 'jedmund'
@ -35,7 +36,7 @@ export const GET: RequestHandler = async ({ url }) => {
return await enrichAlbumWithInfo(client, album)
} catch (error) {
if (error instanceof Error && error.message.includes('Album not found')) {
console.debug(`Skipping album: ${album.name} (Album not found)`)
logger.music('debug', `Skipping album: ${album.name} (Album not found)`)
return null // Skip the album
}
throw error // Re-throw if it's a different error
@ -78,7 +79,7 @@ async function getRecentAlbums(
let recentTracksResponse
if (cached) {
console.log('Using cached Last.fm recent tracks')
logger.music('debug', 'Using cached Last.fm recent tracks')
recentTracksResponse = JSON.parse(cached)
// Convert date strings back to Date objects
if (recentTracksResponse.tracks) {
@ -162,7 +163,7 @@ async function enrichAlbumWithInfo(client: LastClient, album: Album): Promise<Al
const cached = await redis.get(cacheKey)
if (cached) {
console.log(`Using cached album info for "${album.name}"`)
logger.music('debug', `Using cached album info for "${album.name}"`)
const albumInfo = JSON.parse(cached)
return {
...album,
@ -196,7 +197,7 @@ async function searchAppleMusicForAlbum(album: Album): Promise<Album> {
if (cached) {
const cachedData = JSON.parse(cached)
console.log(`Using cached data for "${album.name}":`, {
logger.music('debug', `Using cached data for "${album.name}":`, {
hasPreview: !!cachedData.previewUrl,
trackCount: cachedData.tracks?.length || 0
})

View file

@ -4,6 +4,7 @@ import type { Album, AlbumImages } from '$lib/types/lastfm'
import type { LastfmImage } from '@musicorum/lastfm/dist/types/packages/common'
import { findAlbum, transformAlbumData } from '$lib/server/apple-music-client'
import redis from '../../redis-client'
import { logger } from '$lib/server/logger'
const LASTFM_API_KEY = process.env.LASTFM_API_KEY
const USERNAME = 'jedmund'
@ -41,7 +42,7 @@ export const GET: RequestHandler = async ({ request }) => {
try {
controller.enqueue(encoder.encode('event: connected\ndata: {}\n\n'))
} catch (e) {
console.error('Failed to send initial message:', e)
logger.error('Failed to send initial message:', e as Error, undefined, 'music')
return
}
@ -81,7 +82,7 @@ export const GET: RequestHandler = async ({ request }) => {
return withAppleMusic
} catch (error) {
console.error(`Error enriching album ${album.name}:`, error)
logger.error(`Error enriching album ${album.name}:`, error as Error, undefined, 'music')
return album
}
})
@ -90,7 +91,7 @@ export const GET: RequestHandler = async ({ request }) => {
// Ensure only one album is marked as now playing in the enriched albums
const nowPlayingCount = enrichedAlbums.filter((a) => a.isNowPlaying).length
if (nowPlayingCount > 1) {
console.log(
logger.music('debug',
`Multiple enriched albums marked as now playing (${nowPlayingCount}), keeping only the most recent one`
)
@ -100,11 +101,11 @@ export const GET: RequestHandler = async ({ request }) => {
enrichedAlbums.forEach((album, index) => {
if (album.isNowPlaying) {
if (foundFirst) {
console.log(`Marking album "${album.name}" at position ${index} as not playing`)
logger.music('debug', `Marking album "${album.name}" at position ${index} as not playing`)
album.isNowPlaying = false
album.nowPlayingTrack = undefined
} else {
console.log(`Keeping album "${album.name}" at position ${index} as now playing`)
logger.music('debug', `Keeping album "${album.name}" at position ${index} as now playing`)
foundFirst = true
}
}
@ -148,7 +149,7 @@ export const GET: RequestHandler = async ({ request }) => {
const data = JSON.stringify(enrichedAlbums)
controller.enqueue(encoder.encode(`event: albums\ndata: ${data}\n\n`))
const nowPlayingAlbum = enrichedAlbums.find((a) => a.isNowPlaying)
console.log('Sent album update with now playing status:', {
logger.music('debug', 'Sent album update with now playing status:', {
totalAlbums: enrichedAlbums.length,
nowPlayingAlbum: nowPlayingAlbum
? `${nowPlayingAlbum.artist.name} - ${nowPlayingAlbum.name}`
@ -183,7 +184,7 @@ export const GET: RequestHandler = async ({ request }) => {
(album.isNowPlaying && album.nowPlayingTrack !== lastTrack)
) {
updates.push(album)
console.log(
logger.music('debug',
`Now playing update for non-recent album ${album.albumName}: playing=${album.isNowPlaying}, track=${album.nowPlayingTrack}`
)
}
@ -217,7 +218,7 @@ export const GET: RequestHandler = async ({ request }) => {
}
}
} catch (error) {
console.error('Error checking for updates:', error)
logger.error('Error checking for updates:', error as Error, undefined, 'music')
}
}
@ -243,7 +244,7 @@ export const GET: RequestHandler = async ({ request }) => {
cancel() {
// Cleanup when stream is cancelled
console.log('SSE stream cancelled')
logger.music('debug', 'SSE stream cancelled')
}
})
@ -264,7 +265,7 @@ async function getNowPlayingAlbums(client: LastClient): Promise<NowPlayingUpdate
let recentTracksResponse
if (cached) {
console.log('Using cached Last.fm recent tracks for streaming')
logger.music('debug', 'Using cached Last.fm recent tracks for streaming')
recentTracksResponse = JSON.parse(cached)
// Convert date strings back to Date objects
if (recentTracksResponse.tracks) {
@ -274,7 +275,7 @@ async function getNowPlayingAlbums(client: LastClient): Promise<NowPlayingUpdate
}))
}
} else {
console.log('Fetching fresh Last.fm recent tracks for streaming')
logger.music('debug', 'Fetching fresh Last.fm recent tracks for streaming')
recentTracksResponse = await client.user.getRecentTracks(USERNAME, {
limit: 50,
extended: true
@ -333,7 +334,7 @@ async function getNowPlayingAlbums(client: LastClient): Promise<NowPlayingUpdate
// Ensure only one album is marked as now playing - keep the most recent one
const nowPlayingAlbums = Array.from(albums.values()).filter((a) => a.isNowPlaying)
if (nowPlayingAlbums.length > 1) {
console.log(
logger.music('debug',
`Multiple albums marked as now playing (${nowPlayingAlbums.length}), keeping only the most recent one`
)
@ -393,7 +394,7 @@ async function checkNowPlayingWithDuration(
const appleMusicData = JSON.parse(cached)
return checkWithTracks(albumName, appleMusicData.tracks)
} catch (error) {
console.error(`Error checking duration for ${albumName}:`, error)
logger.error(`Error checking duration for ${albumName}:`, error as Error, undefined, 'music')
return null
}
}
@ -435,7 +436,7 @@ function checkWithTracks(
)
if (now < trackEndTime) {
console.log(
logger.music('debug',
`Track "${mostRecentTrack.trackName}" is still playing (ends at ${trackEndTime.toLocaleTimeString()})`
)
return {
@ -476,7 +477,7 @@ async function enrichAlbumWithInfo(client: LastClient, album: Album): Promise<Al
const cached = await redis.get(cacheKey)
if (cached) {
console.log(`Using cached album info for "${album.name}"`)
logger.music('debug', `Using cached album info for "${album.name}"`)
const albumInfo = JSON.parse(cached)
return {
...album,
@ -485,7 +486,7 @@ async function enrichAlbumWithInfo(client: LastClient, album: Album): Promise<Al
}
}
console.log(`Fetching fresh album info for "${album.name}"`)
logger.music('debug', `Fetching fresh album info for "${album.name}"`)
const albumInfo = await client.album.getInfo(album.name, album.artist.name)
// Cache for 1 hour - album info rarely changes
@ -535,9 +536,11 @@ async function searchAppleMusicForAlbum(album: Album): Promise<Album> {
}
}
} catch (error) {
console.error(
logger.error(
`Failed to fetch Apple Music data for "${album.name}" by "${album.artist.name}":`,
error
error as Error,
undefined,
'music'
)
}
@ -552,7 +555,7 @@ async function getRecentAlbums(client: LastClient, limit: number = 4): Promise<A
let recentTracksResponse
if (cached) {
console.log('Using cached Last.fm recent tracks for album stream')
logger.music('debug', 'Using cached Last.fm recent tracks for album stream')
recentTracksResponse = JSON.parse(cached)
// Convert date strings back to Date objects
if (recentTracksResponse.tracks) {
@ -562,7 +565,7 @@ async function getRecentAlbums(client: LastClient, limit: number = 4): Promise<A
}))
}
} else {
console.log('Fetching fresh Last.fm recent tracks for album stream')
logger.music('debug', 'Fetching fresh Last.fm recent tracks for album stream')
recentTracksResponse = await client.user.getRecentTracks(USERNAME, {
limit: 50,
extended: true