From 4519349cbf554f0ca125a885599746169142fe0d Mon Sep 17 00:00:00 2001 From: Dennis <39654202+dwei30@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:12:16 +1000 Subject: [PATCH 1/6] feat(iTunes): Fall back to Apple Music catalog API for streaming-only releases --- providers/iTunes/amp.test.ts | 65 ++++++++++++ providers/iTunes/amp.ts | 76 ++++++++++++++ providers/iTunes/amp_types.ts | 58 +++++++++++ providers/iTunes/mod.ts | 183 ++++++++++++++++++++++++++++++++-- 4 files changed, 374 insertions(+), 8 deletions(-) create mode 100644 providers/iTunes/amp.test.ts create mode 100644 providers/iTunes/amp.ts create mode 100644 providers/iTunes/amp_types.ts diff --git a/providers/iTunes/amp.test.ts b/providers/iTunes/amp.test.ts new file mode 100644 index 00000000..11725bda --- /dev/null +++ b/providers/iTunes/amp.test.ts @@ -0,0 +1,65 @@ +import { assertEquals } from 'std/assert/assert_equals.ts'; +import { describe, it } from '@std/testing/bdd'; +import { collectAmpTracks, extractAppleMusicJwt, extractScriptUrls, parseJwtExpiry, resolveAmpUrl } from './amp.ts'; + +function makeJwt(exp: number): string { + const encode = (value: object) => + btoa(JSON.stringify(value)).replaceAll('+', '-').replaceAll('/', '_').replaceAll('=', ''); + return `${encode({ alg: 'ES256', typ: 'JWT' })}.${encode({ exp })}.sig`; +} + +describe('Apple Music AMP helpers', () => { + it('extracts the JWT with the latest expiry', () => { + const older = makeJwt(1_700_000_000); + const newer = makeJwt(2_000_000_000); + const source = `const a="${older}"; const b='${newer}';`; + assertEquals(extractAppleMusicJwt(source), newer); + assertEquals(parseJwtExpiry(newer), 2_000_000_000 * 1000); + }); + + it('extracts crossorigin and musickit script URLs', () => { + const html = ` + + + + `; + assertEquals(extractScriptUrls(html), [ + 'https://js-cdn.music.apple.com/musickit/v3/musickit.js', + 'https://music.apple.com/assets/index.js', + ]); + }); + + it('hydrates track stubs from included resources', () => { + const tracks = collectAmpTracks({ + data: [{ + id: '1', + type: 'albums', + attributes: { name: 'Mix', artistName: 'DJ' }, + relationships: { + tracks: { data: [{ id: 't1', type: 'songs' }], next: '/v1/next' }, + }, + }], + included: [{ + id: 't1', + type: 'songs', + attributes: { name: 'Track One', artistName: 'Artist', trackNumber: 1, discNumber: 1 }, + }], + }, { + id: '1', + type: 'albums', + attributes: { name: 'Mix', artistName: 'DJ' }, + relationships: { + tracks: { data: [{ id: 't1', type: 'songs' }] }, + }, + }); + assertEquals(tracks[0].attributes?.name, 'Track One'); + }); + + it('resolves relative AMP pagination URLs', () => { + const next = resolveAmpUrl( + '/v1/catalog/au/albums/1/tracks?offset=10', + 'https://amp-api.music.apple.com', + ); + assertEquals(next.href, 'https://amp-api.music.apple.com/v1/catalog/au/albums/1/tracks?offset=10'); + }); +}); diff --git a/providers/iTunes/amp.ts b/providers/iTunes/amp.ts new file mode 100644 index 00000000..9bebd5f2 --- /dev/null +++ b/providers/iTunes/amp.ts @@ -0,0 +1,76 @@ +import { decodeBase64 } from 'std/encoding/base64.ts'; +import type { AmpAlbum, AmpArtist, AmpDocument, AmpTrack } from './amp_types.ts'; + +/** JWT as embedded in Apple Music / MusicKit assets. */ +const jwtPattern = /["'](eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)["']/g; + +export function extractAppleMusicJwt(source: string): string | undefined { + const matches = source.matchAll(jwtPattern); + let best: string | undefined; + let bestExpiry = 0; + for (const match of matches) { + const token = match[1]; + const expiry = parseJwtExpiry(token) ?? 0; + if (expiry > bestExpiry) { + best = token; + bestExpiry = expiry; + } + } + return best; +} + +export function parseJwtExpiry(token: string): number | undefined { + try { + const payloadPart = token.split('.')[1]; + if (!payloadPart) return undefined; + const padded = payloadPart.replace(/-/g, '+').replace(/_/g, '/') + + '='.repeat((4 - (payloadPart.length % 4)) % 4); + const payload = JSON.parse(new TextDecoder().decode(decodeBase64(padded))) as { exp?: number }; + return typeof payload.exp === 'number' ? payload.exp * 1000 : undefined; + } catch { + return undefined; + } +} + +export function extractScriptUrls(html: string): string[] { + const urls: string[] = []; + const seen = new Set(); + const patterns = [ + /]*\bcrossorigin\b[^>]*\bsrc=["']([^"']+)["'][^>]*>/gi, + /]*\bsrc=["']([^"']+)["'][^>]*\bcrossorigin\b[^>]*>/gi, + /]*\bsrc=["']([^"']*musickit[^"']*)["'][^>]*>/gi, + ]; + for (const pattern of patterns) { + for (const match of html.matchAll(pattern)) { + const src = match[1]; + if (!seen.has(src)) { + seen.add(src); + urls.push(src); + } + } + } + return urls; +} + +export function resolveAmpUrl(next: string, apiBaseUrl: string): URL { + if (next.startsWith('http://') || next.startsWith('https://')) { + return new URL(next); + } + return new URL(next, apiBaseUrl); +} + +export function isAmpTrack(resource: AmpAlbum | AmpTrack | AmpArtist): resource is AmpTrack { + return resource.type === 'songs' || resource.type === 'music-videos'; +} + +export function collectAmpTracks(body: AmpDocument, album?: AmpAlbum): AmpTrack[] { + const includedTracks = (body.included ?? []).filter(isAmpTrack); + const byId = new Map(includedTracks.map((track) => [track.id, track] as const)); + const hydrate = (partial: AmpTrack[]) => + partial.map((track) => track.attributes ? track : (byId.get(track.id) ?? track)); + + if (album) { + return hydrate(album.relationships?.tracks?.data ?? []); + } + return hydrate((body.data ?? []).filter(isAmpTrack)); +} diff --git a/providers/iTunes/amp_types.ts b/providers/iTunes/amp_types.ts new file mode 100644 index 00000000..53350823 --- /dev/null +++ b/providers/iTunes/amp_types.ts @@ -0,0 +1,58 @@ +/** Minimal types for the unofficial Apple Music catalog (AMP) API. */ + +export type AmpResourceType = 'albums' | 'songs' | 'music-videos' | 'artists'; + +export type AmpArtwork = { + url: string; + width?: number; + height?: number; +}; + +export type AmpAlbumAttributes = { + name: string; + artistName: string; + upc?: string; + releaseDate?: string; + copyright?: string; + trackCount?: number; + isComplete?: boolean; + url?: string; + artwork?: AmpArtwork; +}; + +export type AmpTrackAttributes = { + name: string; + artistName: string; + durationInMillis?: number; + trackNumber?: number; + discNumber?: number; + isrc?: string; + url?: string; +}; + +export type AmpResource = { + id: string; + type: Type; + attributes?: Attributes; + relationships?: { + tracks?: AmpRelationship; + artists?: AmpRelationship; + }; +}; + +export type AmpAlbum = AmpResource<'albums', AmpAlbumAttributes>; +export type AmpTrack = AmpResource<'songs' | 'music-videos', AmpTrackAttributes>; +export type AmpArtist = AmpResource<'artists', { name: string; url?: string }>; + +export type AmpRelationship = { + data?: T[]; + next?: string; + href?: string; +}; + +export type AmpDocument = { + data?: Array; + included?: Array; + next?: string; + errors?: Array<{ title?: string; detail?: string; status?: string }>; +}; diff --git a/providers/iTunes/mod.ts b/providers/iTunes/mod.ts index 334e7a59..6d6cddae 100644 --- a/providers/iTunes/mod.ts +++ b/providers/iTunes/mod.ts @@ -1,11 +1,14 @@ import { availableRegions } from './regions.ts'; -import { type ApiQueryOptions, type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; +import { type ApiAccessToken, type ApiQueryOptions, type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { fillMediumsTracklistGaps } from '@/harmonizer/tracklist_gap.ts'; import { parseISODateTime, PartialDate } from '@/utils/date.ts'; +import { ProviderError } from '@/utils/errors.ts'; import { isEqualGTIN, isValidGTIN } from '@/utils/gtin.ts'; +import { collectAmpTracks, extractAppleMusicJwt, extractScriptUrls, parseJwtExpiry, resolveAmpUrl } from './amp.ts'; import type { Collection, Kind, ReleaseResult, Track } from './api_types.ts'; +import type { AmpAlbum, AmpDocument, AmpTrack } from './amp_types.ts'; import type { ArtistCreditName, Artwork, @@ -15,6 +18,7 @@ import type { GTIN, HarmonyMedium, HarmonyRelease, + HarmonyTrack, LinkType, ReleaseGroupType, } from '@/harmonizer/types.ts'; @@ -57,6 +61,8 @@ export default class iTunesProvider extends MetadataApiProvider { readonly apiBaseUrl = 'https://itunes.apple.com'; + readonly ampApiBaseUrl = 'https://amp-api.music.apple.com'; + /** URLs without specified region implicitly query the US iTunes store. */ readonly defaultRegion: CountryCode = 'US'; @@ -78,12 +84,106 @@ export default class iTunesProvider extends MetadataApiProvider { return ['paid streaming']; } - async query(apiUrl: URL, options: ApiQueryOptions): Promise> { + async query(apiUrl: URL, options: ApiQueryOptions = {}): Promise> { const cacheEntry = await this.fetchJSON(apiUrl, { policy: { maxTimestamp: options.snapshotMaxTimestamp }, + offline: options.offline, }); return cacheEntry; } + + async queryAmp(apiUrl: URL, options: ApiQueryOptions = {}): Promise> { + const accessToken = await this.cachedAccessToken(() => this.requestAmpAccessToken()); + return await this.fetchJSON(apiUrl, { + policy: { maxTimestamp: options.snapshotMaxTimestamp }, + offline: options.offline, + requestInit: { + headers: { + 'Authorization': `Bearer ${accessToken}`, + 'Origin': 'https://music.apple.com', + 'Accept': 'application/json', + }, + }, + }); + } + + constructAmpAlbumUrl(albumId: string, region: CountryCode): URL { + const url = new URL(`v1/catalog/${region.toLowerCase()}/albums/${albumId}`, this.ampApiBaseUrl); + url.searchParams.set('include', 'tracks'); + return url; + } + + async fetchAmpAlbum( + albumId: string, + region: CountryCode, + options: ApiQueryOptions = {}, + ): Promise<{ album: AmpAlbum; tracks: AmpTrack[]; timestamp: number }> { + let nextUrl: URL | undefined = this.constructAmpAlbumUrl(albumId, region); + let album: AmpAlbum | undefined; + const tracks: AmpTrack[] = []; + let timestamp = 0; + + while (nextUrl) { + const cacheEntry: CacheEntry = await this.queryAmp(nextUrl, options); + timestamp = Math.max(timestamp, cacheEntry.timestamp); + const body: AmpDocument = cacheEntry.content; + if (body.errors?.length) { + const detail = body.errors.map((error) => error.detail ?? error.title).join('; '); + throw new ProviderError(this.name, `Apple Music catalog API error: ${detail || 'unknown error'}`); + } + + if (!album) { + const albumResource = (body.data ?? []).find((resource): resource is AmpAlbum => resource.type === 'albums'); + if (!albumResource) { + throw new ProviderError(this.name, 'Apple Music catalog API returned no album'); + } + album = albumResource; + tracks.push(...collectAmpTracks(body, album)); + const next = album.relationships?.tracks?.next ?? body.next; + nextUrl = next ? resolveAmpUrl(next, this.ampApiBaseUrl) : undefined; + } else { + tracks.push(...collectAmpTracks(body)); + nextUrl = body.next ? resolveAmpUrl(body.next, this.ampApiBaseUrl) : undefined; + } + } + + return { album: album!, tracks, timestamp }; + } + + private async requestAmpAccessToken(): Promise { + const pageUrl = new URL('https://music.apple.com'); + const page = await this.fetchSnapshot(pageUrl); + const html = await page.content.text(); + const fromHtml = extractAppleMusicJwt(html); + if (fromHtml) { + return this.ampTokenFromJwt(fromHtml); + } + + for (const scriptSrc of extractScriptUrls(html)) { + const scriptUrl = new URL(scriptSrc, pageUrl); + if (!scriptUrl.hostname.endsWith('apple.com') && !scriptUrl.hostname.endsWith('mzstatic.com')) { + continue; + } + try { + const script = await this.fetchSnapshot(scriptUrl); + const fromScript = extractAppleMusicJwt(await script.content.text()); + if (fromScript) { + return this.ampTokenFromJwt(fromScript); + } + } catch { + // Try the next candidate script. + } + } + + throw new ProviderError(this.name, 'Failed to extract Apple Music catalog API token from music.apple.com'); + } + + private ampTokenFromJwt(accessToken: string): ApiAccessToken { + return { + accessToken, + validUntilTimestamp: parseJwtExpiry(accessToken) ?? (Date.now() + 60 * 60 * 1000), + }; + } } export class iTunesReleaseLookup extends ReleaseApiLookup { @@ -118,7 +218,7 @@ export class iTunesReleaseLookup extends ReleaseApiLookup { // API sometimes also returns other release variants for GTIN lookups, only use the first collection result. const collections = data.results.filter((result) => result.wrapperType === 'collection') as Collection[]; let collection = collections[0]; @@ -143,9 +243,30 @@ export class iTunesReleaseLookup extends ReleaseApiLookup track.isStreamable || track.kind === 'music-video')) { + if ( + ampTracks.length || + (tracks.length && tracks.every((track) => track.isStreamable || track.kind === 'music-video')) + ) { // All audio tracks should be streamable, music videos are always streamable but have no `isStreamable` property. linkTypes.push('paid streaming'); } const releaseUrl = this.cleanViewUrl(collection.collectionViewUrl); - const gtin = this.extractGTINFromUrl(collection.artworkUrl100); + const gtin = ampUpc && isValidGTIN(ampUpc) ? ampUpc : this.extractGTINFromUrl(collection.artworkUrl100); - if (!gtin) { + if (ampUpc && isValidGTIN(ampUpc)) { + this.addMessage(`Successfully extracted GTIN ${ampUpc} from Apple Music catalog API`); + } else if (!gtin) { this.addMessage('Failed to extract GTIN from artwork URL', 'warning'); } else if (this.lookup.method === 'gtin' && !isEqualGTIN(gtin, this.lookup.value)) { this.addMessage( @@ -195,7 +321,7 @@ export class iTunesReleaseLookup extends ReleaseApiLookup track.attributes?.discNumber ?? 1)); + const media: HarmonyMedium[] = new Array(discCount).fill(null).map((_, index) => ({ + format: 'Digital Media', + number: index + 1, + tracklist: [], + })); + + for (const track of tracklist) { + const attributes = track.attributes; + if (!attributes) continue; + const discNumber = attributes.discNumber ?? 1; + const medium = media[discNumber - 1]; + if (!medium) continue; + + const linkTypes: LinkType[] = ['paid streaming']; + medium.tracklist.push({ + number: attributes.trackNumber, + title: attributes.name, + length: attributes.durationInMillis, + artists: [this.convertRawArtist(attributes.artistName)], + isrc: attributes.isrc, + type: track.type === 'music-videos' ? 'video' : undefined, + recording: { + externalIds: this.provider.makeExternalIds({ + type: track.type === 'music-videos' ? 'music-video' : 'song', + id: track.id, + region: this.lookup.region, + linkTypes, + }), + }, + }); + } + + return media; + } + private convertRawArtist(name: string, url?: string): ArtistCreditName { const artistId = url ? this.provider.extractEntityFromUrl(new URL(url)) : undefined; return { From 69f764aca40a6efaef8b7022334b4c77aad75911 Mon Sep 17 00:00:00 2001 From: Dennis <39654202+dwei30@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:41:15 +1000 Subject: [PATCH 2/6] docs(iTunes): Comment AMP catalog fallback and token scrape --- providers/iTunes/amp.ts | 9 ++++++++- providers/iTunes/amp_types.ts | 4 +++- providers/iTunes/mod.ts | 11 +++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/providers/iTunes/amp.ts b/providers/iTunes/amp.ts index 9bebd5f2..c78320d2 100644 --- a/providers/iTunes/amp.ts +++ b/providers/iTunes/amp.ts @@ -1,9 +1,12 @@ +// Helpers for the unofficial Apple Music catalog (AMP) API. +// Metadata is loaded from amp-api.music.apple.com; music.apple.com is only used to obtain a JWT. import { decodeBase64 } from 'std/encoding/base64.ts'; import type { AmpAlbum, AmpArtist, AmpDocument, AmpTrack } from './amp_types.ts'; -/** JWT as embedded in Apple Music / MusicKit assets. */ +// JWT as embedded in Apple Music / MusicKit assets. const jwtPattern = /["'](eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)["']/g; +// Returns the JWT with the latest expiry when a page or script embeds more than one token. export function extractAppleMusicJwt(source: string): string | undefined { const matches = source.matchAll(jwtPattern); let best: string | undefined; @@ -32,6 +35,7 @@ export function parseJwtExpiry(token: string): number | undefined { } } +// Script URLs that typically contain the MusicKit developer token. export function extractScriptUrls(html: string): string[] { const urls: string[] = []; const seen = new Set(); @@ -52,6 +56,7 @@ export function extractScriptUrls(html: string): string[] { return urls; } +// AMP pagination `next` is often a path (`/v1/catalog/...`), not an absolute URL. export function resolveAmpUrl(next: string, apiBaseUrl: string): URL { if (next.startsWith('http://') || next.startsWith('https://')) { return new URL(next); @@ -63,6 +68,8 @@ export function isAmpTrack(resource: AmpAlbum | AmpTrack | AmpArtist): resource return resource.type === 'songs' || resource.type === 'music-videos'; } +// Collects tracks from an AMP page. +// Relationships may only list IDs; full objects (name, ISRC, duration) are often in `included`. export function collectAmpTracks(body: AmpDocument, album?: AmpAlbum): AmpTrack[] { const includedTracks = (body.included ?? []).filter(isAmpTrack); const byId = new Map(includedTracks.map((track) => [track.id, track] as const)); diff --git a/providers/iTunes/amp_types.ts b/providers/iTunes/amp_types.ts index 53350823..ccd945e8 100644 --- a/providers/iTunes/amp_types.ts +++ b/providers/iTunes/amp_types.ts @@ -1,4 +1,6 @@ -/** Minimal types for the unofficial Apple Music catalog (AMP) API. */ +// Minimal types for the unofficial Apple Music catalog (AMP) API +// (https://amp-api.music.apple.com/v1/catalog/...). +// Not the public Apple Music AP, used as a fallback when iTunes Search returns no tracks. export type AmpResourceType = 'albums' | 'songs' | 'music-videos' | 'artists'; diff --git a/providers/iTunes/mod.ts b/providers/iTunes/mod.ts index 6d6cddae..d6e92b80 100644 --- a/providers/iTunes/mod.ts +++ b/providers/iTunes/mod.ts @@ -61,6 +61,7 @@ export default class iTunesProvider extends MetadataApiProvider { readonly apiBaseUrl = 'https://itunes.apple.com'; + // Unofficial Apple Music catalog API used when iTunes Search omits streaming-only tracks. readonly ampApiBaseUrl = 'https://amp-api.music.apple.com'; /** URLs without specified region implicitly query the US iTunes store. */ @@ -92,6 +93,8 @@ export default class iTunesProvider extends MetadataApiProvider { return cacheEntry; } + // Queries the Apple Music catalog (AMP) API. + // Requires a MusicKit JWT and Origin: https://music.apple.com or AMP returns 401/403. async queryAmp(apiUrl: URL, options: ApiQueryOptions = {}): Promise> { const accessToken = await this.cachedAccessToken(() => this.requestAmpAccessToken()); return await this.fetchJSON(apiUrl, { @@ -100,6 +103,7 @@ export default class iTunesProvider extends MetadataApiProvider { requestInit: { headers: { 'Authorization': `Bearer ${accessToken}`, + // AMP rejects requests that do not look like they come from the Apple Music web app. 'Origin': 'https://music.apple.com', 'Accept': 'application/json', }, @@ -113,6 +117,7 @@ export default class iTunesProvider extends MetadataApiProvider { return url; } + // Loads an AMP album and follows `next` until the full tracklist is collected. async fetchAmpAlbum( albumId: string, region: CountryCode, @@ -133,6 +138,7 @@ export default class iTunesProvider extends MetadataApiProvider { } if (!album) { + // First page is the album resource; later pages are additional tracks only. const albumResource = (body.data ?? []).find((resource): resource is AmpAlbum => resource.type === 'albums'); if (!albumResource) { throw new ProviderError(this.name, 'Apple Music catalog API returned no album'); @@ -150,6 +156,8 @@ export default class iTunesProvider extends MetadataApiProvider { return { album: album!, tracks, timestamp }; } + // AMP has no public client-credentials flow. The web app embeds a MusicKit JWT in + // music.apple.com HTML or JS; scrape that token (not the album DOM) and cache it until `exp`. private async requestAmpAccessToken(): Promise { const pageUrl = new URL('https://music.apple.com'); const page = await this.fetchSnapshot(pageUrl); @@ -243,6 +251,8 @@ export class iTunesReleaseLookup extends ReleaseApiLookup Date: Fri, 14 Aug 2026 18:22:05 +1000 Subject: [PATCH 3/6] remove unused type --- providers/iTunes/mod.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/providers/iTunes/mod.ts b/providers/iTunes/mod.ts index d6e92b80..695db592 100644 --- a/providers/iTunes/mod.ts +++ b/providers/iTunes/mod.ts @@ -1,5 +1,11 @@ import { availableRegions } from './regions.ts'; -import { type ApiAccessToken, type ApiQueryOptions, type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; +import { + type ApiAccessToken, + type ApiQueryOptions, + type CacheEntry, + MetadataApiProvider, + ReleaseApiLookup, +} from '@/providers/base.ts'; import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { fillMediumsTracklistGaps } from '@/harmonizer/tracklist_gap.ts'; import { parseISODateTime, PartialDate } from '@/utils/date.ts'; @@ -18,7 +24,6 @@ import type { GTIN, HarmonyMedium, HarmonyRelease, - HarmonyTrack, LinkType, ReleaseGroupType, } from '@/harmonizer/types.ts'; From 3762ab5ec48732d10b5fa593c558331a942387ea Mon Sep 17 00:00:00 2001 From: dwei30 Date: Sat, 15 Aug 2026 16:50:31 +1000 Subject: [PATCH 4/6] feat(Apple Music): Add catalog API provider for streaming-only releases Keep iTunes Search as the fallback when Apple Music credentials are unset. --- .env.example | 8 + .../catalog.test.ts} | 28 +- .../{iTunes/amp.ts => AppleMusic/catalog.ts} | 34 +- providers/AppleMusic/catalog_types.ts | 60 +++ providers/AppleMusic/mod.test.ts | 56 +++ providers/AppleMusic/mod.ts | 403 ++++++++++++++++++ providers/iTunes/amp_types.ts | 60 --- providers/iTunes/mod.ts | 199 +-------- providers/mod.ts | 6 + server/components/ProviderIcon.tsx | 1 + server/static/harmony.css | 4 +- 11 files changed, 585 insertions(+), 274 deletions(-) rename providers/{iTunes/amp.test.ts => AppleMusic/catalog.test.ts} (73%) rename providers/{iTunes/amp.ts => AppleMusic/catalog.ts} (64%) create mode 100644 providers/AppleMusic/catalog_types.ts create mode 100644 providers/AppleMusic/mod.test.ts create mode 100644 providers/AppleMusic/mod.ts delete mode 100644 providers/iTunes/amp_types.ts diff --git a/.env.example b/.env.example index 7b02228a..32cf076e 100644 --- a/.env.example +++ b/.env.example @@ -33,3 +33,11 @@ HARMONY_QOBUZ_APP_ID= # Bugs! app config. HARMONY_BUGS_CLIENT_SECRET= + +# Apple Music catalog API (optional). Official developer token is preferred. +# See https://developer.apple.com/documentation/applemusicapi/generating-developer-tokens +HARMONY_APPLE_MUSIC_TOKEN= +# Unofficial AMP API token for self-hosted Harmony (used if HARMONY_APPLE_MUSIC_TOKEN is unset). +HARMONY_APPLE_MUSIC_AMP_TOKEN= +# Scrape a MusicKit JWT from music.apple.com when no token is set (self-hosted only). +HARMONY_APPLE_MUSIC_SCRAPE_TOKEN= diff --git a/providers/iTunes/amp.test.ts b/providers/AppleMusic/catalog.test.ts similarity index 73% rename from providers/iTunes/amp.test.ts rename to providers/AppleMusic/catalog.test.ts index 11725bda..c7aea1b8 100644 --- a/providers/iTunes/amp.test.ts +++ b/providers/AppleMusic/catalog.test.ts @@ -1,6 +1,13 @@ import { assertEquals } from 'std/assert/assert_equals.ts'; import { describe, it } from '@std/testing/bdd'; -import { collectAmpTracks, extractAppleMusicJwt, extractScriptUrls, parseJwtExpiry, resolveAmpUrl } from './amp.ts'; +import { + catalogArtworkUrl, + collectCatalogTracks, + extractAppleMusicJwt, + extractScriptUrls, + parseJwtExpiry, + resolveCatalogUrl, +} from './catalog.ts'; function makeJwt(exp: number): string { const encode = (value: object) => @@ -8,7 +15,7 @@ function makeJwt(exp: number): string { return `${encode({ alg: 'ES256', typ: 'JWT' })}.${encode({ exp })}.sig`; } -describe('Apple Music AMP helpers', () => { +describe('Apple Music catalog helpers', () => { it('extracts the JWT with the latest expiry', () => { const older = makeJwt(1_700_000_000); const newer = makeJwt(2_000_000_000); @@ -30,7 +37,7 @@ describe('Apple Music AMP helpers', () => { }); it('hydrates track stubs from included resources', () => { - const tracks = collectAmpTracks({ + const tracks = collectCatalogTracks({ data: [{ id: '1', type: 'albums', @@ -55,11 +62,18 @@ describe('Apple Music AMP helpers', () => { assertEquals(tracks[0].attributes?.name, 'Track One'); }); - it('resolves relative AMP pagination URLs', () => { - const next = resolveAmpUrl( + it('resolves relative catalog pagination URLs', () => { + const next = resolveCatalogUrl( '/v1/catalog/au/albums/1/tracks?offset=10', - 'https://amp-api.music.apple.com', + 'https://api.music.apple.com', + ); + assertEquals(next.href, 'https://api.music.apple.com/v1/catalog/au/albums/1/tracks?offset=10'); + }); + + it('fills artwork template dimensions', () => { + assertEquals( + catalogArtworkUrl({ url: 'https://example.com/{w}x{h}bb.jpg', width: 3000, height: 3000 }, 250), + 'https://example.com/250x250bb.jpg', ); - assertEquals(next.href, 'https://amp-api.music.apple.com/v1/catalog/au/albums/1/tracks?offset=10'); }); }); diff --git a/providers/iTunes/amp.ts b/providers/AppleMusic/catalog.ts similarity index 64% rename from providers/iTunes/amp.ts rename to providers/AppleMusic/catalog.ts index c78320d2..23f5e14f 100644 --- a/providers/iTunes/amp.ts +++ b/providers/AppleMusic/catalog.ts @@ -1,12 +1,8 @@ -// Helpers for the unofficial Apple Music catalog (AMP) API. -// Metadata is loaded from amp-api.music.apple.com; music.apple.com is only used to obtain a JWT. import { decodeBase64 } from 'std/encoding/base64.ts'; -import type { AmpAlbum, AmpArtist, AmpDocument, AmpTrack } from './amp_types.ts'; +import type { CatalogAlbum, CatalogArtist, CatalogDocument, CatalogTrack } from './catalog_types.ts'; -// JWT as embedded in Apple Music / MusicKit assets. const jwtPattern = /["'](eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)["']/g; -// Returns the JWT with the latest expiry when a page or script embeds more than one token. export function extractAppleMusicJwt(source: string): string | undefined { const matches = source.matchAll(jwtPattern); let best: string | undefined; @@ -35,7 +31,6 @@ export function parseJwtExpiry(token: string): number | undefined { } } -// Script URLs that typically contain the MusicKit developer token. export function extractScriptUrls(html: string): string[] { const urls: string[] = []; const seen = new Set(); @@ -56,28 +51,37 @@ export function extractScriptUrls(html: string): string[] { return urls; } -// AMP pagination `next` is often a path (`/v1/catalog/...`), not an absolute URL. -export function resolveAmpUrl(next: string, apiBaseUrl: string): URL { +export function resolveCatalogUrl(next: string, apiBaseUrl: string): URL { if (next.startsWith('http://') || next.startsWith('https://')) { return new URL(next); } return new URL(next, apiBaseUrl); } -export function isAmpTrack(resource: AmpAlbum | AmpTrack | AmpArtist): resource is AmpTrack { +export function isCatalogTrack( + resource: CatalogAlbum | CatalogTrack | CatalogArtist, +): resource is CatalogTrack { return resource.type === 'songs' || resource.type === 'music-videos'; } -// Collects tracks from an AMP page. -// Relationships may only list IDs; full objects (name, ISRC, duration) are often in `included`. -export function collectAmpTracks(body: AmpDocument, album?: AmpAlbum): AmpTrack[] { - const includedTracks = (body.included ?? []).filter(isAmpTrack); +export function collectCatalogTracks(body: CatalogDocument, album?: CatalogAlbum): CatalogTrack[] { + const includedTracks = (body.included ?? []).filter(isCatalogTrack); const byId = new Map(includedTracks.map((track) => [track.id, track] as const)); - const hydrate = (partial: AmpTrack[]) => + const hydrate = (partial: CatalogTrack[]) => partial.map((track) => track.attributes ? track : (byId.get(track.id) ?? track)); if (album) { return hydrate(album.relationships?.tracks?.data ?? []); } - return hydrate((body.data ?? []).filter(isAmpTrack)); + return hydrate((body.data ?? []).filter(isCatalogTrack)); +} + +export function catalogArtworkUrl( + artwork?: { url: string; width?: number; height?: number }, + size?: number, +): string | undefined { + if (!artwork?.url) return undefined; + const width = size ?? artwork.width ?? 3000; + const height = size ?? artwork.height ?? 3000; + return artwork.url.replace('{w}', String(width)).replace('{h}', String(height)); } diff --git a/providers/AppleMusic/catalog_types.ts b/providers/AppleMusic/catalog_types.ts new file mode 100644 index 00000000..21a4e13a --- /dev/null +++ b/providers/AppleMusic/catalog_types.ts @@ -0,0 +1,60 @@ +// Types shared by the official Apple Music API and AMP +// (https://api.music.apple.com/v1/catalog/... and https://amp-api.music.apple.com/v1/catalog/...). + +export type CatalogResourceType = 'albums' | 'songs' | 'music-videos' | 'artists'; + +export type CatalogArtwork = { + url: string; + width?: number; + height?: number; +}; + +export type CatalogAlbumAttributes = { + name: string; + artistName: string; + upc?: string; + releaseDate?: string; + copyright?: string; + recordLabel?: string; + trackCount?: number; + isComplete?: boolean; + url?: string; + artwork?: CatalogArtwork; +}; + +export type CatalogTrackAttributes = { + name: string; + artistName: string; + durationInMillis?: number; + trackNumber?: number; + discNumber?: number; + isrc?: string; + url?: string; +}; + +export type CatalogResource = { + id: string; + type: Type; + attributes?: Attributes; + relationships?: { + tracks?: CatalogRelationship; + artists?: CatalogRelationship; + }; +}; + +export type CatalogAlbum = CatalogResource<'albums', CatalogAlbumAttributes>; +export type CatalogTrack = CatalogResource<'songs' | 'music-videos', CatalogTrackAttributes>; +export type CatalogArtist = CatalogResource<'artists', { name: string; url?: string }>; + +export type CatalogRelationship = { + data?: T[]; + next?: string; + href?: string; +}; + +export type CatalogDocument = { + data?: Array; + included?: Array; + next?: string; + errors?: Array<{ title?: string; detail?: string; status?: string }>; +}; diff --git a/providers/AppleMusic/mod.test.ts b/providers/AppleMusic/mod.test.ts new file mode 100644 index 00000000..e02124eb --- /dev/null +++ b/providers/AppleMusic/mod.test.ts @@ -0,0 +1,56 @@ +import { describeProvider, makeProviderOptions } from '@/providers/test_spec.ts'; +import { stubProviderLookups, stubTokenRetrieval } from '@/providers/test_stubs.ts'; +import { afterAll, describe } from '@std/testing/bdd'; + +import AppleMusicProvider from './mod.ts'; + +describe('Apple Music provider', () => { + const appleMusic = new AppleMusicProvider(makeProviderOptions()); + const stubs = [stubProviderLookups(appleMusic), stubTokenRetrieval(appleMusic)]; + + describeProvider(appleMusic, { + urls: [{ + description: 'Apple Music album URL', + url: new URL('https://music.apple.com/de/album/1705742568'), + id: { type: 'album', id: '1705742568', region: 'DE' }, + isCanonical: true, + }, { + description: 'Apple Music album URL with implicit region', + url: new URL('https://music.apple.com/album/1705742568'), + id: { type: 'album', id: '1705742568', region: 'US' }, + }, { + description: 'Apple Music album URL with slug', + url: new URL('https://music.apple.com/de/album/all-will-be-changed/1705742568'), + id: { type: 'album', id: '1705742568', region: 'DE', slug: 'all-will-be-changed' }, + }, { + description: 'Apple Music artist URL', + url: new URL('https://music.apple.com/gb/artist/136975'), + id: { type: 'artist', id: '136975', region: 'GB' }, + isCanonical: true, + }, { + description: 'Apple Music song URL', + url: new URL('https://music.apple.com/gb/song/1772318408'), + id: { type: 'song', id: '1772318408', region: 'GB' }, + isCanonical: true, + }, { + description: 'Apple Music video URL', + url: new URL('https://music.apple.com/gb/music-video/1441458100'), + id: { type: 'music-video', id: '1441458100', region: 'GB' }, + isCanonical: true, + }, { + description: 'Apple Music geo. album URL', + url: new URL('https://geo.music.apple.com/album/1135913516'), + id: { type: 'album', id: '1135913516', region: 'US' }, + }, { + description: 'iTunes album URL (handled by iTunes provider)', + url: new URL('https://itunes.apple.com/gb/album/id1722294645'), + id: undefined, + }], + invalidIds: ['text'], + releaseLookup: [], + }); + + afterAll(() => { + stubs.forEach((stub) => stub.restore()); + }); +}); diff --git a/providers/AppleMusic/mod.ts b/providers/AppleMusic/mod.ts new file mode 100644 index 00000000..2ab132b5 --- /dev/null +++ b/providers/AppleMusic/mod.ts @@ -0,0 +1,403 @@ +import { availableRegions } from '../iTunes/regions.ts'; +import { + type ApiAccessToken, + type ApiQueryOptions, + type CacheEntry, + MetadataApiProvider, + ReleaseApiLookup, +} from '@/providers/base.ts'; +import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; +import { parseISODateTime, PartialDate } from '@/utils/date.ts'; +import { getBooleanFromEnv, getFromEnv } from '@/utils/config.ts'; +import { ProviderError } from '@/utils/errors.ts'; +import { isValidGTIN } from '@/utils/gtin.ts'; +import { + catalogArtworkUrl, + collectCatalogTracks, + extractAppleMusicJwt, + extractScriptUrls, + parseJwtExpiry, + resolveCatalogUrl, +} from './catalog.ts'; + +import type { CatalogAlbum, CatalogDocument, CatalogTrack } from './catalog_types.ts'; +import type { + ArtistCreditName, + Artwork, + ArtworkType, + CountryCode, + EntityId, + HarmonyMedium, + HarmonyRelease, + LinkType, + ReleaseGroupType, +} from '@/harmonizer/types.ts'; +import type { ProviderCategory } from '@/providers/categories.ts'; + +// Official catalog: https://developer.apple.com/documentation/applemusicapi +const officialApiBaseUrl = 'https://api.music.apple.com'; +const ampApiBaseUrl = 'https://amp-api.music.apple.com'; + +const officialToken = getFromEnv('HARMONY_APPLE_MUSIC_TOKEN') || ''; +const ampToken = getFromEnv('HARMONY_APPLE_MUSIC_AMP_TOKEN') || ''; +const scrapeToken = getBooleanFromEnv('HARMONY_APPLE_MUSIC_SCRAPE_TOKEN'); + +export type AppleMusicBackend = 'official' | 'amp'; + +export function isAppleMusicConfigured(): boolean { + return Boolean(officialToken || ampToken || scrapeToken); +} + +export function appleMusicBackend(): AppleMusicBackend { + return officialToken ? 'official' : 'amp'; +} + +export default class AppleMusicProvider extends MetadataApiProvider { + readonly name = 'Apple Music'; + + readonly supportedUrls = new URLPattern({ + hostname: '{geo.}?music.apple.com', + pathname: String.raw`/:region(\w{2})?/:type(album|artist|song|music-video)/:slug?/{id}?:id(\d+)`, + }); + + override readonly categories = new Set(['digital']); + + override readonly features: FeatureQualityMap = { + 'cover size': 3000, + 'duration precision': DurationPrecision.MS, + 'GTIN lookup': FeatureQuality.GOOD, + 'MBID resolving': FeatureQuality.EXPENSIVE, + 'release label': FeatureQuality.PRESENT, + }; + + readonly entityTypeMap = { + artist: 'artist', + release: 'album', + recording: ['song', 'music-video'], + }; + + override readonly availableRegions = new Set(availableRegions); + + readonly releaseLookup = AppleMusicReleaseLookup; + + override readonly launchDate: PartialDate = { + year: 2015, + month: 6, + day: 30, + }; + + readonly apiBaseUrl = officialToken ? officialApiBaseUrl : ampApiBaseUrl; + + readonly backend: AppleMusicBackend = appleMusicBackend(); + + readonly defaultRegion: CountryCode = 'US'; + + constructUrl(entity: EntityId): URL { + const region = entity.region ?? this.defaultRegion; + return new URL([region.toLowerCase(), entity.type, entity.id].join('/'), 'https://music.apple.com'); + } + + override extractEntityFromUrl(url: URL): EntityId | undefined { + const entity = super.extractEntityFromUrl(url); + if (entity && !entity.region) { + entity.region = this.defaultRegion; + } + return entity; + } + + override getLinkTypesForEntity(): LinkType[] { + return ['paid streaming']; + } + + async query(apiUrl: URL, options: ApiQueryOptions = {}): Promise> { + const accessToken = await this.cachedAccessToken(() => this.requestAccessToken()); + const headers: Record = { + 'Authorization': `Bearer ${accessToken}`, + 'Accept': 'application/json', + }; + if (this.backend === 'amp') { + headers['Origin'] = 'https://music.apple.com'; + } + return await this.fetchJSON(apiUrl, { + policy: { maxTimestamp: options.snapshotMaxTimestamp }, + offline: options.offline, + requestInit: { headers }, + }); + } + + constructAlbumApiUrl(albumId: string, region: CountryCode): URL { + const url = new URL(`v1/catalog/${region.toLowerCase()}/albums/${albumId}`, this.apiBaseUrl); + url.searchParams.set('include', 'tracks'); + return url; + } + + constructUpcSearchUrl(upc: string, region: CountryCode): URL { + const url = new URL(`v1/catalog/${region.toLowerCase()}/albums`, this.apiBaseUrl); + url.searchParams.set('filter[upc]', upc); + return url; + } + + async fetchCatalogAlbum( + albumId: string, + region: CountryCode, + options: ApiQueryOptions = {}, + ): Promise<{ album: CatalogAlbum; tracks: CatalogTrack[]; timestamp: number }> { + let nextUrl: URL | undefined = this.constructAlbumApiUrl(albumId, region); + let album: CatalogAlbum | undefined; + const tracks: CatalogTrack[] = []; + let timestamp = 0; + + while (nextUrl) { + const cacheEntry: CacheEntry = await this.query(nextUrl, options); + timestamp = Math.max(timestamp, cacheEntry.timestamp); + const body: CatalogDocument = cacheEntry.content; + if (body.errors?.length) { + const detail = body.errors.map((error) => error.detail ?? error.title).join('; '); + throw new ProviderError(this.name, `Catalog API error: ${detail || 'unknown error'}`); + } + + if (!album) { + const albumResource = (body.data ?? []).find((resource): resource is CatalogAlbum => + resource.type === 'albums' + ); + if (!albumResource) { + throw new ProviderError(this.name, 'Catalog API returned no album'); + } + album = albumResource; + tracks.push(...collectCatalogTracks(body, album)); + const next = album.relationships?.tracks?.next ?? body.next; + nextUrl = next ? resolveCatalogUrl(next, this.apiBaseUrl) : undefined; + } else { + tracks.push(...collectCatalogTracks(body)); + nextUrl = body.next ? resolveCatalogUrl(body.next, this.apiBaseUrl) : undefined; + } + } + + return { album: album!, tracks, timestamp }; + } + + private async requestAccessToken(): Promise { + if (officialToken) { + return this.tokenFromJwt(officialToken); + } + if (ampToken) { + return this.tokenFromJwt(ampToken); + } + return await this.scrapeAccessToken(); + } + + private async scrapeAccessToken(): Promise { + const pageUrl = new URL('https://music.apple.com'); + const page = await this.fetchSnapshot(pageUrl); + const html = await page.content.text(); + const fromHtml = extractAppleMusicJwt(html); + if (fromHtml) { + return this.tokenFromJwt(fromHtml); + } + + for (const scriptSrc of extractScriptUrls(html)) { + const scriptUrl = new URL(scriptSrc, pageUrl); + if (!scriptUrl.hostname.endsWith('apple.com') && !scriptUrl.hostname.endsWith('mzstatic.com')) { + continue; + } + try { + const script = await this.fetchSnapshot(scriptUrl); + const fromScript = extractAppleMusicJwt(await script.content.text()); + if (fromScript) { + return this.tokenFromJwt(fromScript); + } + } catch { + // Try the next candidate script. + } + } + + throw new ProviderError(this.name, 'Failed to extract MusicKit token from music.apple.com'); + } + + private tokenFromJwt(accessToken: string): ApiAccessToken { + return { + accessToken, + validUntilTimestamp: parseJwtExpiry(accessToken) ?? (Date.now() + 60 * 60 * 1000), + }; + } +} + +export class AppleMusicReleaseLookup extends ReleaseApiLookup { + private tracks: CatalogTrack[] = []; + + constructReleaseApiUrl(): URL { + const { method, value, region } = this.lookup; + const storefront = region ?? this.provider.defaultRegion; + if (method === 'gtin') { + return this.provider.constructUpcSearchUrl(value, storefront); + } + return this.provider.constructAlbumApiUrl(value, storefront); + } + + protected async getRawRelease(): Promise { + if (!this.options.regions?.size) { + this.options.regions = new Set([this.provider.defaultRegion]); + } + + if (this.lookup.method === 'gtin') { + const albumId = await this.queryAlbumIdByGtin(this.lookup.value); + if (!albumId) { + throw new ProviderError(this.provider.name, 'Catalog API returned no results for this GTIN'); + } + this.lookup.method = 'id'; + this.lookup.value = albumId; + } + + const region = this.lookup.region ?? this.provider.defaultRegion; + const { album, tracks, timestamp } = await this.provider.fetchCatalogAlbum( + this.lookup.value, + region, + { snapshotMaxTimestamp: this.options.snapshotMaxTimestamp }, + ); + this.updateCacheTime(timestamp); + this.tracks = tracks; + this.entity = { + id: album.id, + type: 'album', + region, + }; + return album; + } + + private async queryAlbumIdByGtin(gtin: string): Promise { + for (const region of this.options.regions || []) { + this.lookup.region = region; + const cacheEntry = await this.provider.query( + this.provider.constructUpcSearchUrl(gtin, region), + { snapshotMaxTimestamp: this.options.snapshotMaxTimestamp }, + ); + this.updateCacheTime(cacheEntry.timestamp); + const albums = (cacheEntry.content.data ?? []).filter((resource): resource is CatalogAlbum => + resource.type === 'albums' + ); + if (albums.length) { + if (albums.length > 1) { + this.warnMultipleResults( + albums.slice(1).map((album) => + this.provider.constructUrl({ + type: 'album', + id: album.id, + region, + }) + ), + ); + } + return albums[0].id; + } + } + return undefined; + } + + protected convertRawRelease(album: CatalogAlbum): HarmonyRelease { + const attributes = album.attributes; + if (!attributes) { + throw new ProviderError(this.provider.name, 'Album is missing attributes'); + } + + const { title, types } = this.getTypesFromTitle(attributes.name); + const gtin = attributes.upc && isValidGTIN(attributes.upc) ? attributes.upc : undefined; + const coverUrl = catalogArtworkUrl(attributes.artwork); + const thumbUrl = catalogArtworkUrl(attributes.artwork, 250); + + this.addMessage( + this.provider.backend === 'official' + ? 'Looked up via the official Apple Music API' + : 'Looked up via the Apple Music catalog (AMP) API', + ); + if (gtin) { + this.addMessage(`Catalog UPC ${gtin}`); + } + + const release: HarmonyRelease = { + title, + artists: [this.convertRawArtist(attributes.artistName)], + gtin, + externalLinks: [{ + url: this.provider.constructUrl(this.entity!).href, + types: ['paid streaming'], + }], + media: this.convertTracklist(this.tracks), + releaseDate: this.convertReleaseDate( + attributes.releaseDate ? parseISODateTime(attributes.releaseDate) : undefined, + ), + status: 'Official', + types, + packaging: 'None', + images: coverUrl ? [this.processImage(coverUrl, thumbUrl, ['front'])] : undefined, + copyright: attributes.copyright, + labels: attributes.recordLabel ? [{ name: attributes.recordLabel }] : undefined, + info: this.generateReleaseInfo(), + }; + + return release; + } + + private convertTracklist(tracklist: CatalogTrack[]): HarmonyMedium[] { + if (!tracklist.length) { + return []; + } + + const discCount = Math.max(1, ...tracklist.map((track) => track.attributes?.discNumber ?? 1)); + const media: HarmonyMedium[] = new Array(discCount).fill(null).map((_, index) => ({ + format: 'Digital Media', + number: index + 1, + tracklist: [], + })); + + for (const track of tracklist) { + const attributes = track.attributes; + if (!attributes) continue; + const discNumber = attributes.discNumber ?? 1; + const medium = media[discNumber - 1]; + if (!medium) continue; + + medium.tracklist.push({ + number: attributes.trackNumber, + title: attributes.name, + length: attributes.durationInMillis, + artists: [this.convertRawArtist(attributes.artistName)], + isrc: attributes.isrc, + type: track.type === 'music-videos' ? 'video' : undefined, + recording: { + externalIds: this.provider.makeExternalIds({ + type: track.type === 'music-videos' ? 'music-video' : 'song', + id: track.id, + region: this.lookup.region, + linkTypes: ['paid streaming'], + }), + }, + }); + } + + return media; + } + + private convertRawArtist(name: string, url?: string): ArtistCreditName { + const artistId = url ? this.provider.extractEntityFromUrl(new URL(url)) : undefined; + return { + name, + creditedName: name, + externalIds: artistId?.type === 'artist' ? this.provider.makeExternalIds(artistId) : undefined, + }; + } + + private processImage(url: string, thumbUrl: string | undefined, types?: ArtworkType[]): Artwork { + return { url, thumbUrl, types }; + } + + private getTypesFromTitle(title: string): { title: string; types: ReleaseGroupType[] } { + const re = /\s- (EP|Single)$/; + const match = title.match(re); + const types: ReleaseGroupType[] = []; + if (match) { + title = title.replace(re, ''); + types.push(match[1] as ReleaseGroupType); + } + return { title, types }; + } +} diff --git a/providers/iTunes/amp_types.ts b/providers/iTunes/amp_types.ts deleted file mode 100644 index ccd945e8..00000000 --- a/providers/iTunes/amp_types.ts +++ /dev/null @@ -1,60 +0,0 @@ -// Minimal types for the unofficial Apple Music catalog (AMP) API -// (https://amp-api.music.apple.com/v1/catalog/...). -// Not the public Apple Music AP, used as a fallback when iTunes Search returns no tracks. - -export type AmpResourceType = 'albums' | 'songs' | 'music-videos' | 'artists'; - -export type AmpArtwork = { - url: string; - width?: number; - height?: number; -}; - -export type AmpAlbumAttributes = { - name: string; - artistName: string; - upc?: string; - releaseDate?: string; - copyright?: string; - trackCount?: number; - isComplete?: boolean; - url?: string; - artwork?: AmpArtwork; -}; - -export type AmpTrackAttributes = { - name: string; - artistName: string; - durationInMillis?: number; - trackNumber?: number; - discNumber?: number; - isrc?: string; - url?: string; -}; - -export type AmpResource = { - id: string; - type: Type; - attributes?: Attributes; - relationships?: { - tracks?: AmpRelationship; - artists?: AmpRelationship; - }; -}; - -export type AmpAlbum = AmpResource<'albums', AmpAlbumAttributes>; -export type AmpTrack = AmpResource<'songs' | 'music-videos', AmpTrackAttributes>; -export type AmpArtist = AmpResource<'artists', { name: string; url?: string }>; - -export type AmpRelationship = { - data?: T[]; - next?: string; - href?: string; -}; - -export type AmpDocument = { - data?: Array; - included?: Array; - next?: string; - errors?: Array<{ title?: string; detail?: string; status?: string }>; -}; diff --git a/providers/iTunes/mod.ts b/providers/iTunes/mod.ts index 695db592..334e7a59 100644 --- a/providers/iTunes/mod.ts +++ b/providers/iTunes/mod.ts @@ -1,20 +1,11 @@ import { availableRegions } from './regions.ts'; -import { - type ApiAccessToken, - type ApiQueryOptions, - type CacheEntry, - MetadataApiProvider, - ReleaseApiLookup, -} from '@/providers/base.ts'; +import { type ApiQueryOptions, type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { fillMediumsTracklistGaps } from '@/harmonizer/tracklist_gap.ts'; import { parseISODateTime, PartialDate } from '@/utils/date.ts'; -import { ProviderError } from '@/utils/errors.ts'; import { isEqualGTIN, isValidGTIN } from '@/utils/gtin.ts'; -import { collectAmpTracks, extractAppleMusicJwt, extractScriptUrls, parseJwtExpiry, resolveAmpUrl } from './amp.ts'; import type { Collection, Kind, ReleaseResult, Track } from './api_types.ts'; -import type { AmpAlbum, AmpDocument, AmpTrack } from './amp_types.ts'; import type { ArtistCreditName, Artwork, @@ -66,9 +57,6 @@ export default class iTunesProvider extends MetadataApiProvider { readonly apiBaseUrl = 'https://itunes.apple.com'; - // Unofficial Apple Music catalog API used when iTunes Search omits streaming-only tracks. - readonly ampApiBaseUrl = 'https://amp-api.music.apple.com'; - /** URLs without specified region implicitly query the US iTunes store. */ readonly defaultRegion: CountryCode = 'US'; @@ -90,113 +78,12 @@ export default class iTunesProvider extends MetadataApiProvider { return ['paid streaming']; } - async query(apiUrl: URL, options: ApiQueryOptions = {}): Promise> { + async query(apiUrl: URL, options: ApiQueryOptions): Promise> { const cacheEntry = await this.fetchJSON(apiUrl, { policy: { maxTimestamp: options.snapshotMaxTimestamp }, - offline: options.offline, }); return cacheEntry; } - - // Queries the Apple Music catalog (AMP) API. - // Requires a MusicKit JWT and Origin: https://music.apple.com or AMP returns 401/403. - async queryAmp(apiUrl: URL, options: ApiQueryOptions = {}): Promise> { - const accessToken = await this.cachedAccessToken(() => this.requestAmpAccessToken()); - return await this.fetchJSON(apiUrl, { - policy: { maxTimestamp: options.snapshotMaxTimestamp }, - offline: options.offline, - requestInit: { - headers: { - 'Authorization': `Bearer ${accessToken}`, - // AMP rejects requests that do not look like they come from the Apple Music web app. - 'Origin': 'https://music.apple.com', - 'Accept': 'application/json', - }, - }, - }); - } - - constructAmpAlbumUrl(albumId: string, region: CountryCode): URL { - const url = new URL(`v1/catalog/${region.toLowerCase()}/albums/${albumId}`, this.ampApiBaseUrl); - url.searchParams.set('include', 'tracks'); - return url; - } - - // Loads an AMP album and follows `next` until the full tracklist is collected. - async fetchAmpAlbum( - albumId: string, - region: CountryCode, - options: ApiQueryOptions = {}, - ): Promise<{ album: AmpAlbum; tracks: AmpTrack[]; timestamp: number }> { - let nextUrl: URL | undefined = this.constructAmpAlbumUrl(albumId, region); - let album: AmpAlbum | undefined; - const tracks: AmpTrack[] = []; - let timestamp = 0; - - while (nextUrl) { - const cacheEntry: CacheEntry = await this.queryAmp(nextUrl, options); - timestamp = Math.max(timestamp, cacheEntry.timestamp); - const body: AmpDocument = cacheEntry.content; - if (body.errors?.length) { - const detail = body.errors.map((error) => error.detail ?? error.title).join('; '); - throw new ProviderError(this.name, `Apple Music catalog API error: ${detail || 'unknown error'}`); - } - - if (!album) { - // First page is the album resource; later pages are additional tracks only. - const albumResource = (body.data ?? []).find((resource): resource is AmpAlbum => resource.type === 'albums'); - if (!albumResource) { - throw new ProviderError(this.name, 'Apple Music catalog API returned no album'); - } - album = albumResource; - tracks.push(...collectAmpTracks(body, album)); - const next = album.relationships?.tracks?.next ?? body.next; - nextUrl = next ? resolveAmpUrl(next, this.ampApiBaseUrl) : undefined; - } else { - tracks.push(...collectAmpTracks(body)); - nextUrl = body.next ? resolveAmpUrl(body.next, this.ampApiBaseUrl) : undefined; - } - } - - return { album: album!, tracks, timestamp }; - } - - // AMP has no public client-credentials flow. The web app embeds a MusicKit JWT in - // music.apple.com HTML or JS; scrape that token (not the album DOM) and cache it until `exp`. - private async requestAmpAccessToken(): Promise { - const pageUrl = new URL('https://music.apple.com'); - const page = await this.fetchSnapshot(pageUrl); - const html = await page.content.text(); - const fromHtml = extractAppleMusicJwt(html); - if (fromHtml) { - return this.ampTokenFromJwt(fromHtml); - } - - for (const scriptSrc of extractScriptUrls(html)) { - const scriptUrl = new URL(scriptSrc, pageUrl); - if (!scriptUrl.hostname.endsWith('apple.com') && !scriptUrl.hostname.endsWith('mzstatic.com')) { - continue; - } - try { - const script = await this.fetchSnapshot(scriptUrl); - const fromScript = extractAppleMusicJwt(await script.content.text()); - if (fromScript) { - return this.ampTokenFromJwt(fromScript); - } - } catch { - // Try the next candidate script. - } - } - - throw new ProviderError(this.name, 'Failed to extract Apple Music catalog API token from music.apple.com'); - } - - private ampTokenFromJwt(accessToken: string): ApiAccessToken { - return { - accessToken, - validUntilTimestamp: parseJwtExpiry(accessToken) ?? (Date.now() + 60 * 60 * 1000), - }; - } } export class iTunesReleaseLookup extends ReleaseApiLookup { @@ -231,7 +118,7 @@ export class iTunesReleaseLookup extends ReleaseApiLookup { + protected convertRawRelease(data: ReleaseResult): HarmonyRelease { // API sometimes also returns other release variants for GTIN lookups, only use the first collection result. const collections = data.results.filter((result) => result.wrapperType === 'collection') as Collection[]; let collection = collections[0]; @@ -256,32 +143,9 @@ export class iTunesReleaseLookup extends ReleaseApiLookup track.isStreamable || track.kind === 'music-video')) - ) { + if (tracks.every((track) => track.isStreamable || track.kind === 'music-video')) { // All audio tracks should be streamable, music videos are always streamable but have no `isStreamable` property. linkTypes.push('paid streaming'); } const releaseUrl = this.cleanViewUrl(collection.collectionViewUrl); - const gtin = ampUpc && isValidGTIN(ampUpc) ? ampUpc : this.extractGTINFromUrl(collection.artworkUrl100); + const gtin = this.extractGTINFromUrl(collection.artworkUrl100); - if (ampUpc && isValidGTIN(ampUpc)) { - this.addMessage(`Successfully extracted GTIN ${ampUpc} from Apple Music catalog API`); - } else if (!gtin) { + if (!gtin) { this.addMessage('Failed to extract GTIN from artwork URL', 'warning'); } else if (this.lookup.method === 'gtin' && !isEqualGTIN(gtin, this.lookup.value)) { this.addMessage( @@ -336,7 +195,7 @@ export class iTunesReleaseLookup extends ReleaseApiLookup track.attributes?.discNumber ?? 1)); - const media: HarmonyMedium[] = new Array(discCount).fill(null).map((_, index) => ({ - format: 'Digital Media', - number: index + 1, - tracklist: [], - })); - - for (const track of tracklist) { - const attributes = track.attributes; - if (!attributes) continue; - const discNumber = attributes.discNumber ?? 1; - const medium = media[discNumber - 1]; - if (!medium) continue; - - const linkTypes: LinkType[] = ['paid streaming']; - medium.tracklist.push({ - number: attributes.trackNumber, - title: attributes.name, - length: attributes.durationInMillis, - artists: [this.convertRawArtist(attributes.artistName)], - isrc: attributes.isrc, - type: track.type === 'music-videos' ? 'video' : undefined, - recording: { - externalIds: this.provider.makeExternalIds({ - type: track.type === 'music-videos' ? 'music-video' : 'song', - id: track.id, - region: this.lookup.region, - linkTypes, - }), - }, - }); - } - - return media; - } - private convertRawArtist(name: string, url?: string): ArtistCreditName { const artistId = url ? this.provider.extractEntityFromUrl(new URL(url)) : undefined; return { diff --git a/providers/mod.ts b/providers/mod.ts index 72133da9..7d363f89 100644 --- a/providers/mod.ts +++ b/providers/mod.ts @@ -7,6 +7,7 @@ import BandcampProvider from './Bandcamp/mod.ts'; import BeatportProvider from './Beatport/mod.ts'; import DeezerProvider from './Deezer/mod.ts'; import DiscogsProvider from './Discogs/mod.ts'; +import AppleMusicProvider, { isAppleMusicConfigured } from './AppleMusic/mod.ts'; import iTunesProvider from './iTunes/mod.ts'; import MusicBrainzProvider from './MusicBrainz/mod.ts'; import OtotoyProvider from './Ototoy/mod.ts'; @@ -28,6 +29,11 @@ providers.addMultiple( MusicBrainzProvider, DiscogsProvider, DeezerProvider, +); +if (isAppleMusicConfigured()) { + providers.add(AppleMusicProvider); +} +providers.addMultiple( iTunesProvider, SpotifyProvider, TidalProvider, diff --git a/server/components/ProviderIcon.tsx b/server/components/ProviderIcon.tsx index b0a69b37..4d4b1605 100644 --- a/server/components/ProviderIcon.tsx +++ b/server/components/ProviderIcon.tsx @@ -9,6 +9,7 @@ const providerIconMap: Record = { deezer: 'brand-deezer', discogs: 'brand-discogs', itunes: 'brand-apple', + applemusic: 'brand-apple', melon: 'brand-melon', musicbrainz: 'brand-metabrainz', mora: 'brand-mora', diff --git a/server/static/harmony.css b/server/static/harmony.css index 34e41177..c029291d 100644 --- a/server/static/harmony.css +++ b/server/static/harmony.css @@ -493,7 +493,9 @@ td.discogs { background-color: var(--discogs); } label.itunes, -td.itunes { +td.itunes, +label.applemusic, +td.applemusic { background-color: var(--apple); } label.melon, From 9fd128d762ed06919035df9f2d0229cfd75346d1 Mon Sep 17 00:00:00 2001 From: dwei30 Date: Sat, 15 Aug 2026 17:23:55 +1000 Subject: [PATCH 5/6] update .env example veriable name --- .env.example | 4 ++-- providers/AppleMusic/mod.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 32cf076e..14ba3fb5 100644 --- a/.env.example +++ b/.env.example @@ -39,5 +39,5 @@ HARMONY_BUGS_CLIENT_SECRET= HARMONY_APPLE_MUSIC_TOKEN= # Unofficial AMP API token for self-hosted Harmony (used if HARMONY_APPLE_MUSIC_TOKEN is unset). HARMONY_APPLE_MUSIC_AMP_TOKEN= -# Scrape a MusicKit JWT from music.apple.com when no token is set (self-hosted only). -HARMONY_APPLE_MUSIC_SCRAPE_TOKEN= +# Scrape a MusicKit JWT from music.apple.com when no token is set (self-hosted only). Set true or false. +HARMONY_APPLE_MUSIC_SCRAPE= diff --git a/providers/AppleMusic/mod.ts b/providers/AppleMusic/mod.ts index 2ab132b5..59440431 100644 --- a/providers/AppleMusic/mod.ts +++ b/providers/AppleMusic/mod.ts @@ -40,7 +40,7 @@ const ampApiBaseUrl = 'https://amp-api.music.apple.com'; const officialToken = getFromEnv('HARMONY_APPLE_MUSIC_TOKEN') || ''; const ampToken = getFromEnv('HARMONY_APPLE_MUSIC_AMP_TOKEN') || ''; -const scrapeToken = getBooleanFromEnv('HARMONY_APPLE_MUSIC_SCRAPE_TOKEN'); +const scrapeToken = getBooleanFromEnv('HARMONY_APPLE_MUSIC_SCRAPE'); export type AppleMusicBackend = 'official' | 'amp'; From af3b9d9689c7767ffae381391f2d5d57fd98b53b Mon Sep 17 00:00:00 2001 From: dwei30 Date: Sat, 15 Aug 2026 17:37:28 +1000 Subject: [PATCH 6/6] docs(Apple Music): Comment catalog auth, scrape, and lookup flow --- providers/AppleMusic/catalog.ts | 9 +++++++++ providers/AppleMusic/mod.ts | 11 +++++++++++ providers/mod.ts | 1 + 3 files changed, 21 insertions(+) diff --git a/providers/AppleMusic/catalog.ts b/providers/AppleMusic/catalog.ts index 23f5e14f..30a0a700 100644 --- a/providers/AppleMusic/catalog.ts +++ b/providers/AppleMusic/catalog.ts @@ -1,8 +1,12 @@ +// Helpers for the official Apple Music API and unofficial AMP catalog. +// music.apple.com is only fetched when scraping a MusicKit JWT; album metadata comes from the catalog API. import { decodeBase64 } from 'std/encoding/base64.ts'; import type { CatalogAlbum, CatalogArtist, CatalogDocument, CatalogTrack } from './catalog_types.ts'; +// JWT as embedded in Apple Music / MusicKit assets. const jwtPattern = /["'](eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)["']/g; +// Returns the JWT with the latest expiry when a page or script embeds more than one token. export function extractAppleMusicJwt(source: string): string | undefined { const matches = source.matchAll(jwtPattern); let best: string | undefined; @@ -31,6 +35,7 @@ export function parseJwtExpiry(token: string): number | undefined { } } +// Script URLs that typically contain the MusicKit developer token. export function extractScriptUrls(html: string): string[] { const urls: string[] = []; const seen = new Set(); @@ -51,6 +56,7 @@ export function extractScriptUrls(html: string): string[] { return urls; } +// Catalog pagination `next` is often a path (`/v1/catalog/...`), not an absolute URL. export function resolveCatalogUrl(next: string, apiBaseUrl: string): URL { if (next.startsWith('http://') || next.startsWith('https://')) { return new URL(next); @@ -64,6 +70,8 @@ export function isCatalogTrack( return resource.type === 'songs' || resource.type === 'music-videos'; } +// Collects tracks from a catalog page. +// Relationships may only list IDs; full objects (name, ISRC, duration) are often in `included`. export function collectCatalogTracks(body: CatalogDocument, album?: CatalogAlbum): CatalogTrack[] { const includedTracks = (body.included ?? []).filter(isCatalogTrack); const byId = new Map(includedTracks.map((track) => [track.id, track] as const)); @@ -76,6 +84,7 @@ export function collectCatalogTracks(body: CatalogDocument, album?: CatalogAlbum return hydrate((body.data ?? []).filter(isCatalogTrack)); } +// Artwork URLs use `{w}` / `{h}` placeholders; fill them with the requested or native size. export function catalogArtworkUrl( artwork?: { url: string; width?: number; height?: number }, size?: number, diff --git a/providers/AppleMusic/mod.ts b/providers/AppleMusic/mod.ts index 59440431..c03b847b 100644 --- a/providers/AppleMusic/mod.ts +++ b/providers/AppleMusic/mod.ts @@ -36,8 +36,10 @@ import type { ProviderCategory } from '@/providers/categories.ts'; // Official catalog: https://developer.apple.com/documentation/applemusicapi const officialApiBaseUrl = 'https://api.music.apple.com'; +// Unofficial web catalog (AMP). Same JSON shape as the official API, used when no developer token is set. const ampApiBaseUrl = 'https://amp-api.music.apple.com'; +// Auth is one of: official JWT, AMP JWT, or scrape a MusicKit JWT from music.apple.com. const officialToken = getFromEnv('HARMONY_APPLE_MUSIC_TOKEN') || ''; const ampToken = getFromEnv('HARMONY_APPLE_MUSIC_AMP_TOKEN') || ''; const scrapeToken = getBooleanFromEnv('HARMONY_APPLE_MUSIC_SCRAPE'); @@ -55,6 +57,7 @@ export function appleMusicBackend(): AppleMusicBackend { export default class AppleMusicProvider extends MetadataApiProvider { readonly name = 'Apple Music'; + // music.apple.com only; itunes.apple.com stays on the iTunes Search provider. readonly supportedUrls = new URLPattern({ hostname: '{geo.}?music.apple.com', pathname: String.raw`/:region(\w{2})?/:type(album|artist|song|music-video)/:slug?/{id}?:id(\d+)`, @@ -86,6 +89,7 @@ export default class AppleMusicProvider extends MetadataApiProvider { day: 30, }; + // Official API if HARMONY_APPLE_MUSIC_TOKEN is set, otherwise AMP. readonly apiBaseUrl = officialToken ? officialApiBaseUrl : ampApiBaseUrl; readonly backend: AppleMusicBackend = appleMusicBackend(); @@ -110,12 +114,14 @@ export default class AppleMusicProvider extends MetadataApiProvider { } async query(apiUrl: URL, options: ApiQueryOptions = {}): Promise> { + // Both backends require a Bearer JWT; AMP also needs Origin (set below). const accessToken = await this.cachedAccessToken(() => this.requestAccessToken()); const headers: Record = { 'Authorization': `Bearer ${accessToken}`, 'Accept': 'application/json', }; if (this.backend === 'amp') { + // AMP rejects requests that do not look like they come from the Apple Music web app. headers['Origin'] = 'https://music.apple.com'; } return await this.fetchJSON(apiUrl, { @@ -137,6 +143,7 @@ export default class AppleMusicProvider extends MetadataApiProvider { return url; } + // Loads a catalog album and follows `next` until the full tracklist is collected. async fetchCatalogAlbum( albumId: string, region: CountryCode, @@ -157,6 +164,7 @@ export default class AppleMusicProvider extends MetadataApiProvider { } if (!album) { + // First page is the album resource; later pages are additional tracks only. const albumResource = (body.data ?? []).find((resource): resource is CatalogAlbum => resource.type === 'albums' ); @@ -186,6 +194,8 @@ export default class AppleMusicProvider extends MetadataApiProvider { return await this.scrapeAccessToken(); } + // AMP has no public client-credentials flow. The web app embeds a MusicKit JWT in + // music.apple.com HTML or JS; scrape that token (not the album DOM) and cache it until `exp`. private async scrapeAccessToken(): Promise { const pageUrl = new URL('https://music.apple.com'); const page = await this.fetchSnapshot(pageUrl); @@ -239,6 +249,7 @@ export class AppleMusicReleaseLookup extends ReleaseApiLookup