From e67d2dca228c49a06c0b72cd238ccf260abbbcd3 Mon Sep 17 00:00:00 2001 From: Paul Davis Date: Wed, 11 Mar 2026 15:31:57 +0000 Subject: [PATCH 01/11] Use beehiiv API for content & members --- packages/mg-beehiiv-api-members/.eslintrc.cjs | 16 + packages/mg-beehiiv-api-members/README.md | 3 + packages/mg-beehiiv-api-members/package.json | 43 ++ packages/mg-beehiiv-api-members/src/index.ts | 9 + .../mg-beehiiv-api-members/src/lib/fetch.ts | 112 ++++ .../src/lib/list-pubs.ts | 20 + .../mg-beehiiv-api-members/src/lib/mapper.ts | 106 ++++ .../src/test/fetch.test.ts | 357 +++++++++++ .../src/test/index.test.ts | 17 + .../src/test/mapper.test.ts | 374 +++++++++++ .../mg-beehiiv-api-members/src/types.d.ts | 45 ++ packages/mg-beehiiv-api-members/tsconfig.json | 32 + packages/mg-beehiiv-api/.eslintrc.cjs | 16 + packages/mg-beehiiv-api/.gitignore | 4 + packages/mg-beehiiv-api/LICENSE | 21 + packages/mg-beehiiv-api/README.md | 3 + packages/mg-beehiiv-api/package.json | 51 ++ packages/mg-beehiiv-api/src/index.ts | 9 + packages/mg-beehiiv-api/src/lib/client.ts | 22 + packages/mg-beehiiv-api/src/lib/fetch.ts | 94 +++ packages/mg-beehiiv-api/src/lib/list-pubs.ts | 20 + packages/mg-beehiiv-api/src/lib/mapper.ts | 102 +++ packages/mg-beehiiv-api/src/lib/process.ts | 552 ++++++++++++++++ .../mg-beehiiv-api/src/test/client.test.ts | 57 ++ .../mg-beehiiv-api/src/test/fetch.test.ts | 349 +++++++++++ .../src/test/fixtures/posts.csv | 3 + .../mg-beehiiv-api/src/test/index.test.ts | 17 + .../mg-beehiiv-api/src/test/list-pubs.test.ts | 72 +++ .../mg-beehiiv-api/src/test/mapper.test.ts | 280 +++++++++ .../mg-beehiiv-api/src/test/process.test.ts | 300 +++++++++ packages/mg-beehiiv-api/src/types.d.ts | 76 +++ packages/mg-beehiiv-api/tsconfig.json | 111 ++++ packages/migrate/bin/cli.js | 4 + .../migrate/commands/beehiiv-api-members.js | 140 +++++ packages/migrate/commands/beehiiv-api.js | 192 ++++++ packages/migrate/package.json | 2 + .../migrate/sources/beehiiv-api-members.js | 184 ++++++ packages/migrate/sources/beehiiv-api.js | 203 ++++++ yarn.lock | 592 +++++++++++++++--- 39 files changed, 4520 insertions(+), 90 deletions(-) create mode 100644 packages/mg-beehiiv-api-members/.eslintrc.cjs create mode 100644 packages/mg-beehiiv-api-members/README.md create mode 100644 packages/mg-beehiiv-api-members/package.json create mode 100644 packages/mg-beehiiv-api-members/src/index.ts create mode 100644 packages/mg-beehiiv-api-members/src/lib/fetch.ts create mode 100644 packages/mg-beehiiv-api-members/src/lib/list-pubs.ts create mode 100644 packages/mg-beehiiv-api-members/src/lib/mapper.ts create mode 100644 packages/mg-beehiiv-api-members/src/test/fetch.test.ts create mode 100644 packages/mg-beehiiv-api-members/src/test/index.test.ts create mode 100644 packages/mg-beehiiv-api-members/src/test/mapper.test.ts create mode 100644 packages/mg-beehiiv-api-members/src/types.d.ts create mode 100644 packages/mg-beehiiv-api-members/tsconfig.json create mode 100644 packages/mg-beehiiv-api/.eslintrc.cjs create mode 100644 packages/mg-beehiiv-api/.gitignore create mode 100644 packages/mg-beehiiv-api/LICENSE create mode 100644 packages/mg-beehiiv-api/README.md create mode 100644 packages/mg-beehiiv-api/package.json create mode 100644 packages/mg-beehiiv-api/src/index.ts create mode 100644 packages/mg-beehiiv-api/src/lib/client.ts create mode 100644 packages/mg-beehiiv-api/src/lib/fetch.ts create mode 100644 packages/mg-beehiiv-api/src/lib/list-pubs.ts create mode 100644 packages/mg-beehiiv-api/src/lib/mapper.ts create mode 100644 packages/mg-beehiiv-api/src/lib/process.ts create mode 100644 packages/mg-beehiiv-api/src/test/client.test.ts create mode 100644 packages/mg-beehiiv-api/src/test/fetch.test.ts create mode 100644 packages/mg-beehiiv-api/src/test/fixtures/posts.csv create mode 100644 packages/mg-beehiiv-api/src/test/index.test.ts create mode 100644 packages/mg-beehiiv-api/src/test/list-pubs.test.ts create mode 100644 packages/mg-beehiiv-api/src/test/mapper.test.ts create mode 100644 packages/mg-beehiiv-api/src/test/process.test.ts create mode 100644 packages/mg-beehiiv-api/src/types.d.ts create mode 100644 packages/mg-beehiiv-api/tsconfig.json create mode 100644 packages/migrate/commands/beehiiv-api-members.js create mode 100644 packages/migrate/commands/beehiiv-api.js create mode 100644 packages/migrate/sources/beehiiv-api-members.js create mode 100644 packages/migrate/sources/beehiiv-api.js diff --git a/packages/mg-beehiiv-api-members/.eslintrc.cjs b/packages/mg-beehiiv-api-members/.eslintrc.cjs new file mode 100644 index 000000000..bf11a9cea --- /dev/null +++ b/packages/mg-beehiiv-api-members/.eslintrc.cjs @@ -0,0 +1,16 @@ +module.exports = { + parser: '@typescript-eslint/parser', + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ], + rules: { + 'no-unused-vars': 'off', // doesn't work with typescript + 'no-undef': 'off', // doesn't work with typescript + 'ghost/ghost-custom/no-native-errors': 'off', + 'ghost/ghost-custom/no-native-error': 'off', + 'ghost/ghost-custom/ghost-error-usage': 'off', + // todo: clean this up + 'ghost/filenames/match-regex': 'off' + } +}; diff --git a/packages/mg-beehiiv-api-members/README.md b/packages/mg-beehiiv-api-members/README.md new file mode 100644 index 000000000..04af663d6 --- /dev/null +++ b/packages/mg-beehiiv-api-members/README.md @@ -0,0 +1,3 @@ +# Migrate beehiiv members API + +... diff --git a/packages/mg-beehiiv-api-members/package.json b/packages/mg-beehiiv-api-members/package.json new file mode 100644 index 000000000..f6678467e --- /dev/null +++ b/packages/mg-beehiiv-api-members/package.json @@ -0,0 +1,43 @@ +{ + "name": "@tryghost/mg-beehiiv-api-members", + "version": "0.1.0", + "repository": "https://github.com/TryGhost/migrate/tree/main/packages/mg-beehiiv-api-members", + "author": "Ghost Foundation", + "license": "MIT", + "type": "module", + "main": "build/index.js", + "types": "build/types.d.ts", + "exports": { + ".": { + "development": "./src/index.ts", + "default": "./build/index.js" + } + }, + "scripts": { + "dev": "echo \"Implement me!\"", + "build:watch": "tsc --watch --preserveWatchOutput --sourceMap", + "build": "rm -rf build && rm -rf tsconfig.tsbuildinfo && tsc --build --sourceMap", + "prepare": "yarn build", + "lint": "eslint src/ --ext .ts --cache", + "posttest": "yarn lint", + "test": "rm -rf build && yarn build --force && c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura node --test build/test/*.test.js" + }, + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@typescript-eslint/eslint-plugin": "8.0.0", + "@typescript-eslint/parser": "8.0.0", + "c8": "10.1.3", + "eslint": "8.57.0", + "typescript": "5.9.3" + }, + "dependencies": { + "@tryghost/errors": "1.3.8", + "@tryghost/mg-fs-utils": "0.12.14", + "@tryghost/string": "0.2.21" + } +} diff --git a/packages/mg-beehiiv-api-members/src/index.ts b/packages/mg-beehiiv-api-members/src/index.ts new file mode 100644 index 000000000..19b1e8ed3 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/index.ts @@ -0,0 +1,9 @@ +import {listPublications} from './lib/list-pubs.js'; +import {fetchTasks} from './lib/fetch.js'; +import {mapMembersTasks} from './lib/mapper.js'; + +export default { + listPublications, + fetchTasks, + mapMembersTasks +}; diff --git a/packages/mg-beehiiv-api-members/src/lib/fetch.ts b/packages/mg-beehiiv-api-members/src/lib/fetch.ts new file mode 100644 index 000000000..312f54c53 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/lib/fetch.ts @@ -0,0 +1,112 @@ +const API_LIMIT = 100; + +const authedClient = async (apiKey: string, theUrl: URL) => { + return fetch(theUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}` + } + }); +}; + +const discover = async (key: string, pubId: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications/${pubId}`); + url.searchParams.append('limit', '1'); + url.searchParams.append('expand[]', 'stats'); + + const response = await authedClient(key, url); + + if (!response.ok) { + throw new Error(`Request failed: ${response.status} ${response.statusText}`); + } + + const data: BeehiivPublicationResponse = await response.json(); + + return data.data.stats?.active_subscriptions; +}; + +const cachedFetch = async ({fileCache, key, pubId, cursor, cursorIndex}: { + fileCache: any; + key: string; + pubId: string; + cursor: string | null; + cursorIndex: number; +}) => { + const filename = `beehiiv_api_members_${cursorIndex}.json`; + + if (fileCache.hasFile(filename, 'tmp')) { + return await fileCache.readTmpJSONFile(filename); + } + + const url = new URL(`https://api.beehiiv.com/v2/publications/${pubId}/subscriptions`); + url.searchParams.append('limit', API_LIMIT.toString()); + url.searchParams.append('status', 'active'); + url.searchParams.append('expand[]', 'custom_fields'); + + if (cursor) { + url.searchParams.append('cursor', cursor); + } + + const response = await authedClient(key, url); + + if (!response.ok) { + throw new Error(`Request failed: ${response.status} ${response.statusText}`); + } + + const data: BeehiivSubscriptionsResponse = await response.json(); + + await fileCache.writeTmpFile(data, filename); + + return data; +}; + +export const fetchTasks = async (options: any, ctx: any) => { + const totalSubscriptions = await discover(options.key, options.id); + const estimatedPages = Math.ceil(totalSubscriptions / API_LIMIT); + + const tasks = [ + { + title: `Fetching subscriptions (estimated ${estimatedPages} pages)`, + task: async (_: any, task: any) => { + let cursor: string | null = null; + let hasMore = true; + let cursorIndex = 0; + + ctx.result.subscriptions = []; + + while (hasMore) { + try { + const response: BeehiivSubscriptionsResponse = await cachedFetch({ + fileCache: ctx.fileCache, + key: options.key, + pubId: options.id, + cursor, + cursorIndex + }); + + ctx.result.subscriptions = ctx.result.subscriptions.concat(response.data); + hasMore = response.has_more; + cursor = response.next_cursor; + cursorIndex += 1; + + task.output = `Fetched ${ctx.result.subscriptions.length} of ${totalSubscriptions} subscriptions`; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + task.output = errorMessage; + throw error; + } + } + + task.output = `Fetched ${ctx.result.subscriptions.length} subscriptions`; + } + } + ]; + + return tasks; +}; + +export { + authedClient, + discover, + cachedFetch +}; diff --git a/packages/mg-beehiiv-api-members/src/lib/list-pubs.ts b/packages/mg-beehiiv-api-members/src/lib/list-pubs.ts new file mode 100644 index 000000000..b497f1d34 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/lib/list-pubs.ts @@ -0,0 +1,20 @@ +import {authedClient} from './fetch.js'; + +const listPublications = async (apiKey: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications`); + url.searchParams.append('expand[]', 'stats'); + + const response = await authedClient(apiKey, url); + + if (!response.ok) { + throw new Error(`Request failed: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + + return data.data; +}; + +export { + listPublications +}; diff --git a/packages/mg-beehiiv-api-members/src/lib/mapper.ts b/packages/mg-beehiiv-api-members/src/lib/mapper.ts new file mode 100644 index 000000000..d202e0050 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/lib/mapper.ts @@ -0,0 +1,106 @@ +import {slugify} from '@tryghost/string'; + +const extractName = (customFields: Array<{name: string; value: string}>): string | null => { + const firstNameField = customFields.find(f => f.name.toLowerCase() === 'first_name' || f.name.toLowerCase() === 'firstname'); + const lastNameField = customFields.find(f => f.name.toLowerCase() === 'last_name' || f.name.toLowerCase() === 'lastname'); + + const firstName = firstNameField?.value?.trim() || ''; + const lastName = lastNameField?.value?.trim() || ''; + + const combinedName = [firstName, lastName].filter(name => name.length > 0).join(' '); + + return combinedName.length > 0 ? combinedName : null; +}; + +const mapSubscription = (subscription: BeehiivSubscription): GhostMemberObject => { + const labels: string[] = []; + + // Add status label + labels.push(`beehiiv-status-${subscription.status}`); + + // Add tier label + labels.push(`beehiiv-tier-${subscription.subscription_tier}`); + + // Add premium tier names as labels + if (subscription.subscription_premium_tier_names && subscription.subscription_premium_tier_names.length > 0) { + subscription.subscription_premium_tier_names.forEach((tierName: string) => { + const slugifiedTier = slugify(tierName); + labels.push(`beehiiv-premium-${slugifiedTier}`); + }); + } + + // Add tags as labels + if (subscription.tags && subscription.tags.length > 0) { + subscription.tags.forEach((tag: string) => { + const slugifiedTag = slugify(tag); + labels.push(`beehiiv-tag-${slugifiedTag}`); + }); + } + + // Determine if this is a complimentary plan + // A member is on a complimentary plan if they have premium access but no Stripe customer ID + const isPremium = subscription.subscription_tier === 'premium'; + const hasStripeId = Boolean(subscription.stripe_customer_id); + const complimentaryPlan = isPremium && !hasStripeId; + + return { + email: subscription.email, + name: extractName(subscription.custom_fields || []), + note: null, + subscribed_to_emails: subscription.status === 'active', + stripe_customer_id: subscription.stripe_customer_id || '', + complimentary_plan: complimentaryPlan, + labels, + created_at: new Date(subscription.created * 1000) + }; +}; + +const mapSubscriptions = (subscriptions: BeehiivSubscription[]): MappedMembers => { + const result: MappedMembers = { + free: [], + paid: [] + }; + + subscriptions.forEach((subscription) => { + const member = mapSubscription(subscription); + + if (member.stripe_customer_id) { + result.paid.push(member); + } else { + result.free.push(member); + } + }); + + return result; +}; + +export const mapMembersTasks = (_options: any, ctx: any) => { + const tasks = [ + { + title: 'Mapping subscriptions to Ghost member format', + task: async (_: any, task: any) => { + try { + const subscriptions: BeehiivSubscription[] = ctx.result.subscriptions || []; + ctx.result.members = mapSubscriptions(subscriptions); + + const freeCount = ctx.result.members.free.length; + const paidCount = ctx.result.members.paid.length; + + task.output = `Mapped ${freeCount} free and ${paidCount} paid members`; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + task.output = errorMessage; + throw error; + } + } + } + ]; + + return tasks; +}; + +export { + extractName, + mapSubscription, + mapSubscriptions +}; diff --git a/packages/mg-beehiiv-api-members/src/test/fetch.test.ts b/packages/mg-beehiiv-api-members/src/test/fetch.test.ts new file mode 100644 index 000000000..3ebc1bbdd --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/test/fetch.test.ts @@ -0,0 +1,357 @@ +import assert from 'node:assert/strict'; +import {describe, it, beforeEach, afterEach, mock} from 'node:test'; +import {listPublications} from '../lib/list-pubs.js'; +import {fetchTasks, authedClient, discover, cachedFetch} from '../lib/fetch.js'; + +describe('beehiiv API Members Fetch', () => { + let fetchMock: any; + + beforeEach(() => { + fetchMock = mock.method(global, 'fetch', () => Promise.resolve()); + }); + + afterEach(() => { + mock.restoreAll(); + }); + + describe('authedClient', () => { + it('makes authenticated GET request', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ok: true, json: () => Promise.resolve({data: []})})); + + const url = new URL('https://api.beehiiv.com/v2/publications'); + await authedClient('test-api-key', url); + + assert.equal(fetchMock.mock.callCount(), 1); + const [calledUrl, options] = fetchMock.mock.calls[0].arguments; + assert.equal(calledUrl.toString(), 'https://api.beehiiv.com/v2/publications'); + assert.equal(options.method, 'GET'); + assert.equal(options.headers.Authorization, 'Bearer test-api-key'); + }); + }); + + describe('listPublications', () => { + it('fetches and returns publications', async () => { + const mockPubs = [{id: 'pub-1', name: 'Test Pub'}]; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({data: mockPubs}) + })); + + const result = await listPublications('test-key'); + + assert.deepEqual(result, mockPubs); + }); + + it('throws on API error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 401, + statusText: 'Unauthorized' + })); + + await assert.rejects(async () => { + await listPublications('invalid-key'); + }, /Request failed: 401 Unauthorized/); + }); + }); + + describe('discover', () => { + it('returns total subscription count', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({data: {stats: {active_subscriptions: 1500}}}) + })); + + const result = await discover('test-key', 'pub-123'); + + assert.equal(result, 1500); + }); + + it('throws on API error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 403, + statusText: 'Forbidden' + })); + + await assert.rejects(async () => { + await discover('test-key', 'pub-123'); + }, /Request failed: 403 Forbidden/); + }); + }); + + describe('cachedFetch', () => { + it('returns cached data when available', async () => { + const cachedData = {data: [{id: 'sub-1', email: 'test@example.com'}], has_more: false}; + const fileCache = { + hasFile: () => true, + readTmpJSONFile: () => Promise.resolve(cachedData) + }; + + const result = await cachedFetch({ + fileCache, + key: 'test-key', + pubId: 'pub-123', + cursor: null, + cursorIndex: 0 + }); + + assert.deepEqual(result, cachedData); + assert.equal(fetchMock.mock.callCount(), 0); + }); + + it('fetches from API when not cached', async () => { + const apiData = {data: [{id: 'sub-1', email: 'test@example.com'}], has_more: false, next_cursor: null}; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve(apiData) + })); + + const writeTmpFileMock = mock.fn(() => Promise.resolve()); + const fileCache = { + hasFile: () => false, + writeTmpFile: writeTmpFileMock + }; + + const result = await cachedFetch({ + fileCache, + key: 'test-key', + pubId: 'pub-123', + cursor: null, + cursorIndex: 0 + }); + + assert.deepEqual(result, apiData); + assert.equal(writeTmpFileMock.mock.callCount(), 1); + }); + + it('includes cursor when provided', async () => { + const apiData = {data: [], has_more: false, next_cursor: null}; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve(apiData) + })); + + const fileCache = { + hasFile: () => false, + writeTmpFile: mock.fn(() => Promise.resolve()) + }; + + await cachedFetch({ + fileCache, + key: 'test-key', + pubId: 'pub-123', + cursor: 'cursor-abc', + cursorIndex: 1 + }); + + const [calledUrl] = fetchMock.mock.calls[0].arguments; + assert.ok(calledUrl.toString().includes('cursor=cursor-abc')); + }); + + it('throws on API error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 500, + statusText: 'Internal Server Error' + })); + + const fileCache = { + hasFile: () => false, + writeTmpFile: mock.fn(() => Promise.resolve()) + }; + + await assert.rejects(async () => { + await cachedFetch({ + fileCache, + key: 'test-key', + pubId: 'pub-123', + cursor: null, + cursorIndex: 0 + }); + }, /Request failed: 500 Internal Server Error/); + }); + }); + + describe('fetchTasks', () => { + it('creates a single task that fetches all subscriptions', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 150, data: [], has_more: true}) + }), 0); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + + assert.equal(tasks.length, 1); + assert.ok(tasks[0].title.includes('Fetching subscriptions')); + }); + + it('task fetches all pages using cursor pagination', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 3, data: [], has_more: true}) + }), 0); + + // Mock first page + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [{id: 'sub-1', email: 'a@test.com'}], + has_more: true, + next_cursor: 'cursor-1' + }) + }), 1); + + // Mock second page + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [{id: 'sub-2', email: 'b@test.com'}], + has_more: false, + next_cursor: null + }) + }), 2); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx: any = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.subscriptions.length, 2); + assert.equal(ctx.result.subscriptions[0].id, 'sub-1'); + assert.equal(ctx.result.subscriptions[1].id, 'sub-2'); + }); + + it('task uses cached data when available', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 1, data: [], has_more: false}) + }), 0); + + const cachedData = { + data: [{id: 'cached-sub', email: 'cached@test.com'}], + has_more: false, + next_cursor: null + }; + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx: any = { + fileCache: { + hasFile: () => true, + readTmpJSONFile: () => Promise.resolve(cachedData) + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.subscriptions.length, 1); + assert.equal(ctx.result.subscriptions[0].id, 'cached-sub'); + // fetch should only be called once (for discover) + assert.equal(fetchMock.mock.callCount(), 1); + }); + + it('task throws and sets output on fetch error', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: [], has_more: true}) + }), 0); + + // Mock a failed fetch + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: false, + status: 500, + statusText: 'Internal Server Error' + }), 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }, /Request failed: 500 Internal Server Error/); + + assert.ok(mockTask.output.includes('500')); + }); + + it('task handles non-Error thrown objects', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: [], has_more: true}) + }), 0); + + // Mock fetch that throws a non-Error value + fetchMock.mock.mockImplementationOnce(() => { + // eslint-disable-next-line no-throw-literal + throw 'Network error string'; + }, 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + + assert.equal(mockTask.output, 'Network error string'); + }); + + it('handles discover error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 401, + statusText: 'Unauthorized' + })); + + const options = {key: 'invalid-key', id: 'pub-123'}; + const ctx = { + fileCache: {}, + result: {} + }; + + await assert.rejects(async () => { + await fetchTasks(options, ctx); + }, /Request failed: 401 Unauthorized/); + }); + }); +}); diff --git a/packages/mg-beehiiv-api-members/src/test/index.test.ts b/packages/mg-beehiiv-api-members/src/test/index.test.ts new file mode 100644 index 000000000..e26d11826 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/test/index.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import beehiivApiMembers from '../index.js'; + +describe('beehiiv API Members Package', () => { + it('exports listPublications', () => { + assert.ok(typeof beehiivApiMembers.listPublications === 'function'); + }); + + it('exports fetchTasks', () => { + assert.ok(typeof beehiivApiMembers.fetchTasks === 'function'); + }); + + it('exports mapMembersTasks', () => { + assert.ok(typeof beehiivApiMembers.mapMembersTasks === 'function'); + }); +}); diff --git a/packages/mg-beehiiv-api-members/src/test/mapper.test.ts b/packages/mg-beehiiv-api-members/src/test/mapper.test.ts new file mode 100644 index 000000000..96b5c2912 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/test/mapper.test.ts @@ -0,0 +1,374 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import {extractName, mapSubscription, mapSubscriptions, mapMembersTasks} from '../lib/mapper.js'; + +describe('beehiiv API Members Mapper', () => { + describe('extractName', () => { + it('combines first and last name', () => { + const customFields = [ + {name: 'first_name', value: 'John'}, + {name: 'last_name', value: 'Doe'} + ]; + assert.equal(extractName(customFields), 'John Doe'); + }); + + it('handles only first name', () => { + const customFields = [{name: 'first_name', value: 'Jane'}]; + assert.equal(extractName(customFields), 'Jane'); + }); + + it('handles only last name', () => { + const customFields = [{name: 'last_name', value: 'Smith'}]; + assert.equal(extractName(customFields), 'Smith'); + }); + + it('returns null for empty custom fields', () => { + assert.equal(extractName([]), null); + }); + + it('returns null when name fields are empty strings', () => { + const customFields = [ + {name: 'first_name', value: ''}, + {name: 'last_name', value: ''} + ]; + assert.equal(extractName(customFields), null); + }); + + it('trims whitespace from names', () => { + const customFields = [ + {name: 'first_name', value: ' John '}, + {name: 'last_name', value: ' Doe '} + ]; + assert.equal(extractName(customFields), 'John Doe'); + }); + + it('handles alternate field names (firstname/lastname)', () => { + const customFields = [ + {name: 'firstname', value: 'Jane'}, + {name: 'lastname', value: 'Doe'} + ]; + assert.equal(extractName(customFields), 'Jane Doe'); + }); + + it('handles case-insensitive field names', () => { + const customFields = [ + {name: 'First_Name', value: 'Jane'}, + {name: 'Last_Name', value: 'Doe'} + ]; + assert.equal(extractName(customFields), 'Jane Doe'); + }); + }); + + describe('mapSubscription', () => { + const baseSubscription: BeehiivSubscription = { + id: 'sub-123', + email: 'test@example.com', + status: 'active', + created: 1704067200, // 2024-01-01 00:00:00 UTC + subscription_tier: 'free', + subscription_premium_tier_names: [], + stripe_customer_id: null, + custom_fields: [], + tags: [] + }; + + it('maps basic subscription fields', () => { + const result = mapSubscription(baseSubscription); + + assert.equal(result.email, 'test@example.com'); + assert.equal(result.name, null); + assert.equal(result.note, null); + assert.equal(result.subscribed_to_emails, true); + assert.equal(result.stripe_customer_id, ''); + assert.equal(result.complimentary_plan, false); + assert.deepEqual(result.created_at, new Date(1704067200 * 1000)); + }); + + it('sets subscribed_to_emails true for active status', () => { + const subscription = {...baseSubscription, status: 'active' as const}; + const result = mapSubscription(subscription); + assert.equal(result.subscribed_to_emails, true); + }); + + it('sets subscribed_to_emails false for inactive status', () => { + const subscription = {...baseSubscription, status: 'inactive' as const}; + const result = mapSubscription(subscription); + assert.equal(result.subscribed_to_emails, false); + }); + + it('sets subscribed_to_emails false for validating status', () => { + const subscription = {...baseSubscription, status: 'validating' as const}; + const result = mapSubscription(subscription); + assert.equal(result.subscribed_to_emails, false); + }); + + it('sets subscribed_to_emails false for pending status', () => { + const subscription = {...baseSubscription, status: 'pending' as const}; + const result = mapSubscription(subscription); + assert.equal(result.subscribed_to_emails, false); + }); + + it('maps stripe_customer_id when present', () => { + const subscription = {...baseSubscription, stripe_customer_id: 'cus_abc123'}; + const result = mapSubscription(subscription); + assert.equal(result.stripe_customer_id, 'cus_abc123'); + }); + + it('sets complimentary_plan true for premium without stripe_customer_id', () => { + const subscription = { + ...baseSubscription, + subscription_tier: 'premium' as const, + stripe_customer_id: null + }; + const result = mapSubscription(subscription); + assert.equal(result.complimentary_plan, true); + }); + + it('sets complimentary_plan false for premium with stripe_customer_id', () => { + const subscription = { + ...baseSubscription, + subscription_tier: 'premium' as const, + stripe_customer_id: 'cus_abc123' + }; + const result = mapSubscription(subscription); + assert.equal(result.complimentary_plan, false); + }); + + it('sets complimentary_plan false for free tier', () => { + const subscription = {...baseSubscription, subscription_tier: 'free' as const}; + const result = mapSubscription(subscription); + assert.equal(result.complimentary_plan, false); + }); + + it('extracts name from custom_fields', () => { + const subscription = { + ...baseSubscription, + custom_fields: [ + {name: 'first_name', value: 'John'}, + {name: 'last_name', value: 'Doe'} + ] + }; + const result = mapSubscription(subscription); + assert.equal(result.name, 'John Doe'); + }); + + it('adds status label', () => { + const result = mapSubscription(baseSubscription); + assert.ok(result.labels.includes('beehiiv-status-active')); + }); + + it('adds tier label', () => { + const result = mapSubscription(baseSubscription); + assert.ok(result.labels.includes('beehiiv-tier-free')); + }); + + it('adds premium tier names as labels', () => { + const subscription = { + ...baseSubscription, + subscription_tier: 'premium' as const, + subscription_premium_tier_names: ['Gold Plan', 'VIP Access'] + }; + const result = mapSubscription(subscription); + assert.ok(result.labels.includes('beehiiv-premium-gold-plan')); + assert.ok(result.labels.includes('beehiiv-premium-vip-access')); + }); + + it('adds tags as labels', () => { + const subscription = { + ...baseSubscription, + tags: ['Newsletter', 'Tech Updates'] + }; + const result = mapSubscription(subscription); + assert.ok(result.labels.includes('beehiiv-tag-newsletter')); + assert.ok(result.labels.includes('beehiiv-tag-tech-updates')); + }); + + it('handles empty premium tier names array', () => { + const subscription = { + ...baseSubscription, + subscription_premium_tier_names: [] + }; + const result = mapSubscription(subscription); + assert.ok(!result.labels.some(l => l.startsWith('beehiiv-premium-'))); + }); + + it('handles empty tags array', () => { + const subscription = { + ...baseSubscription, + tags: [] + }; + const result = mapSubscription(subscription); + assert.ok(!result.labels.some(l => l.startsWith('beehiiv-tag-'))); + }); + + it('handles undefined custom_fields', () => { + const subscription = { + ...baseSubscription, + custom_fields: undefined as any + }; + const result = mapSubscription(subscription); + assert.equal(result.name, null); + }); + }); + + describe('mapSubscriptions', () => { + it('splits subscriptions into free and paid based on stripe_customer_id', () => { + const subscriptions: BeehiivSubscription[] = [ + { + id: 'sub-1', + email: 'free@test.com', + status: 'active', + created: 1704067200, + subscription_tier: 'free', + subscription_premium_tier_names: [], + stripe_customer_id: null, + custom_fields: [], + tags: [] + }, + { + id: 'sub-2', + email: 'paid@test.com', + status: 'active', + created: 1704067200, + subscription_tier: 'premium', + subscription_premium_tier_names: [], + stripe_customer_id: 'cus_paid123', + custom_fields: [], + tags: [] + } + ]; + + const result = mapSubscriptions(subscriptions); + + assert.equal(result.free.length, 1); + assert.equal(result.paid.length, 1); + assert.equal(result.free[0].email, 'free@test.com'); + assert.equal(result.paid[0].email, 'paid@test.com'); + }); + + it('returns empty arrays for empty input', () => { + const result = mapSubscriptions([]); + assert.deepEqual(result, {free: [], paid: []}); + }); + + it('categorizes complimentary premium as free', () => { + const subscriptions: BeehiivSubscription[] = [ + { + id: 'sub-1', + email: 'comp@test.com', + status: 'active', + created: 1704067200, + subscription_tier: 'premium', + subscription_premium_tier_names: [], + stripe_customer_id: null, // No stripe ID means complimentary + custom_fields: [], + tags: [] + } + ]; + + const result = mapSubscriptions(subscriptions); + + assert.equal(result.free.length, 1); + assert.equal(result.paid.length, 0); + assert.equal(result.free[0].complimentary_plan, true); + }); + }); + + describe('mapMembersTasks', () => { + it('creates a single mapping task', () => { + const ctx = {result: {subscriptions: []}}; + const tasks = mapMembersTasks({}, ctx); + + assert.equal(tasks.length, 1); + assert.equal(tasks[0].title, 'Mapping subscriptions to Ghost member format'); + }); + + it('task maps subscriptions and stores result', async () => { + const ctx: any = { + result: { + subscriptions: [ + { + id: 'sub-1', + email: 'test@test.com', + status: 'active', + created: 1704067200, + subscription_tier: 'free', + subscription_premium_tier_names: [], + stripe_customer_id: null, + custom_fields: [], + tags: [] + } + ] + } + }; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + await tasks[0].task({}, mockTask); + + assert.ok(ctx.result.members); + assert.equal(ctx.result.members.free.length, 1); + assert.equal(ctx.result.members.paid.length, 0); + assert.ok(mockTask.output.includes('1 free')); + assert.ok(mockTask.output.includes('0 paid')); + }); + + it('task handles empty subscriptions', async () => { + const ctx: any = {result: {subscriptions: []}}; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + await tasks[0].task({}, mockTask); + + assert.deepEqual(ctx.result.members, {free: [], paid: []}); + }); + + it('task handles undefined subscriptions', async () => { + const ctx: any = {result: {}}; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + await tasks[0].task({}, mockTask); + + assert.deepEqual(ctx.result.members, {free: [], paid: []}); + }); + + it('task sets error output on failure with Error instance', async () => { + const ctx = { + result: { + subscriptions: 'not an array' // Invalid data + } + }; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + + assert.ok(mockTask.output.length > 0); + }); + + it('task sets error output on failure with non-Error thrown', async () => { + // Create a context that will trigger an error through a getter that throws a string + const ctx: any = { + result: { + get subscriptions() { + // eslint-disable-next-line no-throw-literal + throw 'Custom string error'; + } + } + }; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + + assert.equal(mockTask.output, 'Custom string error'); + }); + }); +}); diff --git a/packages/mg-beehiiv-api-members/src/types.d.ts b/packages/mg-beehiiv-api-members/src/types.d.ts new file mode 100644 index 000000000..fb4b4bc62 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/types.d.ts @@ -0,0 +1,45 @@ +declare module '@tryghost/mg-fs-utils'; +declare module '@tryghost/string'; + +type BeehiivSubscription = { + id: string; + email: string; + status: 'active' | 'inactive' | 'validating' | 'pending'; + created: number; + subscription_tier: 'free' | 'premium'; + subscription_premium_tier_names: string[]; + stripe_customer_id: string | null; + custom_fields: Array<{name: string; value: string}>; + tags: string[]; +}; + +type BeehiivPublicationResponse = { + data: { + stats: { + active_subscriptions: number; + } + } +} + +type BeehiivSubscriptionsResponse = { + data: BeehiivSubscription[]; + total_results: number; + has_more: boolean; + next_cursor: string | null; +}; + +type GhostMemberObject = { + email: string; + name: string | null; + note: string | null; + subscribed_to_emails: boolean; + stripe_customer_id: string; + complimentary_plan: boolean; + labels: string[]; + created_at: Date; +}; + +type MappedMembers = { + free: GhostMemberObject[]; + paid: GhostMemberObject[]; +}; diff --git a/packages/mg-beehiiv-api-members/tsconfig.json b/packages/mg-beehiiv-api-members/tsconfig.json new file mode 100644 index 000000000..322068b4d --- /dev/null +++ b/packages/mg-beehiiv-api-members/tsconfig.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + + /* Language and Environment */ + "target": "es2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + + /* Modules */ + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": "src", /* Specify the root folder within your source files. */ + + /* Emit */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + "outDir": "build", /* Specify an output folder for all emitted files. */ + + /* Interop Constraints */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + + /* Completeness */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "include": ["src/**/*"] +} diff --git a/packages/mg-beehiiv-api/.eslintrc.cjs b/packages/mg-beehiiv-api/.eslintrc.cjs new file mode 100644 index 000000000..bf11a9cea --- /dev/null +++ b/packages/mg-beehiiv-api/.eslintrc.cjs @@ -0,0 +1,16 @@ +module.exports = { + parser: '@typescript-eslint/parser', + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ], + rules: { + 'no-unused-vars': 'off', // doesn't work with typescript + 'no-undef': 'off', // doesn't work with typescript + 'ghost/ghost-custom/no-native-errors': 'off', + 'ghost/ghost-custom/no-native-error': 'off', + 'ghost/ghost-custom/ghost-error-usage': 'off', + // todo: clean this up + 'ghost/filenames/match-regex': 'off' + } +}; diff --git a/packages/mg-beehiiv-api/.gitignore b/packages/mg-beehiiv-api/.gitignore new file mode 100644 index 000000000..bd7611769 --- /dev/null +++ b/packages/mg-beehiiv-api/.gitignore @@ -0,0 +1,4 @@ +/build +/tsconfig.tsbuildinfo +/tmp +.env diff --git a/packages/mg-beehiiv-api/LICENSE b/packages/mg-beehiiv-api/LICENSE new file mode 100644 index 000000000..efad547e8 --- /dev/null +++ b/packages/mg-beehiiv-api/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013-2026 Ghost Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/mg-beehiiv-api/README.md b/packages/mg-beehiiv-api/README.md new file mode 100644 index 000000000..478ae3d63 --- /dev/null +++ b/packages/mg-beehiiv-api/README.md @@ -0,0 +1,3 @@ +# Migrate beehiiv API + +... diff --git a/packages/mg-beehiiv-api/package.json b/packages/mg-beehiiv-api/package.json new file mode 100644 index 000000000..9fc2e65d4 --- /dev/null +++ b/packages/mg-beehiiv-api/package.json @@ -0,0 +1,51 @@ +{ + "name": "@tryghost/mg-beehiiv-api", + "version": "0.1.0", + "repository": "https://github.com/TryGhost/migrate/tree/main/packages/mg-beehiiv-api", + "author": "Ghost Foundation", + "license": "MIT", + "type": "module", + "main": "build/index.js", + "types": "build/types.d.ts", + "exports": { + ".": { + "development": "./src/index.ts", + "default": "./build/index.js" + } + }, + "scripts": { + "dev": "echo \"Implement me!\"", + "build:watch": "tsc --watch --preserveWatchOutput --sourceMap", + "build": "rm -rf build && rm -rf tsconfig.tsbuildinfo && tsc --build --sourceMap", + "prepare": "yarn build", + "lint": "eslint src/ --ext .ts --cache", + "posttest": "yarn lint", + "test": "rm -rf build && yarn build --force && c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura node --test build/test/*.test.js" + }, + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@types/lodash": "4.17.23", + "@typescript-eslint/eslint-plugin": "^8.0.0", + "@typescript-eslint/parser": "^8.0.0", + "c8": "10.1.3", + "dotenv": "17.2.3", + "eslint": "8.57.0", + "typescript": "5.9.3" + }, + "dependencies": { + "@tryghost/debug": "0.1.35", + "@tryghost/errors": "1.3.8", + "@tryghost/kg-default-cards": "10.2.5", + "@tryghost/mg-fs-utils": "0.12.14", + "@tryghost/mg-utils": "0.1.1", + "@tryghost/string": "0.2.21", + "lodash": "4.17.23", + "sanitize-html": "2.17.0", + "simple-dom": "1.4.0" + } +} diff --git a/packages/mg-beehiiv-api/src/index.ts b/packages/mg-beehiiv-api/src/index.ts new file mode 100644 index 000000000..294946b59 --- /dev/null +++ b/packages/mg-beehiiv-api/src/index.ts @@ -0,0 +1,9 @@ +import {listPublications} from './lib/list-pubs.js'; +import {mapPostsTasks} from './lib/mapper.js'; +import {fetchTasks} from './lib/fetch.js'; + +export default { + listPublications, + fetchTasks, + mapPostsTasks +}; diff --git a/packages/mg-beehiiv-api/src/lib/client.ts b/packages/mg-beehiiv-api/src/lib/client.ts new file mode 100644 index 000000000..21c11bfaf --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/client.ts @@ -0,0 +1,22 @@ +const client = async (apiKey: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications`); + + const response = await fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}` + } + }); + + if (!response.ok) { + throw new Error(`Request failed: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + + return data.data; +}; + +export { + client +}; diff --git a/packages/mg-beehiiv-api/src/lib/fetch.ts b/packages/mg-beehiiv-api/src/lib/fetch.ts new file mode 100644 index 000000000..56ba2cdd3 --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/fetch.ts @@ -0,0 +1,94 @@ +const API_LIMIT = 10; + +const authedClient = async (apiKey: string, theUrl: URL) => { + return fetch(theUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}` + } + }); +}; + +const discover = async (key: string, pubId: string, limit: number) => { + const url = new URL(`https://api.beehiiv.com/v2/publications/${pubId}/posts`); + url.searchParams.append('limit', '1'); + + const response = await authedClient(key, url); + + if (!response.ok) { + throw new Error(`Request failed: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + + return data.total_results; +}; + +const cachedFetch = async ({fileCache, key, pubId, limit = API_LIMIT, page}: {fileCache: any, key: string, pubId: string, limit?: number, page: number}) => { + let filename = `beehiiv_api_${limit}_${page}.json`; + + if (fileCache.hasFile(filename, 'tmp')) { + return await fileCache.readTmpJSONFile(filename); + } + + const url = new URL(`https://api.beehiiv.com/v2/publications/${pubId}/posts`); + url.searchParams.append('limit', limit.toString()); + url.searchParams.append('expand', 'free_web_content'); + url.searchParams.append('expand', 'premium_web_content'); + url.searchParams.append('page', page.toString()); + + const response = await authedClient(key, url); + + if (!response.ok) { + throw new Error(`Request failed: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + + await fileCache.writeTmpFile(data, filename); + + return data; +}; + +export const fetchTasks = async (options: any, ctx: any) => { + const numberOfPosts = await discover(options.key, options.id, API_LIMIT); + const totalPages = Math.ceil(numberOfPosts / API_LIMIT); + + let tasks = []; + + for (let page = 1; page <= totalPages; page++) { + tasks.push({ + title: `Fetching posts page ${page} of ${totalPages}`, + task: async (_: any, task: any) => { + try { + let response = await cachedFetch({fileCache: ctx.fileCache ,key: options.key, pubId: options.id, page: page}); + + ctx.result.posts = ctx.result.posts.concat(response.data); + + if (options.postsAfter) { + const afterTimestamp = new Date(options.postsAfter).getTime() / 1000; + ctx.result.posts = ctx.result.posts.filter( + (post: any) => post.publish_date >= afterTimestamp + ); + } + if (options.postsBefore) { + const beforeTimestamp = new Date(options.postsBefore).getTime() / 1000; + ctx.result.posts = ctx.result.posts.filter( + (post: any) => post.publish_date <= beforeTimestamp + ); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + task.output = errorMessage; + throw error; + } + } + }); + } + + return tasks; +}; + +export { + authedClient +}; diff --git a/packages/mg-beehiiv-api/src/lib/list-pubs.ts b/packages/mg-beehiiv-api/src/lib/list-pubs.ts new file mode 100644 index 000000000..b497f1d34 --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/list-pubs.ts @@ -0,0 +1,20 @@ +import {authedClient} from './fetch.js'; + +const listPublications = async (apiKey: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications`); + url.searchParams.append('expand[]', 'stats'); + + const response = await authedClient(apiKey, url); + + if (!response.ok) { + throw new Error(`Request failed: ${response.status} ${response.statusText}`); + } + + const data = await response.json(); + + return data.data; +}; + +export { + listPublications +}; diff --git a/packages/mg-beehiiv-api/src/lib/mapper.ts b/packages/mg-beehiiv-api/src/lib/mapper.ts new file mode 100644 index 000000000..742516a14 --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/mapper.ts @@ -0,0 +1,102 @@ +// import fsUtils from '@tryghost/mg-fs-utils'; +import {slugify} from '@tryghost/string'; +import {processHTML, removeDuplicateFeatureImage} from './process.js'; + +const mapPost = ({postData, options}: {postData: beehiivPostDataObject, options?: any}) => { + const mappedData: mappedDataObject = { + url: postData.web_url, + data: { + comment_id: postData.id, + slug: postData.slug, + published_at: new Date(postData.publish_date * 1000), + updated_at: new Date(postData.created * 1000), + created_at: new Date(postData.created * 1000), + title: postData.title, + type: 'post', + html: postData.content.premium.web, + status: (postData.status === 'confirmed') ? 'published' : 'draft', + custom_excerpt: postData.subtitle ?? null, + visibility: (postData.audience === 'premium') ? 'paid' : 'public', + authors: [], + tags: [] + } + }; + + if (postData.thumbnail_url) { + mappedData.data.feature_image = postData.thumbnail_url; + } + + if (postData.meta_default_title) { + mappedData.data.og_title = postData.meta_default_title; + } + + if (postData.meta_default_description) { + mappedData.data.og_description = postData.meta_default_description; + } + + mappedData.data.html = processHTML({ + post: mappedData, + options + }); + + postData.authors.forEach((author: string) => { + const authorSlug = slugify(author); + mappedData.data.authors.push({ + url: `migrator-added-author-${authorSlug}`, + data: { + slug: authorSlug, + name: author, + email: `${authorSlug}@example.com` + } + }); + }); + + postData.content_tags.forEach((tag: string) => { + const tagSlug = slugify(tag); + mappedData.data.tags.push({ + url: `migrator-added-tag-${tagSlug}`, + data: { + slug: tagSlug, + name: tag + } + }); + }); + + mappedData.data.tags.push({ + url: 'migrator-added-tag-hash-beehiiv', + data: { + slug: 'hash-beehiiv', + name: '#beehiiv' + } + }); + + return mappedData; +}; + +const mapPostsTasks = async (options: any, ctx: any) => { + let tasks: any = []; + + ctx.result.posts.forEach((postData: any, index: number) => { + tasks.push({ + title: `Mapping post: ${postData.title}`, + task: async (_: any, task: any) => { + try { + const mappedPost = mapPost({postData, options}); + ctx.result.posts[index] = mappedPost; + } catch (error) { + /* c8 ignore next */ + const errorMessage = error instanceof Error ? error.message : String(error); + task.output = errorMessage; + throw error; + } + } + }); + }); + + return tasks; +}; + +export { + mapPost, + mapPostsTasks +}; diff --git a/packages/mg-beehiiv-api/src/lib/process.ts b/packages/mg-beehiiv-api/src/lib/process.ts new file mode 100644 index 000000000..b4b7091e5 --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/process.ts @@ -0,0 +1,552 @@ +import {domUtils} from '@tryghost/mg-utils'; +import _ from 'lodash'; +import sanitizeHtml from 'sanitize-html'; +import SimpleDom from 'simple-dom'; +import imageCard from '@tryghost/kg-default-cards/lib/cards/image.js'; +import embedCard from '@tryghost/kg-default-cards/lib/cards/embed.js'; +import bookmarkCard from '@tryghost/kg-default-cards/lib/cards/bookmark.js'; + +const {parseFragment, serializeChildren, serializeNode, replaceWith, insertAfter, attr, parents} = domUtils; + +const serializer = new SimpleDom.HTMLSerializer(SimpleDom.voidMap); + +const getYouTubeID = (url: string) => { + const arr = url.split(/(vi\/|v%3D|v=|\/v\/|youtu\.be\/|\/embed\/)/); + return undefined !== arr[2] ? arr[2].split(/[^\w-]/i)[0] : arr[0]; +}; + +const isURL = (urlString: string | undefined) => { + if (undefined === urlString) { + return false; + } + + try { + new URL(urlString); + return true; + } catch (err) { + return false; + } +}; + +const processHTML = ({post, options}: {post?: mappedDataObject, options?: any}) => { + // First, clean up the email HTML to remove bits we don't want or change up + + let html = post?.data.html ?? ''; + + // Let's do some regexp magic to remove some beehiiv variables + // https://support.beehiiv.com/hc/en-us/articles/7606088263191 + html = html.replace(/{{subscriber_id}}/g, '#'); + html = html.replace(/{{rp_refer_url}}/g, '#'); + html = html.replace(/{{rp_refer_url_no_params}}/g, '#'); + + const allParsed = parseFragment(html); + const contentBlocksEl = allParsed.$('#content-blocks')[0]; + const contentBlocksHtml = contentBlocksEl ? serializeChildren(contentBlocksEl) : ''; + const parsed = parseFragment(contentBlocksHtml); + + // Convert divs with no content but a border-top to a HR + parsed.$('div').forEach((el) => { + const styleAttr = attr(el, 'style') || ''; + const hasBorderTop = styleAttr.includes('border-top'); + const hasContent = serializeChildren(el).trim().length > 0; + + if (hasBorderTop && !hasContent) { + replaceWith(el, '
'); + } + }); + + // Convert images to Ghost image cards + parsed.$('img').forEach((el) => { + // Skip images inside generic embeds (handled separately as bookmark cards) + if (parents(el, '.generic-embed--root').length > 0) { + return; + } + + const src = attr(el, 'src'); + const altText = attr(el, 'alt') || ''; + const parent = el.parentElement!; + + const cardOpts: any = { + env: {dom: new SimpleDom.Document()}, + payload: { + src, + alt: altText, + caption: null + } + }; + + if (parent.tagName === 'DIV') { + const caption = parsed.$('small', parent).map(s => s.textContent!.trim()).filter(Boolean).join('') || null; + cardOpts.payload.caption = caption; + const outerDiv = parent.parentElement; + const target = outerDiv && outerDiv.tagName === 'DIV' ? outerDiv : parent; + replaceWith(target, serializer.serialize(imageCard.render(cardOpts))); + } else { + el.remove(); + insertAfter(parent, serializer.serialize(imageCard.render(cardOpts))); + } + }); + + // Convert nested padding-left divs to blockquotes + parsed.$('div[style*="padding-left"] > div[style*="padding-left"]').forEach((el) => { + const outerDiv = el.parentElement!; + const paragraphs = parsed.$('p', outerDiv); + + if (paragraphs.length > 0) { + const content = paragraphs.map(p => serializeNode(p)).join(''); + const citation = parsed.$('small', outerDiv).map(s => s.textContent!.trim()).filter(Boolean).join(''); + const cite = citation ? `${citation}` : ''; + replaceWith(outerDiv, `
${content}${cite}
`); + } + }); + + // Convert buttons to Ghost buttons + parsed.$('a > button').forEach((el) => { + const link = el.parentElement!; // guaranteed by 'a > button' selector + const wrapper = link.parentElement; + const buttonText = el.textContent!.trim(); + const buttonHref = attr(link, 'href'); + const target = wrapper && wrapper.tagName === 'DIV' ? wrapper : link; + + replaceWith(target, `
${buttonText}
`); + }); + + // Rewrite subscribe links to Portal signup + parsed.$('a[href*="/subscribe"]').forEach((el) => { + attr(el, 'href', '/#/portal/signup'); + }); + + // Handle link embeds + parsed.$('.generic-embed--root').forEach((el) => { + const href = attr(parsed.$('a', el)[0] as Element, 'href'); + const title = _.unescape((parsed.$('.generic-embed--title p', el)[0] as Element).textContent!); + const description = _.unescape((parsed.$('.generic-embed--description p', el)[0] as Element).textContent!); + const image = attr(parsed.$('.generic-embed--image img', el)[0] as Element, 'src'); + + let cardOpts = { + env: {dom: new SimpleDom.Document()}, + payload: { + url: href, + metadata: { + url: href, + title: title, + description: description, + icon: null, + thumbnail: image, + publisher: null + }, + caption: null + } + }; + + replaceWith(el, serializer.serialize(bookmarkCard.render(cardOpts))); + }); + + // Unwrap audio iframes from tables + parsed.$('table iframe[src^="https://audio.beehiiv.com"]').forEach((el) => { + const tableParent = parents(el, 'table')[0]; + if (tableParent) { + const iframeHtml = serializeNode(el); + replaceWith(tableParent, iframeHtml); + } + }); + + // Convert YouTube iframes to embeds + parsed.$('iframe[src^="https://youtube"]').forEach((el) => { + const videoID = getYouTubeID(attr(el, 'src') as string); + + let cardOpts = { + env: {dom: new SimpleDom.Document()}, + payload: { + caption: null, + html: `` + } + }; + + replaceWith(el, serializer.serialize(embedCard.render(cardOpts))); + }); + + // Remove empty paragraphs + parsed.$('p').forEach((el) => { + const text = (el.textContent || '').trim(); + + if (text.length === 0) { + el.remove(); + } + }); + + // Remove style tags + parsed.$('style').forEach((el) => { + el.remove(); + }); + + // Remove mobile ads + parsed.$('#pad-mobile').forEach((el) => { + el.remove(); + }); + + // Remove style attributes + parsed.$('[style]').forEach((el) => { + el.removeAttribute('style'); + }); + + // Unwrap divs, but preserve Ghost card divs + parsed.$('div:not([class*="kg-card"])').forEach((el) => { + replaceWith(el, serializeChildren(el)); + }); + + // Remove b, strong, i, em tags from headings but allow links inside them + parsed.$('h1, h2, h3, h4, h5, h6').forEach((el) => { + parsed.$('b, strong, i, em', el).forEach((ell) => { + replaceWith(ell, serializeChildren(ell)); + }); + }); + + // if (options?.url) { + // $html('a').each((i: any, el: any) => { + // const theHref = $html(el).attr('href'); + // const isHrefURL = isURL(theHref); + + // if (theHref && isHrefURL) { + // const url = new URL(theHref); + + // const params = new URLSearchParams(url.search); + + // params.delete('utm_source'); + // params.delete('utm_medium'); + // params.delete('utm_campaign'); + // params.delete('last_resource_guid'); + + // url.search = params.toString(); + + // $html(el).attr('href', url.toString()); + // } + // }); + // } + + // Polls + // $html('td[class="e"], td[class="ee e "]').each((i: any, el: any) => { + // $html(el).remove(); + // }); + + // FInd

tag which contains {{rp_personalized_text}} and find the parent table to remove the whole section + // $html('p:contains("{{rp_personalized_text}}")').each((i: any, el: any) => { + // const parentTable = $html(el).parents('table').first(); + // $html(parentTable).remove(); + // }); + + // $html('h1 strong, h1 b, h2 strong, h2 b, h3 strong, h3 b, h4 strong, h4 b, h5 strong, h5 b, h6 strong, h6 b').each((i: any, el: any) => { + // const text = $html(el).html().trim(); + // $html(el).replaceWith(text); + // }); + + // $html('table.j').each((i: any, el: any) => { + // const tdContent = $html(el).find('td').html().trim(); + + // if (tdContent === ' ') { + // $html(el).replaceWith('

'); + // } + // }); + + // $html('table.d3[align="center"]').each((i: any, el: any) => { + // const parent = $html(el).parent('td[align="center"]'); + // const text = $html(el).text().replace(/(\r\n|\n|\r| )/gm, ' ').replace(/(\s{2,})/gm, ' ').trim(); + // $html(parent).replaceWith(`

${text}

`); + // }); + + // // Galleries + // $html('table.mob-w-full').each((i: any, el: any) => { + // let allImages: string[] = []; + + // $html(el).find('td.mob-stack').each((ii: any, ell: any) => { + // const img = $html(ell).find('img'); + // const pText = $html(ell).find('p').text().trim(); + + // let cardOpts: any = { + // env: {dom: new SimpleDom.Document()}, + // payload: { + // src: img.attr('src'), + // alt: img.attr('alt') + // } + // }; + + // if (pText && pText.length > 0) { + // cardOpts.payload.caption = pText; + // } + + // const isInLink = $html(img).parents('a').length; + + // if (isInLink) { + // cardOpts.payload.href = $html(img).parents('a').attr('href'); + // } + + // allImages.push(serializer.serialize(imageCard.render(cardOpts))); + // }); + + // $html(el).replaceWith(allImages.join('')); + // }); + + // // Embeds + // $html('td.embed-img.mob-stack').each((i: any, el: any) => { + // const parent = $html(el).parent().parent(); + + // const href = $html(parent).find('a').attr('href'); + // const image = $html(parent).find('img').attr('src'); + // const title = $html(parent).find('p').eq(0).text(); + // const description = $html(parent).find('p').eq(1).text(); + + // const parentTable = $html(parent).parent().parent().parent().parent().parent(); + + // let cardOpts = { + // env: {dom: new SimpleDom.Document()}, + // payload: { + // url: href, + // metadata: { + // url: href, + // title: title, + // description: description, + // icon: null, + // thumbnail: image, + // publisher: null, + // author: null + // }, + // caption: null + // } + // }; + + // $html(parentTable).replaceWith(serializer.serialize(bookmarkCard.render(cardOpts))); + // }); + + // // Remove hidden elements + // $html('[style*="display:none"]').remove(); + // $html('[style*="display: none"]').remove(); + + // // Remove the share links at the top + // $html('table.mob-block').remove(); + + // // Remove sponsor blocks + // $html('table.rec__content').remove(); + + // // Remove the open tracking pixel element + // $html('div[data-open-tracking="true"]:contains("{{OPEN_TRACKING_PIXEL}}")').remove(); + + // $html('p:contains("{{rp_personalized_text}}")').remove(); + // $html('img[src="{{rp_next_milestone_image_url}}"]').remove(); + // $html(`a[href="{{rp_referral_hub_url}}"]`).remove(); + + // // Remove unsubscribe links, social links, & email footer + // $html('td.b').remove(); + + // // Remove the 'Read online' link + // $html('td.f').remove(); + + // // Remove the post title container, otherwise it would be a duplicate + // if (postData?.data?.title) { + // $html(`h1:contains("${postData.data.title}")`).parentsUntil('table').remove(); + // } + + // // Remove cells that only contain a non-breaking space + // $html('td').each((i: any, el: any) => { + // const text = $html(el).html().trim(); + + // if (text === ' ') { + // $html(el).remove(); + // } + // }); + + // // Convert '...' to
+ // $html('p').each((i: any, el: any) => { + // const text = $html(el).text().trim(); + + // if (text === '...' || text === '…' || text === '…') { + // $html(el).replaceWith('
'); + // } + // }); + + // // If the iframe has no inner HTML, add a non-breaking space so it doesn't get removed later + // // Case: There's a bug somewhere in the chain that causes iframes to become self-closing, and this prevents that. + // $html('iframe').each((i: any, el: any) => { + // const innerHtml = $html(el).html().length; + + // if (innerHtml === 0) { + // $html(el).html(' '); + // } + // }); + + // // Convert linked YouTube thumbnails to embeds + // $html('a[href*="youtube.com"], a[href*="youtu.be"]').each((i: any, el: any) => { + // const imageCount = $html(el).find('img').length; + // const hasPlayIcon = $html(el).find('img[src*="youtube_play_icon.png"]').length; + // const hasThumbnail = $html(el).find('img[src*="i.ytimg.com/vi"]').length; + // const src = $html(el)?.attr('href'); + // const captionText = $html(el).find('p')?.text()?.trim() || false; + // const captionHtml = $html(el).find('p')?.html()?.trim() || false; + + // if (imageCount === 2 && hasPlayIcon && hasThumbnail && src) { + // const videoID = getYouTubeID(src); + + // let cardOpts = { + // env: {dom: new SimpleDom.Document()}, + // payload: { + // caption: null, + // html: `` + // } + // }; + + // if (captionText && captionText.length > 0) { + // cardOpts.payload.caption = captionHtml; + // } + + // $html(el).replaceWith(serializer.serialize(embedCard.render(cardOpts))); + // } + // }); + + // $html('img').each((i: any, el: any) => { + // // Skip if the image is in a figure + // const isInFigure = $html(el).parents('figure').length; + // if (isInFigure) { + // return; + // } + + // const parentTable = $html(el).parent().parent().parent(); + + // const theSrc = $html(el).attr('src'); + // let theAlt = $html(el).attr('alt'); + + // const secondTr = ($html(parentTable).find('tr').eq(1).find('p').length) ? $html(parentTable).find('tr').eq(1).find('p') : false; + // const theText = $html(secondTr)?.html()?.trim() ?? false; + + // if (!theAlt) { + // theAlt = $html(secondTr)?.text()?.trim(); + // } + + // let cardOpts: any = { + // env: {dom: new SimpleDom.Document()}, + // payload: { + // src: theSrc, + // alt: theAlt, + // caption: theText + // } + // }; + + // // Check if the parent element to this is a tag + // const isInLink = $html(el).parents('a').length; + + // if (isInLink) { + // cardOpts.payload.href = $html(el).parents('a').attr('href'); + // } + + // $html(parentTable).replaceWith(serializer.serialize(imageCard.render(cardOpts))); + // }); + + // // Convert buttons to Ghost buttons + // $html('a[style="color:#FFFFFF;font-size:18px;padding:0px 14px;text-decoration:none;"]').each((i: any, el: any) => { + // const buttonText = $html(el).text(); + // const buttonHref = $html(el).attr('href'); + // $html(el).replaceWith(`
${buttonText}
`); + // }); + + // if (options?.url && options?.subscribeLink) { + // $html(`a[href^="${options.url}/subscribe"]`).each((i: any, el: any) => { + // $html(el).attr('href', options.subscribeLink); + // $html(el).removeAttr('target'); + // $html(el).removeAttr('rel'); + // }); + // } + + // if (options?.url && options?.comments && options?.commentLink) { + // $html('a[href*="comments=true"]').each((i: any, el: any) => { + // const href = $html(el).attr('href'); + + // if (href.includes(options.url)) { + // $html(el).attr('href', options.commentLink); + // $html(el).removeAttr('target'); + // $html(el).removeAttr('rel'); + // } + // }); + // } else if (options?.url && options?.comments === false) { + // $html('a[href*="comments=true"]').each((i: any, el: any) => { + // const href = $html(el).attr('href'); + + // if (href.includes(options.url)) { + // $html(el).remove(); + // } + // }); + // } + + // // Remove empty tags + // $html('p, figure').each((i: any, el: any) => { + // const elementHtml = $html(el).html().trim(); + + // if (elementHtml === '') { + // $html(el).remove(); + // } + // }); + + // // Get the cleaned HTML + let bodyHtml = parsed.html(); + + return bodyHtml; + + // // Pass the cleaned HTML through the sanitizer to only include specific elements + // const sanitizedHtml = sanitizeHtml(bodyHtml, { + // allowedTags: [ + // 'b', 'i', 'em', 'strong', 'a', 'p', 'br', 'ul', 'ol', 'li', 'blockquote', + // 'figure', 'figcaption', 'img', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', + // 'div', 'hr', 'iframe', 'span' + // ], + // allowedAttributes: { + // a: ['href', 'title', 'rel', 'target', 'class'], + // img: ['src', 'alt', 'title', 'class', 'width', 'height'], + // iframe: ['width', 'height', 'src', 'title', 'frameborder', 'allow', 'allowfullscreen'], + // figure: ['class'], + // div: ['class'] + // }, + // allowedClasses: { + // '*': ['kg-*'] + // } + // }); + + // return sanitizedHtml.trim(); +}; + +const removeDuplicateFeatureImage = ({html, featureSrc}: {html: string, featureSrc: string}) => { + const parsed = parseFragment(html); + + let firstElement = parsed.$('*')[0]; + + if (firstElement) { + const isImg = firstElement.tagName === 'IMG'; + const hasImg = !isImg && parsed.$('img', firstElement).length > 0; + + if (isImg || hasImg) { + let theElementItself = isImg ? firstElement : parsed.$('img', firstElement)[0]; + let firstImgSrc: any = attr(theElementItself, 'src'); + + // Both images usually end in the same way, so we can split the URL and compare the last part + const firstImageSplit = firstImgSrc.split('/uploads/asset/'); + const featureImageSplit = featureSrc.split('/uploads/asset/'); + + if (firstImageSplit[1] === featureImageSplit[1]) { + theElementItself.remove(); + } + + if (featureSrc.length > 0 && firstImgSrc) { + let normalizedFirstSrc = firstImgSrc.replace('fit=scale-down,format=auto,onerror=redirect,quality=80', 'quality=100'); + + if (featureSrc === normalizedFirstSrc) { + theElementItself.remove(); + } + } + } + } + + return parsed.html(); +}; + +export { + getYouTubeID, + isURL, + processHTML, + removeDuplicateFeatureImage +}; diff --git a/packages/mg-beehiiv-api/src/test/client.test.ts b/packages/mg-beehiiv-api/src/test/client.test.ts new file mode 100644 index 000000000..44ff723d8 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/client.test.ts @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import {describe, it, beforeEach, afterEach, mock} from 'node:test'; +import {client} from '../lib/client.js'; + +describe('beehiiv API Client', () => { + let fetchMock: any; + + beforeEach(() => { + fetchMock = mock.method(global, 'fetch', () => Promise.resolve()); + }); + + afterEach(() => { + mock.restoreAll(); + }); + + describe('client', () => { + it('makes authenticated request to publications endpoint', async () => { + const mockData = {data: [{id: 'pub-1', name: 'Test Publication'}]}; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve(mockData) + })); + + await client('test-api-key'); + + assert.equal(fetchMock.mock.callCount(), 1); + const [calledUrl, options] = fetchMock.mock.calls[0].arguments; + assert.equal(calledUrl.toString(), 'https://api.beehiiv.com/v2/publications'); + assert.equal(options.method, 'GET'); + assert.equal(options.headers.Authorization, 'Bearer test-api-key'); + }); + + it('returns publications data', async () => { + const publications = [{id: 'pub-1', name: 'Test Pub'}]; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({data: publications}) + })); + + const result = await client('test-api-key'); + + assert.deepEqual(result, publications); + }); + + it('throws error on failed request', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 401, + statusText: 'Unauthorized' + })); + + await assert.rejects(async () => { + await client('invalid-key'); + }, /Request failed: 401 Unauthorized/); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/fetch.test.ts b/packages/mg-beehiiv-api/src/test/fetch.test.ts new file mode 100644 index 000000000..2b197fdf4 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/fetch.test.ts @@ -0,0 +1,349 @@ +import assert from 'node:assert/strict'; +import {describe, it, beforeEach, afterEach, mock} from 'node:test'; +import {fetchTasks, authedClient} from '../lib/fetch.js'; + +describe('beehiiv API Fetch', () => { + let fetchMock: any; + + beforeEach(() => { + fetchMock = mock.method(global, 'fetch', () => Promise.resolve()); + }); + + afterEach(() => { + mock.restoreAll(); + }); + + describe('authedClient', () => { + it('makes authenticated GET request', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ok: true, json: () => Promise.resolve({data: []})})); + + const url = new URL('https://api.beehiiv.com/v2/publications'); + await authedClient('test-api-key', url); + + assert.equal(fetchMock.mock.callCount(), 1); + const [calledUrl, options] = fetchMock.mock.calls[0].arguments; + assert.equal(calledUrl.toString(), 'https://api.beehiiv.com/v2/publications'); + assert.equal(options.method, 'GET'); + assert.equal(options.headers.Authorization, 'Bearer test-api-key'); + }); + }); + + describe('fetchTasks', () => { + it('creates correct number of tasks based on total posts', async () => { + // Mock the discover call (limit=1) + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 25, data: []}) + }), 0); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + readTmpJSONFile: () => Promise.resolve({}), + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + + // 25 posts / 10 per page = 3 pages + assert.equal(tasks.length, 3); + assert.equal(tasks[0].title, 'Fetching posts page 1 of 3'); + assert.equal(tasks[1].title, 'Fetching posts page 2 of 3'); + assert.equal(tasks[2].title, 'Fetching posts page 3 of 3'); + }); + + it('task fetches posts and adds to context', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: []}) + }), 0); + + // Mock the actual fetch call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [{id: 'post-1', title: 'Test Post'}] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + + // Execute the first task + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 1); + assert.equal((ctx.result.posts[0] as any).id, 'post-1'); + }); + + it('task uses cached data when available', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: []}) + }), 0); + + const cachedData = {data: [{id: 'cached-post', title: 'Cached Post'}]}; + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => true, + readTmpJSONFile: () => Promise.resolve(cachedData) + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + // Should use cached data, not make another API call + assert.equal(ctx.result.posts.length, 1); + assert.equal((ctx.result.posts[0] as any).id, 'cached-post'); + // fetch should only be called once (for discover), not twice + assert.equal(fetchMock.mock.callCount(), 1); + }); + + it('task throws and sets output on fetch error', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: []}) + }), 0); + + // Mock a failed fetch + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: false, + status: 500, + statusText: 'Internal Server Error' + }), 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }, /Request failed: 500 Internal Server Error/); + + assert.ok(mockTask.output.includes('500')); + }); + + it('task handles non-Error thrown objects', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: []}) + }), 0); + + // Mock fetch that throws a non-Error value (string) + fetchMock.mock.mockImplementationOnce(() => { + // eslint-disable-next-line no-throw-literal + throw 'Network error string'; + }, 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + + // Verify String() conversion was used + assert.equal(mockTask.output, 'Network error string'); + }); + + it('filters posts with postsAfter', async () => { + const jan1 = new Date('2024-01-01').getTime() / 1000; + const feb1 = new Date('2024-02-01').getTime() / 1000; + const mar1 = new Date('2024-03-01').getTime() / 1000; + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 3, data: []}) + }), 0); + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [ + {id: 'post-jan', publish_date: jan1}, + {id: 'post-feb', publish_date: feb1}, + {id: 'post-mar', publish_date: mar1} + ] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123', postsAfter: '2024-02-01'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 2); + assert.equal((ctx.result.posts[0] as any).id, 'post-feb'); + assert.equal((ctx.result.posts[1] as any).id, 'post-mar'); + }); + + it('filters posts with postsBefore', async () => { + const jan1 = new Date('2024-01-01').getTime() / 1000; + const feb1 = new Date('2024-02-01').getTime() / 1000; + const mar1 = new Date('2024-03-01').getTime() / 1000; + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 3, data: []}) + }), 0); + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [ + {id: 'post-jan', publish_date: jan1}, + {id: 'post-feb', publish_date: feb1}, + {id: 'post-mar', publish_date: mar1} + ] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123', postsBefore: '2024-02-01'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 2); + assert.equal((ctx.result.posts[0] as any).id, 'post-jan'); + assert.equal((ctx.result.posts[1] as any).id, 'post-feb'); + }); + + it('filters posts with both postsAfter and postsBefore', async () => { + const jan1 = new Date('2024-01-01').getTime() / 1000; + const feb1 = new Date('2024-02-01').getTime() / 1000; + const mar1 = new Date('2024-03-01').getTime() / 1000; + const apr1 = new Date('2024-04-01').getTime() / 1000; + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 4, data: []}) + }), 0); + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [ + {id: 'post-jan', publish_date: jan1}, + {id: 'post-feb', publish_date: feb1}, + {id: 'post-mar', publish_date: mar1}, + {id: 'post-apr', publish_date: apr1} + ] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123', postsAfter: '2024-02-01', postsBefore: '2024-03-01'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 2); + assert.equal((ctx.result.posts[0] as any).id, 'post-feb'); + assert.equal((ctx.result.posts[1] as any).id, 'post-mar'); + }); + + it('does not filter when neither postsAfter nor postsBefore is set', async () => { + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 2, data: []}) + }), 0); + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [ + {id: 'post-1', publish_date: 1704067200}, + {id: 'post-2', publish_date: 1706745600} + ] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 2); + }); + + it('handles discover error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 401, + statusText: 'Unauthorized' + })); + + const options = {key: 'invalid-key', id: 'pub-123'}; + const ctx = { + fileCache: {}, + result: {posts: [] as any[]} + }; + + await assert.rejects(async () => { + await fetchTasks(options, ctx); + }, /Request failed: 401 Unauthorized/); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/fixtures/posts.csv b/packages/mg-beehiiv-api/src/test/fixtures/posts.csv new file mode 100644 index 000000000..d96f6fc9b --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/fixtures/posts.csv @@ -0,0 +1,3 @@ +id,web_title,status,audience,content_tags,url,web_subtitle,email_subject_line,email_preview_text,content_html,thumbnail_url,created_at +abcd1234-1505-4fbf-9576-f3d1bd3034cc,Sample Post,confirmed,free,Lorem Ipsum; Dolor Simet,https://example.beehiiv.com/p/sample-post,"A website subtitle",Sample Post as a subject line,"Email preview text","

Sample HTML here

",uploads/asset/file/12345678/image.png,2023-01-18 21:25:27 +efgh1234-1505-4fbf-9576-f3d1bd3034cc,Another Post,confirmed,free,"Lorem Ipsum; Dolor Simet",https://example.beehiiv.com/p/another-post,"Another website subtitle",Another sample Post as a subject line,"Email preview text","

More sample HTML here

",uploads/asset/file/12345678/another.png,2023-01-19 21:25:27 diff --git a/packages/mg-beehiiv-api/src/test/index.test.ts b/packages/mg-beehiiv-api/src/test/index.test.ts new file mode 100644 index 000000000..5c5a5a222 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/index.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import beehiivApi from '../index.js'; + +describe('beehiiv API index exports', () => { + it('exports listPublications', () => { + assert.ok(typeof beehiivApi.listPublications === 'function'); + }); + + it('exports fetchTasks', () => { + assert.ok(typeof beehiivApi.fetchTasks === 'function'); + }); + + it('exports mapPostsTasks', () => { + assert.ok(typeof beehiivApi.mapPostsTasks === 'function'); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/list-pubs.test.ts b/packages/mg-beehiiv-api/src/test/list-pubs.test.ts new file mode 100644 index 000000000..2e2f21a50 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/list-pubs.test.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import {describe, it, beforeEach, afterEach, mock} from 'node:test'; +import {listPublications} from '../lib/list-pubs.js'; + +describe('beehiiv API List Publications', () => { + let fetchMock: any; + + beforeEach(() => { + fetchMock = mock.method(global, 'fetch', () => Promise.resolve()); + }); + + afterEach(() => { + mock.restoreAll(); + }); + + describe('listPublications', () => { + it('makes authenticated request to publications endpoint', async () => { + const mockData = {data: [{id: 'pub-1', name: 'Test Publication'}]}; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve(mockData) + })); + + await listPublications('test-api-key'); + + assert.equal(fetchMock.mock.callCount(), 1); + const [calledUrl, options] = fetchMock.mock.calls[0].arguments; + assert.equal(calledUrl.toString(), 'https://api.beehiiv.com/v2/publications?expand%5B%5D=stats'); + assert.equal(options.method, 'GET'); + assert.equal(options.headers.Authorization, 'Bearer test-api-key'); + }); + + it('returns publications data', async () => { + const publications = [ + {id: 'pub-1', name: 'Publication One'}, + {id: 'pub-2', name: 'Publication Two'} + ]; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({data: publications}) + })); + + const result = await listPublications('test-api-key'); + + assert.deepEqual(result, publications); + }); + + it('throws error on failed request', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 403, + statusText: 'Forbidden' + })); + + await assert.rejects(async () => { + await listPublications('invalid-key'); + }, /Request failed: 403 Forbidden/); + }); + + it('throws error on server error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 500, + statusText: 'Internal Server Error' + })); + + await assert.rejects(async () => { + await listPublications('test-key'); + }, /Request failed: 500 Internal Server Error/); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/mapper.test.ts b/packages/mg-beehiiv-api/src/test/mapper.test.ts new file mode 100644 index 000000000..a1a8356c8 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/mapper.test.ts @@ -0,0 +1,280 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import {mapPost, mapPostsTasks} from '../lib/mapper.js'; + +describe('beehiiv API Mapper', () => { + describe('mapPost', () => { + const createMockPostData = (overrides = {}): any => ({ + id: 'post-123', + title: 'Test Post Title', + subtitle: 'Test subtitle', + slug: 'test-post-title', + web_url: 'https://example.beehiiv.com/p/test-post-title', + status: 'confirmed', + audience: 'free', + publish_date: 1700000000, // Unix timestamp + created: 1699900000, + thumbnail_url: 'https://example.com/image.jpg', + meta_default_title: 'OG Title', + meta_default_description: 'OG Description', + authors: ['John Doe', 'Jane Smith'], + content_tags: ['Tech', 'News'], + content: { + premium: { + web: '

Test content

' + } + }, + ...overrides + }); + + it('maps basic post fields correctly', () => { + const postData = createMockPostData(); + const result = mapPost({postData}); + + assert.equal(result.url, 'https://example.beehiiv.com/p/test-post-title'); + assert.equal(result.data.comment_id, 'post-123'); + assert.equal(result.data.slug, 'test-post-title'); + assert.equal(result.data.title, 'Test Post Title'); + assert.equal(result.data.type, 'post'); + assert.equal(result.data.custom_excerpt, 'Test subtitle'); + }); + + it('converts timestamps to Date objects', () => { + const postData = createMockPostData(); + const result = mapPost({postData}); + + assert.ok(result.data.published_at instanceof Date); + assert.ok(result.data.updated_at instanceof Date); + assert.ok(result.data.created_at instanceof Date); + + // Check the dates are converted correctly (Unix timestamp * 1000) + assert.equal(result.data.published_at.getTime(), 1700000000 * 1000); + assert.equal(result.data.created_at.getTime(), 1699900000 * 1000); + }); + + it('sets status to published for confirmed posts', () => { + const postData = createMockPostData({status: 'confirmed'}); + const result = mapPost({postData}); + assert.equal(result.data.status, 'published'); + }); + + it('sets status to draft for non-confirmed posts', () => { + const postData = createMockPostData({status: 'draft'}); + const result = mapPost({postData}); + assert.equal(result.data.status, 'draft'); + }); + + it('sets visibility to paid for premium audience', () => { + const postData = createMockPostData({audience: 'premium'}); + const result = mapPost({postData}); + assert.equal(result.data.visibility, 'paid'); + }); + + it('sets visibility to public for free audience', () => { + const postData = createMockPostData({audience: 'free'}); + const result = mapPost({postData}); + assert.equal(result.data.visibility, 'public'); + }); + + it('sets feature_image when thumbnail_url exists', () => { + const postData = createMockPostData({thumbnail_url: 'https://example.com/thumb.jpg'}); + const result = mapPost({postData}); + assert.equal(result.data.feature_image, 'https://example.com/thumb.jpg'); + }); + + it('does not set feature_image when thumbnail_url is empty', () => { + const postData = createMockPostData({thumbnail_url: ''}); + const result = mapPost({postData}); + assert.equal(result.data.feature_image, undefined); + }); + + it('sets og_title when meta_default_title exists', () => { + const postData = createMockPostData({meta_default_title: 'Custom OG Title'}); + const result = mapPost({postData}); + assert.equal(result.data.og_title, 'Custom OG Title'); + }); + + it('does not set og_title when meta_default_title is falsy', () => { + const postData = createMockPostData({meta_default_title: null}); + const result = mapPost({postData}); + assert.equal(result.data.og_title, undefined); + }); + + it('sets og_description when meta_default_description exists', () => { + const postData = createMockPostData({meta_default_description: 'Custom OG Desc'}); + const result = mapPost({postData}); + assert.equal(result.data.og_description, 'Custom OG Desc'); + }); + + it('does not set og_description when meta_default_description is falsy', () => { + const postData = createMockPostData({meta_default_description: null}); + const result = mapPost({postData}); + assert.equal(result.data.og_description, undefined); + }); + + it('maps authors correctly', () => { + const postData = createMockPostData({authors: ['John Doe', 'Jane Smith']}); + const result = mapPost({postData}); + + assert.equal(result.data.authors.length, 2); + + assert.equal(result.data.authors[0].data.name, 'John Doe'); + assert.equal(result.data.authors[0].data.slug, 'john-doe'); + assert.equal(result.data.authors[0].data.email, 'john-doe@example.com'); + assert.ok(result.data.authors[0].url.includes('john-doe')); + + assert.equal(result.data.authors[1].data.name, 'Jane Smith'); + assert.equal(result.data.authors[1].data.slug, 'jane-smith'); + }); + + it('maps content_tags correctly', () => { + const postData = createMockPostData({content_tags: ['Tech', 'News']}); + const result = mapPost({postData}); + + // Should have 3 tags: 2 content tags + 1 beehiiv source tag + assert.equal(result.data.tags.length, 3); + + assert.equal(result.data.tags[0].data.name, 'Tech'); + assert.equal(result.data.tags[0].data.slug, 'tech'); + + assert.equal(result.data.tags[1].data.name, 'News'); + assert.equal(result.data.tags[1].data.slug, 'news'); + }); + + it('always adds #beehiiv source tag', () => { + const postData = createMockPostData({content_tags: []}); + const result = mapPost({postData}); + + const beehiivTag = result.data.tags.find((t: any) => t.data.slug === 'hash-beehiiv'); + assert.ok(beehiivTag); + assert.equal(beehiivTag.data.name, '#beehiiv'); + }); + + it('handles null subtitle', () => { + const postData = createMockPostData({subtitle: null}); + const result = mapPost({postData}); + assert.equal(result.data.custom_excerpt, null); + }); + + it('processes HTML through processHTML', () => { + const postData = createMockPostData({ + content: { + premium: { + web: '

Processed content

' + } + } + }); + const result = mapPost({postData}); + assert.ok(result.data.html.includes('Processed content')); + }); + }); + + describe('mapPostsTasks', () => { + it('creates tasks for each post', async () => { + const ctx = { + result: { + posts: [ + { + id: 'post-1', + title: 'Post 1', + subtitle: null, + slug: 'post-1', + web_url: 'https://example.com/p/post-1', + status: 'confirmed', + audience: 'free', + publish_date: 1700000000, + created: 1699900000, + thumbnail_url: '', + meta_default_title: null, + meta_default_description: null, + authors: ['Author'], + content_tags: [], + content: {premium: {web: '

Content 1

'}} + }, + { + id: 'post-2', + title: 'Post 2', + subtitle: null, + slug: 'post-2', + web_url: 'https://example.com/p/post-2', + status: 'confirmed', + audience: 'free', + publish_date: 1700000000, + created: 1699900000, + thumbnail_url: '', + meta_default_title: null, + meta_default_description: null, + authors: ['Author'], + content_tags: [], + content: {premium: {web: '

Content 2

'}} + } + ] + } + }; + + const tasks = await mapPostsTasks({}, ctx); + + assert.equal(tasks.length, 2); + assert.equal(tasks[0].title, 'Mapping post: Post 1'); + assert.equal(tasks[1].title, 'Mapping post: Post 2'); + }); + + it('task maps post and updates context', async () => { + const ctx = { + result: { + posts: [ + { + id: 'post-1', + title: 'Post 1', + subtitle: 'Subtitle', + slug: 'post-1', + web_url: 'https://example.com/p/post-1', + status: 'confirmed', + audience: 'free', + publish_date: 1700000000, + created: 1699900000, + thumbnail_url: '', + meta_default_title: null, + meta_default_description: null, + authors: ['Author'], + content_tags: ['Tag1'], + content: {premium: {web: '

Content

'}} + } + ] + } + }; + + const tasks = await mapPostsTasks({}, ctx); + + // Execute the task + await tasks[0].task({}, {output: ''}); + + // Check that the post was mapped + const mappedPost = ctx.result.posts[0] as any; + assert.equal(mappedPost.url, 'https://example.com/p/post-1'); + assert.ok(mappedPost.data); + assert.equal(mappedPost.data.title, 'Post 1'); + }); + + it('task throws error on mapping failure', async () => { + const ctx = { + result: { + posts: [ + { + // Missing required fields to cause an error + id: 'post-1', + title: 'Post 1' + } + ] + } + }; + + const tasks = await mapPostsTasks({}, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/process.test.ts b/packages/mg-beehiiv-api/src/test/process.test.ts new file mode 100644 index 000000000..c2a1a1053 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/process.test.ts @@ -0,0 +1,300 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import {getYouTubeID, isURL, processHTML, removeDuplicateFeatureImage} from '../lib/process.js'; + +describe('beehiiv API processor', () => { + describe('isURL', () => { + it('returns true for valid HTTP URL', () => { + assert.equal(isURL('https://example.com'), true); + }); + + it('returns true for valid HTTP URL with path', () => { + assert.equal(isURL('https://example.com/path/to/resource'), true); + }); + + it('returns false for undefined', () => { + assert.equal(isURL(undefined), false); + }); + + it('returns false for invalid URL string', () => { + assert.equal(isURL('not-a-url'), false); + }); + + it('returns false for empty string', () => { + assert.equal(isURL(''), false); + }); + }); + + describe('getYouTubeID', () => { + it('extracts ID from standard youtube.com URL', () => { + const result = getYouTubeID('https://www.youtube.com/watch?v=dQw4w9WgXcQ'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('extracts ID from youtu.be short URL', () => { + const result = getYouTubeID('https://youtu.be/dQw4w9WgXcQ'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('extracts ID from embed URL', () => { + const result = getYouTubeID('https://www.youtube.com/embed/dQw4w9WgXcQ'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('extracts ID from /v/ URL', () => { + const result = getYouTubeID('https://www.youtube.com/v/dQw4w9WgXcQ'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('extracts ID with additional parameters', () => { + const result = getYouTubeID('https://www.youtube.com/watch?v=dQw4w9WgXcQ&feature=youtu.be'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('returns original string when no ID pattern found', () => { + const result = getYouTubeID('https://example.com/video'); + assert.equal(result, 'https://example.com/video'); + }); + }); + + describe('processHTML', () => { + it('handles post with undefined html by using empty string', () => { + // When post.data.html is undefined, it falls back to empty string via ?? + const result = processHTML({post: {url: 'test', data: {}} as any}); + // Should return empty string without throwing + assert.equal(result, ''); + }); + + it('handles html without content-blocks element', () => { + // When there's no #content-blocks element, contentBlocksHtml falls back to '' + const html = '

No content blocks here

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, ''); + }); + + it('replaces beehiiv subscriber_id variable with #', () => { + const html = '

Hello {{subscriber_id}}

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('#')); + assert.ok(!result.includes('{{subscriber_id}}')); + }); + + it('replaces beehiiv rp_refer_url variable with #', () => { + const html = '
Link
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('#')); + assert.ok(!result.includes('{{rp_refer_url}}')); + }); + + it('replaces beehiiv rp_refer_url_no_params variable with #', () => { + const html = '
Link
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('#')); + assert.ok(!result.includes('{{rp_refer_url_no_params}}')); + }); + + it('converts empty divs with border-top to HR', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes(' { + const html = '
content
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(!result.includes(' { + const html = `
❝

Quote text here.

Author Name
`; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Quote text here.

Author Name
'); + }); + + it('converts nested padding-left divs to blockquote without citation when small is empty', () => { + const html = `
❝

Quote text here.

`; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Quote text here.

'); + }); + + it('does not convert nested padding-left divs without paragraphs', () => { + const html = '
plain text
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(!result.includes('
')); + }); + + it('converts subscribe button wrapper to Portal signup button', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
Subscribe
'); + }); + + it('converts subscribe button without wrapper div to Portal signup', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
Subscribe
'); + }); + + it('converts buttons inside anchors to Ghost buttons', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('kg-btn kg-btn-accent')); + assert.ok(result.includes('Click me')); + }); + + it('converts generic embeds to bookmark cards', () => { + const html = `
+
+ Link +

Title

+

Description

+
+
+
`; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('kg-card kg-bookmark-card')); + }); + + it('unwraps audio iframes from tables', () => { + const html = `
+ + + + +
+ +
+
`; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('iframe')); + assert.ok(result.includes('audio.beehiiv.com')); + }); + + it('converts YouTube iframes to embed cards', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('kg-card kg-embed-card')); + assert.ok(result.includes('dQw4w9WgXcQ')); + }); + + it('converts image with caption to Ghost image card', () => { + const html = `

Caption text here

`; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('kg-card kg-image-card')); + assert.ok(result.includes('https://media.beehiiv.com/image.jpg')); + assert.ok(result.includes('Caption text here')); + }); + + it('converts image without caption to Ghost image card', () => { + const html = '
Test
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('kg-card kg-image-card')); + assert.ok(result.includes('https://media.beehiiv.com/photo.jpg')); + }); + + it('converts image in single div wrapper to Ghost image card', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('kg-card kg-image-card')); + assert.ok(result.includes('https://media.beehiiv.com/single.jpg')); + }); + + it('converts image inside non-div wrapper and places card after parent', () => { + const html = '

Text more text

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('kg-card kg-image-card')); + assert.ok(result.includes('https://example.com/img.jpg')); + assert.ok(result.includes('

Text more text

')); + }); + + it('removes empty paragraphs', () => { + const html = '

Content

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(result.includes('

Content

')); + // Empty paragraphs should be removed + const pCount = (result.match(/

/g) || []).length; + assert.equal(pCount, 1); + }); + + it('removes style tags', () => { + const html = '

Content

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.ok(!result.includes('

Content

'; const result = processHTML({post: {url: 'test', data: {html}} as any}); - assert.ok(!result.includes('