diff --git a/packages/mg-beehiiv-api-members/.eslintrc.cjs b/packages/mg-beehiiv-api-members/.eslintrc.cjs new file mode 100644 index 000000000..cd6f552f4 --- /dev/null +++ b/packages/mg-beehiiv-api-members/.eslintrc.cjs @@ -0,0 +1,12 @@ +module.exports = { + parser: '@typescript-eslint/parser', + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ], + rules: { + 'no-unused-vars': 'off', // doesn't work with typescript + 'no-undef': 'off', // doesn't work with typescript + 'ghost/filenames/match-regex': 'off' + } +}; diff --git a/packages/mg-beehiiv-api-members/README.md b/packages/mg-beehiiv-api-members/README.md new file mode 100644 index 000000000..04af663d6 --- /dev/null +++ b/packages/mg-beehiiv-api-members/README.md @@ -0,0 +1,3 @@ +# Migrate beehiiv members API + +... diff --git a/packages/mg-beehiiv-api-members/package.json b/packages/mg-beehiiv-api-members/package.json new file mode 100644 index 000000000..f6678467e --- /dev/null +++ b/packages/mg-beehiiv-api-members/package.json @@ -0,0 +1,43 @@ +{ + "name": "@tryghost/mg-beehiiv-api-members", + "version": "0.1.0", + "repository": "https://github.com/TryGhost/migrate/tree/main/packages/mg-beehiiv-api-members", + "author": "Ghost Foundation", + "license": "MIT", + "type": "module", + "main": "build/index.js", + "types": "build/types.d.ts", + "exports": { + ".": { + "development": "./src/index.ts", + "default": "./build/index.js" + } + }, + "scripts": { + "dev": "echo \"Implement me!\"", + "build:watch": "tsc --watch --preserveWatchOutput --sourceMap", + "build": "rm -rf build && rm -rf tsconfig.tsbuildinfo && tsc --build --sourceMap", + "prepare": "yarn build", + "lint": "eslint src/ --ext .ts --cache", + "posttest": "yarn lint", + "test": "rm -rf build && yarn build --force && c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura node --test build/test/*.test.js" + }, + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@typescript-eslint/eslint-plugin": "8.0.0", + "@typescript-eslint/parser": "8.0.0", + "c8": "10.1.3", + "eslint": "8.57.0", + "typescript": "5.9.3" + }, + "dependencies": { + "@tryghost/errors": "1.3.8", + "@tryghost/mg-fs-utils": "0.12.14", + "@tryghost/string": "0.2.21" + } +} diff --git a/packages/mg-beehiiv-api-members/src/index.ts b/packages/mg-beehiiv-api-members/src/index.ts new file mode 100644 index 000000000..19b1e8ed3 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/index.ts @@ -0,0 +1,9 @@ +import {listPublications} from './lib/list-pubs.js'; +import {fetchTasks} from './lib/fetch.js'; +import {mapMembersTasks} from './lib/mapper.js'; + +export default { + listPublications, + fetchTasks, + mapMembersTasks +}; diff --git a/packages/mg-beehiiv-api-members/src/lib/fetch.ts b/packages/mg-beehiiv-api-members/src/lib/fetch.ts new file mode 100644 index 000000000..e07b3faac --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/lib/fetch.ts @@ -0,0 +1,114 @@ +import errors from '@tryghost/errors'; + +const API_LIMIT = 100; + +const authedClient = async (apiKey: string, theUrl: URL) => { + return fetch(theUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}` + } + }); +}; + +const discover = async (key: string, pubId: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications/${pubId}`); + url.searchParams.append('limit', '1'); + url.searchParams.append('expand[]', 'stats'); + + const response = await authedClient(key, url); + + if (!response.ok) { + throw new errors.InternalServerError({message: `Request failed: ${response.status} ${response.statusText}`}); + } + + const data: BeehiivPublicationResponse = await response.json(); + + return data.data.stats?.active_subscriptions; +}; + +const cachedFetch = async ({fileCache, key, pubId, cursor, cursorIndex}: { + fileCache: any; + key: string; + pubId: string; + cursor: string | null; + cursorIndex: number; +}) => { + const filename = `beehiiv_api_members_${cursorIndex}.json`; + + if (fileCache.hasFile(filename, 'tmp')) { + return await fileCache.readTmpJSONFile(filename); + } + + const url = new URL(`https://api.beehiiv.com/v2/publications/${pubId}/subscriptions`); + url.searchParams.append('limit', API_LIMIT.toString()); + url.searchParams.append('status', 'active'); + url.searchParams.append('expand[]', 'custom_fields'); + + if (cursor) { + url.searchParams.append('cursor', cursor); + } + + const response = await authedClient(key, url); + + if (!response.ok) { + throw new errors.InternalServerError({message: `Request failed: ${response.status} ${response.statusText}`}); + } + + const data: BeehiivSubscriptionsResponse = await response.json(); + + await fileCache.writeTmpFile(data, filename); + + return data; +}; + +export const fetchTasks = async (options: any, ctx: any) => { + const totalSubscriptions = await discover(options.key, options.id); + const estimatedPages = Math.ceil(totalSubscriptions / API_LIMIT); + + const tasks = [ + { + title: `Fetching subscriptions (estimated ${estimatedPages} pages)`, + task: async (_: any, task: any) => { + let cursor: string | null = null; + let hasMore = true; + let cursorIndex = 0; + + ctx.result.subscriptions = []; + + while (hasMore) { + try { + const response: BeehiivSubscriptionsResponse = await cachedFetch({ + fileCache: ctx.fileCache, + key: options.key, + pubId: options.id, + cursor, + cursorIndex + }); + + ctx.result.subscriptions = ctx.result.subscriptions.concat(response.data); + hasMore = response.has_more; + cursor = response.next_cursor; + cursorIndex += 1; + + task.output = `Fetched ${ctx.result.subscriptions.length} of ${totalSubscriptions} subscriptions`; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + task.output = errorMessage; + throw error; + } + } + + task.output = `Fetched ${ctx.result.subscriptions.length} subscriptions`; + } + } + ]; + + return tasks; +}; + +export { + authedClient, + discover, + cachedFetch +}; diff --git a/packages/mg-beehiiv-api-members/src/lib/list-pubs.ts b/packages/mg-beehiiv-api-members/src/lib/list-pubs.ts new file mode 100644 index 000000000..2197557bd --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/lib/list-pubs.ts @@ -0,0 +1,21 @@ +import errors from '@tryghost/errors'; +import {authedClient} from './fetch.js'; + +const listPublications = async (apiKey: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications`); + url.searchParams.append('expand[]', 'stats'); + + const response = await authedClient(apiKey, url); + + if (!response.ok) { + throw new errors.InternalServerError({message: `Request failed: ${response.status} ${response.statusText}`}); + } + + const data = await response.json(); + + return data.data; +}; + +export { + listPublications +}; diff --git a/packages/mg-beehiiv-api-members/src/lib/mapper.ts b/packages/mg-beehiiv-api-members/src/lib/mapper.ts new file mode 100644 index 000000000..d202e0050 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/lib/mapper.ts @@ -0,0 +1,106 @@ +import {slugify} from '@tryghost/string'; + +const extractName = (customFields: Array<{name: string; value: string}>): string | null => { + const firstNameField = customFields.find(f => f.name.toLowerCase() === 'first_name' || f.name.toLowerCase() === 'firstname'); + const lastNameField = customFields.find(f => f.name.toLowerCase() === 'last_name' || f.name.toLowerCase() === 'lastname'); + + const firstName = firstNameField?.value?.trim() || ''; + const lastName = lastNameField?.value?.trim() || ''; + + const combinedName = [firstName, lastName].filter(name => name.length > 0).join(' '); + + return combinedName.length > 0 ? combinedName : null; +}; + +const mapSubscription = (subscription: BeehiivSubscription): GhostMemberObject => { + const labels: string[] = []; + + // Add status label + labels.push(`beehiiv-status-${subscription.status}`); + + // Add tier label + labels.push(`beehiiv-tier-${subscription.subscription_tier}`); + + // Add premium tier names as labels + if (subscription.subscription_premium_tier_names && subscription.subscription_premium_tier_names.length > 0) { + subscription.subscription_premium_tier_names.forEach((tierName: string) => { + const slugifiedTier = slugify(tierName); + labels.push(`beehiiv-premium-${slugifiedTier}`); + }); + } + + // Add tags as labels + if (subscription.tags && subscription.tags.length > 0) { + subscription.tags.forEach((tag: string) => { + const slugifiedTag = slugify(tag); + labels.push(`beehiiv-tag-${slugifiedTag}`); + }); + } + + // Determine if this is a complimentary plan + // A member is on a complimentary plan if they have premium access but no Stripe customer ID + const isPremium = subscription.subscription_tier === 'premium'; + const hasStripeId = Boolean(subscription.stripe_customer_id); + const complimentaryPlan = isPremium && !hasStripeId; + + return { + email: subscription.email, + name: extractName(subscription.custom_fields || []), + note: null, + subscribed_to_emails: subscription.status === 'active', + stripe_customer_id: subscription.stripe_customer_id || '', + complimentary_plan: complimentaryPlan, + labels, + created_at: new Date(subscription.created * 1000) + }; +}; + +const mapSubscriptions = (subscriptions: BeehiivSubscription[]): MappedMembers => { + const result: MappedMembers = { + free: [], + paid: [] + }; + + subscriptions.forEach((subscription) => { + const member = mapSubscription(subscription); + + if (member.stripe_customer_id) { + result.paid.push(member); + } else { + result.free.push(member); + } + }); + + return result; +}; + +export const mapMembersTasks = (_options: any, ctx: any) => { + const tasks = [ + { + title: 'Mapping subscriptions to Ghost member format', + task: async (_: any, task: any) => { + try { + const subscriptions: BeehiivSubscription[] = ctx.result.subscriptions || []; + ctx.result.members = mapSubscriptions(subscriptions); + + const freeCount = ctx.result.members.free.length; + const paidCount = ctx.result.members.paid.length; + + task.output = `Mapped ${freeCount} free and ${paidCount} paid members`; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + task.output = errorMessage; + throw error; + } + } + } + ]; + + return tasks; +}; + +export { + extractName, + mapSubscription, + mapSubscriptions +}; diff --git a/packages/mg-beehiiv-api-members/src/test/fetch.test.ts b/packages/mg-beehiiv-api-members/src/test/fetch.test.ts new file mode 100644 index 000000000..3ebc1bbdd --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/test/fetch.test.ts @@ -0,0 +1,357 @@ +import assert from 'node:assert/strict'; +import {describe, it, beforeEach, afterEach, mock} from 'node:test'; +import {listPublications} from '../lib/list-pubs.js'; +import {fetchTasks, authedClient, discover, cachedFetch} from '../lib/fetch.js'; + +describe('beehiiv API Members Fetch', () => { + let fetchMock: any; + + beforeEach(() => { + fetchMock = mock.method(global, 'fetch', () => Promise.resolve()); + }); + + afterEach(() => { + mock.restoreAll(); + }); + + describe('authedClient', () => { + it('makes authenticated GET request', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ok: true, json: () => Promise.resolve({data: []})})); + + const url = new URL('https://api.beehiiv.com/v2/publications'); + await authedClient('test-api-key', url); + + assert.equal(fetchMock.mock.callCount(), 1); + const [calledUrl, options] = fetchMock.mock.calls[0].arguments; + assert.equal(calledUrl.toString(), 'https://api.beehiiv.com/v2/publications'); + assert.equal(options.method, 'GET'); + assert.equal(options.headers.Authorization, 'Bearer test-api-key'); + }); + }); + + describe('listPublications', () => { + it('fetches and returns publications', async () => { + const mockPubs = [{id: 'pub-1', name: 'Test Pub'}]; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({data: mockPubs}) + })); + + const result = await listPublications('test-key'); + + assert.deepEqual(result, mockPubs); + }); + + it('throws on API error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 401, + statusText: 'Unauthorized' + })); + + await assert.rejects(async () => { + await listPublications('invalid-key'); + }, /Request failed: 401 Unauthorized/); + }); + }); + + describe('discover', () => { + it('returns total subscription count', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({data: {stats: {active_subscriptions: 1500}}}) + })); + + const result = await discover('test-key', 'pub-123'); + + assert.equal(result, 1500); + }); + + it('throws on API error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 403, + statusText: 'Forbidden' + })); + + await assert.rejects(async () => { + await discover('test-key', 'pub-123'); + }, /Request failed: 403 Forbidden/); + }); + }); + + describe('cachedFetch', () => { + it('returns cached data when available', async () => { + const cachedData = {data: [{id: 'sub-1', email: 'test@example.com'}], has_more: false}; + const fileCache = { + hasFile: () => true, + readTmpJSONFile: () => Promise.resolve(cachedData) + }; + + const result = await cachedFetch({ + fileCache, + key: 'test-key', + pubId: 'pub-123', + cursor: null, + cursorIndex: 0 + }); + + assert.deepEqual(result, cachedData); + assert.equal(fetchMock.mock.callCount(), 0); + }); + + it('fetches from API when not cached', async () => { + const apiData = {data: [{id: 'sub-1', email: 'test@example.com'}], has_more: false, next_cursor: null}; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve(apiData) + })); + + const writeTmpFileMock = mock.fn(() => Promise.resolve()); + const fileCache = { + hasFile: () => false, + writeTmpFile: writeTmpFileMock + }; + + const result = await cachedFetch({ + fileCache, + key: 'test-key', + pubId: 'pub-123', + cursor: null, + cursorIndex: 0 + }); + + assert.deepEqual(result, apiData); + assert.equal(writeTmpFileMock.mock.callCount(), 1); + }); + + it('includes cursor when provided', async () => { + const apiData = {data: [], has_more: false, next_cursor: null}; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve(apiData) + })); + + const fileCache = { + hasFile: () => false, + writeTmpFile: mock.fn(() => Promise.resolve()) + }; + + await cachedFetch({ + fileCache, + key: 'test-key', + pubId: 'pub-123', + cursor: 'cursor-abc', + cursorIndex: 1 + }); + + const [calledUrl] = fetchMock.mock.calls[0].arguments; + assert.ok(calledUrl.toString().includes('cursor=cursor-abc')); + }); + + it('throws on API error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 500, + statusText: 'Internal Server Error' + })); + + const fileCache = { + hasFile: () => false, + writeTmpFile: mock.fn(() => Promise.resolve()) + }; + + await assert.rejects(async () => { + await cachedFetch({ + fileCache, + key: 'test-key', + pubId: 'pub-123', + cursor: null, + cursorIndex: 0 + }); + }, /Request failed: 500 Internal Server Error/); + }); + }); + + describe('fetchTasks', () => { + it('creates a single task that fetches all subscriptions', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 150, data: [], has_more: true}) + }), 0); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + + assert.equal(tasks.length, 1); + assert.ok(tasks[0].title.includes('Fetching subscriptions')); + }); + + it('task fetches all pages using cursor pagination', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 3, data: [], has_more: true}) + }), 0); + + // Mock first page + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [{id: 'sub-1', email: 'a@test.com'}], + has_more: true, + next_cursor: 'cursor-1' + }) + }), 1); + + // Mock second page + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [{id: 'sub-2', email: 'b@test.com'}], + has_more: false, + next_cursor: null + }) + }), 2); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx: any = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.subscriptions.length, 2); + assert.equal(ctx.result.subscriptions[0].id, 'sub-1'); + assert.equal(ctx.result.subscriptions[1].id, 'sub-2'); + }); + + it('task uses cached data when available', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 1, data: [], has_more: false}) + }), 0); + + const cachedData = { + data: [{id: 'cached-sub', email: 'cached@test.com'}], + has_more: false, + next_cursor: null + }; + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx: any = { + fileCache: { + hasFile: () => true, + readTmpJSONFile: () => Promise.resolve(cachedData) + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.subscriptions.length, 1); + assert.equal(ctx.result.subscriptions[0].id, 'cached-sub'); + // fetch should only be called once (for discover) + assert.equal(fetchMock.mock.callCount(), 1); + }); + + it('task throws and sets output on fetch error', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: [], has_more: true}) + }), 0); + + // Mock a failed fetch + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: false, + status: 500, + statusText: 'Internal Server Error' + }), 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }, /Request failed: 500 Internal Server Error/); + + assert.ok(mockTask.output.includes('500')); + }); + + it('task handles non-Error thrown objects', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: [], has_more: true}) + }), 0); + + // Mock fetch that throws a non-Error value + fetchMock.mock.mockImplementationOnce(() => { + // eslint-disable-next-line no-throw-literal + throw 'Network error string'; + }, 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {} + }; + + const tasks = await fetchTasks(options, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + + assert.equal(mockTask.output, 'Network error string'); + }); + + it('handles discover error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 401, + statusText: 'Unauthorized' + })); + + const options = {key: 'invalid-key', id: 'pub-123'}; + const ctx = { + fileCache: {}, + result: {} + }; + + await assert.rejects(async () => { + await fetchTasks(options, ctx); + }, /Request failed: 401 Unauthorized/); + }); + }); +}); diff --git a/packages/mg-beehiiv-api-members/src/test/index.test.ts b/packages/mg-beehiiv-api-members/src/test/index.test.ts new file mode 100644 index 000000000..e26d11826 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/test/index.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import beehiivApiMembers from '../index.js'; + +describe('beehiiv API Members Package', () => { + it('exports listPublications', () => { + assert.ok(typeof beehiivApiMembers.listPublications === 'function'); + }); + + it('exports fetchTasks', () => { + assert.ok(typeof beehiivApiMembers.fetchTasks === 'function'); + }); + + it('exports mapMembersTasks', () => { + assert.ok(typeof beehiivApiMembers.mapMembersTasks === 'function'); + }); +}); diff --git a/packages/mg-beehiiv-api-members/src/test/mapper.test.ts b/packages/mg-beehiiv-api-members/src/test/mapper.test.ts new file mode 100644 index 000000000..96b5c2912 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/test/mapper.test.ts @@ -0,0 +1,374 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import {extractName, mapSubscription, mapSubscriptions, mapMembersTasks} from '../lib/mapper.js'; + +describe('beehiiv API Members Mapper', () => { + describe('extractName', () => { + it('combines first and last name', () => { + const customFields = [ + {name: 'first_name', value: 'John'}, + {name: 'last_name', value: 'Doe'} + ]; + assert.equal(extractName(customFields), 'John Doe'); + }); + + it('handles only first name', () => { + const customFields = [{name: 'first_name', value: 'Jane'}]; + assert.equal(extractName(customFields), 'Jane'); + }); + + it('handles only last name', () => { + const customFields = [{name: 'last_name', value: 'Smith'}]; + assert.equal(extractName(customFields), 'Smith'); + }); + + it('returns null for empty custom fields', () => { + assert.equal(extractName([]), null); + }); + + it('returns null when name fields are empty strings', () => { + const customFields = [ + {name: 'first_name', value: ''}, + {name: 'last_name', value: ''} + ]; + assert.equal(extractName(customFields), null); + }); + + it('trims whitespace from names', () => { + const customFields = [ + {name: 'first_name', value: ' John '}, + {name: 'last_name', value: ' Doe '} + ]; + assert.equal(extractName(customFields), 'John Doe'); + }); + + it('handles alternate field names (firstname/lastname)', () => { + const customFields = [ + {name: 'firstname', value: 'Jane'}, + {name: 'lastname', value: 'Doe'} + ]; + assert.equal(extractName(customFields), 'Jane Doe'); + }); + + it('handles case-insensitive field names', () => { + const customFields = [ + {name: 'First_Name', value: 'Jane'}, + {name: 'Last_Name', value: 'Doe'} + ]; + assert.equal(extractName(customFields), 'Jane Doe'); + }); + }); + + describe('mapSubscription', () => { + const baseSubscription: BeehiivSubscription = { + id: 'sub-123', + email: 'test@example.com', + status: 'active', + created: 1704067200, // 2024-01-01 00:00:00 UTC + subscription_tier: 'free', + subscription_premium_tier_names: [], + stripe_customer_id: null, + custom_fields: [], + tags: [] + }; + + it('maps basic subscription fields', () => { + const result = mapSubscription(baseSubscription); + + assert.equal(result.email, 'test@example.com'); + assert.equal(result.name, null); + assert.equal(result.note, null); + assert.equal(result.subscribed_to_emails, true); + assert.equal(result.stripe_customer_id, ''); + assert.equal(result.complimentary_plan, false); + assert.deepEqual(result.created_at, new Date(1704067200 * 1000)); + }); + + it('sets subscribed_to_emails true for active status', () => { + const subscription = {...baseSubscription, status: 'active' as const}; + const result = mapSubscription(subscription); + assert.equal(result.subscribed_to_emails, true); + }); + + it('sets subscribed_to_emails false for inactive status', () => { + const subscription = {...baseSubscription, status: 'inactive' as const}; + const result = mapSubscription(subscription); + assert.equal(result.subscribed_to_emails, false); + }); + + it('sets subscribed_to_emails false for validating status', () => { + const subscription = {...baseSubscription, status: 'validating' as const}; + const result = mapSubscription(subscription); + assert.equal(result.subscribed_to_emails, false); + }); + + it('sets subscribed_to_emails false for pending status', () => { + const subscription = {...baseSubscription, status: 'pending' as const}; + const result = mapSubscription(subscription); + assert.equal(result.subscribed_to_emails, false); + }); + + it('maps stripe_customer_id when present', () => { + const subscription = {...baseSubscription, stripe_customer_id: 'cus_abc123'}; + const result = mapSubscription(subscription); + assert.equal(result.stripe_customer_id, 'cus_abc123'); + }); + + it('sets complimentary_plan true for premium without stripe_customer_id', () => { + const subscription = { + ...baseSubscription, + subscription_tier: 'premium' as const, + stripe_customer_id: null + }; + const result = mapSubscription(subscription); + assert.equal(result.complimentary_plan, true); + }); + + it('sets complimentary_plan false for premium with stripe_customer_id', () => { + const subscription = { + ...baseSubscription, + subscription_tier: 'premium' as const, + stripe_customer_id: 'cus_abc123' + }; + const result = mapSubscription(subscription); + assert.equal(result.complimentary_plan, false); + }); + + it('sets complimentary_plan false for free tier', () => { + const subscription = {...baseSubscription, subscription_tier: 'free' as const}; + const result = mapSubscription(subscription); + assert.equal(result.complimentary_plan, false); + }); + + it('extracts name from custom_fields', () => { + const subscription = { + ...baseSubscription, + custom_fields: [ + {name: 'first_name', value: 'John'}, + {name: 'last_name', value: 'Doe'} + ] + }; + const result = mapSubscription(subscription); + assert.equal(result.name, 'John Doe'); + }); + + it('adds status label', () => { + const result = mapSubscription(baseSubscription); + assert.ok(result.labels.includes('beehiiv-status-active')); + }); + + it('adds tier label', () => { + const result = mapSubscription(baseSubscription); + assert.ok(result.labels.includes('beehiiv-tier-free')); + }); + + it('adds premium tier names as labels', () => { + const subscription = { + ...baseSubscription, + subscription_tier: 'premium' as const, + subscription_premium_tier_names: ['Gold Plan', 'VIP Access'] + }; + const result = mapSubscription(subscription); + assert.ok(result.labels.includes('beehiiv-premium-gold-plan')); + assert.ok(result.labels.includes('beehiiv-premium-vip-access')); + }); + + it('adds tags as labels', () => { + const subscription = { + ...baseSubscription, + tags: ['Newsletter', 'Tech Updates'] + }; + const result = mapSubscription(subscription); + assert.ok(result.labels.includes('beehiiv-tag-newsletter')); + assert.ok(result.labels.includes('beehiiv-tag-tech-updates')); + }); + + it('handles empty premium tier names array', () => { + const subscription = { + ...baseSubscription, + subscription_premium_tier_names: [] + }; + const result = mapSubscription(subscription); + assert.ok(!result.labels.some(l => l.startsWith('beehiiv-premium-'))); + }); + + it('handles empty tags array', () => { + const subscription = { + ...baseSubscription, + tags: [] + }; + const result = mapSubscription(subscription); + assert.ok(!result.labels.some(l => l.startsWith('beehiiv-tag-'))); + }); + + it('handles undefined custom_fields', () => { + const subscription = { + ...baseSubscription, + custom_fields: undefined as any + }; + const result = mapSubscription(subscription); + assert.equal(result.name, null); + }); + }); + + describe('mapSubscriptions', () => { + it('splits subscriptions into free and paid based on stripe_customer_id', () => { + const subscriptions: BeehiivSubscription[] = [ + { + id: 'sub-1', + email: 'free@test.com', + status: 'active', + created: 1704067200, + subscription_tier: 'free', + subscription_premium_tier_names: [], + stripe_customer_id: null, + custom_fields: [], + tags: [] + }, + { + id: 'sub-2', + email: 'paid@test.com', + status: 'active', + created: 1704067200, + subscription_tier: 'premium', + subscription_premium_tier_names: [], + stripe_customer_id: 'cus_paid123', + custom_fields: [], + tags: [] + } + ]; + + const result = mapSubscriptions(subscriptions); + + assert.equal(result.free.length, 1); + assert.equal(result.paid.length, 1); + assert.equal(result.free[0].email, 'free@test.com'); + assert.equal(result.paid[0].email, 'paid@test.com'); + }); + + it('returns empty arrays for empty input', () => { + const result = mapSubscriptions([]); + assert.deepEqual(result, {free: [], paid: []}); + }); + + it('categorizes complimentary premium as free', () => { + const subscriptions: BeehiivSubscription[] = [ + { + id: 'sub-1', + email: 'comp@test.com', + status: 'active', + created: 1704067200, + subscription_tier: 'premium', + subscription_premium_tier_names: [], + stripe_customer_id: null, // No stripe ID means complimentary + custom_fields: [], + tags: [] + } + ]; + + const result = mapSubscriptions(subscriptions); + + assert.equal(result.free.length, 1); + assert.equal(result.paid.length, 0); + assert.equal(result.free[0].complimentary_plan, true); + }); + }); + + describe('mapMembersTasks', () => { + it('creates a single mapping task', () => { + const ctx = {result: {subscriptions: []}}; + const tasks = mapMembersTasks({}, ctx); + + assert.equal(tasks.length, 1); + assert.equal(tasks[0].title, 'Mapping subscriptions to Ghost member format'); + }); + + it('task maps subscriptions and stores result', async () => { + const ctx: any = { + result: { + subscriptions: [ + { + id: 'sub-1', + email: 'test@test.com', + status: 'active', + created: 1704067200, + subscription_tier: 'free', + subscription_premium_tier_names: [], + stripe_customer_id: null, + custom_fields: [], + tags: [] + } + ] + } + }; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + await tasks[0].task({}, mockTask); + + assert.ok(ctx.result.members); + assert.equal(ctx.result.members.free.length, 1); + assert.equal(ctx.result.members.paid.length, 0); + assert.ok(mockTask.output.includes('1 free')); + assert.ok(mockTask.output.includes('0 paid')); + }); + + it('task handles empty subscriptions', async () => { + const ctx: any = {result: {subscriptions: []}}; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + await tasks[0].task({}, mockTask); + + assert.deepEqual(ctx.result.members, {free: [], paid: []}); + }); + + it('task handles undefined subscriptions', async () => { + const ctx: any = {result: {}}; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + await tasks[0].task({}, mockTask); + + assert.deepEqual(ctx.result.members, {free: [], paid: []}); + }); + + it('task sets error output on failure with Error instance', async () => { + const ctx = { + result: { + subscriptions: 'not an array' // Invalid data + } + }; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + + assert.ok(mockTask.output.length > 0); + }); + + it('task sets error output on failure with non-Error thrown', async () => { + // Create a context that will trigger an error through a getter that throws a string + const ctx: any = { + result: { + get subscriptions() { + // eslint-disable-next-line no-throw-literal + throw 'Custom string error'; + } + } + }; + + const tasks = mapMembersTasks({}, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + + assert.equal(mockTask.output, 'Custom string error'); + }); + }); +}); diff --git a/packages/mg-beehiiv-api-members/src/types.d.ts b/packages/mg-beehiiv-api-members/src/types.d.ts new file mode 100644 index 000000000..3695466f7 --- /dev/null +++ b/packages/mg-beehiiv-api-members/src/types.d.ts @@ -0,0 +1,46 @@ +declare module '@tryghost/errors'; +declare module '@tryghost/mg-fs-utils'; +declare module '@tryghost/string'; + +type BeehiivSubscription = { + id: string; + email: string; + status: 'active' | 'inactive' | 'validating' | 'pending'; + created: number; + subscription_tier: 'free' | 'premium'; + subscription_premium_tier_names: string[]; + stripe_customer_id: string | null; + custom_fields: Array<{name: string; value: string}>; + tags: string[]; +}; + +type BeehiivPublicationResponse = { + data: { + stats: { + active_subscriptions: number; + } + } +} + +type BeehiivSubscriptionsResponse = { + data: BeehiivSubscription[]; + total_results: number; + has_more: boolean; + next_cursor: string | null; +}; + +type GhostMemberObject = { + email: string; + name: string | null; + note: string | null; + subscribed_to_emails: boolean; + stripe_customer_id: string; + complimentary_plan: boolean; + labels: string[]; + created_at: Date; +}; + +type MappedMembers = { + free: GhostMemberObject[]; + paid: GhostMemberObject[]; +}; diff --git a/packages/mg-beehiiv-api-members/tsconfig.json b/packages/mg-beehiiv-api-members/tsconfig.json new file mode 100644 index 000000000..322068b4d --- /dev/null +++ b/packages/mg-beehiiv-api-members/tsconfig.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + + /* Language and Environment */ + "target": "es2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + + /* Modules */ + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": "src", /* Specify the root folder within your source files. */ + + /* Emit */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + "outDir": "build", /* Specify an output folder for all emitted files. */ + + /* Interop Constraints */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + + /* Completeness */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "include": ["src/**/*"] +} diff --git a/packages/mg-beehiiv-api/.eslintrc.cjs b/packages/mg-beehiiv-api/.eslintrc.cjs new file mode 100644 index 000000000..cd6f552f4 --- /dev/null +++ b/packages/mg-beehiiv-api/.eslintrc.cjs @@ -0,0 +1,12 @@ +module.exports = { + parser: '@typescript-eslint/parser', + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ], + rules: { + 'no-unused-vars': 'off', // doesn't work with typescript + 'no-undef': 'off', // doesn't work with typescript + 'ghost/filenames/match-regex': 'off' + } +}; diff --git a/packages/mg-beehiiv-api/.gitignore b/packages/mg-beehiiv-api/.gitignore new file mode 100644 index 000000000..bd7611769 --- /dev/null +++ b/packages/mg-beehiiv-api/.gitignore @@ -0,0 +1,4 @@ +/build +/tsconfig.tsbuildinfo +/tmp +.env diff --git a/packages/mg-beehiiv-api/LICENSE b/packages/mg-beehiiv-api/LICENSE new file mode 100644 index 000000000..efad547e8 --- /dev/null +++ b/packages/mg-beehiiv-api/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013-2026 Ghost Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/mg-beehiiv-api/README.md b/packages/mg-beehiiv-api/README.md new file mode 100644 index 000000000..478ae3d63 --- /dev/null +++ b/packages/mg-beehiiv-api/README.md @@ -0,0 +1,3 @@ +# Migrate beehiiv API + +... diff --git a/packages/mg-beehiiv-api/package.json b/packages/mg-beehiiv-api/package.json new file mode 100644 index 000000000..9fc2e65d4 --- /dev/null +++ b/packages/mg-beehiiv-api/package.json @@ -0,0 +1,51 @@ +{ + "name": "@tryghost/mg-beehiiv-api", + "version": "0.1.0", + "repository": "https://github.com/TryGhost/migrate/tree/main/packages/mg-beehiiv-api", + "author": "Ghost Foundation", + "license": "MIT", + "type": "module", + "main": "build/index.js", + "types": "build/types.d.ts", + "exports": { + ".": { + "development": "./src/index.ts", + "default": "./build/index.js" + } + }, + "scripts": { + "dev": "echo \"Implement me!\"", + "build:watch": "tsc --watch --preserveWatchOutput --sourceMap", + "build": "rm -rf build && rm -rf tsconfig.tsbuildinfo && tsc --build --sourceMap", + "prepare": "yarn build", + "lint": "eslint src/ --ext .ts --cache", + "posttest": "yarn lint", + "test": "rm -rf build && yarn build --force && c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura node --test build/test/*.test.js" + }, + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@types/lodash": "4.17.23", + "@typescript-eslint/eslint-plugin": "^8.0.0", + "@typescript-eslint/parser": "^8.0.0", + "c8": "10.1.3", + "dotenv": "17.2.3", + "eslint": "8.57.0", + "typescript": "5.9.3" + }, + "dependencies": { + "@tryghost/debug": "0.1.35", + "@tryghost/errors": "1.3.8", + "@tryghost/kg-default-cards": "10.2.5", + "@tryghost/mg-fs-utils": "0.12.14", + "@tryghost/mg-utils": "0.1.1", + "@tryghost/string": "0.2.21", + "lodash": "4.17.23", + "sanitize-html": "2.17.0", + "simple-dom": "1.4.0" + } +} diff --git a/packages/mg-beehiiv-api/src/index.ts b/packages/mg-beehiiv-api/src/index.ts new file mode 100644 index 000000000..294946b59 --- /dev/null +++ b/packages/mg-beehiiv-api/src/index.ts @@ -0,0 +1,9 @@ +import {listPublications} from './lib/list-pubs.js'; +import {mapPostsTasks} from './lib/mapper.js'; +import {fetchTasks} from './lib/fetch.js'; + +export default { + listPublications, + fetchTasks, + mapPostsTasks +}; diff --git a/packages/mg-beehiiv-api/src/lib/client.ts b/packages/mg-beehiiv-api/src/lib/client.ts new file mode 100644 index 000000000..4ca754071 --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/client.ts @@ -0,0 +1,24 @@ +import errors from '@tryghost/errors'; + +const client = async (apiKey: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications`); + + const response = await fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}` + } + }); + + if (!response.ok) { + throw new errors.InternalServerError({message: `Request failed: ${response.status} ${response.statusText}`}); + } + + const data = await response.json(); + + return data.data; +}; + +export { + client +}; diff --git a/packages/mg-beehiiv-api/src/lib/fetch.ts b/packages/mg-beehiiv-api/src/lib/fetch.ts new file mode 100644 index 000000000..a0e3a9336 --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/fetch.ts @@ -0,0 +1,96 @@ +import errors from '@tryghost/errors'; + +const API_LIMIT = 10; + +const authedClient = async (apiKey: string, theUrl: URL) => { + return fetch(theUrl, { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}` + } + }); +}; + +const discover = async (key: string, pubId: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications/${pubId}/posts`); + url.searchParams.append('limit', '1'); + + const response = await authedClient(key, url); + + if (!response.ok) { + throw new errors.InternalServerError({message: `Request failed: ${response.status} ${response.statusText}`}); + } + + const data = await response.json(); + + return data.total_results; +}; + +const cachedFetch = async ({fileCache, key, pubId, limit = API_LIMIT, page}: {fileCache: any, key: string, pubId: string, limit?: number, page: number}) => { + let filename = `beehiiv_api_${limit}_${page}.json`; + + if (fileCache.hasFile(filename, 'tmp')) { + return await fileCache.readTmpJSONFile(filename); + } + + const url = new URL(`https://api.beehiiv.com/v2/publications/${pubId}/posts`); + url.searchParams.append('limit', limit.toString()); + url.searchParams.append('expand', 'free_web_content'); + url.searchParams.append('expand', 'premium_web_content'); + url.searchParams.append('page', page.toString()); + + const response = await authedClient(key, url); + + if (!response.ok) { + throw new errors.InternalServerError({message: `Request failed: ${response.status} ${response.statusText}`}); + } + + const data = await response.json(); + + await fileCache.writeTmpFile(data, filename); + + return data; +}; + +export const fetchTasks = async (options: any, ctx: any) => { + const numberOfPosts = await discover(options.key, options.id); + const totalPages = Math.ceil(numberOfPosts / API_LIMIT); + + let tasks = []; + + for (let page = 1; page <= totalPages; page++) { + tasks.push({ + title: `Fetching posts page ${page} of ${totalPages}`, + task: async (_: any, task: any) => { + try { + let response = await cachedFetch({fileCache: ctx.fileCache ,key: options.key, pubId: options.id, page: page}); + + ctx.result.posts = ctx.result.posts.concat(response.data); + + if (options.postsAfter) { + const afterTimestamp = new Date(options.postsAfter).getTime() / 1000; + ctx.result.posts = ctx.result.posts.filter( + (post: any) => post.publish_date >= afterTimestamp + ); + } + if (options.postsBefore) { + const beforeTimestamp = new Date(options.postsBefore).getTime() / 1000; + ctx.result.posts = ctx.result.posts.filter( + (post: any) => post.publish_date <= beforeTimestamp + ); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + task.output = errorMessage; + throw error; + } + } + }); + } + + return tasks; +}; + +export { + authedClient +}; diff --git a/packages/mg-beehiiv-api/src/lib/list-pubs.ts b/packages/mg-beehiiv-api/src/lib/list-pubs.ts new file mode 100644 index 000000000..2197557bd --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/list-pubs.ts @@ -0,0 +1,21 @@ +import errors from '@tryghost/errors'; +import {authedClient} from './fetch.js'; + +const listPublications = async (apiKey: string) => { + const url = new URL(`https://api.beehiiv.com/v2/publications`); + url.searchParams.append('expand[]', 'stats'); + + const response = await authedClient(apiKey, url); + + if (!response.ok) { + throw new errors.InternalServerError({message: `Request failed: ${response.status} ${response.statusText}`}); + } + + const data = await response.json(); + + return data.data; +}; + +export { + listPublications +}; diff --git a/packages/mg-beehiiv-api/src/lib/mapper.ts b/packages/mg-beehiiv-api/src/lib/mapper.ts new file mode 100644 index 000000000..774089f99 --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/mapper.ts @@ -0,0 +1,102 @@ +// import fsUtils from '@tryghost/mg-fs-utils'; +import {slugify} from '@tryghost/string'; +import {processHTML, removeDuplicateFeatureImage} from './process.js'; + +const mapPost = ({postData, options}: {postData: beehiivPostDataObject, options?: any}) => { + const mappedData: mappedDataObject = { + url: postData.web_url, + data: { + comment_id: postData.id, + slug: postData.slug, + published_at: new Date(postData.publish_date * 1000), + updated_at: new Date(postData.publish_date * 1000), + created_at: new Date(postData.created * 1000), + title: postData.title, + type: 'post', + html: postData.content.premium.web, + status: (postData.status === 'confirmed') ? 'published' : 'draft', + custom_excerpt: postData.subtitle ?? null, + visibility: (postData.audience === 'premium') ? 'paid' : 'public', + authors: [], + tags: [] + } + }; + + if (postData.thumbnail_url) { + mappedData.data.feature_image = postData.thumbnail_url; + } + + if (postData.meta_default_title) { + mappedData.data.og_title = postData.meta_default_title; + } + + if (postData.meta_default_description) { + mappedData.data.og_description = postData.meta_default_description; + } + + mappedData.data.html = processHTML({ + post: mappedData, + options + }); + + postData.authors.forEach((author: string) => { + const authorSlug = slugify(author); + mappedData.data.authors.push({ + url: `migrator-added-author-${authorSlug}`, + data: { + slug: authorSlug, + name: author, + email: `${authorSlug}@example.com` + } + }); + }); + + postData.content_tags.forEach((tag: string) => { + const tagSlug = slugify(tag); + mappedData.data.tags.push({ + url: `migrator-added-tag-${tagSlug}`, + data: { + slug: tagSlug, + name: tag + } + }); + }); + + mappedData.data.tags.push({ + url: 'migrator-added-tag-hash-beehiiv', + data: { + slug: 'hash-beehiiv', + name: '#beehiiv' + } + }); + + return mappedData; +}; + +const mapPostsTasks = async (options: any, ctx: any) => { + let tasks: any = []; + + ctx.result.posts.forEach((postData: any, index: number) => { + tasks.push({ + title: `Mapping post: ${postData.title}`, + task: async (_: any, task: any) => { + try { + const mappedPost = mapPost({postData, options}); + ctx.result.posts[index] = mappedPost; + } catch (error) { + /* c8 ignore next */ + const errorMessage = error instanceof Error ? error.message : String(error); + task.output = errorMessage; + throw error; + } + } + }); + }); + + return tasks; +}; + +export { + mapPost, + mapPostsTasks +}; diff --git a/packages/mg-beehiiv-api/src/lib/process.ts b/packages/mg-beehiiv-api/src/lib/process.ts new file mode 100644 index 000000000..134f249e0 --- /dev/null +++ b/packages/mg-beehiiv-api/src/lib/process.ts @@ -0,0 +1,557 @@ +import {domUtils} from '@tryghost/mg-utils'; +import _ from 'lodash'; +import sanitizeHtml from 'sanitize-html'; +import SimpleDom from 'simple-dom'; +import imageCard from '@tryghost/kg-default-cards/lib/cards/image.js'; +import embedCard from '@tryghost/kg-default-cards/lib/cards/embed.js'; +import bookmarkCard from '@tryghost/kg-default-cards/lib/cards/bookmark.js'; + +const {parseFragment, serializeChildren, serializeNode, replaceWith, insertAfter, attr, parents} = domUtils; + +const serializer = new SimpleDom.HTMLSerializer(SimpleDom.voidMap); + +const getYouTubeID = (url: string) => { + const arr = url.split(/(vi\/|v%3D|v=|\/v\/|youtu\.be\/|\/embed\/)/); + return undefined !== arr[2] ? arr[2].split(/[^\w-]/i)[0] : arr[0]; +}; + +const isURL = (urlString: string | undefined) => { + if (undefined === urlString) { + return false; + } + + try { + new URL(urlString); + return true; + } catch (err) { + return false; + } +}; + +const processHTML = ({post, options}: {post?: mappedDataObject, options?: any}) => { + // First, clean up the email HTML to remove bits we don't want or change up + + let html = post?.data.html ?? ''; + + // Let's do some regexp magic to remove some beehiiv variables + // https://support.beehiiv.com/hc/en-us/articles/7606088263191 + html = html.replace(/{{subscriber_id}}/g, ' '); + html = html.replace(/{{rp_refer_url}}/g, '#'); + html = html.replace(/{{rp_refer_url_no_params}}/g, '#'); + + const allParsed = parseFragment(html); + const contentBlocksEl = allParsed.$('#content-blocks')[0]; + const contentBlocksHtml = contentBlocksEl ? serializeChildren(contentBlocksEl) : ''; + const parsed = parseFragment(contentBlocksHtml); + + // Convert divs with no content but a border-top to a HR + parsed.$('div').forEach((el) => { + const styleAttr = attr(el, 'style') || ''; + const hasBorderTop = styleAttr.includes('border-top'); + const hasContent = serializeChildren(el).trim().length > 0; + + if (hasBorderTop && !hasContent) { + replaceWith(el, '
'); + } + }); + + // Convert images to Ghost image cards + parsed.$('img').forEach((el) => { + // Skip images inside generic embeds (handled separately as bookmark cards) + if (parents(el, '.generic-embed--root').length > 0) { + return; + } + + const src = attr(el, 'src'); + const altText = attr(el, 'alt') || ''; + const parent = el.parentElement!; + + const cardOpts: any = { + env: {dom: new SimpleDom.Document()}, + payload: { + src, + alt: altText, + caption: null + } + }; + + if (parent.tagName === 'DIV') { + const caption = parsed.$('small', parent).map(s => s.textContent!.trim()).filter(Boolean).join('') || null; + cardOpts.payload.caption = caption; + const outerDiv = parent.parentElement; + const target = outerDiv && outerDiv.tagName === 'DIV' ? outerDiv : parent; + replaceWith(target, serializer.serialize(imageCard.render(cardOpts))); + } else { + el.remove(); + insertAfter(parent, serializer.serialize(imageCard.render(cardOpts))); + } + }); + + // Convert nested padding-left divs to blockquotes + parsed.$('div[style*="padding-left"] > div[style*="padding-left"]').forEach((el) => { + const outerDiv = el.parentElement!; + const paragraphs = parsed.$('p', outerDiv); + + if (paragraphs.length > 0) { + const content = paragraphs.map(p => serializeNode(p)).join(''); + const citation = parsed.$('small', outerDiv).map(s => s.textContent!.trim()).filter(Boolean).join(''); + const cite = citation ? `${citation}` : ''; + replaceWith(outerDiv, `
${content}${cite}
`); + } + }); + + // Convert buttons to Ghost buttons + parsed.$('a > button').forEach((el) => { + const link = el.parentElement!; // guaranteed by 'a > button' selector + const wrapper = link.parentElement; + const buttonText = el.textContent!.trim(); + const buttonHref = attr(link, 'href'); + const target = wrapper && wrapper.tagName === 'DIV' ? wrapper : link; + + replaceWith(target, `
${buttonText}
`); + }); + + // Rewrite subscribe links to Portal signup + parsed.$('a[href*="/subscribe"]').forEach((el) => { + attr(el, 'href', '/#/portal/signup'); + }); + + // Handle link embeds + parsed.$('.generic-embed--root').forEach((el) => { + const anchorEl = parsed.$('a', el)[0] as Element | undefined; + const titleEl = parsed.$('.generic-embed--title p', el)[0] as Element | undefined; + const descEl = parsed.$('.generic-embed--description p', el)[0] as Element | undefined; + const imageEl = parsed.$('.generic-embed--image img', el)[0] as Element | undefined; + + const href = anchorEl ? attr(anchorEl, 'href') : null; + const title = titleEl ? _.unescape(titleEl.textContent!) : ''; + const description = descEl ? _.unescape(descEl.textContent!) : ''; + const image = imageEl ? attr(imageEl, 'src') : null; + + let cardOpts = { + env: {dom: new SimpleDom.Document()}, + payload: { + url: href, + metadata: { + url: href, + title: title, + description: description, + icon: null, + thumbnail: image, + publisher: null + }, + caption: null + } + }; + + replaceWith(el, serializer.serialize(bookmarkCard.render(cardOpts))); + }); + + // Unwrap audio iframes from tables + parsed.$('table iframe[src^="https://audio.beehiiv.com"]').forEach((el) => { + const tableParent = parents(el, 'table')[0]; + if (tableParent) { + const iframeHtml = serializeNode(el); + replaceWith(tableParent, iframeHtml); + } + }); + + // Convert YouTube iframes to embeds + parsed.$('iframe[src*="youtube.com/embed"], iframe[src*="youtu.be"]').forEach((el) => { + const videoID = getYouTubeID(attr(el, 'src') as string); + + let cardOpts = { + env: {dom: new SimpleDom.Document()}, + payload: { + caption: null, + html: `` + } + }; + + replaceWith(el, serializer.serialize(embedCard.render(cardOpts))); + }); + + // Remove empty paragraphs + parsed.$('p').forEach((el) => { + const text = (el.textContent || '').trim(); + + if (text.length === 0) { + el.remove(); + } + }); + + // Remove style tags + parsed.$('style').forEach((el) => { + el.remove(); + }); + + // Remove mobile ads + parsed.$('#pad-mobile').forEach((el) => { + el.remove(); + }); + + // Remove style attributes + parsed.$('[style]').forEach((el) => { + el.removeAttribute('style'); + }); + + // Unwrap divs, but preserve Ghost card divs + parsed.$('div:not([class*="kg-card"])').forEach((el) => { + replaceWith(el, serializeChildren(el)); + }); + + // Remove b, strong, i, em tags from headings but allow links inside them + parsed.$('h1, h2, h3, h4, h5, h6').forEach((el) => { + parsed.$('b, strong, i, em', el).forEach((ell) => { + replaceWith(ell, serializeChildren(ell)); + }); + }); + + // if (options?.url) { + // $html('a').each((i: any, el: any) => { + // const theHref = $html(el).attr('href'); + // const isHrefURL = isURL(theHref); + + // if (theHref && isHrefURL) { + // const url = new URL(theHref); + + // const params = new URLSearchParams(url.search); + + // params.delete('utm_source'); + // params.delete('utm_medium'); + // params.delete('utm_campaign'); + // params.delete('last_resource_guid'); + + // url.search = params.toString(); + + // $html(el).attr('href', url.toString()); + // } + // }); + // } + + // Polls + // $html('td[class="e"], td[class="ee e "]').each((i: any, el: any) => { + // $html(el).remove(); + // }); + + // FInd

tag which contains {{rp_personalized_text}} and find the parent table to remove the whole section + // $html('p:contains("{{rp_personalized_text}}")').each((i: any, el: any) => { + // const parentTable = $html(el).parents('table').first(); + // $html(parentTable).remove(); + // }); + + // $html('h1 strong, h1 b, h2 strong, h2 b, h3 strong, h3 b, h4 strong, h4 b, h5 strong, h5 b, h6 strong, h6 b').each((i: any, el: any) => { + // const text = $html(el).html().trim(); + // $html(el).replaceWith(text); + // }); + + // $html('table.j').each((i: any, el: any) => { + // const tdContent = $html(el).find('td').html().trim(); + + // if (tdContent === ' ') { + // $html(el).replaceWith('

'); + // } + // }); + + // $html('table.d3[align="center"]').each((i: any, el: any) => { + // const parent = $html(el).parent('td[align="center"]'); + // const text = $html(el).text().replace(/(\r\n|\n|\r| )/gm, ' ').replace(/(\s{2,})/gm, ' ').trim(); + // $html(parent).replaceWith(`

${text}

`); + // }); + + // // Galleries + // $html('table.mob-w-full').each((i: any, el: any) => { + // let allImages: string[] = []; + + // $html(el).find('td.mob-stack').each((ii: any, ell: any) => { + // const img = $html(ell).find('img'); + // const pText = $html(ell).find('p').text().trim(); + + // let cardOpts: any = { + // env: {dom: new SimpleDom.Document()}, + // payload: { + // src: img.attr('src'), + // alt: img.attr('alt') + // } + // }; + + // if (pText && pText.length > 0) { + // cardOpts.payload.caption = pText; + // } + + // const isInLink = $html(img).parents('a').length; + + // if (isInLink) { + // cardOpts.payload.href = $html(img).parents('a').attr('href'); + // } + + // allImages.push(serializer.serialize(imageCard.render(cardOpts))); + // }); + + // $html(el).replaceWith(allImages.join('')); + // }); + + // // Embeds + // $html('td.embed-img.mob-stack').each((i: any, el: any) => { + // const parent = $html(el).parent().parent(); + + // const href = $html(parent).find('a').attr('href'); + // const image = $html(parent).find('img').attr('src'); + // const title = $html(parent).find('p').eq(0).text(); + // const description = $html(parent).find('p').eq(1).text(); + + // const parentTable = $html(parent).parent().parent().parent().parent().parent(); + + // let cardOpts = { + // env: {dom: new SimpleDom.Document()}, + // payload: { + // url: href, + // metadata: { + // url: href, + // title: title, + // description: description, + // icon: null, + // thumbnail: image, + // publisher: null, + // author: null + // }, + // caption: null + // } + // }; + + // $html(parentTable).replaceWith(serializer.serialize(bookmarkCard.render(cardOpts))); + // }); + + // // Remove hidden elements + // $html('[style*="display:none"]').remove(); + // $html('[style*="display: none"]').remove(); + + // // Remove the share links at the top + // $html('table.mob-block').remove(); + + // // Remove sponsor blocks + // $html('table.rec__content').remove(); + + // // Remove the open tracking pixel element + // $html('div[data-open-tracking="true"]:contains("{{OPEN_TRACKING_PIXEL}}")').remove(); + + // $html('p:contains("{{rp_personalized_text}}")').remove(); + // $html('img[src="{{rp_next_milestone_image_url}}"]').remove(); + // $html(`a[href="{{rp_referral_hub_url}}"]`).remove(); + + // // Remove unsubscribe links, social links, & email footer + // $html('td.b').remove(); + + // // Remove the 'Read online' link + // $html('td.f').remove(); + + // // Remove the post title container, otherwise it would be a duplicate + // if (postData?.data?.title) { + // $html(`h1:contains("${postData.data.title}")`).parentsUntil('table').remove(); + // } + + // // Remove cells that only contain a non-breaking space + // $html('td').each((i: any, el: any) => { + // const text = $html(el).html().trim(); + + // if (text === ' ') { + // $html(el).remove(); + // } + // }); + + // // Convert '...' to
+ // $html('p').each((i: any, el: any) => { + // const text = $html(el).text().trim(); + + // if (text === '...' || text === '…' || text === '…') { + // $html(el).replaceWith('
'); + // } + // }); + + // // If the iframe has no inner HTML, add a non-breaking space so it doesn't get removed later + // // Case: There's a bug somewhere in the chain that causes iframes to become self-closing, and this prevents that. + // $html('iframe').each((i: any, el: any) => { + // const innerHtml = $html(el).html().length; + + // if (innerHtml === 0) { + // $html(el).html(' '); + // } + // }); + + // // Convert linked YouTube thumbnails to embeds + // $html('a[href*="youtube.com"], a[href*="youtu.be"]').each((i: any, el: any) => { + // const imageCount = $html(el).find('img').length; + // const hasPlayIcon = $html(el).find('img[src*="youtube_play_icon.png"]').length; + // const hasThumbnail = $html(el).find('img[src*="i.ytimg.com/vi"]').length; + // const src = $html(el)?.attr('href'); + // const captionText = $html(el).find('p')?.text()?.trim() || false; + // const captionHtml = $html(el).find('p')?.html()?.trim() || false; + + // if (imageCount === 2 && hasPlayIcon && hasThumbnail && src) { + // const videoID = getYouTubeID(src); + + // let cardOpts = { + // env: {dom: new SimpleDom.Document()}, + // payload: { + // caption: null, + // html: `` + // } + // }; + + // if (captionText && captionText.length > 0) { + // cardOpts.payload.caption = captionHtml; + // } + + // $html(el).replaceWith(serializer.serialize(embedCard.render(cardOpts))); + // } + // }); + + // $html('img').each((i: any, el: any) => { + // // Skip if the image is in a figure + // const isInFigure = $html(el).parents('figure').length; + // if (isInFigure) { + // return; + // } + + // const parentTable = $html(el).parent().parent().parent(); + + // const theSrc = $html(el).attr('src'); + // let theAlt = $html(el).attr('alt'); + + // const secondTr = ($html(parentTable).find('tr').eq(1).find('p').length) ? $html(parentTable).find('tr').eq(1).find('p') : false; + // const theText = $html(secondTr)?.html()?.trim() ?? false; + + // if (!theAlt) { + // theAlt = $html(secondTr)?.text()?.trim(); + // } + + // let cardOpts: any = { + // env: {dom: new SimpleDom.Document()}, + // payload: { + // src: theSrc, + // alt: theAlt, + // caption: theText + // } + // }; + + // // Check if the parent element to this is a tag + // const isInLink = $html(el).parents('a').length; + + // if (isInLink) { + // cardOpts.payload.href = $html(el).parents('a').attr('href'); + // } + + // $html(parentTable).replaceWith(serializer.serialize(imageCard.render(cardOpts))); + // }); + + // // Convert buttons to Ghost buttons + // $html('a[style="color:#FFFFFF;font-size:18px;padding:0px 14px;text-decoration:none;"]').each((i: any, el: any) => { + // const buttonText = $html(el).text(); + // const buttonHref = $html(el).attr('href'); + // $html(el).replaceWith(`
${buttonText}
`); + // }); + + // if (options?.url && options?.subscribeLink) { + // $html(`a[href^="${options.url}/subscribe"]`).each((i: any, el: any) => { + // $html(el).attr('href', options.subscribeLink); + // $html(el).removeAttr('target'); + // $html(el).removeAttr('rel'); + // }); + // } + + // if (options?.url && options?.comments && options?.commentLink) { + // $html('a[href*="comments=true"]').each((i: any, el: any) => { + // const href = $html(el).attr('href'); + + // if (href.includes(options.url)) { + // $html(el).attr('href', options.commentLink); + // $html(el).removeAttr('target'); + // $html(el).removeAttr('rel'); + // } + // }); + // } else if (options?.url && options?.comments === false) { + // $html('a[href*="comments=true"]').each((i: any, el: any) => { + // const href = $html(el).attr('href'); + + // if (href.includes(options.url)) { + // $html(el).remove(); + // } + // }); + // } + + // // Remove empty tags + // $html('p, figure').each((i: any, el: any) => { + // const elementHtml = $html(el).html().trim(); + + // if (elementHtml === '') { + // $html(el).remove(); + // } + // }); + + // // Get the cleaned HTML + let bodyHtml = parsed.html(); + + return bodyHtml; + + // // Pass the cleaned HTML through the sanitizer to only include specific elements + // const sanitizedHtml = sanitizeHtml(bodyHtml, { + // allowedTags: [ + // 'b', 'i', 'em', 'strong', 'a', 'p', 'br', 'ul', 'ol', 'li', 'blockquote', + // 'figure', 'figcaption', 'img', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', + // 'div', 'hr', 'iframe', 'span' + // ], + // allowedAttributes: { + // a: ['href', 'title', 'rel', 'target', 'class'], + // img: ['src', 'alt', 'title', 'class', 'width', 'height'], + // iframe: ['width', 'height', 'src', 'title', 'frameborder', 'allow', 'allowfullscreen'], + // figure: ['class'], + // div: ['class'] + // }, + // allowedClasses: { + // '*': ['kg-*'] + // } + // }); + + // return sanitizedHtml.trim(); +}; + +const removeDuplicateFeatureImage = ({html, featureSrc}: {html: string, featureSrc: string}) => { + const parsed = parseFragment(html); + + let firstElement = parsed.$('*')[0]; + + if (firstElement) { + const isImg = firstElement.tagName === 'IMG'; + const hasImg = !isImg && parsed.$('img', firstElement).length > 0; + + if (isImg || hasImg) { + let theElementItself = isImg ? firstElement : parsed.$('img', firstElement)[0]; + let firstImgSrc: any = attr(theElementItself, 'src'); + + // Both images usually end in the same way, so we can split the URL and compare the last part + const firstImageSplit = firstImgSrc.split('/uploads/asset/'); + const featureImageSplit = featureSrc.split('/uploads/asset/'); + + if (firstImageSplit[1] !== undefined && featureImageSplit[1] !== undefined && firstImageSplit[1] === featureImageSplit[1]) { + theElementItself.remove(); + } + + if (featureSrc.length > 0 && firstImgSrc) { + let normalizedFirstSrc = firstImgSrc.replace('fit=scale-down,format=auto,onerror=redirect,quality=80', 'quality=100'); + + if (featureSrc === normalizedFirstSrc) { + theElementItself.remove(); + } + } + } + } + + return parsed.html(); +}; + +export { + getYouTubeID, + isURL, + processHTML, + removeDuplicateFeatureImage +}; diff --git a/packages/mg-beehiiv-api/src/test/client.test.ts b/packages/mg-beehiiv-api/src/test/client.test.ts new file mode 100644 index 000000000..44ff723d8 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/client.test.ts @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import {describe, it, beforeEach, afterEach, mock} from 'node:test'; +import {client} from '../lib/client.js'; + +describe('beehiiv API Client', () => { + let fetchMock: any; + + beforeEach(() => { + fetchMock = mock.method(global, 'fetch', () => Promise.resolve()); + }); + + afterEach(() => { + mock.restoreAll(); + }); + + describe('client', () => { + it('makes authenticated request to publications endpoint', async () => { + const mockData = {data: [{id: 'pub-1', name: 'Test Publication'}]}; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve(mockData) + })); + + await client('test-api-key'); + + assert.equal(fetchMock.mock.callCount(), 1); + const [calledUrl, options] = fetchMock.mock.calls[0].arguments; + assert.equal(calledUrl.toString(), 'https://api.beehiiv.com/v2/publications'); + assert.equal(options.method, 'GET'); + assert.equal(options.headers.Authorization, 'Bearer test-api-key'); + }); + + it('returns publications data', async () => { + const publications = [{id: 'pub-1', name: 'Test Pub'}]; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({data: publications}) + })); + + const result = await client('test-api-key'); + + assert.deepEqual(result, publications); + }); + + it('throws error on failed request', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 401, + statusText: 'Unauthorized' + })); + + await assert.rejects(async () => { + await client('invalid-key'); + }, /Request failed: 401 Unauthorized/); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/fetch.test.ts b/packages/mg-beehiiv-api/src/test/fetch.test.ts new file mode 100644 index 000000000..2b197fdf4 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/fetch.test.ts @@ -0,0 +1,349 @@ +import assert from 'node:assert/strict'; +import {describe, it, beforeEach, afterEach, mock} from 'node:test'; +import {fetchTasks, authedClient} from '../lib/fetch.js'; + +describe('beehiiv API Fetch', () => { + let fetchMock: any; + + beforeEach(() => { + fetchMock = mock.method(global, 'fetch', () => Promise.resolve()); + }); + + afterEach(() => { + mock.restoreAll(); + }); + + describe('authedClient', () => { + it('makes authenticated GET request', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ok: true, json: () => Promise.resolve({data: []})})); + + const url = new URL('https://api.beehiiv.com/v2/publications'); + await authedClient('test-api-key', url); + + assert.equal(fetchMock.mock.callCount(), 1); + const [calledUrl, options] = fetchMock.mock.calls[0].arguments; + assert.equal(calledUrl.toString(), 'https://api.beehiiv.com/v2/publications'); + assert.equal(options.method, 'GET'); + assert.equal(options.headers.Authorization, 'Bearer test-api-key'); + }); + }); + + describe('fetchTasks', () => { + it('creates correct number of tasks based on total posts', async () => { + // Mock the discover call (limit=1) + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 25, data: []}) + }), 0); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + readTmpJSONFile: () => Promise.resolve({}), + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + + // 25 posts / 10 per page = 3 pages + assert.equal(tasks.length, 3); + assert.equal(tasks[0].title, 'Fetching posts page 1 of 3'); + assert.equal(tasks[1].title, 'Fetching posts page 2 of 3'); + assert.equal(tasks[2].title, 'Fetching posts page 3 of 3'); + }); + + it('task fetches posts and adds to context', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: []}) + }), 0); + + // Mock the actual fetch call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [{id: 'post-1', title: 'Test Post'}] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + + // Execute the first task + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 1); + assert.equal((ctx.result.posts[0] as any).id, 'post-1'); + }); + + it('task uses cached data when available', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: []}) + }), 0); + + const cachedData = {data: [{id: 'cached-post', title: 'Cached Post'}]}; + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => true, + readTmpJSONFile: () => Promise.resolve(cachedData) + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + // Should use cached data, not make another API call + assert.equal(ctx.result.posts.length, 1); + assert.equal((ctx.result.posts[0] as any).id, 'cached-post'); + // fetch should only be called once (for discover), not twice + assert.equal(fetchMock.mock.callCount(), 1); + }); + + it('task throws and sets output on fetch error', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: []}) + }), 0); + + // Mock a failed fetch + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: false, + status: 500, + statusText: 'Internal Server Error' + }), 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }, /Request failed: 500 Internal Server Error/); + + assert.ok(mockTask.output.includes('500')); + }); + + it('task handles non-Error thrown objects', async () => { + // Mock the discover call + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 5, data: []}) + }), 0); + + // Mock fetch that throws a non-Error value (string) + fetchMock.mock.mockImplementationOnce(() => { + // eslint-disable-next-line no-throw-literal + throw 'Network error string'; + }, 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + + // Verify String() conversion was used + assert.equal(mockTask.output, 'Network error string'); + }); + + it('filters posts with postsAfter', async () => { + const jan1 = new Date('2024-01-01').getTime() / 1000; + const feb1 = new Date('2024-02-01').getTime() / 1000; + const mar1 = new Date('2024-03-01').getTime() / 1000; + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 3, data: []}) + }), 0); + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [ + {id: 'post-jan', publish_date: jan1}, + {id: 'post-feb', publish_date: feb1}, + {id: 'post-mar', publish_date: mar1} + ] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123', postsAfter: '2024-02-01'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 2); + assert.equal((ctx.result.posts[0] as any).id, 'post-feb'); + assert.equal((ctx.result.posts[1] as any).id, 'post-mar'); + }); + + it('filters posts with postsBefore', async () => { + const jan1 = new Date('2024-01-01').getTime() / 1000; + const feb1 = new Date('2024-02-01').getTime() / 1000; + const mar1 = new Date('2024-03-01').getTime() / 1000; + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 3, data: []}) + }), 0); + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [ + {id: 'post-jan', publish_date: jan1}, + {id: 'post-feb', publish_date: feb1}, + {id: 'post-mar', publish_date: mar1} + ] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123', postsBefore: '2024-02-01'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 2); + assert.equal((ctx.result.posts[0] as any).id, 'post-jan'); + assert.equal((ctx.result.posts[1] as any).id, 'post-feb'); + }); + + it('filters posts with both postsAfter and postsBefore', async () => { + const jan1 = new Date('2024-01-01').getTime() / 1000; + const feb1 = new Date('2024-02-01').getTime() / 1000; + const mar1 = new Date('2024-03-01').getTime() / 1000; + const apr1 = new Date('2024-04-01').getTime() / 1000; + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 4, data: []}) + }), 0); + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [ + {id: 'post-jan', publish_date: jan1}, + {id: 'post-feb', publish_date: feb1}, + {id: 'post-mar', publish_date: mar1}, + {id: 'post-apr', publish_date: apr1} + ] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123', postsAfter: '2024-02-01', postsBefore: '2024-03-01'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 2); + assert.equal((ctx.result.posts[0] as any).id, 'post-feb'); + assert.equal((ctx.result.posts[1] as any).id, 'post-mar'); + }); + + it('does not filter when neither postsAfter nor postsBefore is set', async () => { + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({total_results: 2, data: []}) + }), 0); + + fetchMock.mock.mockImplementationOnce(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + data: [ + {id: 'post-1', publish_date: 1704067200}, + {id: 'post-2', publish_date: 1706745600} + ] + }) + }), 1); + + const options = {key: 'test-key', id: 'pub-123'}; + const ctx = { + fileCache: { + hasFile: () => false, + writeTmpFile: () => Promise.resolve() + }, + result: {posts: [] as any[]} + }; + + const tasks = await fetchTasks(options, ctx); + await tasks[0].task({}, {output: ''}); + + assert.equal(ctx.result.posts.length, 2); + }); + + it('handles discover error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 401, + statusText: 'Unauthorized' + })); + + const options = {key: 'invalid-key', id: 'pub-123'}; + const ctx = { + fileCache: {}, + result: {posts: [] as any[]} + }; + + await assert.rejects(async () => { + await fetchTasks(options, ctx); + }, /Request failed: 401 Unauthorized/); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/fixtures/posts.csv b/packages/mg-beehiiv-api/src/test/fixtures/posts.csv new file mode 100644 index 000000000..d96f6fc9b --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/fixtures/posts.csv @@ -0,0 +1,3 @@ +id,web_title,status,audience,content_tags,url,web_subtitle,email_subject_line,email_preview_text,content_html,thumbnail_url,created_at +abcd1234-1505-4fbf-9576-f3d1bd3034cc,Sample Post,confirmed,free,Lorem Ipsum; Dolor Simet,https://example.beehiiv.com/p/sample-post,"A website subtitle",Sample Post as a subject line,"Email preview text","

Sample HTML here

",uploads/asset/file/12345678/image.png,2023-01-18 21:25:27 +efgh1234-1505-4fbf-9576-f3d1bd3034cc,Another Post,confirmed,free,"Lorem Ipsum; Dolor Simet",https://example.beehiiv.com/p/another-post,"Another website subtitle",Another sample Post as a subject line,"Email preview text","

More sample HTML here

",uploads/asset/file/12345678/another.png,2023-01-19 21:25:27 diff --git a/packages/mg-beehiiv-api/src/test/index.test.ts b/packages/mg-beehiiv-api/src/test/index.test.ts new file mode 100644 index 000000000..5c5a5a222 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/index.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import beehiivApi from '../index.js'; + +describe('beehiiv API index exports', () => { + it('exports listPublications', () => { + assert.ok(typeof beehiivApi.listPublications === 'function'); + }); + + it('exports fetchTasks', () => { + assert.ok(typeof beehiivApi.fetchTasks === 'function'); + }); + + it('exports mapPostsTasks', () => { + assert.ok(typeof beehiivApi.mapPostsTasks === 'function'); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/list-pubs.test.ts b/packages/mg-beehiiv-api/src/test/list-pubs.test.ts new file mode 100644 index 000000000..2e2f21a50 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/list-pubs.test.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import {describe, it, beforeEach, afterEach, mock} from 'node:test'; +import {listPublications} from '../lib/list-pubs.js'; + +describe('beehiiv API List Publications', () => { + let fetchMock: any; + + beforeEach(() => { + fetchMock = mock.method(global, 'fetch', () => Promise.resolve()); + }); + + afterEach(() => { + mock.restoreAll(); + }); + + describe('listPublications', () => { + it('makes authenticated request to publications endpoint', async () => { + const mockData = {data: [{id: 'pub-1', name: 'Test Publication'}]}; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve(mockData) + })); + + await listPublications('test-api-key'); + + assert.equal(fetchMock.mock.callCount(), 1); + const [calledUrl, options] = fetchMock.mock.calls[0].arguments; + assert.equal(calledUrl.toString(), 'https://api.beehiiv.com/v2/publications?expand%5B%5D=stats'); + assert.equal(options.method, 'GET'); + assert.equal(options.headers.Authorization, 'Bearer test-api-key'); + }); + + it('returns publications data', async () => { + const publications = [ + {id: 'pub-1', name: 'Publication One'}, + {id: 'pub-2', name: 'Publication Two'} + ]; + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: true, + json: () => Promise.resolve({data: publications}) + })); + + const result = await listPublications('test-api-key'); + + assert.deepEqual(result, publications); + }); + + it('throws error on failed request', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 403, + statusText: 'Forbidden' + })); + + await assert.rejects(async () => { + await listPublications('invalid-key'); + }, /Request failed: 403 Forbidden/); + }); + + it('throws error on server error', async () => { + fetchMock.mock.mockImplementation(() => Promise.resolve({ + ok: false, + status: 500, + statusText: 'Internal Server Error' + })); + + await assert.rejects(async () => { + await listPublications('test-key'); + }, /Request failed: 500 Internal Server Error/); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/mapper.test.ts b/packages/mg-beehiiv-api/src/test/mapper.test.ts new file mode 100644 index 000000000..a1a8356c8 --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/mapper.test.ts @@ -0,0 +1,280 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import {mapPost, mapPostsTasks} from '../lib/mapper.js'; + +describe('beehiiv API Mapper', () => { + describe('mapPost', () => { + const createMockPostData = (overrides = {}): any => ({ + id: 'post-123', + title: 'Test Post Title', + subtitle: 'Test subtitle', + slug: 'test-post-title', + web_url: 'https://example.beehiiv.com/p/test-post-title', + status: 'confirmed', + audience: 'free', + publish_date: 1700000000, // Unix timestamp + created: 1699900000, + thumbnail_url: 'https://example.com/image.jpg', + meta_default_title: 'OG Title', + meta_default_description: 'OG Description', + authors: ['John Doe', 'Jane Smith'], + content_tags: ['Tech', 'News'], + content: { + premium: { + web: '

Test content

' + } + }, + ...overrides + }); + + it('maps basic post fields correctly', () => { + const postData = createMockPostData(); + const result = mapPost({postData}); + + assert.equal(result.url, 'https://example.beehiiv.com/p/test-post-title'); + assert.equal(result.data.comment_id, 'post-123'); + assert.equal(result.data.slug, 'test-post-title'); + assert.equal(result.data.title, 'Test Post Title'); + assert.equal(result.data.type, 'post'); + assert.equal(result.data.custom_excerpt, 'Test subtitle'); + }); + + it('converts timestamps to Date objects', () => { + const postData = createMockPostData(); + const result = mapPost({postData}); + + assert.ok(result.data.published_at instanceof Date); + assert.ok(result.data.updated_at instanceof Date); + assert.ok(result.data.created_at instanceof Date); + + // Check the dates are converted correctly (Unix timestamp * 1000) + assert.equal(result.data.published_at.getTime(), 1700000000 * 1000); + assert.equal(result.data.created_at.getTime(), 1699900000 * 1000); + }); + + it('sets status to published for confirmed posts', () => { + const postData = createMockPostData({status: 'confirmed'}); + const result = mapPost({postData}); + assert.equal(result.data.status, 'published'); + }); + + it('sets status to draft for non-confirmed posts', () => { + const postData = createMockPostData({status: 'draft'}); + const result = mapPost({postData}); + assert.equal(result.data.status, 'draft'); + }); + + it('sets visibility to paid for premium audience', () => { + const postData = createMockPostData({audience: 'premium'}); + const result = mapPost({postData}); + assert.equal(result.data.visibility, 'paid'); + }); + + it('sets visibility to public for free audience', () => { + const postData = createMockPostData({audience: 'free'}); + const result = mapPost({postData}); + assert.equal(result.data.visibility, 'public'); + }); + + it('sets feature_image when thumbnail_url exists', () => { + const postData = createMockPostData({thumbnail_url: 'https://example.com/thumb.jpg'}); + const result = mapPost({postData}); + assert.equal(result.data.feature_image, 'https://example.com/thumb.jpg'); + }); + + it('does not set feature_image when thumbnail_url is empty', () => { + const postData = createMockPostData({thumbnail_url: ''}); + const result = mapPost({postData}); + assert.equal(result.data.feature_image, undefined); + }); + + it('sets og_title when meta_default_title exists', () => { + const postData = createMockPostData({meta_default_title: 'Custom OG Title'}); + const result = mapPost({postData}); + assert.equal(result.data.og_title, 'Custom OG Title'); + }); + + it('does not set og_title when meta_default_title is falsy', () => { + const postData = createMockPostData({meta_default_title: null}); + const result = mapPost({postData}); + assert.equal(result.data.og_title, undefined); + }); + + it('sets og_description when meta_default_description exists', () => { + const postData = createMockPostData({meta_default_description: 'Custom OG Desc'}); + const result = mapPost({postData}); + assert.equal(result.data.og_description, 'Custom OG Desc'); + }); + + it('does not set og_description when meta_default_description is falsy', () => { + const postData = createMockPostData({meta_default_description: null}); + const result = mapPost({postData}); + assert.equal(result.data.og_description, undefined); + }); + + it('maps authors correctly', () => { + const postData = createMockPostData({authors: ['John Doe', 'Jane Smith']}); + const result = mapPost({postData}); + + assert.equal(result.data.authors.length, 2); + + assert.equal(result.data.authors[0].data.name, 'John Doe'); + assert.equal(result.data.authors[0].data.slug, 'john-doe'); + assert.equal(result.data.authors[0].data.email, 'john-doe@example.com'); + assert.ok(result.data.authors[0].url.includes('john-doe')); + + assert.equal(result.data.authors[1].data.name, 'Jane Smith'); + assert.equal(result.data.authors[1].data.slug, 'jane-smith'); + }); + + it('maps content_tags correctly', () => { + const postData = createMockPostData({content_tags: ['Tech', 'News']}); + const result = mapPost({postData}); + + // Should have 3 tags: 2 content tags + 1 beehiiv source tag + assert.equal(result.data.tags.length, 3); + + assert.equal(result.data.tags[0].data.name, 'Tech'); + assert.equal(result.data.tags[0].data.slug, 'tech'); + + assert.equal(result.data.tags[1].data.name, 'News'); + assert.equal(result.data.tags[1].data.slug, 'news'); + }); + + it('always adds #beehiiv source tag', () => { + const postData = createMockPostData({content_tags: []}); + const result = mapPost({postData}); + + const beehiivTag = result.data.tags.find((t: any) => t.data.slug === 'hash-beehiiv'); + assert.ok(beehiivTag); + assert.equal(beehiivTag.data.name, '#beehiiv'); + }); + + it('handles null subtitle', () => { + const postData = createMockPostData({subtitle: null}); + const result = mapPost({postData}); + assert.equal(result.data.custom_excerpt, null); + }); + + it('processes HTML through processHTML', () => { + const postData = createMockPostData({ + content: { + premium: { + web: '

Processed content

' + } + } + }); + const result = mapPost({postData}); + assert.ok(result.data.html.includes('Processed content')); + }); + }); + + describe('mapPostsTasks', () => { + it('creates tasks for each post', async () => { + const ctx = { + result: { + posts: [ + { + id: 'post-1', + title: 'Post 1', + subtitle: null, + slug: 'post-1', + web_url: 'https://example.com/p/post-1', + status: 'confirmed', + audience: 'free', + publish_date: 1700000000, + created: 1699900000, + thumbnail_url: '', + meta_default_title: null, + meta_default_description: null, + authors: ['Author'], + content_tags: [], + content: {premium: {web: '

Content 1

'}} + }, + { + id: 'post-2', + title: 'Post 2', + subtitle: null, + slug: 'post-2', + web_url: 'https://example.com/p/post-2', + status: 'confirmed', + audience: 'free', + publish_date: 1700000000, + created: 1699900000, + thumbnail_url: '', + meta_default_title: null, + meta_default_description: null, + authors: ['Author'], + content_tags: [], + content: {premium: {web: '

Content 2

'}} + } + ] + } + }; + + const tasks = await mapPostsTasks({}, ctx); + + assert.equal(tasks.length, 2); + assert.equal(tasks[0].title, 'Mapping post: Post 1'); + assert.equal(tasks[1].title, 'Mapping post: Post 2'); + }); + + it('task maps post and updates context', async () => { + const ctx = { + result: { + posts: [ + { + id: 'post-1', + title: 'Post 1', + subtitle: 'Subtitle', + slug: 'post-1', + web_url: 'https://example.com/p/post-1', + status: 'confirmed', + audience: 'free', + publish_date: 1700000000, + created: 1699900000, + thumbnail_url: '', + meta_default_title: null, + meta_default_description: null, + authors: ['Author'], + content_tags: ['Tag1'], + content: {premium: {web: '

Content

'}} + } + ] + } + }; + + const tasks = await mapPostsTasks({}, ctx); + + // Execute the task + await tasks[0].task({}, {output: ''}); + + // Check that the post was mapped + const mappedPost = ctx.result.posts[0] as any; + assert.equal(mappedPost.url, 'https://example.com/p/post-1'); + assert.ok(mappedPost.data); + assert.equal(mappedPost.data.title, 'Post 1'); + }); + + it('task throws error on mapping failure', async () => { + const ctx = { + result: { + posts: [ + { + // Missing required fields to cause an error + id: 'post-1', + title: 'Post 1' + } + ] + } + }; + + const tasks = await mapPostsTasks({}, ctx); + const mockTask = {output: ''}; + + await assert.rejects(async () => { + await tasks[0].task({}, mockTask); + }); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/test/process.test.ts b/packages/mg-beehiiv-api/src/test/process.test.ts new file mode 100644 index 000000000..24a4ed20c --- /dev/null +++ b/packages/mg-beehiiv-api/src/test/process.test.ts @@ -0,0 +1,276 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; +import {getYouTubeID, isURL, processHTML, removeDuplicateFeatureImage} from '../lib/process.js'; + +describe('beehiiv API processor', () => { + describe('isURL', () => { + it('returns true for valid HTTP URL', () => { + assert.equal(isURL('https://example.com'), true); + }); + + it('returns true for valid HTTP URL with path', () => { + assert.equal(isURL('https://example.com/path/to/resource'), true); + }); + + it('returns false for undefined', () => { + assert.equal(isURL(undefined), false); + }); + + it('returns false for invalid URL string', () => { + assert.equal(isURL('not-a-url'), false); + }); + + it('returns false for empty string', () => { + assert.equal(isURL(''), false); + }); + }); + + describe('getYouTubeID', () => { + it('extracts ID from standard youtube.com URL', () => { + const result = getYouTubeID('https://www.youtube.com/watch?v=dQw4w9WgXcQ'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('extracts ID from youtu.be short URL', () => { + const result = getYouTubeID('https://youtu.be/dQw4w9WgXcQ'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('extracts ID from embed URL', () => { + const result = getYouTubeID('https://www.youtube.com/embed/dQw4w9WgXcQ'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('extracts ID from /v/ URL', () => { + const result = getYouTubeID('https://www.youtube.com/v/dQw4w9WgXcQ'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('extracts ID with additional parameters', () => { + const result = getYouTubeID('https://www.youtube.com/watch?v=dQw4w9WgXcQ&feature=youtu.be'); + assert.equal(result, 'dQw4w9WgXcQ'); + }); + + it('returns original string when no ID pattern found', () => { + const result = getYouTubeID('https://example.com/video'); + assert.equal(result, 'https://example.com/video'); + }); + }); + + describe('processHTML', () => { + it('handles post with undefined html by using empty string', () => { + // When post.data.html is undefined, it falls back to empty string via ?? + const result = processHTML({post: {url: 'test', data: {}} as any}); + // Should return empty string without throwing + assert.equal(result, ''); + }); + + it('handles html without content-blocks element', () => { + // When there's no #content-blocks element, contentBlocksHtml falls back to '' + const html = '

No content blocks here

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, ''); + }); + + it('replaces beehiiv subscriber_id variable with a space', () => { + const html = '

Hello {{subscriber_id}}

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Hello

'); + }); + + it('replaces beehiiv rp_refer_url variable with #', () => { + const html = '
Link
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, 'Link'); + }); + + it('replaces beehiiv rp_refer_url_no_params variable with #', () => { + const html = '
Link
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, 'Link'); + }); + + it('converts empty divs with border-top to HR', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
'); + }); + + it('does not convert divs with content to HR', () => { + const html = '
content
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, 'content'); + }); + + it('converts nested padding-left divs to blockquote with paragraph content', () => { + const html = `
❝

Quote text here.

Author Name
`; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Quote text here.

Author Name
'); + }); + + it('converts nested padding-left divs to blockquote without citation when small is empty', () => { + const html = `
❝

Quote text here.

`; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Quote text here.

'); + }); + + it('does not convert nested padding-left divs without paragraphs', () => { + const html = '
plain text
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
plain text
'); + }); + + it('converts subscribe button wrapper to Portal signup button', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
Subscribe
'); + }); + + it('converts subscribe button without wrapper div to Portal signup', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
Subscribe
'); + }); + + it('converts buttons inside anchors to Ghost buttons', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
Click me
'); + }); + + it('converts generic embeds to bookmark cards', () => { + const html = '
Link

Title

Description

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
Title
Description
'); + }); + + it('handles generic embeds with missing sub-elements without crashing', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, ''); + }); + + it('unwraps audio iframes from tables', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, ''); + }); + + it('converts YouTube iframes to embed cards', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
'); + }); + + it('converts youtu.be iframes to embed cards', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
'); + }); + + it('converts image with caption to Ghost image card', () => { + const html = '

Caption text here

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
Caption text here
'); + }); + + it('converts image without caption to Ghost image card', () => { + const html = '
Test
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
Test
'); + }); + + it('converts image in single div wrapper to Ghost image card', () => { + const html = '
'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '
'); + }); + + it('converts image inside non-div wrapper and places card after parent', () => { + const html = '

Text more text

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Text more text

'); + }); + + it('removes empty paragraphs', () => { + const html = '

Content

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Content

'); + }); + + it('removes style tags', () => { + const html = '

Content

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Content

'); + }); + + it('removes mobile ad elements', () => { + const html = '
Ad content

Content

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Content

'); + }); + + it('removes b and strong tags from headings', () => { + const html = '

Bold Heading

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Bold Heading

'); + }); + + it('removes style attributes', () => { + const html = '

Content

'; + const result = processHTML({post: {url: 'test', data: {html}} as any}); + assert.equal(result, '

Content

'); + }); + }); + + describe('removeDuplicateFeatureImage', () => { + it('removes first image when it matches feature image via asset path', () => { + const html = ''; + const featureSrc = 'https://cdn.example.com/uploads/asset/file/123/image.jpg'; + const result = removeDuplicateFeatureImage({html, featureSrc}); + assert.equal(result, ''); + }); + + it('removes first image when normalized src matches feature image', () => { + const html = ''; + const featureSrc = 'https://example.com/quality=100/image.jpg'; + const result = removeDuplicateFeatureImage({html, featureSrc}); + assert.equal(result, ''); + }); + + it('keeps image when it does not match feature image', () => { + const html = ''; + const featureSrc = 'https://cdn.example.com/uploads/asset/file/123/image.jpg'; + const result = removeDuplicateFeatureImage({html, featureSrc}); + assert.equal(result, ''); + }); + + it('handles image wrapped in another element', () => { + const html = '
'; + const featureSrc = 'https://cdn.example.com/uploads/asset/file/123/image.jpg'; + const result = removeDuplicateFeatureImage({html, featureSrc}); + assert.equal(result, '
'); + }); + + it('handles html without images', () => { + const html = '

No images here

'; + const featureSrc = 'https://cdn.example.com/image.jpg'; + const result = removeDuplicateFeatureImage({html, featureSrc}); + assert.equal(result, '

No images here

'); + }); + + it('keeps image when neither URL contains /uploads/asset/', () => { + const html = ''; + const featureSrc = ''; + const result = removeDuplicateFeatureImage({html, featureSrc}); + assert.equal(result, ''); + }); + + it('keeps image when URLs have different asset paths', () => { + const html = ''; + const featureSrc = 'https://cdn.example.com/uploads/asset/file/456/different.jpg'; + const result = removeDuplicateFeatureImage({html, featureSrc}); + assert.equal(result, ''); + }); + }); +}); diff --git a/packages/mg-beehiiv-api/src/types.d.ts b/packages/mg-beehiiv-api/src/types.d.ts new file mode 100644 index 000000000..f6a939c3f --- /dev/null +++ b/packages/mg-beehiiv-api/src/types.d.ts @@ -0,0 +1,77 @@ +declare module '@tryghost/errors'; +declare module '@tryghost/kg-default-cards/lib/cards/image.js'; +declare module '@tryghost/kg-default-cards/lib/cards/embed.js'; +declare module '@tryghost/kg-default-cards/lib/cards/bookmark.js'; +declare module '@tryghost/mg-fs-utils'; +declare module '@tryghost/string'; +declare module 'sanitize-html'; + +type beehiivPostDataObject = { + id: string; + title: string; + subtitle: string; + authors: string[]; + created: number; + status: 'archived' | 'confirmed' | 'draft'; + publish_date: number; + displayed_date: number | null; + split_tested: boolean; + subject_line: string; + preview_text: string; + slug: string; + thumbnail_url: string; + web_url: string; + audience: 'free' | 'premium' | 'both'; + platform: 'web' | 'email' | 'both'; + content_tags: string[]; + meta_default_description: string | null; + meta_default_title: string | null; + hidden_from_feed: boolean; + enforce_gated_content: boolean; + email_capture_popup: boolean; + content: { + premium: { + web: string; + } + } + // content: { premium: [Object] } +}; + +type tagsObject = { + url: string; + data: { + slug: string; + name: string; + } +}; + +type authorsObject = { + url: string; + data: { + slug: string; + name: string; + email: string; + } +}; + +type mappedDataObject = { + url: string; + data: { + comment_id: string; + slug: string; + published_at: Date; + updated_at: Date; + created_at: Date; + title: string; + type: string; + html: string; + status: 'published' | 'draft'; + custom_excerpt: string | null; + visibility: 'public' | 'members' | 'paid'; + tags: tagsObject[]; + feature_image?: string; + authors: authorsObject[]; + og_title?: string; + og_description?: string; + }; +}; diff --git a/packages/mg-beehiiv-api/tsconfig.json b/packages/mg-beehiiv-api/tsconfig.json new file mode 100644 index 000000000..17f8ac0a9 --- /dev/null +++ b/packages/mg-beehiiv-api/tsconfig.json @@ -0,0 +1,111 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": ["es2019"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": "src", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + "outDir": "build", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "include": ["src"] +} diff --git a/packages/migrate/bin/cli.js b/packages/migrate/bin/cli.js index 90705f78b..659a2a872 100755 --- a/packages/migrate/bin/cli.js +++ b/packages/migrate/bin/cli.js @@ -9,6 +9,8 @@ const __dirname = new URL('.', import.meta.url).pathname; const packageJSON = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf-8')); import beehiivCommands from '../commands/beehiiv.js'; +import beehiivApiCommands from '../commands/beehiiv-api.js'; +import beehiivApiMembersCommands from '../commands/beehiiv-api-members.js'; import beehiivMembersCommands from '../commands/beehiiv-members.js'; import bloggerCommands from '../commands/blogger.js'; import buttondownCommands from '../commands/buttondown.js'; @@ -49,6 +51,8 @@ const prettyCLI = Api.get() prettyCLI.preface('Command line utilities for migrating content to Ghost.'); prettyCLI.command(beehiivCommands); +prettyCLI.command(beehiivApiCommands); +prettyCLI.command(beehiivApiMembersCommands); prettyCLI.command(beehiivMembersCommands); prettyCLI.command(bloggerCommands); prettyCLI.command(buttondownCommands); diff --git a/packages/migrate/commands/beehiiv-api-members.js b/packages/migrate/commands/beehiiv-api-members.js new file mode 100644 index 000000000..6824079c0 --- /dev/null +++ b/packages/migrate/commands/beehiiv-api-members.js @@ -0,0 +1,140 @@ +import {inspect} from 'node:util'; +import {ui} from '@tryghost/pretty-cli'; +import mgBeehiivApiMembers from '@tryghost/mg-beehiiv-api-members'; +import beehiivApiMembers from '../sources/beehiiv-api-members.js'; +import {convertOptionsToSywac, convertOptionsToDefaults} from '../lib/utilties/options-to-sywac.js'; + +// Internal ID in case we need one. +const id = 'beehiiv-api-members'; + +const group = 'Sources:'; + +// The command to run and any params +const flags = 'beehiiv-api-members'; + +// Description for the top level command +const desc = 'Migrate members from beehiiv using the API'; + +// Configure all the options +const options = [ + { + type: 'string', + flags: '--key', + defaultValue: null, + desc: 'beehiiv API key', + required: true + }, + { + type: 'string', + flags: '--id', + defaultValue: null, + desc: 'beehiiv publication ID' + }, + { + type: 'boolean', + flags: '--outputSingleCSV', + defaultValue: false, + desc: 'Choose whether to export a single CSV or one for each type.' + }, + { + type: 'boolean', + flags: '--writeCSV', + defaultValue: false, + desc: 'Create a final CSV file' + }, + { + type: 'boolean', + flags: '--cache', + defaultValue: true, + desc: 'Persist local cache after migration is complete (Only if `--zip` is `true`)' + }, + { + type: 'string', + flags: '--tmpPath', + defaultValue: null, + desc: 'Specify the full path where the temporary files will be stored (Defaults a hidden tmp dir)' + }, + { + type: 'string', + flags: '--outputPath', + defaultValue: null, + desc: 'Specify the full path where the final zip file will be saved to (Defaults to CWD)' + }, + { + type: 'string', + flags: '--cacheName', + defaultValue: null, + desc: 'Provide a unique name for the cache directory (defaults to a UUID)' + }, + { + type: 'boolean', + flags: '-V --verbose', + defaultValue: Boolean(process?.env?.DEBUG), + desc: 'Show verbose output' + }, + { + type: 'boolean', + flags: '--zip', + defaultValue: false, + desc: 'Create a zip file (set to false to skip)' + } +]; + +// Build an object of defaults to be exported - Not used here, but needs to be provided +const defaults = convertOptionsToDefaults(options); + +// Convert `options` into a list of Sywac types +const setup = sywac => convertOptionsToSywac(options, sywac); + +// What to do when this command is executed +const run = async (argv) => { + let context = { + errors: [], + warnings: [] + }; + + try { + // If no publication ID is provided, list publications and exit + if (!argv.id) { + const getPubs = await mgBeehiivApiMembers.listPublications(argv.key); + + if (!getPubs || !getPubs.length) { + ui.log.error('Error fetching publications:', getPubs?.errors || 'No publications found'); + return; + } + + console.table(getPubs.map(pub => ({ // eslint-disable-line no-console + name: pub.name, + id: pub.id, + created: new Date(pub.created * 1000), + subscribers: pub.stats?.active_subscriptions || '-' + }))); + + ui.log.warn('No publication ID provided. Please provide an ID using the --id flag to run the migration.'); + + process.exit(0); + } + + // Fetch the tasks, configured correctly according to the options passed in + let migrate = beehiivApiMembers.getTaskRunner(argv); + + // Run the migration + await migrate.run(context); + + if (argv.verbose && context.result) { + ui.log.info('Done', inspect(context.result.data, false, 2)); + } + } catch (error) { + ui.log.error(error); + } +}; + +export default { + id, + group, + flags, + desc, + setup, + run, + defaults +}; diff --git a/packages/migrate/commands/beehiiv-api.js b/packages/migrate/commands/beehiiv-api.js new file mode 100644 index 000000000..e88f1079b --- /dev/null +++ b/packages/migrate/commands/beehiiv-api.js @@ -0,0 +1,192 @@ +import {inspect} from 'node:util'; +import {ui} from '@tryghost/pretty-cli'; +import mgBeehiiv from '@tryghost/mg-beehiiv-api'; +import beehiiv from '../sources/beehiiv-api.js'; +import {convertOptionsToSywac, convertOptionsToDefaults} from '../lib/utilties/options-to-sywac.js'; + +// Internal ID in case we need one. +const id = 'beehiiv-api'; + +const group = 'Sources:'; + +// The command to run and any params +const flags = 'beehiiv-api'; + +// Description for the top level command +const desc = 'Migrate from beehiiv using CSV files'; + +// Configure all the options +const options = [ + { + type: 'string', + flags: '--key', + defaultValue: null, + desc: 'beehiiv API key', + required: true + }, + { + type: 'string', + flags: '--id', + defaultValue: null, + desc: 'beehiiv publication ID' + }, + // { + // type: 'string', + // flags: '--defaultAuthorName', + // defaultValue: null, + // desc: 'The full name of the default author to assign to posts, if one cannot be found' + // }, + { + type: 'string', + flags: '--postsAfter', + defaultValue: null, + desc: 'Only migrate posts published on or after this date (YYYY-MM-DD)' + }, + { + type: 'string', + flags: '--postsBefore', + defaultValue: null, + desc: 'Only migrate posts published on or before this date (YYYY-MM-DD)' + }, + { + type: 'array', + flags: '--scrape', + choices: ['all', 'web', 'assets', 'none', 'img', 'media', 'files'], + defaultValue: ['all'], + desc: 'Configure scraping tasks (all = web + assets, web = metadata only, assets = download assets only). Legacy aliases for assets: img, media, files' + }, + // { + // type: 'string', + // flags: '--subscribeLink', + // defaultValue: '#/portal/signup', + // desc: 'Provide a path that existing "subscribe" anchors will link to e.g. "/join-us" or "#/portal/signup" (# characters need to be escaped with a \\)' + // }, + // { + // type: 'boolean', + // flags: '--comments', + // defaultValue: true, + // desc: 'Keep comment buttons' + // }, + // { + // type: 'string', + // flags: '--commentLink', + // defaultValue: '#ghost-comments-root', + // desc: 'Provide a path that existing "comment" anchors will link to e.g. "#comments" or "#ghost-comments-root" (# characters need to be escaped with a \\)' + // }, + // { + // type: 'boolean', + // flags: '--fallBackHTMLCard', + // defaultValue: true, + // desc: 'Fall back to convert to HTMLCard, if standard Mobiledoc convert fails' + // }, + { + type: 'boolean', + flags: '--cache', + defaultValue: true, + desc: 'Persist local cache after migration is complete (Only if `--zip` is `true`)' + }, + { + type: 'string', + flags: '--tmpPath', + defaultValue: null, + desc: 'Specify the full path where the temporary files will be stored (Defaults a hidden tmp dir)' + }, + { + type: 'string', + flags: '--outputPath', + defaultValue: null, + desc: 'Specify the full path where the final zip file will be saved to (Defaults to CWD)' + }, + { + type: 'string', + flags: '--cacheName', + defaultValue: null, + desc: 'Provide a unique name for the cache directory (defaults to a UUID)' + }, + { + type: 'boolean', + flags: '-V --verbose', + defaultValue: Boolean(process?.env?.DEBUG), + desc: 'Show verbose output' + }, + { + type: 'boolean', + flags: '--veryVerbose', + defaultValue: false, + desc: 'Show very verbose output (implies --verbose)' + }, + { + type: 'boolean', + flags: '--zip', + defaultValue: true, + desc: 'Create a zip file (set to false to skip)' + } +]; + +// Build an object of defaults to be exported - Not used here, but needs to be provided +const defaults = convertOptionsToDefaults(options); + +// Convert `options` into a list of Sywac types +const setup = sywac => convertOptionsToSywac(options, sywac); + +// What to do when this command is executed +const run = async (argv) => { + let context = { + errors: [], + warnings: [] + }; + + // Remove trailing slash from URL + if (argv.url && argv.url.endsWith('/')) { + argv.url = argv.url.slice(0, -1); + } + + try { + // If no publication ID is provided, list publications and exit + if (!argv.id) { + const getPubs = await mgBeehiiv.listPublications(argv.key); + + if (!getPubs || !getPubs.length) { + ui.log.error('Error fetching publications:', getPubs.errors); + return; + } + + console.table(getPubs.map(pub => ({ // eslint-disable-line no-console + name: pub.name, + id: pub.id, + created: new Date(pub.created * 1000), + subscribers: pub.stats?.active_subscriptions || '-' + }))); + + ui.log.warn('No publication ID provided. Please provide an ID using the --id flag to run the migration.'); + + process.exit(0); + } + + // Fetch the tasks, configured correctly according to the options passed in + let migrate = beehiiv.getTaskRunner(argv); + + // Run the migration + await migrate.run(context); + + if (argv.verbose && context.result) { + ui.log.info('Done'); + } + + if (argv.veryVerbose && context.result) { + ui.log.info(inspect(context.result.data, false, 2)); + } + } catch (error) { + ui.log.error(error); + } +}; + +export default { + id, + group, + flags, + desc, + setup, + run, + defaults +}; diff --git a/packages/migrate/package.json b/packages/migrate/package.json index 347fba1ee..3f96f0435 100644 --- a/packages/migrate/package.json +++ b/packages/migrate/package.json @@ -33,6 +33,8 @@ "@tryghost/listr-smart-renderer": "0.5.24", "@tryghost/mg-assetscraper-db": "0.3.0", "@tryghost/mg-beehiiv": "0.3.15", + "@tryghost/mg-beehiiv-api": "0.1.0", + "@tryghost/mg-beehiiv-api-members": "0.1.0", "@tryghost/mg-beehiiv-members": "0.1.24", "@tryghost/mg-blogger": "0.1.31", "@tryghost/mg-buttondown": "0.1.5", diff --git a/packages/migrate/sources/beehiiv-api-members.js b/packages/migrate/sources/beehiiv-api-members.js new file mode 100644 index 000000000..fe295fdb0 --- /dev/null +++ b/packages/migrate/sources/beehiiv-api-members.js @@ -0,0 +1,184 @@ +import {join} from 'node:path'; +import {readFileSync} from 'node:fs'; +import fsUtils from '@tryghost/mg-fs-utils'; +import beehiivApiMembers from '@tryghost/mg-beehiiv-api-members'; +import {makeTaskRunner} from '@tryghost/listr-smart-renderer'; +import prettyMilliseconds from 'pretty-ms'; + +const getTaskRunner = (options) => { + let tasks = [ + { + title: 'Initializing', + task: (ctx, task) => { + ctx.options = options; + ctx.allMembers = []; + + // Prep a file cache for the work we are about to do. + ctx.options.cacheName = options.cacheName || options.id; + ctx.fileCache = new fsUtils.FileCache(`beehiiv-api-members-${ctx.options.cacheName}`, { + tmpPath: ctx.options.tmpPath, + contentDir: false + }); + + ctx.result = {}; + + task.output = `Workspace initialized at ${ctx.fileCache.cacheDir}`; + } + }, + { + title: 'Fetching subscriptions from beehiiv API', + task: async (ctx) => { + const fetchTasks = await beehiivApiMembers.fetchTasks(options, ctx); + return makeTaskRunner(fetchTasks, options); + } + }, + { + title: 'Mapping subscriptions to Ghost member format', + task: async (ctx) => { + const mapTasks = beehiivApiMembers.mapMembersTasks(options, ctx); + return makeTaskRunner(mapTasks, options); + } + }, + { + title: 'Create batches and write CSV files', + enabled: () => !options.outputSingleCSV, + task: async (ctx) => { + try { + const types = Object.keys(ctx.result.members); + const files = []; + + types.forEach(async (type) => { + files.push({ + data: ctx.result.members[type], + fileName: `gh-members-${type}.csv`, + tmpFilename: `gh-members-${type}-${Date.now()}.csv` + }); + }); + + await Promise.all(files.map(async ({data, fileName, tmpFilename}) => { + if (!data || data.length === 0) { + return; + } + + data = await fsUtils.csv.jsonToCSV(data); + + // write the members import file for each batch + await ctx.fileCache.writeGhostImportFile(data, {isJSON: false, filename: fileName, tmpFilename: tmpFilename}); + })); + + if (ctx.logs) { + await ctx.fileCache.writeErrorJSONFile(ctx.logs, {filename: `gh-members-updated-${Date.now()}.logs.json`}); + } + } catch (error) { + ctx.errors.push({message: 'Failed to batch files', error}); + throw error; + } + } + }, + { + title: 'Create singular members list', + enabled: () => options.outputSingleCSV, + task: async (ctx) => { + Object.keys(ctx.result.members).forEach((type) => { + ctx.allMembers.push(...ctx.result.members[type]); + }); + + let tmpFilename = `gh-beehiiv-api-members-${Date.now()}.csv`; + + let data = await fsUtils.csv.jsonToCSV(ctx.allMembers); + await ctx.fileCache.writeTmpFile(data, tmpFilename, false); + } + }, + { + title: 'Write zip file', + enabled: () => !options.outputSingleCSV, + skip: () => !options.zip, + task: async (ctx, task) => { + const isStorage = (options?.outputStorage && typeof options.outputStorage === 'object') ?? false; + + try { + let timer = Date.now(); + const zipFinalPath = options.outputPath || process.cwd(); + + // zip the file and save it temporarily + ctx.outputFile = await fsUtils.zip.write(zipFinalPath, ctx.fileCache.zipDir, ctx.fileCache.defaultZipFileName); + + if (isStorage) { + const storage = options.outputStorage; + const localFilePath = ctx.outputFile.path; + + // read the file buffer + const fileBuffer = await readFileSync(ctx.outputFile.path); + // Upload the file to the storage + ctx.outputFile.path = await storage.upload({body: fileBuffer, fileName: `gh-beehiiv-api-members-${ctx.options.cacheName}.zip`}); + // now that the file is uploaded to the storage, delete the local zip file + await fsUtils.zip.deleteFile(localFilePath); + } + + task.output = `Successfully written zip to ${ctx.outputFile.path} in ${prettyMilliseconds(Date.now() - timer)}`; + } catch (error) { + ctx.errors.push({message: 'Failed to write and upload ZIP file', error}); + throw error; + } + } + }, + { + title: 'Write CSV file', + enabled: () => !options.zip && options.outputSingleCSV && options.writeCSV, + task: async (ctx, task) => { + const isStorage = (options?.outputStorage && typeof options.outputStorage === 'object') ?? false; + + try { + let timer = Date.now(); + const csvFinalPath = options.outputPath || process.cwd(); + + let fileName = `gh-beehiiv-api-members.csv`; + let filePath = join(csvFinalPath, fileName); + let data = await fsUtils.csv.jsonToCSV(ctx.allMembers); + + // save the file + await ctx.fileCache.saveFile(filePath, data); + ctx.outputFile = { + path: filePath + }; + + if (isStorage) { + const storage = options.outputStorage; + const localFilePath = ctx.outputFile.path; + + // read the file buffer + const fileBuffer = await readFileSync(ctx.outputFile.path); + // Upload the file to the storage + ctx.outputFile.path = await storage.upload({body: fileBuffer, fileName: `gh-beehiiv-api-members-${ctx.options.cacheName}.csv`}); + // now that the file is uploaded to the storage, delete the local zip file + await ctx.fileCache.deleteFileOrDir(localFilePath); + } + + task.output = `Successfully written output to ${ctx.outputFile.path} in ${prettyMilliseconds(Date.now() - timer)}`; + } catch (error) { + ctx.errors.push({message: 'Failed to write and upload output file', error}); + throw error; + } + } + }, + { + title: 'Clearing cached files', + enabled: () => !options.cache && options.zip, + task: async (ctx) => { + try { + await ctx.fileCache.emptyCurrentCacheDir(); + } catch (error) { + ctx.errors.push({message: 'Failed to clear cache', error}); + throw error; + } + } + } + ]; + + // Configure a new Listr task manager, we can use different renderers for different configs + return makeTaskRunner(tasks, Object.assign({topLevel: true}, options)); +}; + +export default { + getTaskRunner +}; diff --git a/packages/migrate/sources/beehiiv-api.js b/packages/migrate/sources/beehiiv-api.js new file mode 100644 index 000000000..ea71b2ade --- /dev/null +++ b/packages/migrate/sources/beehiiv-api.js @@ -0,0 +1,203 @@ +import {readFileSync} from 'node:fs'; +import mgBeehiiv from '@tryghost/mg-beehiiv-api'; +import {toGhostJSON} from '@tryghost/mg-json'; +import mgHtmlLexical from '@tryghost/mg-html-lexical'; +import MgAssetScraper from '@tryghost/mg-assetscraper-db'; +import MgLinkFixer from '@tryghost/mg-linkfixer'; +import fsUtils from '@tryghost/mg-fs-utils'; +import {makeTaskRunner} from '@tryghost/listr-smart-renderer'; +import prettyMilliseconds from 'pretty-ms'; + +const initialize = (options) => { + return { + title: 'Initializing Workspace', + task: async (ctx, task) => { + ctx.options = options; + ctx.allowScrape = { + all: ctx.options.scrape.includes('all'), + assets: ctx.options.scrape.includes('all') || ctx.options.scrape.includes('assets') || ctx.options.scrape.includes('img') || ctx.options.scrape.includes('media') || ctx.options.scrape.includes('files'), + web: ctx.options.scrape.includes('web') || ctx.options.scrape.includes('all') + }; + + // 0. Prep a file cache, scrapers, etc, to prepare for the work we are about to do. + ctx.options.cacheName = options.cacheName || fsUtils.utils.cacheNameFromPath(ctx.options.id); + ctx.fileCache = new fsUtils.FileCache(`beehiiv-api-${ctx.options.cacheName}`, { + tmpPath: ctx.options.tmpPath + }); + ctx.assetScraper = new MgAssetScraper(ctx.fileCache, { + domains: [ + 'http://media.beehiiv.com', + 'https://media.beehiiv.com' + ] + }, ctx); + await ctx.assetScraper.init(); + ctx.linkFixer = new MgLinkFixer(); + + ctx.result = { + posts: [] + }; + + task.output = `Workspace initialized at ${ctx.fileCache.cacheDir}`; + } + }; +}; + +const getFullTaskList = (options) => { + return [ + initialize(options), + { + title: 'Fetch beehiiv content', + task: async (ctx) => { + try { + let tasks = await mgBeehiiv.fetchTasks(options, ctx); + // return makeTaskRunner(tasks, options); + return makeTaskRunner(tasks, {...options}); + } catch (error) { + ctx.errors.push({message: 'Failed to fetch content from beehiiv API', error}); + throw error; + } + } + }, + { + title: 'Map & process content', + task: async (ctx) => { + try { + let tasks = await mgBeehiiv.mapPostsTasks(options, ctx); + return makeTaskRunner(tasks, {...options}); + } catch (error) { + ctx.errors.push({message: 'Failed to process beehiiv content', error}); + throw error; + } + } + }, + { + title: 'Build Link Map', + task: async (ctx) => { + // 4. Create a map of all known links for use later + try { + ctx.linkFixer.buildMap(ctx); + } catch (error) { + ctx.errors.push({message: 'Failed to build link map', error}); + throw error; + } + } + }, + { + title: 'Format data as Ghost JSON', + task: async (ctx) => { + // 5. Format the data as a valid Ghost JSON file + try { + ctx.result = await toGhostJSON(ctx.result, ctx.options, ctx); + } catch (error) { + ctx.errors.push({message: 'Failed to format data as Ghost JSON', error}); + throw error; + } + } + }, + { + title: 'Fetch images via AssetScraper', + skip: ctx => !ctx.allowScrape.assets, + task: async (ctx) => { + // 6. Format the data as a valid Ghost JSON file + let tasks = ctx.assetScraper.getTasks(); + return makeTaskRunner(tasks, { + verbose: options.verbose, + exitOnError: false, + concurrent: false + }); + } + }, + { + title: 'Update links in content via LinkFixer', + task: async (ctx, task) => { + // 7. Process the content looking for known links, and update them to new links + let tasks = ctx.linkFixer.fix(ctx, task); + return makeTaskRunner(tasks, options); + } + }, + { + title: 'Convert HTML -> Lexical', + task: (ctx) => { + // 8. Convert post HTML -> Lexical + try { + let tasks = mgHtmlLexical.convert(ctx); + return makeTaskRunner(tasks, options); + } catch (error) { + ctx.errors.push({message: 'Failed to convert HTML -> Lexical', error}); + throw error; + } + } + }, + { + title: 'Write Ghost import JSON File', + task: async (ctx) => { + // 9. Write a valid Ghost import zip + try { + await ctx.fileCache.writeGhostImportFile(ctx.result); + } catch (error) { + ctx.errors.push({message: 'Failed to write Ghost import JSON File', error}); + throw error; + } + } + }, + { + title: 'Write Ghost import zip', + skip: () => !options.zip, + task: async (ctx, task) => { + // 10. Write a valid Ghost import zip + const isStorage = (options?.outputStorage && typeof options.outputStorage === 'object') ?? false; + + try { + let timer = Date.now(); + const zipFinalPath = options.outputPath || process.cwd(); + // zip the file and save it temporarily + ctx.outputFile = await fsUtils.zip.write(zipFinalPath, ctx.fileCache.zipDir, ctx.fileCache.defaultZipFileName); + + if (isStorage) { + const storage = options.outputStorage; + const localFilePath = ctx.outputFile.path; + + // read the file buffer + const fileBuffer = await readFileSync(ctx.outputFile.path); + // Upload the file to the storage + ctx.outputFile.path = await storage.upload({body: fileBuffer, fileName: `gh-beehiiv-${ctx.options.cacheName}.zip`}); + // now that the file is uploaded to the storage, delete the local zip file + await fsUtils.zip.deleteFile(localFilePath); + } + + task.output = `Successfully written zip to ${ctx.outputFile.path} in ${prettyMilliseconds(Date.now() - timer)}`; + } catch (error) { + ctx.errors.push({message: 'Failed to write and upload ZIP file', error}); + throw error; + } + } + }, + { + title: 'Clearing cached files', + enabled: () => !options.cache && options.zip, + task: async (ctx) => { + try { + await ctx.fileCache.emptyCurrentCacheDir(); + } catch (error) { + ctx.errors.push({message: 'Failed to clear temporary cached files', error}); + throw error; + } + } + } + ]; +}; + +const getTaskRunner = (options) => { + let tasks = []; + + tasks = getFullTaskList(options); + + // Configure a new Listr task manager, we can use different renderers for different configs + return makeTaskRunner(tasks, Object.assign({topLevel: true}, options)); +}; + +export default { + initialize, + getFullTaskList, + getTaskRunner +};