diff --git a/.gitignore b/.gitignore index ff3babd..c9e85bb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules .temp .cache .env -dist/ \ No newline at end of file +dist/ +.idea/ \ No newline at end of file diff --git a/README.md b/README.md index 041eada..9be1e42 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ The following [Notion API block object types](https://developers.notion.com/refe | Video | ❌ Missing | | | File | ❌ Missing | | | PDF | ❌ Missing | | -| Bookmark | ❌ Missing | | +| Bookmark | ✅ Yes | use a caption as a link name | | | Equation | ❌ Missing | | | Divider | ✅ Yes | | | Table Of Contents | ❌ not planned | static site generators have their own ToC implementations | @@ -77,7 +77,7 @@ The following [Notion API page property types](https://developers.notion.com/ref | Propety type | Supported | Notes | | ---------------- | --------- | ----------------------------- | -| Rich text | ✅ Yes | rendered as markdown string | +| Rich text | ✅ Yes | rendered as plaintext string | | Number | ✅ Yes | | | Select | ✅ Yes | rendered as name | | Multi Select | ✅ Yes | rendered as array of names | @@ -86,7 +86,7 @@ The following [Notion API page property types](https://developers.notion.com/ref | Relation | ✅ Yes | rendered as array of page ids | | Rollup | ❌ missing | | | Title | ✅ Yes | used as page title | -| People | ❌ missing | | +| People | ✅ Yes | rendered as comma-separated list of names | | Files | ❌ missing | | | Checkbox | ❌ missing | | | Url | ✅ Yes | rendered as string | diff --git a/package.json b/package.json index d4d0bc7..7d64c0f 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "0.9.0", + "version": "0.11.1", "name": "@meshcloud/notion-markdown-cms", "engines": { "node": ">=14" diff --git a/src/AssetWriter.ts b/src/AssetWriter.ts index 6d01ff5..d9dcdf9 100644 --- a/src/AssetWriter.ts +++ b/src/AssetWriter.ts @@ -1,27 +1,25 @@ -import { promises as fs } from 'fs'; -import got from 'got'; -import { KeyvFile } from 'keyv-file'; -import * as mime from 'mime-types'; - -import { RenderingLoggingContext } from './logger'; +import { promises as fs } from "fs"; +import got from "got"; +import { KeyvFile } from "keyv-file"; +import * as mime from "mime-types"; +import { RenderingContextLogger } from "./RenderingContextLogger"; const cache = new KeyvFile({ filename: ".cache/keyv.json", }); export class AssetWriter { - constructor(readonly dir: string) {} + constructor( + private readonly dir: string, + private readonly logger: RenderingContextLogger + ) {} async store(name: string, buffer: Buffer) { await fs.mkdir(this.dir, { recursive: true }); await fs.writeFile(`${this.dir}/${name}`, buffer); } - async download( - url: string, - fileName: string, - context: RenderingLoggingContext - ) { + async download(url: string, fileName: string) { // the got http lib promises to do proper user-agent compliant http caching // see https://github.com/sindresorhus/got/blob/main/documentation/cache.md @@ -35,7 +33,7 @@ export class AssetWriter { const imageFile = fileName + "." + ext; const cacheInfo = response.isFromCache ? " (from cache)" : ""; - context.info(`downloading ${imageFile}` + cacheInfo); + this.logger.info(`downloading ${imageFile}` + cacheInfo); await this.store(imageFile, response.rawBody); return imageFile; diff --git a/src/BlockRenderer.ts b/src/BlockRenderer.ts index c273765..6f98ea9 100644 --- a/src/BlockRenderer.ts +++ b/src/BlockRenderer.ts @@ -1,12 +1,20 @@ -import { RichText } from '@notionhq/client/build/src/api-types'; +import { RichText } from "@notionhq/client/build/src/api-types"; -import { AssetWriter } from './AssetWriter'; +import { AssetWriter } from "./AssetWriter"; import { - Block, Emoji, ExternalFile, ExternalFileWithCaption, File, FileWithCaption, ImageBlock -} from './Blocks'; -import { DeferredRenderer } from './DeferredRenderer'; -import { RenderingLoggingContext } from './logger'; -import { RichTextRenderer } from './RichTextRenderer'; + Block, + Emoji, + ExternalFile, + ExternalFileWithCaption, + File, + FileWithCaption, + ImageBlock, +} from "./Blocks"; +import { DeferredRenderer } from "./DeferredRenderer"; +import { RenderingContextLogger } from "./RenderingContextLogger"; +import { RichTextRenderer } from "./RichTextRenderer"; +import { RenderingContext } from "./RenderingContext"; +import { LinkRenderer } from "./LinkRenderer"; const debug = require("debug")("blocks"); @@ -17,13 +25,13 @@ export interface BlockRenderResult { export class BlockRenderer { constructor( private readonly richText: RichTextRenderer, - private readonly deferredRenderer: DeferredRenderer + private readonly deferredRenderer: DeferredRenderer, + private readonly link: LinkRenderer ) {} async renderBlock( block: Block, - assets: AssetWriter, - context: RenderingLoggingContext + context: RenderingContext ): Promise { const renderMarkdown = async (text: RichText[]) => { return await this.richText.renderMarkdown(text, context); @@ -63,7 +71,7 @@ export class BlockRenderer { }; case "image": return { - lines: await this.renderImage(block, assets, context), + lines: await this.renderImage(block, context.assetWriter), }; case "quote": { // it's legal for a notion block to be cmoposed of multiple lines @@ -85,7 +93,7 @@ export class BlockRenderer { case "callout": { // render emoji as bold, this enables css to target it as `blockquote > strong:first-child` const content = - `**${this.renderIcon(block.callout.icon, context)}** ` + + `**${this.renderIcon(block.callout.icon, context.logger)}** ` + (await renderMarkdown(block.callout.text)); return { @@ -96,16 +104,27 @@ export class BlockRenderer { return { lines: "---" }; case "child_database": const msg = `\n`; - const db = await this.deferredRenderer.renderChildDatabase(block.id); + const db = await this.deferredRenderer.renderChildDatabase( + block.id, + context.linkResolver + ); return { lines: msg + db.markdown }; case "synced_block": // nothing to render, only the contents of the synced block are relevant // however, these are children nöpcl, and thus retrieved by recursion in RecusivveBodyRenderer return null; + case "bookmark": + // render caption (if provided) as a link name + const caption = block.bookmark.caption || []; + let title = block.bookmark.url; + if (caption.length > 0) + title = await this.richText.renderPlainText(caption); + return { + lines: this.link.renderUrlLink(title, block.bookmark.url), + }; case "toggle": case "child_page": case "embed": - case "bookmark": case "video": case "file": case "pdf": @@ -116,7 +135,7 @@ export class BlockRenderer { lines: this.renderUnsupported( `unsupported block type: ${block.type}`, block, - context + context.logger ), }; } @@ -124,7 +143,7 @@ export class BlockRenderer { private renderIcon( icon: File | ExternalFile | Emoji, - context: RenderingLoggingContext + logger: RenderingContextLogger ): string { switch (icon.type) { case "emoji": @@ -134,19 +153,15 @@ export class BlockRenderer { return this.renderUnsupported( `unsupported icon type: ${icon.type}`, icon, - context + logger ); } } - async renderImage( - block: ImageBlock, - assets: AssetWriter, - context: RenderingLoggingContext - ): Promise { + async renderImage(block: ImageBlock, assets: AssetWriter): Promise { const url = this.parseUrl(block.image); - const imageFile = await assets.download(url, block.id, context); + const imageFile = await assets.download(url, block.id); // todo: caption support const markdown = `![image-${block.id}](./${imageFile})`; @@ -172,7 +187,7 @@ export class BlockRenderer { private renderUnsupported( msg: string, obj: any, - context: RenderingLoggingContext + context: RenderingContextLogger ): string { context.warn(msg); debug(msg + "\n%O", obj); diff --git a/src/ChildDatabaseRenderer.ts b/src/ChildDatabaseRenderer.ts index 16ce4c8..ae52645 100644 --- a/src/ChildDatabaseRenderer.ts +++ b/src/ChildDatabaseRenderer.ts @@ -1,13 +1,18 @@ -import { Page } from '@notionhq/client/build/src/api-types'; - -import { SyncConfig } from './'; -import { lookupDatabaseConfig } from './config'; -import { Database } from './Database'; -import { DatabaseViewRenderer } from './DatabaseViewRenderer'; -import { DeferredRenderer } from './DeferredRenderer'; -import { NotionApiFacade } from './NotionApiFacade'; -import { RenderDatabasePageTask } from './RenderDatabasePageTask'; -import { DatabaseConfig, DatabaseConfigRenderPages, DatabaseConfigRenderTable } from './SyncConfig'; +import { Page } from "@notionhq/client/build/src/api-types"; + +import { SyncConfig } from "./"; +import { lookupDatabaseConfig } from "./config"; +import { Database } from "./Database"; +import { DatabaseViewRenderer } from "./DatabaseViewRenderer"; +import { DeferredRenderer } from "./DeferredRenderer"; +import { NotionApiFacade } from "./NotionApiFacade"; +import { PageLinkResolver } from "./PageLinkResolver"; +import { RenderDatabasePageTask } from "./RenderDatabasePageTask"; +import { + DatabaseConfig, + DatabaseConfigRenderPages, + DatabaseConfigRenderTable, +} from "./SyncConfig"; const debug = require("debug")("child-database"); @@ -17,22 +22,31 @@ export class ChildDatabaseRenderer { private readonly publicApi: NotionApiFacade, private readonly deferredRenderer: DeferredRenderer, private readonly viewRenderer: DatabaseViewRenderer - ) { } + ) {} - async renderChildDatabase(databaseId: string): Promise { + async renderChildDatabase( + databaseId: string, + linkResolver: PageLinkResolver + ): Promise { const dbConfig = lookupDatabaseConfig(this.config, databaseId); // no view was defined for this database, render as a plain inline table const allPages = await this.fetchPages(databaseId, dbConfig); - const renderPages = dbConfig.renderAs === "pages+views" + const renderPages = dbConfig.renderAs === "pages+views"; - debug("rendering child database " + databaseId + " as " + dbConfig.renderAs); + debug( + "rendering child database " + databaseId + " as " + dbConfig.renderAs + ); if (renderPages) { const pageConfig = dbConfig as DatabaseConfigRenderPages; const entries = await this.queuePageRendering(allPages, pageConfig); - const markdown = await this.viewRenderer.renderViews(entries, dbConfig as DatabaseConfigRenderPages); + const markdown = await this.viewRenderer.renderViews( + entries, + dbConfig as DatabaseConfigRenderPages, + linkResolver + ); return { config: dbConfig, @@ -42,15 +56,18 @@ export class ChildDatabaseRenderer { } else { // render table const entries = await this.queueEntryRendering(allPages, dbConfig); - const markdown = this.viewRenderer.renderViews(entries, dbConfig); - + const markdown = this.viewRenderer.renderViews( + entries, + dbConfig, + linkResolver + ); + return { config: dbConfig, entries, markdown, }; } - } private async queueEntryRendering( @@ -89,6 +106,6 @@ export class ChildDatabaseRenderer { page_size: 100, }); - return allPages.results; + return allPages; } } diff --git a/src/DatabasePageRenderer.ts b/src/DatabasePageRenderer.ts index fe15bc5..d3ea572 100644 --- a/src/DatabasePageRenderer.ts +++ b/src/DatabasePageRenderer.ts @@ -2,14 +2,13 @@ import * as fsc from "fs"; import { Page } from "@notionhq/client/build/src/api-types"; -import { AssetWriter } from "./AssetWriter"; import { FrontmatterRenderer } from "./FrontmatterRenderer"; -import { RenderingLoggingContext } from "./logger"; import { PropertiesParser } from "./PropertiesParser"; import { RecursiveBodyRenderer } from "./RecursiveBodyRenderer"; import { RenderDatabasePageTask as RenderDatabasePageTask } from "./RenderDatabasePageTask"; import { DatabaseConfigRenderPages } from "./SyncConfig"; import { slugify } from "./slugify"; +import { RenderingContext } from "./RenderingContext"; const fs = fsc.promises; @@ -41,35 +40,33 @@ export class DatabasePageRenderer { frontmatter: frontmatterProperties, properties: props, render: async () => { - const context = new RenderingLoggingContext(page.url, file); - + + const context = new RenderingContext(page.url, file) + if (page.archived) { // have to skip rendering archived pages as attempting to retrieve the block will result in a HTTP 404 - context.warn(`page is archived - skipping`); + context.logger.warn(`page is archived - skipping`); return; } try { - const assetWriter = new AssetWriter(destDir); - const frontmatter = this.frontmatterRenderer.renderFrontmatter(frontmatterProperties); const body = await this.bodyRenderer.renderBody( page, - assetWriter, context ); await fs.mkdir(destDir, { recursive: true }); await fs.writeFile(file, frontmatter + body); - context.complete(); + context.logger.complete(); } catch (error) { // While catch-log-throw is usually an antipattern, it is the renderes job to orchestrate the rendering // job with concerns like logging and writing to the outside world. Hence this place is appropriate. // We need to throw the error here so that the rendering process can crash with a proper error message, since // an error at this point here is unrecoverable. - context.error(error); + context.logger.error(error); throw error; } }, diff --git a/src/DatabaseViewRenderer.ts b/src/DatabaseViewRenderer.ts index 5736c87..16cc20e 100644 --- a/src/DatabaseViewRenderer.ts +++ b/src/DatabaseViewRenderer.ts @@ -1,6 +1,7 @@ import { DatabaseConfigRenderTable } from "."; import { LinkRenderer } from "./LinkRenderer"; import * as markdownTable from "./markdown-table"; +import { PageLinkResolver } from "./PageLinkResolver"; import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask"; import { RenderDatabasePageTask } from "./RenderDatabasePageTask"; import { DatabaseConfigRenderPages, DatabaseView } from "./SyncConfig"; @@ -11,7 +12,8 @@ export class DatabaseViewRenderer { public renderViews( entries: (RenderDatabasePageTask | RenderDatabaseEntryTask)[], - config: DatabaseConfigRenderPages | DatabaseConfigRenderTable + config: DatabaseConfigRenderPages | DatabaseConfigRenderTable, + linkResolver: PageLinkResolver ): string { const configuredViews = config.views || [{}]; @@ -19,7 +21,7 @@ export class DatabaseViewRenderer { const groupByProperty = view?.properties?.groupBy; if (!groupByProperty) { - return this.renderView(entries, null, view); + return this.renderView(entries, null, view, linkResolver); } else { const grouped = new Array( ...groupBy(entries, (p) => @@ -28,7 +30,7 @@ export class DatabaseViewRenderer { ); return grouped - .map(([key, pages]) => this.renderView(pages, key, view)) + .map(([key, pages]) => this.renderView(pages, key, view, linkResolver)) .join("\n\n"); } }); @@ -39,7 +41,8 @@ export class DatabaseViewRenderer { private renderView( pages: (RenderDatabasePageTask | RenderDatabaseEntryTask)[], titleAppendix: string | null, - view: DatabaseView + view: DatabaseView, + linkResolver: PageLinkResolver ): string { if (!pages[0]) { return ""; @@ -60,7 +63,7 @@ export class DatabaseViewRenderer { cols.map((c, i) => { const content = escapeTableCell(r.properties.properties.get(c)); return i == 0 && isRenderPageTask(r) - ? this.linkRenderer.renderPageLink(content, r) // make the first cell a relative link to the page + ? this.linkRenderer.renderPageLink(content, r, linkResolver) // make the first cell a relative link to the page : content; }) ) @@ -68,7 +71,9 @@ export class DatabaseViewRenderer { const tableMd = markdownTable.markdownTable(table); if (view.title) { - const formattedTitle = [view.title, titleAppendix].filter(x => !!x).join(" - "); + const formattedTitle = [view.title, titleAppendix] + .filter((x) => !!x) + .join(" - "); return `## ${formattedTitle}\n\n` + tableMd; } else { return tableMd; diff --git a/src/DeferredRenderer.ts b/src/DeferredRenderer.ts index 75756b6..228e9f1 100644 --- a/src/DeferredRenderer.ts +++ b/src/DeferredRenderer.ts @@ -4,6 +4,7 @@ import { ChildDatabaseRenderer } from "./ChildDatabaseRenderer"; import { Database } from "./Database"; import { DatabaseEntryRenderer } from "./DatabaseEntryRenderer"; import { DatabasePageRenderer } from "./DatabasePageRenderer"; +import { PageLinkResolver } from "./PageLinkResolver"; import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask"; import { RenderDatabasePageTask as RenderDatabasePageTask } from "./RenderDatabasePageTask"; import { RenderedDatabaseEntry } from "./RenderedDatabaseEntry"; @@ -35,8 +36,11 @@ export class DeferredRenderer { this.entryRenderer = entryRenderer; } - public async renderChildDatabase(databaseId: string): Promise { - return await this.dbRenderer.renderChildDatabase(databaseId); + public async renderChildDatabase( + databaseId: string, + linkResolver: PageLinkResolver + ): Promise { + return await this.dbRenderer.renderChildDatabase(databaseId, linkResolver); } public async renderPage( diff --git a/src/LinkRenderer.spec.ts b/src/LinkRenderer.spec.ts index 7bcd2b1..938ddc3 100644 --- a/src/LinkRenderer.spec.ts +++ b/src/LinkRenderer.spec.ts @@ -1,14 +1,17 @@ import { LinkRenderer } from './LinkRenderer'; +import { PageLinkResolver } from './PageLinkResolver'; import { RenderDatabasePageTask } from './RenderDatabasePageTask'; describe("LinkRenderer", () => { + test("renderPageLink strips outDir from link", async () => { + const resolver = new PageLinkResolver("out"); const sut = new LinkRenderer(); const page: Partial = { file: "out/test.md", }; - const link = sut.renderPageLink("text", page as any); - expect(link).toEqual("[text](/out/test.md)"); + const link = sut.renderPageLink("text", page as any, resolver); + expect(link).toEqual("[text](./test.md)"); }); }); diff --git a/src/LinkRenderer.ts b/src/LinkRenderer.ts index 42da12c..978ae38 100644 --- a/src/LinkRenderer.ts +++ b/src/LinkRenderer.ts @@ -1,4 +1,5 @@ -import { RenderDatabasePageTask } from './RenderDatabasePageTask'; +import { PageLinkResolver } from "./PageLinkResolver"; +import { RenderDatabasePageTask } from "./RenderDatabasePageTask"; export class LinkRenderer { constructor() {} @@ -7,9 +8,13 @@ export class LinkRenderer { return `[${text}](${url})`; } - renderPageLink(text: string, page: RenderDatabasePageTask): string { - const url = "/" + page.file; - - return this.renderUrlLink(text, url); + renderPageLink( + text: string, + toPage: RenderDatabasePageTask, + linkResolver: PageLinkResolver + ): string { + const link = linkResolver.resolveRelativeLinkTo(toPage.file); + + return this.renderUrlLink(text, link); } } diff --git a/src/NotionApiFacade.ts b/src/NotionApiFacade.ts index 2cbd92b..721c2d2 100644 --- a/src/NotionApiFacade.ts +++ b/src/NotionApiFacade.ts @@ -1,7 +1,11 @@ import { - APIErrorCode, APIResponseError, Client, RequestTimeoutError, UnknownHTTPResponseError -} from '@notionhq/client'; -import { DatabasesQueryParameters } from '@notionhq/client/build/src/api-endpoints'; + APIErrorCode, + APIResponseError, + Client, + RequestTimeoutError, + UnknownHTTPResponseError, +} from "@notionhq/client"; +import { DatabasesQueryParameters } from "@notionhq/client/build/src/api-endpoints"; const debug = require("debug")("notion-api"); @@ -34,17 +38,25 @@ export class NotionApiFacade { } async queryDatabase(query: DatabasesQueryParameters) { - const result = await this.withRetry( - async () => await this.client.databases.query(query) - ); // todo: paging + const results = []; - if (result.next_cursor) { - throw new Error( - `Paging not implemented, db ${query.database_id} has more than 100 entries` + let next_cursor: string | null = null; + + do { + const response = await this.withRetry( + async () => + await this.client.databases.query({ + ...query, + start_cursor: next_cursor || undefined, + }) ); - } - return result; + results.push(...response.results); + + next_cursor = response.next_cursor; + } while (next_cursor); + + return results; } async retrievePage(pageId: string) { @@ -54,20 +66,25 @@ export class NotionApiFacade { } async listBlockChildren(blockId: string) { - const result = await this.withRetry( - async () => - await this.client.blocks.children.list({ - block_id: blockId, - }) - ); // todo: paging here? - - if (result.next_cursor) { - throw new Error( - `Paging not implemented, block ${blockId} has more children than returned in a single request` + const results = []; + + let next_cursor: string | null = null; + + do { + const response = await this.withRetry( + async () => + await this.client.blocks.children.list({ + block_id: blockId, + start_cursor: next_cursor || undefined, + }) ); - } - return result; + results.push(...response.results); + + next_cursor = response.next_cursor; + } while (next_cursor); + + return results; } printStats() { diff --git a/src/PageLinkResolver.ts b/src/PageLinkResolver.ts new file mode 100644 index 0000000..21406ec --- /dev/null +++ b/src/PageLinkResolver.ts @@ -0,0 +1,19 @@ +import * as path from "path"; + +export class PageLinkResolver { + private readonly absoluteFromDir: string; + + constructor(sourceDir: string) { + this.absoluteFromDir = path.resolve(sourceDir); + } + + resolveRelativeLinkTo(file: string) { + const absoluteToDir = path.resolve(file); + + const relativePath = path.relative(this.absoluteFromDir, absoluteToDir); + + // normalize the rleative path to start with a ./ as that's the markdown convention for relative links + // at least how it's most commonly interpreted (e.g. by textlint) + return relativePath.startsWith(".") ? relativePath : "./" + relativePath; + } +} diff --git a/src/PropertiesParser.ts b/src/PropertiesParser.ts index e5aab1b..333716e 100644 --- a/src/PropertiesParser.ts +++ b/src/PropertiesParser.ts @@ -1,7 +1,7 @@ import { Page, PropertyValue } from "@notionhq/client/build/src/api-types"; import { DatabasePageProperties } from "./DatabasePageProperties"; -import { RenderingLoggingContext } from "./logger"; +import { RenderingContextLogger } from "./RenderingContextLogger"; import { RichTextRenderer } from "./RichTextRenderer"; const debug = require("debug")("properties"); @@ -44,7 +44,7 @@ export class PropertiesParser { let title: string | null = null; let titleProperty: string | null = null; - const context = new RenderingLoggingContext(page.url); + const context = new RenderingContextLogger(page.url); for (const [name, value] of Object.entries(page.properties)) { const parsedValue = await this.parsePropertyValue(value, context); @@ -78,15 +78,15 @@ export class PropertiesParser { private async parsePropertyValue( value: PropertyValue, - context: RenderingLoggingContext + context: RenderingContextLogger ): Promise { switch (value.type) { case "number": return value.number; case "title": - return await this.richText.renderMarkdown(value.title, context); + return await this.richText.renderPlainText(value.title); case "rich_text": - return await this.richText.renderMarkdown(value.rich_text, context); + return await this.richText.renderPlainText(value.rich_text); case "select": return value.select?.name; case "multi_select": @@ -109,9 +109,10 @@ export class PropertiesParser { return value.last_edited_time; case "last_edited_by": return value.last_edited_by.name; + case "people": + return value.people.map((person) => person.name).join(", "); case "formula": case "rollup": - case "people": case "files": case "checkbox": const notSupported = "unsupported property type: " + value.type; diff --git a/src/RecursiveBodyRenderer.ts b/src/RecursiveBodyRenderer.ts index 654a894..251b9e3 100644 --- a/src/RecursiveBodyRenderer.ts +++ b/src/RecursiveBodyRenderer.ts @@ -1,9 +1,8 @@ import { Block, Page } from '@notionhq/client/build/src/api-types'; -import { AssetWriter } from './AssetWriter'; import { BlockRenderer } from './BlockRenderer'; -import { RenderingLoggingContext } from './logger'; import { NotionApiFacade } from './NotionApiFacade'; +import { RenderingContext } from './RenderingContext'; const debug = require("debug")("body"); @@ -15,16 +14,14 @@ export class RecursiveBodyRenderer { async renderBody( page: Page, - assets: AssetWriter, - context: RenderingLoggingContext + context: RenderingContext ): Promise { debug("begin rendering body of page " + page.id, page.properties); const childs = await this.publicApi.listBlockChildren(page.id); - // todo: paging - const renderChilds = childs.results.map( - async (x) => await this.renderBlock(x, "", assets, context) + const renderChilds = childs.map( + async (x) => await this.renderBlock(x, "", context) ); const blocks = await Promise.all(renderChilds); const body = blocks.join("\n\n"); @@ -37,12 +34,10 @@ export class RecursiveBodyRenderer { async renderBlock( block: Block, indent: string, - assets: AssetWriter, - context: RenderingLoggingContext + context: RenderingContext ): Promise { const parentBlock = await this.blockRenderer.renderBlock( block, - assets, context ); const parentLines = parentBlock && this.indent(parentBlock.lines, indent); @@ -51,12 +46,12 @@ export class RecursiveBodyRenderer { // blocks, see https://developers.notion.com/reference/retrieve-a-block // "If a block contains the key has_children: true, use the Retrieve block children endpoint to get the list of children" const children = block.has_children - ? (await this.publicApi.listBlockChildren(block.id)).results + ? (await this.publicApi.listBlockChildren(block.id)) : []; const childIndent = indent + " ".repeat(parentBlock?.childIndent || 0); const renderChilds = children.map( - async (x) => await this.renderBlock(x, childIndent, assets, context) + async (x) => await this.renderBlock(x, childIndent, context) ); const childLines = await Promise.all(renderChilds); diff --git a/src/RenderingContext.ts b/src/RenderingContext.ts new file mode 100644 index 0000000..99468e1 --- /dev/null +++ b/src/RenderingContext.ts @@ -0,0 +1,27 @@ +import * as path from "path"; + +import { AssetWriter } from "./AssetWriter"; +import { PageLinkResolver } from "./PageLinkResolver"; +import { RenderingContextLogger as RenderingContextLogger } from "./RenderingContextLogger"; + +/** + * Unit of work for rendering a specific page. + * Note: this is a bit of a service locator, be careful about not breaking SRP + */ +export class RenderingContext { + readonly assetWriter: AssetWriter; + readonly logger: RenderingContextLogger; + readonly linkResolver: PageLinkResolver; + + constructor(notionUrl: string, file: string) { + this.logger = new RenderingContextLogger(notionUrl, file); + + const dir = path.dirname(file); + + // write all assets right next to the page's markdown file + this.assetWriter = new AssetWriter(dir, this.logger); + + // resolve all links relative to the page's markdown file dir + this.linkResolver = new PageLinkResolver(dir); + } +} diff --git a/src/RenderingContextLogger.ts b/src/RenderingContextLogger.ts new file mode 100644 index 0000000..7a085d0 --- /dev/null +++ b/src/RenderingContextLogger.ts @@ -0,0 +1,34 @@ +import * as chalk from "chalk"; +import { performance } from "perf_hooks"; +import { logger } from "./logger"; + + +export class RenderingContextLogger { + private readonly start = performance.now(); + + constructor( + public readonly notionUrl: string, + public readonly file?: string + ) { } + + info(message: string) { + return logger.info(this.garnish(message)); + } + + warn(message: string) { + return logger.warn(this.garnish(message)); + } + + error(err: unknown) { + return logger.error(this.garnish(err as any)); // bah + } + + complete() { + const elapsed = performance.now() - this.start; + this.info("rendered page in " + Math.round(elapsed) + "ms"); + } + + private garnish(message: string) { + return `${message} ${chalk.gray(this.file || this.notionUrl)}`; + } +} diff --git a/src/RichTextRenderer.spec.ts b/src/RichTextRenderer.spec.ts index 423310f..34e31f6 100644 --- a/src/RichTextRenderer.spec.ts +++ b/src/RichTextRenderer.spec.ts @@ -1,6 +1,6 @@ import { Annotations, RichText } from '@notionhq/client/build/src/api-types'; +import { RenderingContext } from './RenderingContext'; -import { RenderingLoggingContext } from './logger'; import { RichTextRenderer } from './RichTextRenderer'; function annotations(x: Partial): Annotations { @@ -15,7 +15,7 @@ function annotations(x: Partial): Annotations { }; } -const context = new RenderingLoggingContext(""); +const context = new RenderingContext("", ""); describe("RichTextRenderer", () => { let sut: RichTextRenderer; diff --git a/src/RichTextRenderer.ts b/src/RichTextRenderer.ts index 34f6e1e..eeb0531 100644 --- a/src/RichTextRenderer.ts +++ b/src/RichTextRenderer.ts @@ -1,8 +1,9 @@ -import { RichText } from '@notionhq/client/build/src/api-types'; +import { RichText } from "@notionhq/client/build/src/api-types"; -import { LinkRenderer } from './LinkRenderer'; -import { RenderingLoggingContext } from './logger'; -import { MentionedPageRenderer } from './MentionedPageRenderer'; +import { LinkRenderer } from "./LinkRenderer"; +import { RenderingContextLogger } from "./RenderingContextLogger"; +import { MentionedPageRenderer } from "./MentionedPageRenderer"; +import { RenderingContext } from "./RenderingContext"; const debug = require("debug")("richtext"); @@ -15,12 +16,12 @@ export class RichTextRenderer { ) {} public async renderPlainText(text: RichText[]): Promise { - return text.map((rt) => rt.plain_text).join(" "); + return text.map((rt) => rt.plain_text).join(""); } public async renderMarkdown( text: RichText[], - context: RenderingLoggingContext + context: RenderingContext ): Promise { const result: string[] = []; @@ -35,7 +36,7 @@ export class RichTextRenderer { private async renderMarkdownCode( rt: RichText, - context: RenderingLoggingContext + context: RenderingContext ) { const mod = this.modifier(rt); @@ -44,7 +45,7 @@ export class RichTextRenderer { return this.renderUnsupported( `unsupported rich text type: ${rt.type}`, rt, - context + context.logger ); case "mention": switch (rt.mention.type) { @@ -54,14 +55,14 @@ export class RichTextRenderer { rt.plain_text ); const text = this.wrap(mod, page.properties.meta.title); - return this.linkRenderer.renderPageLink(text, page); + return this.linkRenderer.renderPageLink(text, page, context.linkResolver); case "database": case "date": case "user": return this.renderUnsupported( `unsupported rich text mention type: ${rt.mention.type}`, rt, - context + context.logger ); } case "text": @@ -135,7 +136,7 @@ export class RichTextRenderer { private renderUnsupported( msg: string, obj: any, - context: RenderingLoggingContext + context: RenderingContextLogger ): string { context.warn(msg); debug(msg + "\n%O", obj); diff --git a/src/logger.ts b/src/logger.ts index 182103a..875cdc2 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,5 +1,4 @@ import * as chalk from 'chalk'; -import { performance } from 'perf_hooks'; const info = (...args: any[]): void => { console.log(chalk.cyan("info"), ...args); @@ -13,38 +12,10 @@ const error = (...args: any[]): void => { console.error(chalk.red("error"), ...args); }; -const logger = { +export const logger = { info, warn, error, }; -export class RenderingLoggingContext { - private readonly start = performance.now(); - constructor( - public readonly notionUrl: string, - public readonly file?: string - ) {} - - info(message: string) { - return logger.info(this.garnish(message)); - } - - warn(message: string) { - return logger.warn(this.garnish(message)); - } - - error(err: unknown) { - return logger.error(this.garnish(err as any)); // bah - } - - complete() { - const elapsed = performance.now() - this.start; - this.info("rendered page in " + Math.round(elapsed) + "ms"); - } - - private garnish(message: string) { - return `${message} ${chalk.gray(this.file || this.notionUrl)}`; - } -} diff --git a/src/sync.ts b/src/sync.ts index 968038d..4bbeda9 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -1,17 +1,18 @@ -import { BlockRenderer } from './BlockRenderer'; -import { ChildDatabaseRenderer } from './ChildDatabaseRenderer'; -import { DatabaseEntryRenderer } from './DatabaseEntryRenderer'; -import { DatabasePageRenderer } from './DatabasePageRenderer'; -import { DatabaseViewRenderer } from './DatabaseViewRenderer'; -import { DeferredRenderer } from './DeferredRenderer'; -import { FrontmatterRenderer } from './FrontmatterRenderer'; -import { LinkRenderer } from './LinkRenderer'; -import { MentionedPageRenderer } from './MentionedPageRenderer'; -import { NotionApiFacade } from './NotionApiFacade'; -import { PropertiesParser } from './PropertiesParser'; -import { RecursiveBodyRenderer } from './RecursiveBodyRenderer'; -import { RichTextRenderer } from './RichTextRenderer'; -import { SyncConfig } from './SyncConfig'; +import { BlockRenderer } from "./BlockRenderer"; +import { ChildDatabaseRenderer } from "./ChildDatabaseRenderer"; +import { DatabaseEntryRenderer } from "./DatabaseEntryRenderer"; +import { DatabasePageRenderer } from "./DatabasePageRenderer"; +import { DatabaseViewRenderer } from "./DatabaseViewRenderer"; +import { DeferredRenderer } from "./DeferredRenderer"; +import { FrontmatterRenderer } from "./FrontmatterRenderer"; +import { LinkRenderer } from "./LinkRenderer"; +import { MentionedPageRenderer } from "./MentionedPageRenderer"; +import { NotionApiFacade } from "./NotionApiFacade"; +import { PageLinkResolver } from "./PageLinkResolver"; +import { PropertiesParser } from "./PropertiesParser"; +import { RecursiveBodyRenderer } from "./RecursiveBodyRenderer"; +import { RichTextRenderer } from "./RichTextRenderer"; +import { SyncConfig } from "./SyncConfig"; export async function sync(notionApiToken: string, config: SyncConfig) { const publicApi = new NotionApiFacade(notionApiToken); @@ -31,7 +32,11 @@ export async function sync(notionApiToken: string, config: SyncConfig) { linkRenderer ); const propertiesParser = new PropertiesParser(richTextRenderer); - const blockRenderer = new BlockRenderer(richTextRenderer, deferredRenderer); + const blockRenderer = new BlockRenderer( + richTextRenderer, + deferredRenderer, + linkRenderer + ); const bodyRenderer = new RecursiveBodyRenderer(publicApi, blockRenderer); const entryRenderer = new DatabaseEntryRenderer(propertiesParser); const pageRenderer = new DatabasePageRenderer( @@ -50,7 +55,11 @@ export async function sync(notionApiToken: string, config: SyncConfig) { deferredRenderer.initialize(dbRenderer, pageRenderer, entryRenderer); // seed it with the root database - await deferredRenderer.renderChildDatabase(config.cmsDatabaseId); + const rootLinkResolver = new PageLinkResolver("."); + await deferredRenderer.renderChildDatabase( + config.cmsDatabaseId, + rootLinkResolver + ); await deferredRenderer.process(); const rendered = deferredRenderer.getRenderedPages();