diff --git a/.gitignore b/.gitignore index ff3babd..c9e85bb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules .temp .cache .env -dist/ \ No newline at end of file +dist/ +.idea/ \ No newline at end of file diff --git a/README.md b/README.md index 2346f21..9be1e42 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ The following [Notion API block object types](https://developers.notion.com/refe | Video | ❌ Missing | | | File | ❌ Missing | | | PDF | ❌ Missing | | -| Bookmark | ❌ Missing | | +| Bookmark | ✅ Yes | use a caption as a link name | | | Equation | ❌ Missing | | | Divider | ✅ Yes | | | Table Of Contents | ❌ not planned | static site generators have their own ToC implementations | @@ -77,7 +77,7 @@ The following [Notion API page property types](https://developers.notion.com/ref | Propety type | Supported | Notes | | ---------------- | --------- | ----------------------------- | -| Rich text | ✅ Yes | rendered as markdown string | +| Rich text | ✅ Yes | rendered as plaintext string | | Number | ✅ Yes | | | Select | ✅ Yes | rendered as name | | Multi Select | ✅ Yes | rendered as array of names | @@ -86,7 +86,7 @@ The following [Notion API page property types](https://developers.notion.com/ref | Relation | ✅ Yes | rendered as array of page ids | | Rollup | ❌ missing | | | Title | ✅ Yes | used as page title | -| People | ❌ missing | | +| People | ✅ Yes | rendered as comma-separated list of names | | Files | ❌ missing | | | Checkbox | ❌ missing | | | Url | ✅ Yes | rendered as string | @@ -112,15 +112,20 @@ Consult the [SyncConfig](./src/SyncConfig.ts) reference for documentation of ava > A CLI tool could be made available later. ```typescript -import { SyncConfig, sync } from "notion-markdown-cms"; - +import { slugify, SyncConfig, sync } from "notion-markdown-cms"; const config: SyncConfig = { cmsDatabaseId: "8f1de8c578fb4590ad6fbb0dbe283338", - outDir: "docs/", - indexPath: "docs/.vuepress/index.ts", + pages: { + destinationDirBuilder: (page) => slugify(page.properties.get("Category")), + frontmatterBuilder: (page) => ({ + id: page.meta.id, + url: page.meta.url, + title: page.meta.title, + category: page.properties.get("Category") + }), + }, databases: { "fe9836a9-6557-4f17-8adb-a93d2584f35f": { - parentCategory: "cfmm/", sorts: [ { property: "Scope", @@ -131,9 +136,23 @@ const config: SyncConfig = { direction: "ascending", }, ], - properties: { - category: "scope", - include: ["Name", "Scope", "Cluster", "Journey Stage", "Summary"], + renderAs: "pages+views", + pages: { + destinationDirBuilder: (page) => slugify(page.properties.get("Scope")), + frontmatterBuilder: (page) => ({ + id: page.meta.id, + url: page.meta.url, + title: page.meta.title, + cluster: page.properties.get("Cluster") + }), + }, + views: [ + { + title: "By Scope", + properties: { + groupBy: "Scope", + include: ["Name", "Scope", "Cluster", "Summary"], + }, }, }, }, @@ -147,10 +166,20 @@ async function main() { ); } - await sync(notionApiToken, config); + rimraf.sync("docs/!(README.md)**/*"); + + // change into the docs dir, this simplifies handling relative paths + process.chdir("docs/"); + + const rendered = await sync(notionApiToken, config); + + // do something with the rendered index, e.g. writing it to a file or building a nav structure } -main(); +main().catch((e) => { + console.error(e); + process.exit(1); +}); ``` ## Credits, Related Projects and Inspiration diff --git a/jest.config.js b/jest.config.js index bd27b33..693577e 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,6 +1,12 @@ module.exports = { transform: { "^.+\\.ts?$": "ts-jest", + "^.+\\markdown-table.js?$": "ts-jest", + }, + globals: { + 'ts-jest': { + isolatedModules: true + } }, testEnvironment: "node", testRegex: "./src/.*\\.(test|spec)?\\.(ts|ts)$", diff --git a/package.json b/package.json index 73a5daa..7d64c0f 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "0.8.0", + "version": "0.11.1", "name": "@meshcloud/notion-markdown-cms", "engines": { "node": ">=14" diff --git a/src/AssetWriter.ts b/src/AssetWriter.ts index 6d01ff5..d9dcdf9 100644 --- a/src/AssetWriter.ts +++ b/src/AssetWriter.ts @@ -1,27 +1,25 @@ -import { promises as fs } from 'fs'; -import got from 'got'; -import { KeyvFile } from 'keyv-file'; -import * as mime from 'mime-types'; - -import { RenderingLoggingContext } from './logger'; +import { promises as fs } from "fs"; +import got from "got"; +import { KeyvFile } from "keyv-file"; +import * as mime from "mime-types"; +import { RenderingContextLogger } from "./RenderingContextLogger"; const cache = new KeyvFile({ filename: ".cache/keyv.json", }); export class AssetWriter { - constructor(readonly dir: string) {} + constructor( + private readonly dir: string, + private readonly logger: RenderingContextLogger + ) {} async store(name: string, buffer: Buffer) { await fs.mkdir(this.dir, { recursive: true }); await fs.writeFile(`${this.dir}/${name}`, buffer); } - async download( - url: string, - fileName: string, - context: RenderingLoggingContext - ) { + async download(url: string, fileName: string) { // the got http lib promises to do proper user-agent compliant http caching // see https://github.com/sindresorhus/got/blob/main/documentation/cache.md @@ -35,7 +33,7 @@ export class AssetWriter { const imageFile = fileName + "." + ext; const cacheInfo = response.isFromCache ? " (from cache)" : ""; - context.info(`downloading ${imageFile}` + cacheInfo); + this.logger.info(`downloading ${imageFile}` + cacheInfo); await this.store(imageFile, response.rawBody); return imageFile; diff --git a/src/BlockRenderer.ts b/src/BlockRenderer.ts index c273765..6f98ea9 100644 --- a/src/BlockRenderer.ts +++ b/src/BlockRenderer.ts @@ -1,12 +1,20 @@ -import { RichText } from '@notionhq/client/build/src/api-types'; +import { RichText } from "@notionhq/client/build/src/api-types"; -import { AssetWriter } from './AssetWriter'; +import { AssetWriter } from "./AssetWriter"; import { - Block, Emoji, ExternalFile, ExternalFileWithCaption, File, FileWithCaption, ImageBlock -} from './Blocks'; -import { DeferredRenderer } from './DeferredRenderer'; -import { RenderingLoggingContext } from './logger'; -import { RichTextRenderer } from './RichTextRenderer'; + Block, + Emoji, + ExternalFile, + ExternalFileWithCaption, + File, + FileWithCaption, + ImageBlock, +} from "./Blocks"; +import { DeferredRenderer } from "./DeferredRenderer"; +import { RenderingContextLogger } from "./RenderingContextLogger"; +import { RichTextRenderer } from "./RichTextRenderer"; +import { RenderingContext } from "./RenderingContext"; +import { LinkRenderer } from "./LinkRenderer"; const debug = require("debug")("blocks"); @@ -17,13 +25,13 @@ export interface BlockRenderResult { export class BlockRenderer { constructor( private readonly richText: RichTextRenderer, - private readonly deferredRenderer: DeferredRenderer + private readonly deferredRenderer: DeferredRenderer, + private readonly link: LinkRenderer ) {} async renderBlock( block: Block, - assets: AssetWriter, - context: RenderingLoggingContext + context: RenderingContext ): Promise { const renderMarkdown = async (text: RichText[]) => { return await this.richText.renderMarkdown(text, context); @@ -63,7 +71,7 @@ export class BlockRenderer { }; case "image": return { - lines: await this.renderImage(block, assets, context), + lines: await this.renderImage(block, context.assetWriter), }; case "quote": { // it's legal for a notion block to be cmoposed of multiple lines @@ -85,7 +93,7 @@ export class BlockRenderer { case "callout": { // render emoji as bold, this enables css to target it as `blockquote > strong:first-child` const content = - `**${this.renderIcon(block.callout.icon, context)}** ` + + `**${this.renderIcon(block.callout.icon, context.logger)}** ` + (await renderMarkdown(block.callout.text)); return { @@ -96,16 +104,27 @@ export class BlockRenderer { return { lines: "---" }; case "child_database": const msg = `\n`; - const db = await this.deferredRenderer.renderChildDatabase(block.id); + const db = await this.deferredRenderer.renderChildDatabase( + block.id, + context.linkResolver + ); return { lines: msg + db.markdown }; case "synced_block": // nothing to render, only the contents of the synced block are relevant // however, these are children nöpcl, and thus retrieved by recursion in RecusivveBodyRenderer return null; + case "bookmark": + // render caption (if provided) as a link name + const caption = block.bookmark.caption || []; + let title = block.bookmark.url; + if (caption.length > 0) + title = await this.richText.renderPlainText(caption); + return { + lines: this.link.renderUrlLink(title, block.bookmark.url), + }; case "toggle": case "child_page": case "embed": - case "bookmark": case "video": case "file": case "pdf": @@ -116,7 +135,7 @@ export class BlockRenderer { lines: this.renderUnsupported( `unsupported block type: ${block.type}`, block, - context + context.logger ), }; } @@ -124,7 +143,7 @@ export class BlockRenderer { private renderIcon( icon: File | ExternalFile | Emoji, - context: RenderingLoggingContext + logger: RenderingContextLogger ): string { switch (icon.type) { case "emoji": @@ -134,19 +153,15 @@ export class BlockRenderer { return this.renderUnsupported( `unsupported icon type: ${icon.type}`, icon, - context + logger ); } } - async renderImage( - block: ImageBlock, - assets: AssetWriter, - context: RenderingLoggingContext - ): Promise { + async renderImage(block: ImageBlock, assets: AssetWriter): Promise { const url = this.parseUrl(block.image); - const imageFile = await assets.download(url, block.id, context); + const imageFile = await assets.download(url, block.id); // todo: caption support const markdown = `![image-${block.id}](./${imageFile})`; @@ -172,7 +187,7 @@ export class BlockRenderer { private renderUnsupported( msg: string, obj: any, - context: RenderingLoggingContext + context: RenderingContextLogger ): string { context.warn(msg); debug(msg + "\n%O", obj); diff --git a/src/ChildDatabaseRenderer.ts b/src/ChildDatabaseRenderer.ts index bc91a12..ae52645 100644 --- a/src/ChildDatabaseRenderer.ts +++ b/src/ChildDatabaseRenderer.ts @@ -1,14 +1,18 @@ -import { Page } from '@notionhq/client/build/src/api-types'; - -import { SyncConfig } from './'; -import { lookupDatabaseConfig } from './config'; -import { Database } from './Database'; -import { DatabaseTableRenderer } from './DatabaseTableRenderer'; -import { DatabaseViewRenderer } from './DatabaseViewRenderer'; -import { DeferredRenderer } from './DeferredRenderer'; -import { NotionApiFacade } from './NotionApiFacade'; -import { RenderDatabasePageTask } from './RenderDatabasePageTask'; -import { DatabaseConfig, DatabaseConfigRenderPages, DatabaseConfigRenderTable } from './SyncConfig'; +import { Page } from "@notionhq/client/build/src/api-types"; + +import { SyncConfig } from "./"; +import { lookupDatabaseConfig } from "./config"; +import { Database } from "./Database"; +import { DatabaseViewRenderer } from "./DatabaseViewRenderer"; +import { DeferredRenderer } from "./DeferredRenderer"; +import { NotionApiFacade } from "./NotionApiFacade"; +import { PageLinkResolver } from "./PageLinkResolver"; +import { RenderDatabasePageTask } from "./RenderDatabasePageTask"; +import { + DatabaseConfig, + DatabaseConfigRenderPages, + DatabaseConfigRenderTable, +} from "./SyncConfig"; const debug = require("debug")("child-database"); @@ -17,40 +21,53 @@ export class ChildDatabaseRenderer { private readonly config: SyncConfig, private readonly publicApi: NotionApiFacade, private readonly deferredRenderer: DeferredRenderer, - private readonly tableRenderer: DatabaseTableRenderer, private readonly viewRenderer: DatabaseViewRenderer - ) { } + ) {} - async renderChildDatabase(databaseId: string): Promise { + async renderChildDatabase( + databaseId: string, + linkResolver: PageLinkResolver + ): Promise { const dbConfig = lookupDatabaseConfig(this.config, databaseId); // no view was defined for this database, render as a plain inline table const allPages = await this.fetchPages(databaseId, dbConfig); - const renderPages = dbConfig.renderAs === "pages+views" + const renderPages = dbConfig.renderAs === "pages+views"; - debug("rendering child database " + databaseId + " as " + dbConfig.renderAs); + debug( + "rendering child database " + databaseId + " as " + dbConfig.renderAs + ); if (renderPages) { const pageConfig = dbConfig as DatabaseConfigRenderPages; const entries = await this.queuePageRendering(allPages, pageConfig); - const markdown = await this.viewRenderer.renderViews(entries, dbConfig as DatabaseConfigRenderPages); + const markdown = await this.viewRenderer.renderViews( + entries, + dbConfig as DatabaseConfigRenderPages, + linkResolver + ); return { config: dbConfig, entries, markdown, }; - } - - const entries = await this.queueEntryRendering(allPages, dbConfig); - const markdown = this.tableRenderer.renderTable(entries); + } else { + // render table + const entries = await this.queueEntryRendering(allPages, dbConfig); + const markdown = this.viewRenderer.renderViews( + entries, + dbConfig, + linkResolver + ); - return { - config: dbConfig, - entries, - markdown, - }; + return { + config: dbConfig, + entries, + markdown, + }; + } } private async queueEntryRendering( @@ -89,6 +106,6 @@ export class ChildDatabaseRenderer { page_size: 100, }); - return allPages.results; + return allPages; } } diff --git a/src/DatabaseEntryRenderer.ts b/src/DatabaseEntryRenderer.ts index 0b0819f..0943080 100644 --- a/src/DatabaseEntryRenderer.ts +++ b/src/DatabaseEntryRenderer.ts @@ -1,25 +1,22 @@ import { Page } from "@notionhq/client/build/src/api-types"; +import { DatabaseConfigRenderTable } from "."; import { PropertiesParser } from "./PropertiesParser"; import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask"; -import { DatabaseConfig } from "./SyncConfig"; export class DatabaseEntryRenderer { constructor(private readonly propertiesParser: PropertiesParser) {} async renderEntry( page: Page, - config: DatabaseConfig + config: DatabaseConfigRenderTable ): Promise { - const props = await this.propertiesParser.parseProperties(page, config); + const props = await this.propertiesParser.parsePageProperties(page); + const frontmatterProperties = config.entries?.frontmatterBuilder(props); return { - id: page.id, - url: page.url, - properties: { - keys: props.keys, - values: props.properties, - }, + properties: props, + frontmatter: frontmatterProperties, }; } } diff --git a/src/DatabasePageMeta.ts b/src/DatabasePageMeta.ts index 9714e48..12496fb 100644 --- a/src/DatabasePageMeta.ts +++ b/src/DatabasePageMeta.ts @@ -6,9 +6,6 @@ import { DatabaseEntryMeta } from './DatabaseEntryMeta'; */ export interface DatabasePageMeta extends DatabaseEntryMeta { title: string; - category: string; - order?: number; - layout?: string; } diff --git a/src/DatabasePageProperties.ts b/src/DatabasePageProperties.ts index a559c4d..f0f39b3 100644 --- a/src/DatabasePageProperties.ts +++ b/src/DatabasePageProperties.ts @@ -1,4 +1,4 @@ -import { DatabasePageMeta } from './DatabasePageMeta'; +import { DatabasePageMeta } from "./DatabasePageMeta"; export interface DatabasePageProperties { /** @@ -7,12 +7,7 @@ export interface DatabasePageProperties { meta: DatabasePageMeta; /** - * A mapping of property object keys -> property values + * A mapping of Notion property names -> parsed property values */ - values: Record; - - /** - * A mapping of Notion API property names -> property object keys - */ - keys: Map; + properties: Map; } diff --git a/src/DatabasePageRenderer.ts b/src/DatabasePageRenderer.ts index edbee3c..d3ea572 100644 --- a/src/DatabasePageRenderer.ts +++ b/src/DatabasePageRenderer.ts @@ -1,15 +1,14 @@ -import * as fsc from 'fs'; +import * as fsc from "fs"; -import { Page } from '@notionhq/client/build/src/api-types'; +import { Page } from "@notionhq/client/build/src/api-types"; -import { AssetWriter } from './AssetWriter'; -import { FrontmatterRenderer } from './FrontmatterRenderer'; -import { RenderingLoggingContext } from './logger'; -import { PropertiesParser } from './PropertiesParser'; -import { RecursiveBodyRenderer } from './RecursiveBodyRenderer'; -import { RenderDatabasePageTask as RenderDatabasePageTask } from './RenderDatabasePageTask'; -import { slugify } from './slugify'; -import { DatabaseConfigRenderPages } from './SyncConfig'; +import { FrontmatterRenderer } from "./FrontmatterRenderer"; +import { PropertiesParser } from "./PropertiesParser"; +import { RecursiveBodyRenderer } from "./RecursiveBodyRenderer"; +import { RenderDatabasePageTask as RenderDatabasePageTask } from "./RenderDatabasePageTask"; +import { DatabaseConfigRenderPages } from "./SyncConfig"; +import { slugify } from "./slugify"; +import { RenderingContext } from "./RenderingContext"; const fs = fsc.promises; @@ -24,49 +23,50 @@ export class DatabasePageRenderer { page: Page, config: DatabaseConfigRenderPages ): Promise { - const props = await this.propertiesParser.parsePageProperties(page, config); + const props = await this.propertiesParser.parsePageProperties(page); - const categorySlug = slugify(props.meta.category); - const destDir = `${config.outDir}/${categorySlug}`; - - const nameSlug = slugify(props.meta.title); - const file = `${destDir}/${nameSlug}.md`; + const destDir = config.pages.destinationDirBuilder(props); + const filenameBuilder = + config.pages.filenameBuilder || ((x) => slugify(x.meta.title)); + const file = `${destDir}/${filenameBuilder(props)}.md`; // Design: all the rendering performance could be greatly enhanced writing directly to output streams instead // of concatenating all in memory. OTOH naively concatenatic strings is straightforward, easier to debug and rendering // performance probably not the bottleneck compared to the IO cost of notion API invocations. + const frontmatterProperties = config.pages.frontmatterBuilder(props); return { - id: page.id, file, + frontmatter: frontmatterProperties, properties: props, render: async () => { - const context = new RenderingLoggingContext(page.url, file); - + + const context = new RenderingContext(page.url, file) + if (page.archived) { - context.warn(`page is arvhied`); + // have to skip rendering archived pages as attempting to retrieve the block will result in a HTTP 404 + context.logger.warn(`page is archived - skipping`); + + return; } try { - const assetWriter = new AssetWriter(destDir); - - const frontmatter = this.frontmatterRenderer.renderFrontmatter(props); + const frontmatter = this.frontmatterRenderer.renderFrontmatter(frontmatterProperties); const body = await this.bodyRenderer.renderBody( page, - assetWriter, context ); await fs.mkdir(destDir, { recursive: true }); await fs.writeFile(file, frontmatter + body); - context.complete(); + context.logger.complete(); } catch (error) { // While catch-log-throw is usually an antipattern, it is the renderes job to orchestrate the rendering // job with concerns like logging and writing to the outside world. Hence this place is appropriate. // We need to throw the error here so that the rendering process can crash with a proper error message, since // an error at this point here is unrecoverable. - context.error(error); + context.logger.error(error); throw error; } }, diff --git a/src/DatabaseTableRenderer.ts b/src/DatabaseTableRenderer.ts deleted file mode 100644 index ff8c212..0000000 --- a/src/DatabaseTableRenderer.ts +++ /dev/null @@ -1,34 +0,0 @@ -import * as markdownTable from "./markdown-table"; -import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask"; - -export class DatabaseTableRenderer { - public renderTable(entries: RenderDatabaseEntryTask[]): string { - const table: any[][] = []; - - for (const page of entries) { - if (table.length === 0) { - const headers = Array.from(page.properties.keys.keys()); - table[0] = headers; - } - - const cols = Array.from(page.properties.keys.values()).map((c, i) => - DatabaseTableRenderer.escapeTableCell( - page.properties.values[c] - ) - ); - - table.push(cols); - } - - return markdownTable.markdownTable(table); - } - - static escapeTableCell(content: string | number | any): string { - // markdown table cells do not support newlines, however we can insert
elements instead - if (typeof content === "string") { - return content.replace(/\n/g, "
"); - } - - return content.toString(); - } -} diff --git a/src/DatabaseViewRenderer.spec.ts b/src/DatabaseViewRenderer.spec.ts new file mode 100644 index 0000000..be7f26b --- /dev/null +++ b/src/DatabaseViewRenderer.spec.ts @@ -0,0 +1,92 @@ +import { DatabaseViewRenderer } from "./DatabaseViewRenderer"; +import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask"; + +describe("DatabaseViewRenderer", () => { + test("renders with grouping", async () => { + const linkRenderer = {}; + const sut = new DatabaseViewRenderer(linkRenderer as any); + + const entries: RenderDatabaseEntryTask[] = [ + { + properties: { + meta: { id: "a", title: " A", url: "http://a" }, + properties: new Map([ + ["Name", "A"], + ["Foo", "Bar"], + ]), + }, + }, + { + properties: { + meta: { id: "b", title: " B", url: "http://b" }, + properties: new Map([ + ["Name", "B"], + ["Foo", "Baz"], + ]), + }, + }, + ]; + + const result = sut.renderViews(entries, { + renderAs: "table", + views: [ + { + title: "By Foo", + properties: { + groupBy: "Foo", + }, + }, + ], + }); + + const expected = `## By Foo - Bar + +| Name | Foo | +| ---- | --- | +| A | Bar | + +## By Foo - Baz + +| Name | Foo | +| ---- | --- | +| B | Baz |`; + expect(result).toEqual(expected); + }); + + test("filters columns", async () => { + const linkRenderer = {}; + const sut = new DatabaseViewRenderer(linkRenderer as any); + + const entries: RenderDatabaseEntryTask[] = [ + { + properties: { + meta: { id: "a", title: " A", url: "http://a" }, + properties: new Map([ + ["Name", "A"], + ["Foo", "Bar"], + ["Alice", "Bob"], + ]), + }, + }, + ]; + + const result = sut.renderViews(entries, { + renderAs: "table", + views: [ + { + title: "By Foo", + properties: { + include: ["Name", "Foo"], + }, + }, + ], + }); + + const expected = `## By Foo + +| Name | Foo | +| ---- | --- | +| A | Bar |`; + expect(result).toEqual(expected); + }); +}); diff --git a/src/DatabaseViewRenderer.ts b/src/DatabaseViewRenderer.ts index 2cf5cf8..16cc20e 100644 --- a/src/DatabaseViewRenderer.ts +++ b/src/DatabaseViewRenderer.ts @@ -1,73 +1,83 @@ -import { DatabaseTableRenderer } from './DatabaseTableRenderer'; -import { LinkRenderer } from './LinkRenderer'; -import * as markdownTable from './markdown-table'; -import { PropertiesParser } from './PropertiesParser'; -import { RenderDatabasePageTask } from './RenderDatabasePageTask'; -import { DatabaseConfigRenderPages, DatabaseView } from './SyncConfig'; - -const debug = require("debug")("database-views"); +import { DatabaseConfigRenderTable } from "."; +import { LinkRenderer } from "./LinkRenderer"; +import * as markdownTable from "./markdown-table"; +import { PageLinkResolver } from "./PageLinkResolver"; +import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask"; +import { RenderDatabasePageTask } from "./RenderDatabasePageTask"; +import { DatabaseConfigRenderPages, DatabaseView } from "./SyncConfig"; // todo: name afte what it renders, not to where export class DatabaseViewRenderer { constructor(private readonly linkRenderer: LinkRenderer) {} - public renderViews(entries: RenderDatabasePageTask[], config: DatabaseConfigRenderPages): string { - const views = config.views?.map((view) => { - const propKeys = entries[0].properties.keys; - const propKey = propKeys.get(view.properties.groupBy); + public renderViews( + entries: (RenderDatabasePageTask | RenderDatabaseEntryTask)[], + config: DatabaseConfigRenderPages | DatabaseConfigRenderTable, + linkResolver: PageLinkResolver + ): string { + const configuredViews = config.views || [{}]; - if (!propKey) { - const msg = `Could not render view ${view.title}, groupBy property ${view.properties.groupBy} not found`; - debug(msg + "%O", view); - throw new Error(msg); - } + const views = configuredViews?.map((view) => { + const groupByProperty = view?.properties?.groupBy; - const grouped = new Array( - ...groupBy(entries, (p) => p.properties.values[propKey]) - ); + if (!groupByProperty) { + return this.renderView(entries, null, view, linkResolver); + } else { + const grouped = new Array( + ...groupBy(entries, (p) => + p.properties.properties.get(groupByProperty) + ) + ); - return grouped - .map(([key, pages]) => this.renderView(pages, key, view)) - .join("\n\n"); + return grouped + .map(([key, pages]) => this.renderView(pages, key, view, linkResolver)) + .join("\n\n"); + } }); return views?.join("\n\n") || ""; } - public renderView( - pages: RenderDatabasePageTask[], - titleAppendix: string, - view: DatabaseView + private renderView( + pages: (RenderDatabasePageTask | RenderDatabaseEntryTask)[], + titleAppendix: string | null, + view: DatabaseView, + linkResolver: PageLinkResolver ): string { - // todo: handle empty page - const props = pages[0].properties; + if (!pages[0]) { + return ""; + } + const pageProps = pages[0].properties; - const keys = PropertiesParser.filterIncludedKeys( - props.keys, - view.properties.include, - ); + const includedProps = + view?.properties?.include || Array.from(pageProps.properties.keys()); const table: any[][] = []; - const headers = Array.from(keys.keys()); + const headers = includedProps; table[0] = headers; - const cols = Array.from(keys.values()); + const cols = includedProps; pages.forEach((r) => table.push( cols.map((c, i) => { - const content = DatabaseTableRenderer.escapeTableCell(r.properties.values[c]); - return i == 0 - ? this.linkRenderer.renderPageLink(content, r) // make the first cell a relative link to the page + const content = escapeTableCell(r.properties.properties.get(c)); + return i == 0 && isRenderPageTask(r) + ? this.linkRenderer.renderPageLink(content, r, linkResolver) // make the first cell a relative link to the page : content; }) ) ); - return ( - `## ${view.title} - ${titleAppendix}\n\n` + - markdownTable.markdownTable(table) - ); + const tableMd = markdownTable.markdownTable(table); + if (view.title) { + const formattedTitle = [view.title, titleAppendix] + .filter((x) => !!x) + .join(" - "); + return `## ${formattedTitle}\n\n` + tableMd; + } else { + return tableMd; + } } } @@ -98,3 +108,18 @@ export function groupBy( }); return map; } + +function escapeTableCell(content: string | number | any): string { + // markdown table cells do not support newlines, however we can insert
elements instead + if (typeof content === "string") { + return content.replace(/\n/g, "
"); + } + + return content?.toString() || ""; +} + +function isRenderPageTask( + task: RenderDatabasePageTask | RenderDatabaseEntryTask +): task is RenderDatabasePageTask { + return (task as RenderDatabasePageTask).render !== undefined; +} diff --git a/src/DeferredRenderer.ts b/src/DeferredRenderer.ts index 331e70f..228e9f1 100644 --- a/src/DeferredRenderer.ts +++ b/src/DeferredRenderer.ts @@ -1,14 +1,18 @@ -import { Page } from '@notionhq/client/build/src/api-types'; - -import { ChildDatabaseRenderer } from './ChildDatabaseRenderer'; -import { Database } from './Database'; -import { DatabaseEntryRenderer } from './DatabaseEntryRenderer'; -import { DatabasePageRenderer } from './DatabasePageRenderer'; -import { RenderDatabaseEntryTask } from './RenderDatabaseEntryTask'; -import { RenderDatabasePageTask as RenderDatabasePageTask } from './RenderDatabasePageTask'; -import { RenderedDatabaseEntry } from './RenderedDatabaseEntry'; -import { RenderedDatabasePage } from './RenderedDatabasePage'; -import { DatabaseConfigRenderPages, DatabaseConfigRenderTable } from './SyncConfig'; +import { Page } from "@notionhq/client/build/src/api-types"; + +import { ChildDatabaseRenderer } from "./ChildDatabaseRenderer"; +import { Database } from "./Database"; +import { DatabaseEntryRenderer } from "./DatabaseEntryRenderer"; +import { DatabasePageRenderer } from "./DatabasePageRenderer"; +import { PageLinkResolver } from "./PageLinkResolver"; +import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask"; +import { RenderDatabasePageTask as RenderDatabasePageTask } from "./RenderDatabasePageTask"; +import { RenderedDatabaseEntry } from "./RenderedDatabaseEntry"; +import { RenderedDatabasePage } from "./RenderedDatabasePage"; +import { + DatabaseConfigRenderPages, + DatabaseConfigRenderTable, +} from "./SyncConfig"; const debug = require("debug")("rendering"); @@ -32,8 +36,11 @@ export class DeferredRenderer { this.entryRenderer = entryRenderer; } - public async renderChildDatabase(databaseId: string): Promise { - return await this.dbRenderer.renderChildDatabase(databaseId); + public async renderChildDatabase( + databaseId: string, + linkResolver: PageLinkResolver + ): Promise { + return await this.dbRenderer.renderChildDatabase(databaseId, linkResolver); } public async renderPage( @@ -63,7 +70,7 @@ export class DeferredRenderer { // entries are complete the moment they are retrieved, there's no more deferred processing necessary on them // also there should be no duplicate entries, so we do not cache/lookup any of them - if (config.entries.emitToIndex) { + if (task.frontmatter) { this.renderedEntries.push(task); } @@ -100,15 +107,15 @@ export class DeferredRenderer { ).map((x) => ({ file: x.file, meta: x.properties.meta, - properties: x.properties.values, + frontmatter: x.frontmatter, })); const entries: RenderedDatabaseEntry[] = this.renderedEntries.map((x) => ({ meta: { - id: x.id, - url: x.url, + id: x.properties.meta.id, + url: x.properties.meta.url, }, - properties: x.properties.values, + frontmatter: x.frontmatter, })); return pages.concat(entries); diff --git a/src/FrontmatterRenderer.ts b/src/FrontmatterRenderer.ts index 38c7b21..008c4d3 100644 --- a/src/FrontmatterRenderer.ts +++ b/src/FrontmatterRenderer.ts @@ -1,16 +1,9 @@ import * as yaml from 'js-yaml'; -import { DatabasePageProperties } from './DatabasePageProperties'; - export class FrontmatterRenderer { constructor() {} - public renderFrontmatter(props: DatabasePageProperties) { - const obj = { - ...props.meta, - properties: props.values, - }; - + public renderFrontmatter(obj: Record) { const frontmatter = `---\n${yaml.dump(obj)}---\n\n`; return frontmatter; diff --git a/src/LinkRenderer.spec.ts b/src/LinkRenderer.spec.ts index b6a347b..938ddc3 100644 --- a/src/LinkRenderer.spec.ts +++ b/src/LinkRenderer.spec.ts @@ -1,19 +1,17 @@ import { LinkRenderer } from './LinkRenderer'; +import { PageLinkResolver } from './PageLinkResolver'; import { RenderDatabasePageTask } from './RenderDatabasePageTask'; describe("LinkRenderer", () => { - const config = { - outDir: "out/", - }; - + test("renderPageLink strips outDir from link", async () => { - const sut = new LinkRenderer(config as any); + const resolver = new PageLinkResolver("out"); + const sut = new LinkRenderer(); const page: Partial = { - id: "id", file: "out/test.md", }; - const link = sut.renderPageLink("text", page as any); - expect(link).toEqual("[text](test.md)"); + const link = sut.renderPageLink("text", page as any, resolver); + expect(link).toEqual("[text](./test.md)"); }); }); diff --git a/src/LinkRenderer.ts b/src/LinkRenderer.ts index e627bf1..978ae38 100644 --- a/src/LinkRenderer.ts +++ b/src/LinkRenderer.ts @@ -1,16 +1,20 @@ -import { SyncConfig } from './'; -import { RenderDatabasePageTask } from './RenderDatabasePageTask'; +import { PageLinkResolver } from "./PageLinkResolver"; +import { RenderDatabasePageTask } from "./RenderDatabasePageTask"; export class LinkRenderer { - constructor(private readonly config: SyncConfig) {} + constructor() {} renderUrlLink(text: string, url: string): string { return `[${text}](${url})`; } - renderPageLink(text: string, page: RenderDatabasePageTask): string { - const url = page.file.substring(this.config.outDir.length); + renderPageLink( + text: string, + toPage: RenderDatabasePageTask, + linkResolver: PageLinkResolver + ): string { + const link = linkResolver.resolveRelativeLinkTo(toPage.file); - return this.renderUrlLink(text, url); + return this.renderUrlLink(text, link); } } diff --git a/src/MentionedPageRenderer.ts b/src/MentionedPageRenderer.ts index 1bab586..7be6c59 100644 --- a/src/MentionedPageRenderer.ts +++ b/src/MentionedPageRenderer.ts @@ -65,7 +65,7 @@ export class MentionedPageRenderer { } } } - + private formatMentionedPage(pageId: string, mentionPlaintext: string) { const formattedId = pageId.replace(/-/g, ""); return `mentioned page '${mentionPlaintext}' with url https://notion.so/${formattedId}`; diff --git a/src/NotionApiFacade.ts b/src/NotionApiFacade.ts index 2cbd92b..721c2d2 100644 --- a/src/NotionApiFacade.ts +++ b/src/NotionApiFacade.ts @@ -1,7 +1,11 @@ import { - APIErrorCode, APIResponseError, Client, RequestTimeoutError, UnknownHTTPResponseError -} from '@notionhq/client'; -import { DatabasesQueryParameters } from '@notionhq/client/build/src/api-endpoints'; + APIErrorCode, + APIResponseError, + Client, + RequestTimeoutError, + UnknownHTTPResponseError, +} from "@notionhq/client"; +import { DatabasesQueryParameters } from "@notionhq/client/build/src/api-endpoints"; const debug = require("debug")("notion-api"); @@ -34,17 +38,25 @@ export class NotionApiFacade { } async queryDatabase(query: DatabasesQueryParameters) { - const result = await this.withRetry( - async () => await this.client.databases.query(query) - ); // todo: paging + const results = []; - if (result.next_cursor) { - throw new Error( - `Paging not implemented, db ${query.database_id} has more than 100 entries` + let next_cursor: string | null = null; + + do { + const response = await this.withRetry( + async () => + await this.client.databases.query({ + ...query, + start_cursor: next_cursor || undefined, + }) ); - } - return result; + results.push(...response.results); + + next_cursor = response.next_cursor; + } while (next_cursor); + + return results; } async retrievePage(pageId: string) { @@ -54,20 +66,25 @@ export class NotionApiFacade { } async listBlockChildren(blockId: string) { - const result = await this.withRetry( - async () => - await this.client.blocks.children.list({ - block_id: blockId, - }) - ); // todo: paging here? - - if (result.next_cursor) { - throw new Error( - `Paging not implemented, block ${blockId} has more children than returned in a single request` + const results = []; + + let next_cursor: string | null = null; + + do { + const response = await this.withRetry( + async () => + await this.client.blocks.children.list({ + block_id: blockId, + start_cursor: next_cursor || undefined, + }) ); - } - return result; + results.push(...response.results); + + next_cursor = response.next_cursor; + } while (next_cursor); + + return results; } printStats() { diff --git a/src/PageLinkResolver.ts b/src/PageLinkResolver.ts new file mode 100644 index 0000000..21406ec --- /dev/null +++ b/src/PageLinkResolver.ts @@ -0,0 +1,19 @@ +import * as path from "path"; + +export class PageLinkResolver { + private readonly absoluteFromDir: string; + + constructor(sourceDir: string) { + this.absoluteFromDir = path.resolve(sourceDir); + } + + resolveRelativeLinkTo(file: string) { + const absoluteToDir = path.resolve(file); + + const relativePath = path.relative(this.absoluteFromDir, absoluteToDir); + + // normalize the rleative path to start with a ./ as that's the markdown convention for relative links + // at least how it's most commonly interpreted (e.g. by textlint) + return relativePath.startsWith(".") ? relativePath : "./" + relativePath; + } +} diff --git a/src/PropertiesParser.spec.ts b/src/PropertiesParser.spec.ts index 860ed8e..511b27b 100644 --- a/src/PropertiesParser.spec.ts +++ b/src/PropertiesParser.spec.ts @@ -1,8 +1,8 @@ -import { Page } from '@notionhq/client/build/src/api-types'; +import { Page } from "@notionhq/client/build/src/api-types"; +import { DatabasePageProperties } from "./DatabasePageProperties"; -import { PropertiesParser } from './PropertiesParser'; -import { RichTextRenderer } from './RichTextRenderer'; -import { DatabaseConfig } from './SyncConfig'; +import { PropertiesParser } from "./PropertiesParser"; +import { RichTextRenderer } from "./RichTextRenderer"; const page: Partial = { id: "123", @@ -33,72 +33,34 @@ const page: Partial = { }; describe("PropertiesParser", () => { - describe("parse", () => { + describe("parsePageProperties", () => { + test("preserves all properties and adds conventional sort with title coming first", async () => { + const sut = new PropertiesParser( + new RichTextRenderer({} as any, {} as any) + ); - test("preserves all properties and adds conventional with no include filter", async () => { - const sut = new PropertiesParser(new RichTextRenderer({} as any, {} as any)); + const result = await sut.parsePageProperties(page as any); - const config: DatabaseConfig = { - outDir: "db/", - renderAs: 'table', - entries: { - emitToIndex: false - } - }; - - const result = await sut.parseProperties(page as any, config); - - const expected = { - category: null, - order: 30, - title: "Terraform", - keys: new Map([ - ["order", "order"], - ["Category", "category"], - ["Name", "name"], - ]), - properties: { - order: 30, - category: "Tools", - name: "Terraform", + const expected: DatabasePageProperties = { + meta: { + id: page.id!!, + url: page.url!!, + title: "Terraform", }, + properties: new Map([ + ["Name", "Terraform"], + ["order", 30], + ["Category", "Tools"], + ]), }; expect(result).toEqual(expected); // explicitly test key ordering - expect(Array.from(result.keys.keys())).toEqual(["Name", "order", "Category"]); - }); - - test("filters according to include filter", async () => { - const sut = new PropertiesParser(new RichTextRenderer({} as any, {} as any)); - - const config: DatabaseConfig = { - outDir: "db/", - renderAs: "table", - properties: { - include: ["Name", "Category"], - }, - entries: { - emitToIndex: false - } - }; - - const result = await sut.parseProperties(page as any, config); - - const expected = { - category: null, - order: 30, - title: "Terraform", - keys: new Map([ - ["Category", "category"], - ["Name", "name"], - ]), - properties: { - category: "Tools", - name: "Terraform", - }, - }; - expect(result).toEqual(expected); + expect(Array.from(result.properties.keys())).toEqual([ + "Name", + "order", + "Category", + ]); }); }); }); diff --git a/src/PropertiesParser.ts b/src/PropertiesParser.ts index b3a5312..333716e 100644 --- a/src/PropertiesParser.ts +++ b/src/PropertiesParser.ts @@ -1,10 +1,8 @@ -import { Page, PropertyValue } from '@notionhq/client/build/src/api-types'; +import { Page, PropertyValue } from "@notionhq/client/build/src/api-types"; -import { DatabasePageProperties } from './DatabasePageProperties'; -import { RenderingLoggingContext } from './logger'; -import { RichTextRenderer } from './RichTextRenderer'; -import { slugify } from './slugify'; -import { DatabaseConfig, DatabaseConfigRenderPages } from './SyncConfig'; +import { DatabasePageProperties } from "./DatabasePageProperties"; +import { RenderingContextLogger } from "./RenderingContextLogger"; +import { RichTextRenderer } from "./RichTextRenderer"; const debug = require("debug")("properties"); @@ -12,131 +10,83 @@ export class PropertiesParser { constructor(private readonly richText: RichTextRenderer) {} public async parsePageProperties( - page: Page, - config: DatabaseConfigRenderPages + page: Page ): Promise { - const { title, category, order, properties, keys } = - await this.parseProperties(page, config); + const { title, properties } = await this.parseProperties(page); if (!title) { throw this.errorMissingRequiredProperty("of type 'title'", page); } - const theCategory = category || config.pages.frontmatter.category.static; - - if (!theCategory) { - throw this.errorMissingRequiredProperty( - config.pages.frontmatter.category.property || "static category", - page - ); - } - return { meta: { id: page.id, url: page.url, - title: title, // notion API always calls it name - category: theCategory, - order: order, - ...config.pages.frontmatter.extra, + title: title, }, - values: properties, - keys: keys, + properties, }; } - public async parseProperties(page: Page, config: DatabaseConfig) { - /** - * Design: we always lookup the properties on the page object itself. - * This way we only parse properties once and avoid any problems coming from - * e.g. category properties being filtered via include filters. - */ - + private async parseProperties(page: Page) { /** * Terminology: * * property: Notion API property name - * key: slugified Notion API property name, used to later build frontmatter * value: Notion API property value */ /** - * A record of key->value + * A record of property->value */ - const properties: Record = {}; - - /** - * A map of proprety -> key - */ - const keys = new Map(); + const properties: Map = new Map(); let title: string | null = null; let titleProperty: string | null = null; - let category: string | null = null; - let order: number | undefined = undefined; - const categoryProperty = - config.renderAs === "pages+views" && - config.pages.frontmatter.category.property; - - const context = new RenderingLoggingContext(page.url); + const context = new RenderingContextLogger(page.url); for (const [name, value] of Object.entries(page.properties)) { const parsedValue = await this.parsePropertyValue(value, context); - - if ( - !config.properties?.include || - config.properties.include.indexOf(name) >= 0 - ) { - const slug = slugify(name); - properties[slug] = parsedValue; - keys.set(name, slug); - } + properties.set(name, parsedValue); if (value.type === "title") { title = parsedValue; titleProperty = name; } - - if (categoryProperty && name === categoryProperty) { - category = parsedValue; - } - - if (name === "order") { - order = parsedValue; - } } - if (!titleProperty) { + if (!title || !titleProperty) { throw this.errorMissingRequiredProperty("of type 'title'", page); } // no explicit ordering specified, so we make sure to put the title property first - const includes = config.properties?.include || [ + const keyOrder = [ titleProperty, - ...Array.from(keys.keys()).filter((x) => x != titleProperty), + ...Array.from(properties.keys()).filter((x) => x != titleProperty), ]; + // maps preserve insertion order + const sortedProperties = new Map(); + keyOrder.forEach(x => sortedProperties.set(x, properties.get(x))); + return { title, - category, - order, - properties, - keys: PropertiesParser.filterIncludedKeys(keys, includes), + properties: sortedProperties, }; } private async parsePropertyValue( value: PropertyValue, - context: RenderingLoggingContext + context: RenderingContextLogger ): Promise { switch (value.type) { case "number": return value.number; case "title": - return await this.richText.renderMarkdown(value.title, context); + return await this.richText.renderPlainText(value.title); case "rich_text": - return await this.richText.renderMarkdown(value.rich_text, context); + return await this.richText.renderPlainText(value.rich_text); case "select": return value.select?.name; case "multi_select": @@ -159,9 +109,10 @@ export class PropertiesParser { return value.last_edited_time; case "last_edited_by": return value.last_edited_by.name; + case "people": + return value.people.map((person) => person.name).join(", "); case "formula": case "rollup": - case "people": case "files": case "checkbox": const notSupported = "unsupported property type: " + value.type; @@ -172,21 +123,6 @@ export class PropertiesParser { } } - public static filterIncludedKeys( - keys: Map, - includes: string[] | undefined - ): Map { - if (!includes) { - return keys; - } - - // Maps iterate in insertion order, so preserve the correct ordering of keys according to includes ordering - const filtered = new Map(); - includes.forEach((i) => filtered.set(i, keys.get(i)!!)); // todo: should probably handle undefined here - - return filtered; - } - private errorMissingRequiredProperty(propertyName: string, page: Page) { // todo: should this use context? const msg = `Page ${page.url} is missing required property ${propertyName}`; diff --git a/src/RecursiveBodyRenderer.ts b/src/RecursiveBodyRenderer.ts index 654a894..251b9e3 100644 --- a/src/RecursiveBodyRenderer.ts +++ b/src/RecursiveBodyRenderer.ts @@ -1,9 +1,8 @@ import { Block, Page } from '@notionhq/client/build/src/api-types'; -import { AssetWriter } from './AssetWriter'; import { BlockRenderer } from './BlockRenderer'; -import { RenderingLoggingContext } from './logger'; import { NotionApiFacade } from './NotionApiFacade'; +import { RenderingContext } from './RenderingContext'; const debug = require("debug")("body"); @@ -15,16 +14,14 @@ export class RecursiveBodyRenderer { async renderBody( page: Page, - assets: AssetWriter, - context: RenderingLoggingContext + context: RenderingContext ): Promise { debug("begin rendering body of page " + page.id, page.properties); const childs = await this.publicApi.listBlockChildren(page.id); - // todo: paging - const renderChilds = childs.results.map( - async (x) => await this.renderBlock(x, "", assets, context) + const renderChilds = childs.map( + async (x) => await this.renderBlock(x, "", context) ); const blocks = await Promise.all(renderChilds); const body = blocks.join("\n\n"); @@ -37,12 +34,10 @@ export class RecursiveBodyRenderer { async renderBlock( block: Block, indent: string, - assets: AssetWriter, - context: RenderingLoggingContext + context: RenderingContext ): Promise { const parentBlock = await this.blockRenderer.renderBlock( block, - assets, context ); const parentLines = parentBlock && this.indent(parentBlock.lines, indent); @@ -51,12 +46,12 @@ export class RecursiveBodyRenderer { // blocks, see https://developers.notion.com/reference/retrieve-a-block // "If a block contains the key has_children: true, use the Retrieve block children endpoint to get the list of children" const children = block.has_children - ? (await this.publicApi.listBlockChildren(block.id)).results + ? (await this.publicApi.listBlockChildren(block.id)) : []; const childIndent = indent + " ".repeat(parentBlock?.childIndent || 0); const renderChilds = children.map( - async (x) => await this.renderBlock(x, childIndent, assets, context) + async (x) => await this.renderBlock(x, childIndent, context) ); const childLines = await Promise.all(renderChilds); diff --git a/src/RenderDatabaseEntryTask.ts b/src/RenderDatabaseEntryTask.ts index 34d0538..23e5b87 100644 --- a/src/RenderDatabaseEntryTask.ts +++ b/src/RenderDatabaseEntryTask.ts @@ -1,18 +1,7 @@ +import { DatabasePageProperties } from "./DatabasePageProperties"; export interface RenderDatabaseEntryTask { - id: string; - url: string; - properties: { - /** - * A mapping of property object keys -> property values - */ - values: Record; - - /** - * A mapping of Notion API property names -> property object keys - */ - keys: Map; - }; - + properties: DatabasePageProperties + frontmatter?: Record // note: there's nothing to do to render an individual database entry, they are always rendered as part of tables } diff --git a/src/RenderDatabasePageTask.ts b/src/RenderDatabasePageTask.ts index a39b5b3..cda9757 100644 --- a/src/RenderDatabasePageTask.ts +++ b/src/RenderDatabasePageTask.ts @@ -1,8 +1,7 @@ -import { DatabasePageProperties } from './DatabasePageProperties'; +import { RenderDatabaseEntryTask } from "./RenderDatabaseEntryTask"; -export interface RenderDatabasePageTask { - id: string; - file: string; - properties: DatabasePageProperties; - render: () => Promise; +export interface RenderDatabasePageTask extends RenderDatabaseEntryTask { + file: string; + frontmatter: Record + render: () => Promise; } diff --git a/src/RenderedDatabaseEntry.ts b/src/RenderedDatabaseEntry.ts index 0223692..eda71ab 100644 --- a/src/RenderedDatabaseEntry.ts +++ b/src/RenderedDatabaseEntry.ts @@ -2,5 +2,5 @@ import { DatabaseEntryMeta } from './DatabaseEntryMeta'; export interface RenderedDatabaseEntry { meta: DatabaseEntryMeta, - properties: Record; + frontmatter?: Record; } diff --git a/src/RenderedDatabasePage.ts b/src/RenderedDatabasePage.ts index b4acb82..671f5ea 100644 --- a/src/RenderedDatabasePage.ts +++ b/src/RenderedDatabasePage.ts @@ -4,4 +4,5 @@ import { RenderedDatabaseEntry } from './RenderedDatabaseEntry'; export interface RenderedDatabasePage extends RenderedDatabaseEntry { meta: DatabasePageMeta; file: string; + frontmatter: Record; } diff --git a/src/RenderingContext.ts b/src/RenderingContext.ts new file mode 100644 index 0000000..99468e1 --- /dev/null +++ b/src/RenderingContext.ts @@ -0,0 +1,27 @@ +import * as path from "path"; + +import { AssetWriter } from "./AssetWriter"; +import { PageLinkResolver } from "./PageLinkResolver"; +import { RenderingContextLogger as RenderingContextLogger } from "./RenderingContextLogger"; + +/** + * Unit of work for rendering a specific page. + * Note: this is a bit of a service locator, be careful about not breaking SRP + */ +export class RenderingContext { + readonly assetWriter: AssetWriter; + readonly logger: RenderingContextLogger; + readonly linkResolver: PageLinkResolver; + + constructor(notionUrl: string, file: string) { + this.logger = new RenderingContextLogger(notionUrl, file); + + const dir = path.dirname(file); + + // write all assets right next to the page's markdown file + this.assetWriter = new AssetWriter(dir, this.logger); + + // resolve all links relative to the page's markdown file dir + this.linkResolver = new PageLinkResolver(dir); + } +} diff --git a/src/RenderingContextLogger.ts b/src/RenderingContextLogger.ts new file mode 100644 index 0000000..7a085d0 --- /dev/null +++ b/src/RenderingContextLogger.ts @@ -0,0 +1,34 @@ +import * as chalk from "chalk"; +import { performance } from "perf_hooks"; +import { logger } from "./logger"; + + +export class RenderingContextLogger { + private readonly start = performance.now(); + + constructor( + public readonly notionUrl: string, + public readonly file?: string + ) { } + + info(message: string) { + return logger.info(this.garnish(message)); + } + + warn(message: string) { + return logger.warn(this.garnish(message)); + } + + error(err: unknown) { + return logger.error(this.garnish(err as any)); // bah + } + + complete() { + const elapsed = performance.now() - this.start; + this.info("rendered page in " + Math.round(elapsed) + "ms"); + } + + private garnish(message: string) { + return `${message} ${chalk.gray(this.file || this.notionUrl)}`; + } +} diff --git a/src/RichTextRenderer.spec.ts b/src/RichTextRenderer.spec.ts index 423310f..34e31f6 100644 --- a/src/RichTextRenderer.spec.ts +++ b/src/RichTextRenderer.spec.ts @@ -1,6 +1,6 @@ import { Annotations, RichText } from '@notionhq/client/build/src/api-types'; +import { RenderingContext } from './RenderingContext'; -import { RenderingLoggingContext } from './logger'; import { RichTextRenderer } from './RichTextRenderer'; function annotations(x: Partial): Annotations { @@ -15,7 +15,7 @@ function annotations(x: Partial): Annotations { }; } -const context = new RenderingLoggingContext(""); +const context = new RenderingContext("", ""); describe("RichTextRenderer", () => { let sut: RichTextRenderer; diff --git a/src/RichTextRenderer.ts b/src/RichTextRenderer.ts index 34f6e1e..eeb0531 100644 --- a/src/RichTextRenderer.ts +++ b/src/RichTextRenderer.ts @@ -1,8 +1,9 @@ -import { RichText } from '@notionhq/client/build/src/api-types'; +import { RichText } from "@notionhq/client/build/src/api-types"; -import { LinkRenderer } from './LinkRenderer'; -import { RenderingLoggingContext } from './logger'; -import { MentionedPageRenderer } from './MentionedPageRenderer'; +import { LinkRenderer } from "./LinkRenderer"; +import { RenderingContextLogger } from "./RenderingContextLogger"; +import { MentionedPageRenderer } from "./MentionedPageRenderer"; +import { RenderingContext } from "./RenderingContext"; const debug = require("debug")("richtext"); @@ -15,12 +16,12 @@ export class RichTextRenderer { ) {} public async renderPlainText(text: RichText[]): Promise { - return text.map((rt) => rt.plain_text).join(" "); + return text.map((rt) => rt.plain_text).join(""); } public async renderMarkdown( text: RichText[], - context: RenderingLoggingContext + context: RenderingContext ): Promise { const result: string[] = []; @@ -35,7 +36,7 @@ export class RichTextRenderer { private async renderMarkdownCode( rt: RichText, - context: RenderingLoggingContext + context: RenderingContext ) { const mod = this.modifier(rt); @@ -44,7 +45,7 @@ export class RichTextRenderer { return this.renderUnsupported( `unsupported rich text type: ${rt.type}`, rt, - context + context.logger ); case "mention": switch (rt.mention.type) { @@ -54,14 +55,14 @@ export class RichTextRenderer { rt.plain_text ); const text = this.wrap(mod, page.properties.meta.title); - return this.linkRenderer.renderPageLink(text, page); + return this.linkRenderer.renderPageLink(text, page, context.linkResolver); case "database": case "date": case "user": return this.renderUnsupported( `unsupported rich text mention type: ${rt.mention.type}`, rt, - context + context.logger ); } case "text": @@ -135,7 +136,7 @@ export class RichTextRenderer { private renderUnsupported( msg: string, obj: any, - context: RenderingLoggingContext + context: RenderingContextLogger ): string { context.warn(msg); debug(msg + "\n%O", obj); diff --git a/src/SyncConfig.ts b/src/SyncConfig.ts index 9f36b06..5f96d5e 100644 --- a/src/SyncConfig.ts +++ b/src/SyncConfig.ts @@ -1,4 +1,5 @@ import { Sort } from "@notionhq/client/build/src/api-types"; +import { DatabasePageProperties } from "./DatabasePageProperties"; export interface SyncConfig { /** @@ -14,11 +15,9 @@ export interface SyncConfig { cmsDatabaseId: string; /** - * The output directory where the sync will place pages. - * - * Example: "docs/" + * Configuration options for the rendered pages */ - outDir: string; + pages: PagesConfig; /** * Configuration options for any database encountered while traversing the block graph. @@ -32,66 +31,64 @@ export type DatabaseConfig = | DatabaseConfigRenderTable; export interface DatabaseConfigBase { - /** - * The output directory where the sync will place pages of this database. - * - * Example: docs/mydb" - */ - outDir: string; - /** * Notion API https://developers.notion.com/reference/post-database-query#post-database-query-sort */ sorts?: Sort[]; - /** - * Configuration options for Notion API page properties - */ - properties?: { - /** - * A whitelist of Notion API page property names to include in the markdown page properties. - * Use this to select properties for export and control their ordering in rendered tables. - */ - include?: string[]; - }; - renderAs: "table" | "pages+views"; } export interface DatabaseConfigRenderTable extends DatabaseConfigBase { renderAs: "table"; - entries: { + /** + * Customize rendering of the table as one or multiple views. + * + * If not defined, will render a single view of the table with the Notion database property marked "title" + * as the first column. + * + * An empty array will supress rendering of the table (useful if you want to only emit the table to the index). + */ + views?: DatabaseView[]; + + entries?: { /** - * Controls whether to emit database entries to the index of rendered pages/entries + * Optional: Build frontmatter onject for index entries. + * If omitted, no index entries will be rendered for this table */ - emitToIndex: boolean; + frontmatterBuilder: (props: DatabasePageProperties) => Record; }; } +interface PagesConfig { + /** + * Build frontmatter object + */ + frontmatterBuilder: (props: DatabasePageProperties) => Record; + + /** + * Build the destination directory where to store the page. + * Final path will be $destinationDir/$filename.md + */ + destinationDirBuilder: (props: DatabasePageProperties) => string; + + /** + * Build the filename to store the page. + * Final path will be $destinationDir/$filename.md. + * + * default: slug of the meta.title + */ + filenameBuilder?: (props: DatabasePageProperties) => string; +} + export interface DatabaseConfigRenderPages extends DatabaseConfigBase { renderAs: "pages+views"; + /** - * Add custom data to the page frontmatter + * Configuration options for the rendered pages */ - pages: { - frontmatter: { - category: { - /** - * The Notion API page property that provides an optional sub-category value to use for the markdown page category. - * - * Example: "Cluster" - */ - property?: string; - /** - * A static category value to assign to every page - */ - static?: string; - }; - - extra?: Record; - }; - }; + pages: PagesConfig; /** * Configure "views" to render on the page where the child_database is encountered. @@ -108,10 +105,11 @@ export interface DatabaseConfigRenderPages extends DatabaseConfigBase { */ views: DatabaseView[]; } + export interface DatabaseView { - title: string; - properties: { - groupBy: string; + title?: string; + properties?: { + groupBy?: string; include?: string[]; }; } diff --git a/src/config.ts b/src/config.ts index 67244b3..31f2809 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,23 +10,12 @@ export function lookupDatabaseConfig( databaseId: string | null ): DatabaseConfig { const rootCmsDbConfig: DatabaseConfigRenderPages = { - outDir: config.outDir, renderAs: "pages+views", - pages: { - frontmatter: { - category: { - property: "Category", - }, - }, - }, + pages: config.pages, views: [], }; const defaultDbConfig: DatabaseConfigRenderTable = { - outDir: config.outDir + "/" + databaseId, renderAs: "table", - entries: { - emitToIndex: false, - }, }; const fallbackDbConfig: DatabaseConfig = diff --git a/src/index.ts b/src/index.ts index de86d5b..3469419 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,12 @@ -export { RenderedDatabaseEntry } from './RenderedDatabaseEntry'; -export { RenderedDatabasePage } from './RenderedDatabasePage'; -export { sync } from './sync'; +export { RenderedDatabaseEntry } from "./RenderedDatabaseEntry"; +export { RenderedDatabasePage } from "./RenderedDatabasePage"; +export { sync } from "./sync"; +export { slugify } from "./slugify"; export { - DatabaseConfig, DatabaseConfigRenderPages, DatabaseConfigRenderTable, SyncConfig -} from './SyncConfig'; - + DatabaseConfig, + DatabaseConfigRenderPages, + DatabaseConfigRenderTable, + SyncConfig, +} from "./SyncConfig"; +export { DatabasePageProperties } from "./DatabasePageProperties"; +export { DatabasePageMeta } from "./DatabasePageMeta"; diff --git a/src/logger.ts b/src/logger.ts index 182103a..875cdc2 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,5 +1,4 @@ import * as chalk from 'chalk'; -import { performance } from 'perf_hooks'; const info = (...args: any[]): void => { console.log(chalk.cyan("info"), ...args); @@ -13,38 +12,10 @@ const error = (...args: any[]): void => { console.error(chalk.red("error"), ...args); }; -const logger = { +export const logger = { info, warn, error, }; -export class RenderingLoggingContext { - private readonly start = performance.now(); - constructor( - public readonly notionUrl: string, - public readonly file?: string - ) {} - - info(message: string) { - return logger.info(this.garnish(message)); - } - - warn(message: string) { - return logger.warn(this.garnish(message)); - } - - error(err: unknown) { - return logger.error(this.garnish(err as any)); // bah - } - - complete() { - const elapsed = performance.now() - this.start; - this.info("rendered page in " + Math.round(elapsed) + "ms"); - } - - private garnish(message: string) { - return `${message} ${chalk.gray(this.file || this.notionUrl)}`; - } -} diff --git a/src/sync.ts b/src/sync.ts index b551dae..4bbeda9 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -1,18 +1,18 @@ -import { BlockRenderer } from './BlockRenderer'; -import { ChildDatabaseRenderer } from './ChildDatabaseRenderer'; -import { DatabaseEntryRenderer } from './DatabaseEntryRenderer'; -import { DatabasePageRenderer } from './DatabasePageRenderer'; -import { DatabaseTableRenderer } from './DatabaseTableRenderer'; -import { DatabaseViewRenderer } from './DatabaseViewRenderer'; -import { DeferredRenderer } from './DeferredRenderer'; -import { FrontmatterRenderer } from './FrontmatterRenderer'; -import { LinkRenderer } from './LinkRenderer'; -import { MentionedPageRenderer } from './MentionedPageRenderer'; -import { NotionApiFacade } from './NotionApiFacade'; -import { PropertiesParser } from './PropertiesParser'; -import { RecursiveBodyRenderer } from './RecursiveBodyRenderer'; -import { RichTextRenderer } from './RichTextRenderer'; -import { SyncConfig } from './SyncConfig'; +import { BlockRenderer } from "./BlockRenderer"; +import { ChildDatabaseRenderer } from "./ChildDatabaseRenderer"; +import { DatabaseEntryRenderer } from "./DatabaseEntryRenderer"; +import { DatabasePageRenderer } from "./DatabasePageRenderer"; +import { DatabaseViewRenderer } from "./DatabaseViewRenderer"; +import { DeferredRenderer } from "./DeferredRenderer"; +import { FrontmatterRenderer } from "./FrontmatterRenderer"; +import { LinkRenderer } from "./LinkRenderer"; +import { MentionedPageRenderer } from "./MentionedPageRenderer"; +import { NotionApiFacade } from "./NotionApiFacade"; +import { PageLinkResolver } from "./PageLinkResolver"; +import { PropertiesParser } from "./PropertiesParser"; +import { RecursiveBodyRenderer } from "./RecursiveBodyRenderer"; +import { RichTextRenderer } from "./RichTextRenderer"; +import { SyncConfig } from "./SyncConfig"; export async function sync(notionApiToken: string, config: SyncConfig) { const publicApi = new NotionApiFacade(notionApiToken); @@ -20,20 +20,23 @@ export async function sync(notionApiToken: string, config: SyncConfig) { const deferredRenderer = new DeferredRenderer(); const frontmatterRenderer = new FrontmatterRenderer(); - const tableRenderer = new DatabaseTableRenderer(); const mentionedPageRenderer = new MentionedPageRenderer( publicApi, deferredRenderer, config ); - const linkRenderer = new LinkRenderer(config); + const linkRenderer = new LinkRenderer(); const viewRenderer = new DatabaseViewRenderer(linkRenderer); const richTextRenderer = new RichTextRenderer( mentionedPageRenderer, linkRenderer ); const propertiesParser = new PropertiesParser(richTextRenderer); - const blockRenderer = new BlockRenderer(richTextRenderer, deferredRenderer); + const blockRenderer = new BlockRenderer( + richTextRenderer, + deferredRenderer, + linkRenderer + ); const bodyRenderer = new RecursiveBodyRenderer(publicApi, blockRenderer); const entryRenderer = new DatabaseEntryRenderer(propertiesParser); const pageRenderer = new DatabasePageRenderer( @@ -46,14 +49,17 @@ export async function sync(notionApiToken: string, config: SyncConfig) { config, publicApi, deferredRenderer, - tableRenderer, viewRenderer ); deferredRenderer.initialize(dbRenderer, pageRenderer, entryRenderer); // seed it with the root database - await deferredRenderer.renderChildDatabase(config.cmsDatabaseId); + const rootLinkResolver = new PageLinkResolver("."); + await deferredRenderer.renderChildDatabase( + config.cmsDatabaseId, + rootLinkResolver + ); await deferredRenderer.process(); const rendered = deferredRenderer.getRenderedPages();