diff --git a/README.md b/README.md index a09b534e..b6585322 120000 --- a/README.md +++ b/README.md @@ -1 +1 @@ -packages/contentlayer/README.md \ No newline at end of file +packages/@contentlayer/source-notion/README.md \ No newline at end of file diff --git a/docs/integration_granular_permissions.gif b/docs/integration_granular_permissions.gif new file mode 100644 index 00000000..8414aade Binary files /dev/null and b/docs/integration_granular_permissions.gif differ diff --git a/docs/table.png b/docs/table.png new file mode 100644 index 00000000..bf2c962e Binary files /dev/null and b/docs/table.png differ diff --git a/examples/node-script-notion/contentlayer.config.ts b/examples/node-script-notion/contentlayer.config.ts new file mode 100644 index 00000000..ef200b08 --- /dev/null +++ b/examples/node-script-notion/contentlayer.config.ts @@ -0,0 +1,13 @@ +import { makeSource, defineDatabase } from 'contentlayer-source-notion' + +export const Post = defineDatabase(() => ({ + name: 'Post', + databaseId: 'fe26b972ec3f4b32a1882230915fe111', +})) + +export default makeSource({ + client: { + auth: process.env.NOTION_TOKEN, + }, + databaseTypes: [Post], +}) diff --git a/examples/node-script-notion/my-script.mjs b/examples/node-script-notion/my-script.mjs new file mode 100644 index 00000000..553cc6ff --- /dev/null +++ b/examples/node-script-notion/my-script.mjs @@ -0,0 +1,6 @@ +import { allPosts } from './.contentlayer/generated/index.mjs' + +const postUrls = allPosts.map(post => post.url) + +console.log(`Found ${postUrls.length} posts:`); +console.log(postUrls) diff --git a/examples/node-script-notion/package.json b/examples/node-script-notion/package.json new file mode 100644 index 00000000..dcc2e0fa --- /dev/null +++ b/examples/node-script-notion/package.json @@ -0,0 +1,12 @@ +{ + "name": "node-script-notion-example", + "private": true, + "scripts": { + "start": "contentlayer build && node --experimental-json-modules my-script.mjs", + "dev": "contentlayer dev" + }, + "dependencies": { + "@notionhq/client": "^2.2.3", + "contentlayer": "latest" + } +} \ No newline at end of file diff --git a/packages/@contentlayer/core/src/plugin.ts b/packages/@contentlayer/core/src/plugin.ts index 0ed6d384..ff041052 100644 --- a/packages/@contentlayer/core/src/plugin.ts +++ b/packages/@contentlayer/core/src/plugin.ts @@ -10,7 +10,7 @@ import type { DataCache } from './DataCache.js' import type { SourceFetchDataError, SourceProvideSchemaError } from './errors.js' import type { SchemaDef, StackbitExtension } from './schema/index.js' -export type SourcePluginType = LiteralUnion<'local' | 'contentful' | 'sanity', string> +export type SourcePluginType = LiteralUnion<'local' | 'contentful' | 'notion' | 'sanity', string> export type PluginExtensions = { // TODO decentralized extension definitions + logic diff --git a/packages/@contentlayer/source-files/tsconfig.json b/packages/@contentlayer/source-files/tsconfig.json index f8f6e568..234d8e45 100644 --- a/packages/@contentlayer/source-files/tsconfig.json +++ b/packages/@contentlayer/source-files/tsconfig.json @@ -6,6 +6,6 @@ "outDir": "./dist", "tsBuildInfoFile": "./dist/.tsbuildinfo.json" }, - "include": ["./src"], + "include": ["./src", "../source-contentful/src/fetchData/types"], "references": [{ "path": "../utils" }, { "path": "../core" }] } diff --git a/packages/@contentlayer/source-notion/README.md b/packages/@contentlayer/source-notion/README.md new file mode 100644 index 00000000..c584077f --- /dev/null +++ b/packages/@contentlayer/source-notion/README.md @@ -0,0 +1,211 @@ +#   Contentlayer Source Notion [![Discord](https://badgen.net/badge/icon/discord?icon=discord&label)](https://discord.gg/fk83HNECYJ) + +[⚠️ Alpha test](#⚠️-alpha-test) • +[🎲 Features](#🎲-features) • +[🚀 Get started](#🚀-getting-started) • +[🔧 Configure](#🔧-configure) • +[✍ Contribute](#✍-contribute) + +Contentlayer Source Notion is a [Contentlayer](https://www.contentlayer.dev/) plugin to use [Notion.so](https://notion.so) as a content source. + +## ⚠️ Alpha test + +**This plugin is actually under heavy-development, use it at your own risks or for testing purposes only.** + +You can contribute to this project by listing bugs and giving ideas in the [issues of this repository](https://github.com/kerwanp/contentlayer-source-notion/issues). + +> Do not report bugs related to `contentlayer-source-notion` in the official repository. + +## 🎲 Features + +- [x] Generate your content from Notion Databases +- [x] Automatically infer the type of your properties +- [x] Render HTML from your Rich Text properties and pages content ([@notion-render/client](https://github.com/kerwanp/notion-render)) +- [x] Filter and sorts pages queried from your databases +- [x] Use Rollup and Relation properties with ease +- [ ] Recompute values to create new fields (computed fields) +- [ ] Iteration and cache system to work safely with ton of pages + +## 🚀 Getting started + +### 1. Install Contentlayer, Notion source plugin and dependencies + +```bash +$ npm install contentlayer contentlayer-source-notion @notionhq/client +$ yarn add contentlayer contentlayer-source-notion @notionhq/client +``` + +### 2. Create a database + +Save your database ID, it should be available in the url: **/myworkspace/fe26b972ec3f4b32a1882230915fe111?v=b56e97ee99a74f3f8c3ee80543fe22c6** + +![My integrations](docs/table.png) + +### 3. Get a Notion Token + +To interact with [Notion](https://notion.so) you need to create an integration and give it the correct permissions. +Create a new integration by heading to the [following link](https://www.notion.so/my-integrations). + +You should then have your Notion Token, also called **Internal Integration Token**. + +![My integrations](docs/integration_granular_permissions.gif) + +### 4. Add the integration to your databases + +By default, your integration does not have any permissions. +On each databases you want to query, click on the `•••` in the top right corner. + +Click on **Add connection** and select your Integration. Your token should now have access to your database. + +### 5. Configure and build Contentlayer + +You can configure your content source in the `contentlayer.config.js` file and then run `contentlayer build`. + +```typescript +import { makeSource, defineDatabase } from 'contentlayer-source-notion' +import * as notion from '@notionhq/client' + +const client = new notion.Client({ + auth: '', +}) + +const Post = defineDatabase(() => ({ + name: 'Post', + databaseId: '', +})) + +export default makeSource({ + client, + databaseTypes: [Category, Post], +}) +``` + +> ℹ Read more on how to configure `contentlayer-source-notion` [here](#🔧-configure) + +### 6. Use generated content + +Contentlayer will generate your content and typings in the `.contentlayer` folder. + +```typescript +import { allPosts } from './.contentlayer/generated/index.mjs' + +const postIds = allPosts.map((post) => post._id) +``` + +## 🔧 Configure + +### Source plugin options + +```typescript +import { makeSource } from 'contentlayer-source-notion' + +export default makeSource({ + client, + renderer, + databaseTypes: [], +}) +``` + +The `PluginOptions` supports the following parameters. Thoses options are defined when using `makeSource`. + +| Option | Default value | Type | Description | +| --------------- | ------------- | ------------------------------------------------ | ------------------------------------------------------ | +| `client` | `undefined` | `Client` | The Notion Client used to query the Notion API. | +| `renderer` | `undefined` | `NotionRenderer` | The renderer used to transform Notion Blocks into HTML | +| `databaseTypes` | | `DatabaseType[] \| Record` | The databases definitions. | + +### Database definition options + +```typescript +import { defineDatabase } from 'contentlayer-source-notion' + +const Post = defineDatabase(() => ({ + name: 'Post', + databaseId: '', + importContent: false, + automaticImport: true, + properties: { + email: { + name: 'Email', + description: 'The author email', + isRequired: true, + }, + }, +})) +``` + +The `DatabaseTypeDef` supports the following parameters. Thoses options are defined when using `defineDatabase`. + +| Option | Default value | Type | Description | +| ----------------- | ------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | | `string` | The name of this content used to generate types and constants names. | +| `description` | `undefined` | `string` | The description of this content used to generate comments | +| `databaseId` | | `string` | The database ID where your pages will be queried from | +| `automaticImport` | `undefined` | `bool` | By default, all your properties will be generated. By disabling automatic import you can whitelist the properties you want to use. Useful when you have sensitive content in your page properties | +| `importContent` | `undefined` | `bool` | By default, your page content will be generated. Disable it if you only want to use the properties. | +| `query` | `undefined` | `QueryDatabaseParameters` | Filter and sorts the page queried from the Notion API. More information on the [@notionhq/client repository](https://github.com/makenotion/notion-sdk-js) | +| `properties` | `undefined` | `Record \| DatabasePropertyTypeDef[]` | The properties definitions. When using `Record` the key will be used as the `key` option. | +| `computedFields` | `undefined` | `Record` | Create computed fields by using existing database properties. | + +#### Field definition options + +The `DatabasePropertyTypeDef` supports the following parameters. Thoses options are defined when using `defineDatabase`. + +| Option | Default value | Type | Description | +| ------------- | ------------- | -------- | ------------------------------------------------------------------------- | +| `id\|name` | | `string` | The id or name of the property you want to configure. | +| `key` | `undefined` | `string` | Map this property to a specific key. Defaults to the property name. | +| `description` | `undefined` | `string` | Field description used to generate comments. | +| `isRequired` | `false` | `bool` | When required, pages without this property defined will not be generated. | + +### Configure the Notion Client + +This plugin depends on the official Notion JS SDK, you can find more information on how to configure it on the [following repository](https://github.com/makenotion/notion-sdk-js). + +```typescript +import { makeSource, defineDatabase } from 'contentlayer-source-notion' +import * as notion from '@notionhq/client' + +const client = new notion.Client({ + auth: '', +}) + +export default makeSource({ + client, + renderer, + databaseTypes: [], +}) +``` + +### Configure the Notion Renderer + +Rich text properties and your pages content must be renderer, you can find more information on how to configure it on the [following repository](https://github.com/kerwanp/notion-render). + +```typescript +import { makeSource, defineDatabase } from 'contentlayer-source-notion' +import { NotionRenderer } from '@notion-render/client' +import * as notion from '@notionhq/client' + +const client = new notion.Client({ + auth: '', +}) + +const renderer = new NotionRenderer({ client }) + +export default makeSource({ + client, + renderer, + databaseTypes: [], +}) +``` + +## ✍ Contribute + +Wether it is a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from the community. + +You can : + +- Report Bugs/Reafture Requests directly in the [issues](https://github.com/kerwanp/contentlayer-source-notion/issues). +- Contribute via Pull Requests + +If you have any questions, feel free to join the [Official Contentlayer Discord](https://discord.gg/fk83HNECYJ). diff --git a/packages/@contentlayer/source-notion/package.json b/packages/@contentlayer/source-notion/package.json new file mode 100644 index 00000000..3c69ec8e --- /dev/null +++ b/packages/@contentlayer/source-notion/package.json @@ -0,0 +1,28 @@ +{ + "name": "contentlayer-source-notion", + "version": "0.0.1-alpha.26", + "type": "module", + "exports": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "./dist/*.{js,ts,map}", + "./dist/!(__test__)/**/*.{js,ts,map}", + "./src", + "./package.json" + ], + "scripts": { + "test": "echo No tests yet" + }, + "devDependencies": { + "@types/node": "^18.13.0" + }, + "dependencies": { + "@contentlayer/core": "workspace:*", + "@contentlayer/utils": "workspace:*", + "@notion-render/client": "0.0.1-alpha.3", + "@notionhq/client": "^2.2.3", + "p-queue": "^7.3.4", + "p-retry": "^5.1.2", + "slugify": "^1.6.5" + } +} \ No newline at end of file diff --git a/packages/@contentlayer/source-notion/src/fetchData/errors.ts b/packages/@contentlayer/source-notion/src/fetchData/errors.ts new file mode 100644 index 00000000..75900200 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/fetchData/errors.ts @@ -0,0 +1,8 @@ +import type * as core from '@contentlayer/core' +import { Tagged } from '@contentlayer/utils/effect' + +export class ComputedValueError extends Tagged('ComputedValueError')<{ + readonly error: unknown + readonly documentTypeDef: core.DocumentTypeDef + readonly document: core.Document +}> {} diff --git a/packages/@contentlayer/source-notion/src/fetchData/fetchAllDocuments.ts b/packages/@contentlayer/source-notion/src/fetchData/fetchAllDocuments.ts new file mode 100644 index 00000000..6fed6855 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/fetchData/fetchAllDocuments.ts @@ -0,0 +1,57 @@ +import * as os from 'node:os' + +import type { DataCache } from '@contentlayer/core' +import * as core from '@contentlayer/core' +import { Chunk, OT, pipe, S, T } from '@contentlayer/utils/effect' + +import { fetchDatabasePages } from '../notion/fetchDatabasePages.js' +import type { DatabaseTypeDef } from '../schema/types/database.js' +import { makeCacheItem } from './makeCacheItem.js' + +export type FetchAllDocumentsArgs = { + databaseTypeDefs: DatabaseTypeDef[] + previousCache: core.DataCache.Cache | undefined + schemaDef: core.SchemaDef + options: core.PluginOptions +} + +export const fetchAllDocuments = ({ databaseTypeDefs, previousCache, schemaDef, options }: FetchAllDocumentsArgs) => + pipe( + T.forEachPar_(databaseTypeDefs, (databaseTypeDef) => + pipe( + fetchDatabasePages({ databaseTypeDef }), + S.chain((pages) => + pipe( + S.effect( + pipe( + T.forEachParN_(pages, os.cpus().length, (page) => + makeCacheItem({ + page, + documentTypeDef: schemaDef.documentTypeDefMap[databaseTypeDef.name]!, + databaseTypeDef, + previousCache, + options, + }), + ), + ), + ), + OT.withStreamSpan('@contentlayer/source-notion/fetchData:makeCacheItems'), + ), + ), + S.runCollect, + T.chain((chunks) => + T.reduce_(chunks, [] as { fromCache: boolean; cacheItem: DataCache.CacheItem }[], (z, a) => + T.succeed([...z, ...a]), + ), + ), + ), + ), + T.map((chunks) => + Chunk.reduce_(chunks, [] as { fromCache: boolean; cacheItem: DataCache.CacheItem }[], (z, a) => [...z, ...a]), + ), + T.map((documents) => ({ + cacheItemsMap: Object.fromEntries(documents.map((_) => [_.cacheItem.document._id, _.cacheItem])), + })), + OT.withSpan('@contentlayer/source-notion/fetchData:fetchAllDocuments'), + T.mapError((error) => new core.SourceFetchDataError({ error, alreadyHandled: false })), + ) diff --git a/packages/@contentlayer/source-notion/src/fetchData/fetchData.ts b/packages/@contentlayer/source-notion/src/fetchData/fetchData.ts new file mode 100644 index 00000000..6bd73ae2 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/fetchData/fetchData.ts @@ -0,0 +1,27 @@ +import * as core from '@contentlayer/core' +import { OT, pipe, T } from '@contentlayer/utils/effect' + +import type { DatabaseTypeDef } from '../schema/types/database.js' +import { fetchAllDocuments } from './fetchAllDocuments.js' + +export type FetchDataArgs = { + databaseTypeDefs: DatabaseTypeDef[] + schemaDef: core.SchemaDef + options: core.PluginOptions +} + +export const fetchData = ({ schemaDef, databaseTypeDefs, options }: FetchDataArgs) => { + const resolveParams = pipe(core.DataCache.loadPreviousCacheFromDisk({ schemaHash: schemaDef.hash }), T.either) + return pipe( + T.rightOrFail(resolveParams), + T.chain((cache) => + pipe( + fetchAllDocuments({ schemaDef, databaseTypeDefs, previousCache: cache, options }), + T.tap((cache_) => T.succeedWith(() => (cache = cache_))), + T.tap((cache_) => core.DataCache.writeCacheToDisk({ cache: cache_, schemaHash: schemaDef.hash })), + ), + ), + OT.withSpan('@contentlayer/source-notion/fetchData:fetchData'), + T.mapError((error) => new core.SourceFetchDataError({ error, alreadyHandled: false })), + ) +} diff --git a/packages/@contentlayer/source-notion/src/fetchData/getComputedValues.ts b/packages/@contentlayer/source-notion/src/fetchData/getComputedValues.ts new file mode 100644 index 00000000..43239490 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/fetchData/getComputedValues.ts @@ -0,0 +1,22 @@ +import type * as core from '@contentlayer/core' +import { OT, pipe, T } from '@contentlayer/utils/effect' + +import { ComputedValueError } from './errors.js' + +export type GetComputedValuesArgs = { + document: core.Document + documentTypeDef: core.DocumentTypeDef +} + +export const getComputedValues = ({ document, documentTypeDef }: GetComputedValuesArgs) => + pipe( + T.forEachParDict_(documentTypeDef.computedFields ?? {}, { + mapKey: (field) => T.succeed(field.name), + mapValue: (field) => + T.tryCatchPromise( + async () => field.resolve(document), + (error) => new ComputedValueError({ error, documentTypeDef, document }), + ), + }), + OT.withSpan('@contentlayer/source-notion/fetchData:getComputedValues'), + ) diff --git a/packages/@contentlayer/source-notion/src/fetchData/makeCacheItem.ts b/packages/@contentlayer/source-notion/src/fetchData/makeCacheItem.ts new file mode 100644 index 00000000..3475c9c7 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/fetchData/makeCacheItem.ts @@ -0,0 +1,50 @@ +import type * as core from '@contentlayer/core' +import { OT, pipe, T } from '@contentlayer/utils/effect' +import type { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +import type { DatabaseTypeDef } from '../schema/types/database.js' +import { getComputedValues } from './getComputedValues.js' +import { makeDocument } from './makeDocument.js' + +export type MakeCacheItemArgs = { + databaseTypeDef: DatabaseTypeDef + documentTypeDef: core.DocumentTypeDef + page: PageObjectResponse + previousCache: core.DataCache.Cache | undefined + options: core.PluginOptions +} + +export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, previousCache, page, options }: MakeCacheItemArgs) => + pipe( + T.gen(function* ($) { + const documentHash = new Date(page.last_edited_time).getTime().toString() + + if ( + previousCache && + previousCache.cacheItemsMap[page.id] && + previousCache.cacheItemsMap[page.id]!.documentHash === documentHash && + previousCache.cacheItemsMap[page.id]!.hasWarnings === false + ) { + const cacheItem = previousCache.cacheItemsMap[page.id]! + return { cacheItem, fromCache: true } + } + + const document = yield* $(makeDocument({ documentTypeDef, databaseTypeDef, page, options })) + const computedValues = yield* $(getComputedValues({ document, documentTypeDef })) + + Object.entries(computedValues).forEach(([fieldName, value]) => { + document[fieldName] = value + }) + + return { + cacheItem: { + document, + documentHash, + hasWarnings: false, + documentTypeName: documentTypeDef.name, + }, + fromCache: false, + } + }), + OT.withSpan('@contentlayer/source-notion/fetchData:makeCacheItem'), + ) diff --git a/packages/@contentlayer/source-notion/src/fetchData/makeDocument.ts b/packages/@contentlayer/source-notion/src/fetchData/makeDocument.ts new file mode 100644 index 00000000..f9e90595 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/fetchData/makeDocument.ts @@ -0,0 +1,54 @@ +import type * as core from '@contentlayer/core' +import { OT, pipe, T } from '@contentlayer/utils/effect' +import type { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +import { getFieldData } from '../mapping/index.js' +import { fetchPageContent } from '../notion/fetchPageContent.js' +import type { PageProperties } from '../notion/types.js' +import type { DatabaseTypeDef } from '../schema/types/database.js' +import type { FieldDef } from '../types.js' + +export type MakeDocument = { + databaseTypeDef: DatabaseTypeDef + documentTypeDef: core.DocumentTypeDef + page: PageObjectResponse + options: core.PluginOptions +} + +export const makeDocument = ({ documentTypeDef, databaseTypeDef, page, options }: MakeDocument) => + pipe( + T.forEachParDict_(documentTypeDef.fieldDefs.filter((f) => !f.isSystemField) as FieldDef[], { + mapKey: (fieldDef) => T.succeed(fieldDef.name), + mapValue: (fieldDef) => { + const databaseFieldTypeDef = databaseTypeDef.properties?.find((field) => field.key === fieldDef.propertyKey) + + return getFieldData({ + fieldDef, + property: page.properties[fieldDef.propertyKey!] as PageProperties, + databaseFieldTypeDef, + databaseTypeDef, + documentTypeDef, + }) + }, + }), + T.chain((docValues) => + T.gen(function* ($) { + const document: core.Document = { + ...docValues, + [options.fieldOptions.typeFieldName]: documentTypeDef.name, + _id: page.id, + _raw: {}, + ...(databaseTypeDef.importContent !== false + ? { + [options.fieldOptions.bodyFieldName]: { + html: yield* $(fetchPageContent({ page })), + }, + } + : {}), + } + + return document + }), + ), + OT.withSpan('@contentlayer/source-notion/fetchData:makeDocument'), + ) diff --git a/packages/@contentlayer/source-notion/src/index.ts b/packages/@contentlayer/source-notion/src/index.ts new file mode 100644 index 00000000..2cc5bd1e --- /dev/null +++ b/packages/@contentlayer/source-notion/src/index.ts @@ -0,0 +1,67 @@ +import type * as core from '@contentlayer/core' +import { processArgs } from '@contentlayer/core' +import { pipe, S, SC, T } from '@contentlayer/utils/effect' +import { NotionRenderer } from '@notion-render/client' +import * as notion from '@notionhq/client' + +import { fetchData } from './fetchData/fetchData.js' +import { fetchNotion } from './notion/fetchNotion.js' +import { provideSchema } from './schema/provideSchema.js' +import { flattendDatabaseTypeDef } from './schema/utils/flattenDatabaseTypeDef.js' +import { NotionClient, NotionRenderer as NotionRendererTag } from './services.js' +import type { PluginOptions } from './types.js' + +export * from './schema/types/database.js' + +export const makeSource: core.MakeSourcePlugin = (args) => async (sourceKey) => { + const { + options, + extensions, + restArgs: { databaseTypes, dev, ...rest }, + } = await processArgs(args, sourceKey) + + const databaseTypeDefs = (Array.isArray(databaseTypes) ? databaseTypes : Object.values(databaseTypes)).map((_) => + _.def(), + ) + + const polling = dev && dev.polling === false ? false : dev?.polling ?? 5_000 + + const client = + rest.client instanceof notion.Client + ? rest.client + : new notion.Client({ + fetch: fetchNotion, + ...rest.client, + }) + + const renderer = + rest.renderer instanceof NotionRenderer ? rest.renderer : new NotionRenderer({ client, ...rest.renderer }) + + return { + type: 'notion', + extensions, + options, + provideSchema: () => + pipe( + provideSchema({ databaseTypeDefs, options }), + T.provideService(NotionClient)(client), + T.provideService(NotionRendererTag)(renderer), + ), + fetchData: ({ schemaDef }) => + pipe( + S.fromEffect( + pipe( + fetchData({ + databaseTypeDefs: databaseTypeDefs.map((databaseTypeDef) => flattendDatabaseTypeDef(databaseTypeDef)), + schemaDef, + options, + }), + T.either, + T.provideService(NotionClient)(client), + T.provideService(NotionRendererTag)(renderer), + ), + ), + S.repeatSchedule(polling ? SC.spaced(polling) : SC.stop), + ), + } +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-bool.ts b/packages/@contentlayer/source-notion/src/mapping/field-bool.ts new file mode 100644 index 00000000..48497f6b --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-bool.ts @@ -0,0 +1,11 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldBool: FieldFunctions<'checkbox'> = { + getFieldDef: () => + T.succeed({ + type: 'boolean', + }), + getFieldData: ({ propertyData }) => T.succeed(propertyData), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-created-by.ts b/packages/@contentlayer/source-notion/src/mapping/field-created-by.ts new file mode 100644 index 00000000..acd6b309 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-created-by.ts @@ -0,0 +1,31 @@ +import { T } from '@contentlayer/utils/effect' +import type { UserObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +import type { FieldFunctions } from '.' + +export const fieldCreatedBy: FieldFunctions<'created_by'> = { + getFieldDef: () => + T.succeed({ + type: 'nested', + nestedTypeName: 'User', + }), + getFieldData: ({ propertyData }) => { + const user = propertyData as UserObjectResponse + + return T.succeed({ + type: user.type, + name: user.name, + avatarUrl: user.avatar_url, + ...('person' in user + ? { + email: user.person.email, + } + : {}), + ...('bot' in user + ? { + workspace: user.bot.workspace_name, + } + : {}), + }) + }, +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-date-range.ts b/packages/@contentlayer/source-notion/src/mapping/field-date-range.ts new file mode 100644 index 00000000..58bb87f7 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-date-range.ts @@ -0,0 +1,19 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldDateRange: FieldFunctions<'date'> = { + getFieldDef: () => + T.succeed({ + type: 'nested', + nestedTypeName: 'DateRange', + }), + getFieldData: ({ propertyData }) => { + if (!propertyData) return T.succeed(undefined) + return T.succeed({ + start: new Date(propertyData.start), + end: propertyData.end ? new Date(propertyData.end) : undefined, + timezone: propertyData.time_zone ?? undefined, + }) + }, +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-date.ts b/packages/@contentlayer/source-notion/src/mapping/field-date.ts new file mode 100644 index 00000000..de1b73c8 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-date.ts @@ -0,0 +1,12 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldDate: FieldFunctions<'created_time' | 'last_edited_time'> = { + getFieldDef: () => + T.succeed({ + type: 'date', + required: true, + }), + getFieldData: ({ propertyData }) => T.succeed(new Date(propertyData)), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-files.ts b/packages/@contentlayer/source-notion/src/mapping/field-files.ts new file mode 100644 index 00000000..ebc1f7a2 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-files.ts @@ -0,0 +1,14 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldFiles: FieldFunctions<'files'> = { + getFieldDef: () => + T.succeed({ + type: 'list', + of: { type: 'string' }, + default: [], + }), + getFieldData: ({ propertyData }) => + T.succeed(propertyData.map((file) => ('file' in file ? file.file.url : file.external.url))), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-formula.ts b/packages/@contentlayer/source-notion/src/mapping/field-formula.ts new file mode 100644 index 00000000..8b93e4bd --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-formula.ts @@ -0,0 +1,15 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldFormula: FieldFunctions<'formula'> = { + getFieldDef: () => + T.succeed({ + type: 'string', + }), + getFieldData: ({ propertyData }) => { + const type = propertyData.type + if (type in propertyData) return T.succeed((propertyData as any)[type]) + return T.succeed(undefined) + }, +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-last-edited-by.ts b/packages/@contentlayer/source-notion/src/mapping/field-last-edited-by.ts new file mode 100644 index 00000000..f2dd4cc7 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-last-edited-by.ts @@ -0,0 +1,30 @@ +import { T } from '@contentlayer/utils/effect' +import type { UserObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +import type { FieldFunctions } from '.' + +export const fieldLastEditedBy: FieldFunctions<'last_edited_by'> = { + getFieldDef: () => + T.succeed({ + type: 'nested', + nestedTypeName: 'User', + }), + getFieldData: ({ propertyData }) => { + const user = propertyData as UserObjectResponse + return T.succeed({ + type: user.type, + name: user.name, + avatarUrl: user.avatar_url, + ...('person' in user + ? { + email: user.person.email, + } + : {}), + ...('bot' in user + ? { + workspace: user.bot.workspace_name, + } + : {}), + }) + }, +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-multi-select.ts b/packages/@contentlayer/source-notion/src/mapping/field-multi-select.ts new file mode 100644 index 00000000..fef30d74 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-multi-select.ts @@ -0,0 +1,16 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldMultiSelect: FieldFunctions<'multi_select'> = { + getFieldDef: ({ propertyData }) => + T.succeed({ + type: 'list', + of: { + type: 'enum', + options: propertyData.options.map((o) => o.name), + }, + default: [], + }), + getFieldData: ({ propertyData }) => T.succeed(propertyData.map((d) => d.name)), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-number.ts b/packages/@contentlayer/source-notion/src/mapping/field-number.ts new file mode 100644 index 00000000..113091ee --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-number.ts @@ -0,0 +1,11 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldNumber: FieldFunctions<'number'> = { + getFieldDef: () => + T.succeed({ + type: 'number', + }), + getFieldData: ({ propertyData }) => T.succeed(propertyData), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-people.ts b/packages/@contentlayer/source-notion/src/mapping/field-people.ts new file mode 100644 index 00000000..fcbccb6b --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-people.ts @@ -0,0 +1,34 @@ +import { T } from '@contentlayer/utils/effect' +import type { UserObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +import type { FieldFunctions } from '.' + +export const fieldPeople: FieldFunctions<'people'> = { + getFieldDef: () => + T.succeed({ + type: 'list', + of: { + type: 'nested', + nestedTypeName: 'User', + }, + default: [], + }), + getFieldData: ({ propertyData }) => + T.succeed( + (propertyData as UserObjectResponse[]).map((user) => ({ + type: user.type, + name: user.name, + avatarUrl: user.avatar_url, + ...('person' in user + ? { + email: user.person.email, + } + : {}), + ...('bot' in user + ? { + workspace: user.bot.workspace_name, + } + : {}), + })), + ), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-relation.ts b/packages/@contentlayer/source-notion/src/mapping/field-relation.ts new file mode 100644 index 00000000..7b9465f2 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-relation.ts @@ -0,0 +1,36 @@ +import { T } from '@contentlayer/utils/effect' + +import type { DatabasePropertyTypeDef } from '../schema/types/property' +import type { FieldFunctions } from '.' + +const isSingle = (databaseFieldTypeDef: DatabasePropertyTypeDef | undefined) => { + return ( + databaseFieldTypeDef && + 'type' in databaseFieldTypeDef && + databaseFieldTypeDef.type === 'relation' && + databaseFieldTypeDef.single + ) +} + +export const fieldRelation: FieldFunctions<'relation'> = { + getFieldDef: ({ databaseFieldTypeDef }) => { + if (isSingle(databaseFieldTypeDef)) { + return T.succeed({ + type: 'string', + }) + } + + return T.succeed({ + type: 'list', + of: { type: 'string' }, + default: [], + }) + }, + getFieldData: ({ propertyData, databaseFieldTypeDef }) => { + if (isSingle(databaseFieldTypeDef)) { + return T.succeed(propertyData[0]?.id) + } + + return T.succeed(propertyData.map((r) => r.id)) + }, +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-rich-text.ts b/packages/@contentlayer/source-notion/src/mapping/field-rich-text.ts new file mode 100644 index 00000000..58ec372a --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-rich-text.ts @@ -0,0 +1,16 @@ +import { pipe, T } from '@contentlayer/utils/effect' + +import { NotionRenderer } from '../services.js' +import type { FieldFunctions } from '.' + +export const fieldRichText: FieldFunctions<'rich_text' | 'title'> = { + getFieldDef: () => + T.succeed({ + type: 'string', + }), + getFieldData: ({ propertyData }) => + pipe( + T.service(NotionRenderer), + T.chain((renderer) => T.tryPromise(() => renderer.render(...propertyData))), + ), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-rollup.ts b/packages/@contentlayer/source-notion/src/mapping/field-rollup.ts new file mode 100644 index 00000000..c627c01c --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-rollup.ts @@ -0,0 +1,84 @@ +import { pipe, T } from '@contentlayer/utils/effect' + +import { findDatabaseFieldDef } from '../schema/utils/findDatabaseFieldDef.js' +import type { FieldDef } from '../types.js' +import type { FieldFunctions } from '.' +import { getFieldData } from './index.js' + +export const fieldRollup: FieldFunctions<'rollup'> = { + getFieldDef: ({ propertyData, databaseTypeDef, getDocumentTypeDef }) => + T.gen(function* ($) { + const relationFieldDef = findDatabaseFieldDef({ + databaseTypeDef, + property: { id: propertyData.relation_property_id, name: propertyData.relation_property_name }, + }) + + if (!relationFieldDef || !('type' in relationFieldDef) || relationFieldDef.type !== 'relation') + throw new Error('Field not configured properly') + + const relationDatabaseTypeDef = yield* $(getDocumentTypeDef(relationFieldDef.relation.def())) + const fieldDef = (relationDatabaseTypeDef.fieldDefs as FieldDef[]).find( + (fieldDef) => fieldDef.propertyKey === propertyData.rollup_property_name, + ) + + if (!fieldDef) throw new Error('Field not configured properly') + + if (['show_original', 'show_unique'].includes(propertyData.function)) { + if ( + fieldDef.type === 'reference' || + fieldDef.type === 'list_polymorphic' || + fieldDef.type === 'nested_polymorphic' || + fieldDef.type === 'reference_polymorphic' + ) { + throw new Error(`Rollup field of type ${fieldDef.type}`) + } + + if (fieldDef.type === 'list') { + return { + type: 'list', + of: { + ...fieldDef.of, + }, + } + } + + return { + type: 'list', + of: { + ...fieldDef, + }, + } + } + + return fieldDef + }), + getFieldData: ({ propertyData, ...args }) => + T.gen(function* ($) { + if (propertyData.type === 'array') { + const res = yield* $( + pipe( + T.forEach_(propertyData.array, (property) => + getFieldData({ ...args, property: { ...property, id: 'unknown' } }), + ), + + // As Contentlayer does not support list of list, we have to reduce in case the case occurs (eg. people) + T.map((res) => + [...res].length > 1 ? [...res].reduce((a, b) => (Array.isArray(a) ? [...a, ...b] : [a, b])) : res, + ), + ), + ) + + return res + } + + return yield* $( + getFieldData({ + ...args, + property: { + ...propertyData, + id: 'unkown', + }, + }), + ) + }), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-select.ts b/packages/@contentlayer/source-notion/src/mapping/field-select.ts new file mode 100644 index 00000000..4f91a77e --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-select.ts @@ -0,0 +1,12 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldSelect: FieldFunctions<'select'> = { + getFieldDef: ({ propertyData }) => + T.succeed({ + type: 'enum', + options: propertyData.options.map((o) => o.name), + }), + getFieldData: ({ propertyData }) => T.succeed(propertyData?.name), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-status.ts b/packages/@contentlayer/source-notion/src/mapping/field-status.ts new file mode 100644 index 00000000..7895b5eb --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-status.ts @@ -0,0 +1,12 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldStatus: FieldFunctions<'status'> = { + getFieldDef: ({ propertyData }) => + T.succeed({ + type: 'enum', + options: propertyData.options.map((o) => o.name), + }), + getFieldData: ({ propertyData }) => T.succeed(propertyData?.name), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/field-string.ts b/packages/@contentlayer/source-notion/src/mapping/field-string.ts new file mode 100644 index 00000000..9198a9af --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/field-string.ts @@ -0,0 +1,11 @@ +import { T } from '@contentlayer/utils/effect' + +import type { FieldFunctions } from '.' + +export const fieldString: FieldFunctions<'phone_number'> = { + getFieldDef: () => + T.succeed({ + type: 'string', + }), + getFieldData: ({ propertyData }) => T.succeed(propertyData), +} diff --git a/packages/@contentlayer/source-notion/src/mapping/index.ts b/packages/@contentlayer/source-notion/src/mapping/index.ts new file mode 100644 index 00000000..f027a389 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/mapping/index.ts @@ -0,0 +1,109 @@ +import type * as core from '@contentlayer/core' +import type { Has } from '@contentlayer/utils/effect' +import { T } from '@contentlayer/utils/effect' +import type { NotionRenderer } from '@notion-render/client' +import type * as notion from '@notionhq/client' + +import type { + DatabaseProperties, + DatabasePropertyData, + DatabasePropertyTypes, + PageProperties, + PagePropertyData, + PagePropertyTypes, +} from '../notion/types.js' +import type { DatabaseTypeDef } from '../schema/types/database.js' +import type { DatabasePropertyTypeDef } from '../schema/types/property.js' +import { getDatabasePropertyData, getPagePropertyData } from '../schema/utils/getPropertyData.js' +import type { DistributiveOmit, FieldDef } from '../types.js' +import { fieldBool } from './field-bool.js' +import { fieldCreatedBy } from './field-created-by.js' +import { fieldDate } from './field-date.js' +import { fieldDateRange } from './field-date-range.js' +import { fieldFiles } from './field-files.js' +import { fieldFormula } from './field-formula.js' +import { fieldLastEditedBy } from './field-last-edited-by.js' +import { fieldMultiSelect } from './field-multi-select.js' +import { fieldNumber } from './field-number.js' +import { fieldPeople } from './field-people.js' +import { fieldRelation } from './field-relation.js' +import { fieldRichText } from './field-rich-text.js' +import { fieldRollup } from './field-rollup.js' +import { fieldSelect } from './field-select.js' +import { fieldStatus } from './field-status.js' +import { fieldString } from './field-string.js' + +export type GetFieldDefArgs = { + propertyData: DatabasePropertyData + databaseFieldTypeDef: DatabasePropertyTypeDef | undefined + databaseTypeDef: DatabaseTypeDef + getDocumentTypeDef: (databaseTypeDef: DatabaseTypeDef) => T.Effect +} + +export type GetFieldDef = ( + args: GetFieldDefArgs, +) => T.Effect< + Has & Has, + unknown, + DistributiveOmit +> + +export type GetFieldDataArgs = { + propertyData: PagePropertyData + databaseFieldTypeDef: DatabasePropertyTypeDef | undefined + databaseTypeDef: DatabaseTypeDef + fieldDef: FieldDef + documentTypeDef: core.DocumentTypeDef +} + +export type GetFieldData = ( + args: GetFieldDataArgs, +) => T.Effect & Has, unknown, any> + +export type FieldFunctions = { + getFieldDef: GetFieldDef + getFieldData: GetFieldData +} + +type FieldMappingType = { + [key in DatabasePropertyTypes]: FieldFunctions +} + +const FieldMapping: FieldMappingType = { + checkbox: fieldBool, + email: fieldString, + phone_number: fieldString, + select: fieldSelect, + multi_select: fieldMultiSelect, + url: fieldString, + number: fieldNumber, + title: fieldRichText, + created_time: fieldDate, + status: fieldStatus, + date: fieldDateRange, + last_edited_time: fieldDate, + rich_text: fieldRichText, + files: fieldFiles, + people: fieldPeople, + last_edited_by: fieldLastEditedBy, + created_by: fieldCreatedBy, + formula: fieldFormula, + relation: fieldRelation, + rollup: fieldRollup, +} + +export const getFieldDef = ( + args: { property: DatabaseProperties } & Omit, 'propertyData'>, +) => + FieldMapping[args.property.type].getFieldDef({ + propertyData: getDatabasePropertyData(args.property), + ...args, + }) + +export const getFieldData = ( + args: { property: PageProperties } & Omit, 'propertyData'>, +) => + FieldMapping[args.property.type].getFieldData({ + propertyData: getPagePropertyData(args.property), + ...args, + }) diff --git a/packages/@contentlayer/source-notion/src/notion/errors.ts b/packages/@contentlayer/source-notion/src/notion/errors.ts new file mode 100644 index 00000000..918e28da --- /dev/null +++ b/packages/@contentlayer/source-notion/src/notion/errors.ts @@ -0,0 +1,6 @@ +import { errorToString } from '@contentlayer/utils' +import { Tagged } from '@contentlayer/utils/effect' + +export class UnknownNotionError extends Tagged('UnknownNotionError')<{ readonly error: unknown }> { + toString = () => `UnknowNotionError: ${errorToString(this.error)}` +} diff --git a/packages/@contentlayer/source-notion/src/notion/fetchDatabasePages.ts b/packages/@contentlayer/source-notion/src/notion/fetchDatabasePages.ts new file mode 100644 index 00000000..b5b77b57 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/notion/fetchDatabasePages.ts @@ -0,0 +1,36 @@ +import { OT, pipe, S } from '@contentlayer/utils/effect' +import type { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +import type { DatabaseTypeDef } from '../schema/types/database.js' +import { NotionClient } from '../services.js' +import { UnknownNotionError } from './errors.js' + +export type FetchDatabasePagesArgs = { + databaseTypeDef: DatabaseTypeDef +} + +export const fetchDatabasePages = ({ databaseTypeDef }: FetchDatabasePagesArgs) => + pipe( + S.service(NotionClient), + S.chain((client) => + S.async(async (emit) => { + let nextCursor: string | undefined = undefined + + do { + const res = await client.databases.query({ + database_id: databaseTypeDef.databaseId, + start_cursor: nextCursor ?? undefined, + filter: databaseTypeDef.query?.filter, + sorts: databaseTypeDef.query?.sorts, + }) + + nextCursor = res.next_cursor as string | undefined // NOTE: Throw type error and make res any if not typed, why??? + emit.single(res.results as PageObjectResponse[]) + } while (nextCursor) + + emit.end() + }), + ), + OT.withStreamSpan('@contentlayer/source-notion/fetchData:fetchDatabasePages'), + S.mapError((error) => new UnknownNotionError({ error })), + ) diff --git a/packages/@contentlayer/source-notion/src/notion/fetchNotion.ts b/packages/@contentlayer/source-notion/src/notion/fetchNotion.ts new file mode 100644 index 00000000..139d171b --- /dev/null +++ b/packages/@contentlayer/source-notion/src/notion/fetchNotion.ts @@ -0,0 +1,31 @@ +import { buildRequestError } from '@notionhq/client/build/src/errors.js' +import PQueue from 'p-queue' +import pRetry, { AbortError } from 'p-retry' + +const queue = new PQueue({ interval: 1000, intervalCap: 4 }) + +export const fetchNotion = (...args: Parameters) => { + return queue.add(() => + pRetry( + async () => { + const response = await fetch(...args) + + if (!response.ok) { + const error = buildRequestError(response, await response.text()) + + if ([401, 404].includes(response.status)) { + throw new AbortError(error) + } + + throw error + } + + return response + }, + { + retries: 5, + onFailedAttempt: async () => new Promise((res) => setTimeout(res, 1000)), + }, + ), + ) as Promise +} diff --git a/packages/@contentlayer/source-notion/src/notion/fetchPageContent.ts b/packages/@contentlayer/source-notion/src/notion/fetchPageContent.ts new file mode 100644 index 00000000..bed60cf9 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/notion/fetchPageContent.ts @@ -0,0 +1,12 @@ +import { pipe, T } from '@contentlayer/utils/effect' +import type { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +import { NotionRenderer } from '../services.js' +import { UnknownNotionError } from './errors.js' + +export const fetchPageContent = ({ page }: { page: PageObjectResponse }) => + pipe( + T.service(NotionRenderer), + T.chain((renderer) => T.tryPromise(() => renderer.renderBlock(page.id))), + T.mapError((error) => new UnknownNotionError({ error })), + ) diff --git a/packages/@contentlayer/source-notion/src/notion/types.ts b/packages/@contentlayer/source-notion/src/notion/types.ts new file mode 100644 index 00000000..82aac095 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/notion/types.ts @@ -0,0 +1,17 @@ +import type { DatabaseObjectResponse, PageObjectResponse } from '@notionhq/client/build/src/api-endpoints' + +import type { DiscriminateUnion, DiscriminateUnionValue } from '../types' + +export type DatabaseProperties = DatabaseObjectResponse['properties'][number] +export type DatabasePropertyTypes = DatabaseProperties['type'] +export type DatabaseProperty = DiscriminateUnion +export type DatabasePropertyData = DiscriminateUnionValue< + DatabaseProperties, + 'type', + T +> + +export type PageProperties = PageObjectResponse['properties'][number] +export type PagePropertyTypes = PageProperties['type'] +export type PageProperty = DiscriminateUnion +export type PagePropertyData = DiscriminateUnionValue diff --git a/packages/@contentlayer/source-notion/src/schema/provideDocumentTypeDef.ts b/packages/@contentlayer/source-notion/src/schema/provideDocumentTypeDef.ts new file mode 100644 index 00000000..69cdf979 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/provideDocumentTypeDef.ts @@ -0,0 +1,64 @@ +import type * as core from '@contentlayer/core' +import { OT, pipe, T } from '@contentlayer/utils/effect' + +import { NotionClient } from '../services.js' +import type { FieldDef } from '../types.js' +import { provideFieldDef } from './provideFieldDef.js' +import type { DatabaseTypeDef } from './types/database.js' + +export type ProvideDocumentTypeDefArgs = { + databaseTypeDef: DatabaseTypeDef + getDocumentTypeDef: (databaseTypeDef: DatabaseTypeDef) => T.Effect + options: core.PluginOptions +} + +export const provideDocumentTypeDef = ({ databaseTypeDef, getDocumentTypeDef, options }: ProvideDocumentTypeDefArgs) => + pipe( + T.service(NotionClient), + T.chain((client) => + pipe( + T.tryPromise(() => client.databases.retrieve({ database_id: databaseTypeDef.databaseId })), + T.map(({ properties }) => Object.values(properties)), + T.chain((properties) => + T.forEachPar_(properties, (property) => provideFieldDef({ databaseTypeDef, property, getDocumentTypeDef })), + ), + ), + ), + T.map((fieldDefsChunk) => { + const fieldDefs = [...fieldDefsChunk].filter((fd) => fd) as FieldDef[] + + if (databaseTypeDef.importContent !== false) { + fieldDefs.push({ + name: options.fieldOptions.bodyFieldName, + type: 'nested', + nestedTypeName: 'Body', + description: 'The page content', + isRequired: true, + isSystemField: true, + default: undefined, + }) + } + + const computedFields = Object.entries(databaseTypeDef.computedFields ?? {}).map( + ([name, computedField]) => ({ + description: computedField.description, + type: computedField.type, + name, + // NOTE we need to flip the variance here (casting a core.Document to a LocalDocument) + resolve: computedField.resolve as core.ComputedFieldResolver, + }), + ) + + return { + _tag: 'DocumentTypeDef' as const, + name: databaseTypeDef.name, + description: databaseTypeDef.description, + isSingleton: false, + fieldDefs: fieldDefs, + computedFields: computedFields, + extensions: {}, + } as core.DocumentTypeDef + }), + + OT.withSpan('@contentlayer/source-notion/schema:provideDocumentTypeDef'), + ) diff --git a/packages/@contentlayer/source-notion/src/schema/provideDocumentTypeDefMap.ts b/packages/@contentlayer/source-notion/src/schema/provideDocumentTypeDefMap.ts new file mode 100644 index 00000000..c439fc2a --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/provideDocumentTypeDefMap.ts @@ -0,0 +1,38 @@ +import type * as core from '@contentlayer/core' +import { OT, pipe, T } from '@contentlayer/utils/effect' + +import { provideDocumentTypeDef } from './provideDocumentTypeDef.js' +import type { DatabaseTypeDef } from './types/database.js' +import { flattendDatabaseTypeDef } from './utils/flattenDatabaseTypeDef.js' + +export type ProvideDocumentTypeDefMapArgs = { + databaseTypeDefs: DatabaseTypeDef[] + options: core.PluginOptions +} + +export const provideDocumentTypeDefMap = ({ databaseTypeDefs, options }: ProvideDocumentTypeDefMapArgs) => + pipe( + T.gen(function* ($) { + const documentTypeDefMap: core.DocumentTypeDefMap = {} + + const getDocumentTypeDef = (databaseTypeDef: DatabaseTypeDef) => { + return databaseTypeDef.name in documentTypeDefMap + ? T.succeed(documentTypeDefMap[databaseTypeDef.name]) + : pipe( + provideDocumentTypeDef({ + databaseTypeDef: flattendDatabaseTypeDef(databaseTypeDef), + getDocumentTypeDef, + options, + }), + T.tap((documentTypeDef) => T.succeed((documentTypeDefMap[databaseTypeDef.name] = documentTypeDef))), + ) + } + + for (const databaseTypeDef of databaseTypeDefs) { + yield* $(getDocumentTypeDef(databaseTypeDef)) + } + + return documentTypeDefMap + }), + OT.withSpan('@contentlayer/source-notion/schema:provideDocumentTypeDefMap'), + ) diff --git a/packages/@contentlayer/source-notion/src/schema/provideFieldDef.ts b/packages/@contentlayer/source-notion/src/schema/provideFieldDef.ts new file mode 100644 index 00000000..a6fd78b9 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/provideFieldDef.ts @@ -0,0 +1,35 @@ +import type * as core from '@contentlayer/core' +import { OT, pipe, T } from '@contentlayer/utils/effect' + +import { getFieldDef } from '../mapping/index.js' +import type { DatabaseProperties } from '../notion/types.js' +import type { FieldDef } from '../types.js' +import type { DatabaseTypeDef } from './types/database.js' +import { findDatabaseFieldDef } from './utils/findDatabaseFieldDef.js' +import { normalizeKey } from './utils/normalizeKey.js' + +export type ProvideFieldDefArgs = { + property: DatabaseProperties + databaseTypeDef: DatabaseTypeDef + getDocumentTypeDef: (databaseTypeDef: DatabaseTypeDef) => T.Effect +} + +export const provideFieldDef = ({ property, databaseTypeDef, getDocumentTypeDef }: ProvideFieldDefArgs) => + pipe( + T.succeed(findDatabaseFieldDef({ databaseTypeDef, property })), + T.chain((databaseFieldTypeDef) => + T.gen(function* ($) { + const name = databaseFieldTypeDef?.key ?? normalizeKey(property.name) + + return { + ...(yield* $(getFieldDef({ property, databaseFieldTypeDef, databaseTypeDef, getDocumentTypeDef }))), + name, + propertyKey: property.name, + isSystemField: false, + description: databaseFieldTypeDef?.description, + isRequired: databaseFieldTypeDef?.required ?? false, + } as FieldDef + }), + ), + OT.withSpan('@contentlayer/source-notion/schema:provideFieldDef'), + ) diff --git a/packages/@contentlayer/source-notion/src/schema/provideNestedTypeDefMap.ts b/packages/@contentlayer/source-notion/src/schema/provideNestedTypeDefMap.ts new file mode 100644 index 00000000..de9c5fb3 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/provideNestedTypeDefMap.ts @@ -0,0 +1,107 @@ +import type * as core from '@contentlayer/core' +import { OT, pipe, T } from '@contentlayer/utils/effect' + +export const provideNestedTypeDefMap = (): T.Effect => + pipe( + T.succeed({ + body: { + _tag: 'NestedTypeDef' as const, + name: 'Body', + description: 'The body generated using page content', + fieldDefs: [ + { + name: 'html', + type: 'string' as const, + description: 'The HTML body', + isSystemField: false, + default: undefined, + isRequired: true, + }, + ], + extensions: {}, + }, + dateRange: { + _tag: 'NestedTypeDef' as const, + name: 'DateRange', + description: 'Nested type definition for Notion date properties', + fieldDefs: [ + { + name: 'start', + type: 'date' as const, + description: undefined, + isSystemField: false, + default: undefined, + isRequired: true, + }, + { + name: 'end', + type: 'date' as const, + description: undefined, + isSystemField: false, + default: undefined, + isRequired: false, + }, + { + name: 'timezone', + type: 'string' as const, + description: undefined, + isSystemField: false, + default: undefined, + isRequired: false, + }, + ], + extensions: {}, + }, + user: { + _tag: 'NestedTypeDef' as const, + name: 'User', + description: 'Nested type definition for Notion people properties', + fieldDefs: [ + { + name: 'type', + type: 'enum' as const, + options: ['person', 'bot'], + description: 'The user type', + isSystemField: false, + default: undefined, + isRequired: false, + }, + { + name: 'name', + type: 'string' as const, + description: 'The user name', + isSystemField: false, + default: undefined, + isRequired: false, + }, + { + name: 'avatarUrl', + type: 'string' as const, + description: 'The user avatar', + isSystemField: false, + default: undefined, + isRequired: false, + }, + { + name: 'email', + type: 'string' as const, + description: + 'User email address, only if the user is a person and the integration has user capabilities to access email addresses.', + isSystemField: false, + default: undefined, + isRequired: false, + }, + { + name: 'workspace', + type: 'string' as const, + description: 'User workspace owner, only if the user is a bot', + isSystemField: false, + default: undefined, + isRequired: false, + }, + ], + extensions: {}, + }, + }), + OT.withSpan('@contentlayer/source-notion/schema:provideNestedTypeDefMap'), + ) diff --git a/packages/@contentlayer/source-notion/src/schema/provideSchema.ts b/packages/@contentlayer/source-notion/src/schema/provideSchema.ts new file mode 100644 index 00000000..56d8d63a --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/provideSchema.ts @@ -0,0 +1,38 @@ +import * as core from '@contentlayer/core' +import * as utils from '@contentlayer/utils' +import { OT, pipe, T } from '@contentlayer/utils/effect' + +import { provideDocumentTypeDefMap } from './provideDocumentTypeDefMap.js' +import { provideNestedTypeDefMap } from './provideNestedTypeDefMap.js' +import type { DatabaseTypeDef } from './types/database.js' + +export type ProvideSchemaArgs = { + databaseTypeDefs: DatabaseTypeDef[] + options: core.PluginOptions +} + +export const provideSchema = ({ databaseTypeDefs, options }: ProvideSchemaArgs) => + pipe( + T.gen(function* ($) { + return { + documentTypeDefMap: yield* $(provideDocumentTypeDefMap({ databaseTypeDefs, options })), + nestedTypeDefMap: yield* $(provideNestedTypeDefMap()), + } + }), + + T.chain((schemaDef) => + pipe( + utils.hashObject(schemaDef), + T.map( + (hash): core.SchemaDef => ({ + ...schemaDef, + hash, + }), + ), + T.tap((schemaDef) => T.succeed(core.validateSchema(schemaDef))), + ), + ), + + T.mapError((error) => new core.SourceProvideSchemaError({ error })), + OT.withSpan('@contentlayer/source-notion/schema:provideSchema'), + ) diff --git a/packages/@contentlayer/source-notion/src/schema/types/computed-field.ts b/packages/@contentlayer/source-notion/src/schema/types/computed-field.ts new file mode 100644 index 00000000..a97c87d5 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/types/computed-field.ts @@ -0,0 +1,20 @@ +import type { FieldDefType, GetDocumentTypeMapGen } from '@contentlayer/core' + +import type { LocalDocument } from '../../types.js' + +export type ComputedField = { + description?: string + type: FieldDefType + resolve: ComputedFieldResolver +} + +// TODO come up with a way to hide computed fields from passed in document +type ComputedFieldResolver = ( + _: GetDocumentTypeGen, +) => any | Promise + +type GetDocumentTypeGen = Name extends keyof GetDocumentTypeMapGen + ? GetDocumentTypeMapGen[Name] + : LocalDocument + +// type GetDocumentTypeGen = GetDocumentTypeMapGen[Name] diff --git a/packages/@contentlayer/source-notion/src/schema/types/database.ts b/packages/@contentlayer/source-notion/src/schema/types/database.ts new file mode 100644 index 00000000..a9dc0dc8 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/types/database.ts @@ -0,0 +1,63 @@ +import type { Thunk } from '@contentlayer/utils' +import type { QueryDatabaseParameters } from '@notionhq/client/build/src/api-endpoints' + +import type { ComputedField } from './computed-field' +import type { DatabasePropertyTypeDef } from './property' + +export type DatabaseTypeDef = { + /** + * The database name. + */ + name: DefName + + /** + * The database description used to generate comments. + */ + description?: string + + /** + * The database ID used as the content source. + */ + databaseId: string + + /** + * By disabling automatic imports, properties must be defined in `fields` property to be present in generated content. + * Useful when you have page properties containing sensitive data. + */ + automaticImport?: boolean + + /** + * By disabling content import, the page content will not be fetched. + * Useful when you only want to use page properties for this database. + */ + importContent?: boolean + + /** + * Sort and filter pages queried from the database. + * More information on the Notion API documentation https://developers.notion.com/reference/post-database-query-filter + */ + query?: Omit + + /** + * The fields configuration, usefull to remap keys and configure complex properties. + */ + properties?: Flattened extends false + ? Record | DatabasePropertyTypeDef[] + : DatabasePropertyTypeDef[] + + computedFields?: Record> +} + +export type DatabaseType = { + type: 'database' + def: Thunk> +} + +export type DatabaseTypes = DatabaseType[] | Record> + +export const defineDatabase = ( + def: Thunk>, +): DatabaseType => ({ + type: 'database', + def, +}) diff --git a/packages/@contentlayer/source-notion/src/schema/types/property.ts b/packages/@contentlayer/source-notion/src/schema/types/property.ts new file mode 100644 index 00000000..2e1cd7d5 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/types/property.ts @@ -0,0 +1,41 @@ +import type { DatabaseType } from './database' + +export type DatabasePropertyTypeDefBase = { + /** + * Map this property to a specific key. + * Defaults to the property name. + */ + key?: string + + /** + * Field description used to generate comments. + */ + description?: string + + /** + * When required, pages without this property defined will not be generated. + */ + required?: boolean +} & ({ id: string } | { name: string }) + +export type DatabasePropertyFieldTypeDef = DatabasePropertyTypeDefBase & { + /** + * Type of the property. + */ + type: 'relation' + + /** + * Database related to this relation. + * + * TODO : Will be used for Rollup properties. + */ + relation: DatabaseType + + /** + * If true, the property will be of type `string` instead of type `string[]` + * and only the first item will be taken. + */ + single?: boolean +} + +export type DatabasePropertyTypeDef = DatabasePropertyTypeDefBase | DatabasePropertyFieldTypeDef diff --git a/packages/@contentlayer/source-notion/src/schema/utils/findDatabaseFieldDef.ts b/packages/@contentlayer/source-notion/src/schema/utils/findDatabaseFieldDef.ts new file mode 100644 index 00000000..1c9a74d3 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/utils/findDatabaseFieldDef.ts @@ -0,0 +1,19 @@ +import type { DatabaseTypeDef } from '../types/database' +import type { DatabasePropertyTypeDef } from '../types/property' + +export type FindDatabaseFieldDefArgs = { + property: { id: string; name: string } + databaseTypeDef: DatabaseTypeDef +} + +export const findDatabaseFieldDef = ({ + databaseTypeDef, + property, +}: FindDatabaseFieldDefArgs): DatabasePropertyTypeDef | undefined => { + if (!databaseTypeDef.properties) return + + return databaseTypeDef.properties.find((fieldDef) => { + if ('name' in fieldDef) return fieldDef.name === property.name + if ('id' in fieldDef) return fieldDef.id === property.id + }) +} diff --git a/packages/@contentlayer/source-notion/src/schema/utils/flattenDatabaseTypeDef.ts b/packages/@contentlayer/source-notion/src/schema/utils/flattenDatabaseTypeDef.ts new file mode 100644 index 00000000..60948a4f --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/utils/flattenDatabaseTypeDef.ts @@ -0,0 +1,15 @@ +import type { DatabaseTypeDef } from '../types/database' + +export const flattendDatabaseTypeDef = (databaseTypeDef: DatabaseTypeDef): DatabaseTypeDef => { + return { + ...databaseTypeDef, + properties: databaseTypeDef.properties + ? Array.isArray(databaseTypeDef.properties) + ? databaseTypeDef.properties + : Object.entries(databaseTypeDef.properties).map(([key, field]) => ({ + key, + ...field, + })) + : [], + } +} diff --git a/packages/@contentlayer/source-notion/src/schema/utils/getPropertyData.ts b/packages/@contentlayer/source-notion/src/schema/utils/getPropertyData.ts new file mode 100644 index 00000000..3f94cde8 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/utils/getPropertyData.ts @@ -0,0 +1,22 @@ +import type { + DatabaseProperty, + DatabasePropertyData, + DatabasePropertyTypes, + PageProperty, + PagePropertyData, + PagePropertyTypes, +} from '../../notion/types' + +export const getPagePropertyData = (property: PageProperty): PagePropertyData => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + return property[property.type] +} + +export const getDatabasePropertyData = ( + property: DatabaseProperty, +): DatabasePropertyData => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + return property[property.type] +} diff --git a/packages/@contentlayer/source-notion/src/schema/utils/normalizeKey.ts b/packages/@contentlayer/source-notion/src/schema/utils/normalizeKey.ts new file mode 100644 index 00000000..92260bf2 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/schema/utils/normalizeKey.ts @@ -0,0 +1,6 @@ +import slugify from 'slugify' + +export const normalizeKey = (key: string) => { + const slugified = slugify(key) + return slugified.toLowerCase().replace(/[-_][a-z0-9]/g, (group) => group.slice(-1).toUpperCase()) +} diff --git a/packages/@contentlayer/source-notion/src/services.ts b/packages/@contentlayer/source-notion/src/services.ts new file mode 100644 index 00000000..f94c51a2 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/services.ts @@ -0,0 +1,6 @@ +import { tag } from '@contentlayer/utils/effect' +import type { NotionRenderer as ONotionRenderer } from '@notion-render/client' +import type * as notion from '@notionhq/client' + +export const NotionClient = tag('NotionClient') +export const NotionRenderer = tag('NotionRenderer') diff --git a/packages/@contentlayer/source-notion/src/types.ts b/packages/@contentlayer/source-notion/src/types.ts new file mode 100644 index 00000000..8e2e3564 --- /dev/null +++ b/packages/@contentlayer/source-notion/src/types.ts @@ -0,0 +1,30 @@ +import type * as core from '@contentlayer/core' +import type { NotionRenderer } from '@notion-render/client' +import type * as notion from '@notionhq/client' + +import type { DatabaseTypes } from './schema/types/database.js' + +export type PluginOptions = { + client?: ConstructorParameters[0] | notion.Client + renderer?: ConstructorParameters[0] | NotionRenderer + databaseTypes: DatabaseTypes + dev?: { + polling: false | number + } +} + +export type FieldDef = core.FieldDef & { propertyKey?: string } + +export type LocalDocument = Record & { _raw: core.RawDocumentData; _id: string } + +export type DiscriminateUnion = T extends Record ? T : never + +export type DiscriminateUnionValue = T extends Record + ? V extends string + ? T extends Record + ? T[V] + : never + : never + : never + +export type DistributiveOmit = T extends any ? Omit : never diff --git a/packages/@contentlayer/source-notion/tsconfig.json b/packages/@contentlayer/source-notion/tsconfig.json new file mode 100644 index 00000000..f8f6e568 --- /dev/null +++ b/packages/@contentlayer/source-notion/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "module": "ES2020", + "rootDir": "./src", + "outDir": "./dist", + "tsBuildInfoFile": "./dist/.tsbuildinfo.json" + }, + "include": ["./src"], + "references": [{ "path": "../utils" }, { "path": "../core" }] +} diff --git a/tsconfig.all.json b/tsconfig.all.json index a0f9184b..6bb1c85b 100644 --- a/tsconfig.all.json +++ b/tsconfig.all.json @@ -7,6 +7,7 @@ { "path": "./packages/@contentlayer/core" }, { "path": "./packages/@contentlayer/source-contentful" }, { "path": "./packages/@contentlayer/source-files" }, + { "path": "./packages/@contentlayer/source-notion" }, { "path": "./packages/@contentlayer/experimental-source-files-stackbit" }, { "path": "./packages/@contentlayer/utils" }, { "path": "./packages/contentlayer" }, diff --git a/yarn.lock b/yarn.lock index 1b15531e..cc5ce575 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1083,6 +1083,25 @@ __metadata: languageName: node linkType: hard +"@notion-render/client@npm:0.0.1-alpha.3": + version: 0.0.1-alpha.3 + resolution: "@notion-render/client@npm:0.0.1-alpha.3" + dependencies: + sanitize-html: ^2.10.0 + checksum: 1994f3e582adac945b3ff94f1c20c79cd1db1aaa86dae0064bfe2d935fa6f57866b9bc80c6f051bcb7428c00cec4108df274510a5d25c9b6418fa689a2fc21b7 + languageName: node + linkType: hard + +"@notionhq/client@npm:^2.2.3": + version: 2.2.3 + resolution: "@notionhq/client@npm:2.2.3" + dependencies: + "@types/node-fetch": ^2.5.10 + node-fetch: ^2.6.1 + checksum: 356e889012d36bf2045099d28c65fa2e41defa849f5bf34c6b82fbc7a028ce66a17aa2bd63b2451cfecc8378a49c2d179357896d211d3d92648c425157f628fe + languageName: node + linkType: hard + "@npmcli/fs@npm:^2.1.0": version: 2.1.2 resolution: "@npmcli/fs@npm:2.1.2" @@ -1802,7 +1821,17 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*, @types/node@npm:>=12.12.47, @types/node@npm:>=13.7.0": +"@types/node-fetch@npm:^2.5.10": + version: 2.6.2 + resolution: "@types/node-fetch@npm:2.6.2" + dependencies: + "@types/node": "*" + form-data: ^3.0.0 + checksum: 6f73b1470000d303d25a6fb92875ea837a216656cb7474f66cdd67bb014aa81a5a11e7ac9c21fe19bee9ecb2ef87c1962bceeaec31386119d1ac86e4c30ad7a6 + languageName: node + linkType: hard + +"@types/node@npm:*, @types/node@npm:>=12.12.47, @types/node@npm:>=13.7.0, @types/node@npm:^18.13.0": version: 18.13.0 resolution: "@types/node@npm:18.13.0" checksum: 4ea10f8802848b01672bce938f678b6774ca2cee0c9774f12275ab064ae07818419c3e2e41d6257ce7ba846d1ea26c63214aa1dfa4166fa3746291752b8c6416 @@ -1889,6 +1918,13 @@ __metadata: languageName: node linkType: hard +"@types/retry@npm:0.12.1": + version: 0.12.1 + resolution: "@types/retry@npm:0.12.1" + checksum: 5f46b2556053655f78262bb33040dc58417c900457cc63ff37d6c35349814471453ef511af0cec76a540c601296cd2b22f64bab1ab649c0dacc0223765ba876c + languageName: node + linkType: hard + "@types/scheduler@npm:*": version: 0.16.2 resolution: "@types/scheduler@npm:0.16.2" @@ -2612,6 +2648,13 @@ __metadata: languageName: node linkType: hard +"asynckit@npm:^0.4.0": + version: 0.4.0 + resolution: "asynckit@npm:0.4.0" + checksum: 7b78c451df768adba04e2d02e63e2d0bf3b07adcd6e42b4cf665cb7ce899bedd344c69a1dcbce355b5f972d597b25aaa1c1742b52cffd9caccb22f348114f6be + languageName: node + linkType: hard + "at-least-node@npm:^1.0.0": version: 1.0.0 resolution: "at-least-node@npm:1.0.0" @@ -3236,6 +3279,15 @@ __metadata: languageName: node linkType: hard +"combined-stream@npm:^1.0.8": + version: 1.0.8 + resolution: "combined-stream@npm:1.0.8" + dependencies: + delayed-stream: ~1.0.0 + checksum: 49fa4aeb4916567e33ea81d088f6584749fc90c7abec76fd516bf1c5aa5c79f3584b5ba3de6b86d26ddd64bae5329c4c7479343250cfe71c75bb366eae53bb7c + languageName: node + linkType: hard + "comma-separated-tokens@npm:^2.0.0": version: 2.0.3 resolution: "comma-separated-tokens@npm:2.0.3" @@ -3398,6 +3450,21 @@ __metadata: languageName: unknown linkType: soft +"contentlayer-source-notion@workspace:packages/@contentlayer/source-notion": + version: 0.0.0-use.local + resolution: "contentlayer-source-notion@workspace:packages/@contentlayer/source-notion" + dependencies: + "@contentlayer/core": "workspace:*" + "@contentlayer/utils": "workspace:*" + "@notion-render/client": 0.0.1-alpha.3 + "@notionhq/client": ^2.2.3 + "@types/node": ^18.13.0 + p-queue: ^7.3.4 + p-retry: ^5.1.2 + slugify: ^1.6.5 + languageName: unknown + linkType: soft + "contentlayer-stackbit-yaml-generator@workspace:packages/contentlayer-stackbit-yaml-generator": version: 0.0.0-use.local resolution: "contentlayer-stackbit-yaml-generator@workspace:packages/contentlayer-stackbit-yaml-generator" @@ -3620,6 +3687,13 @@ __metadata: languageName: node linkType: hard +"deepmerge@npm:^4.2.2": + version: 4.3.1 + resolution: "deepmerge@npm:4.3.1" + checksum: 2024c6a980a1b7128084170c4cf56b0fd58a63f2da1660dcfe977415f27b17dbe5888668b59d0b063753f3220719d5e400b7f113609489c90160bb9a5518d052 + languageName: node + linkType: hard + "defaults@npm:^1.0.3": version: 1.0.4 resolution: "defaults@npm:1.0.4" @@ -3646,6 +3720,13 @@ __metadata: languageName: node linkType: hard +"delayed-stream@npm:~1.0.0": + version: 1.0.0 + resolution: "delayed-stream@npm:1.0.0" + checksum: 46fe6e83e2cb1d85ba50bd52803c68be9bd953282fa7096f51fc29edd5d67ff84ff753c51966061e5ba7cb5e47ef6d36a91924eddb7f3f3483b1c560f77a0020 + languageName: node + linkType: hard + "delegates@npm:^1.0.0": version: 1.0.0 resolution: "delegates@npm:1.0.0" @@ -3767,10 +3848,48 @@ __metadata: languageName: node linkType: hard +"dom-serializer@npm:^2.0.0": + version: 2.0.0 + resolution: "dom-serializer@npm:2.0.0" + dependencies: + domelementtype: ^2.3.0 + domhandler: ^5.0.2 + entities: ^4.2.0 + checksum: cd1810544fd8cdfbd51fa2c0c1128ec3a13ba92f14e61b7650b5de421b88205fd2e3f0cc6ace82f13334114addb90ed1c2f23074a51770a8e9c1273acbc7f3e6 + languageName: node + linkType: hard + +"domelementtype@npm:^2.3.0": + version: 2.3.0 + resolution: "domelementtype@npm:2.3.0" + checksum: ee837a318ff702622f383409d1f5b25dd1024b692ef64d3096ff702e26339f8e345820f29a68bcdcea8cfee3531776b3382651232fbeae95612d6f0a75efb4f6 + languageName: node + linkType: hard + +"domhandler@npm:^5.0.2, domhandler@npm:^5.0.3": + version: 5.0.3 + resolution: "domhandler@npm:5.0.3" + dependencies: + domelementtype: ^2.3.0 + checksum: 0f58f4a6af63e6f3a4320aa446d28b5790a009018707bce2859dcb1d21144c7876482b5188395a188dfa974238c019e0a1e610d2fc269a12b2c192ea2b0b131c + languageName: node + linkType: hard + +"domutils@npm:^3.0.1": + version: 3.1.0 + resolution: "domutils@npm:3.1.0" + dependencies: + dom-serializer: ^2.0.0 + domelementtype: ^2.3.0 + domhandler: ^5.0.3 + checksum: e5757456ddd173caa411cfc02c2bb64133c65546d2c4081381a3bafc8a57411a41eed70494551aa58030be9e58574fcc489828bebd673863d39924fb4878f416 + languageName: node + linkType: hard + "electron-to-chromium@npm:^1.4.284": - version: 1.4.295 - resolution: "electron-to-chromium@npm:1.4.295" - checksum: 66fff1341d3c94c2ccd1f2a39cffdb92118304f4b949d3194427e7022d6a6bd8c482b5c4afd9dce210117ba20cac01c1a1466089f5a862fe9c563113b86ff829 + version: 1.4.352 + resolution: "electron-to-chromium@npm:1.4.352" + checksum: 225a7494040015372a27f83e2e06e8d13895a8bb7901c44cce029162fb678b51c0864268228ef5d4705d17b78940a1329798295e2070c569fdf97fe1ad85f457 languageName: node linkType: hard @@ -3818,6 +3937,13 @@ __metadata: languageName: node linkType: hard +"entities@npm:^4.2.0, entities@npm:^4.4.0": + version: 4.5.0 + resolution: "entities@npm:4.5.0" + checksum: 853f8ebd5b425d350bffa97dd6958143179a5938352ccae092c62d1267c4e392a039be1bae7d51b6e4ffad25f51f9617531fedf5237f15df302ccfb452cbf2d7 + languageName: node + linkType: hard + "env-paths@npm:^2.2.0": version: 2.2.1 resolution: "env-paths@npm:2.2.1" @@ -4312,6 +4438,13 @@ __metadata: languageName: node linkType: hard +"eventemitter3@npm:^4.0.7": + version: 4.0.7 + resolution: "eventemitter3@npm:4.0.7" + checksum: 1875311c42fcfe9c707b2712c32664a245629b42bb0a5a84439762dd0fd637fc54d078155ea83c2af9e0323c9ac13687e03cfba79b03af9f40c89b4960099374 + languageName: node + linkType: hard + "events@npm:^3.2.0": version: 3.3.0 resolution: "events@npm:3.3.0" @@ -4551,6 +4684,17 @@ __metadata: languageName: node linkType: hard +"form-data@npm:^3.0.0": + version: 3.0.1 + resolution: "form-data@npm:3.0.1" + dependencies: + asynckit: ^0.4.0 + combined-stream: ^1.0.8 + mime-types: ^2.1.12 + checksum: b019e8d35c8afc14a2bd8a7a92fa4f525a4726b6d5a9740e8d2623c30e308fbb58dc8469f90415a856698933c8479b01646a9dff33c87cc4e76d72aedbbf860d + languageName: node + linkType: hard + "format@npm:^0.2.0": version: 0.2.2 resolution: "format@npm:0.2.2" @@ -5108,6 +5252,18 @@ __metadata: languageName: node linkType: hard +"htmlparser2@npm:^8.0.0": + version: 8.0.2 + resolution: "htmlparser2@npm:8.0.2" + dependencies: + domelementtype: ^2.3.0 + domhandler: ^5.0.3 + domutils: ^3.0.1 + entities: ^4.4.0 + checksum: 29167a0f9282f181da8a6d0311b76820c8a59bc9e3c87009e21968264c2987d2723d6fde5a964d4b7b6cba663fca96ffb373c06d8223a85f52a6089ced942700 + languageName: node + linkType: hard + "http-cache-semantics@npm:^4.1.0": version: 4.1.1 resolution: "http-cache-semantics@npm:4.1.1" @@ -5825,9 +5981,9 @@ __metadata: linkType: hard "lilconfig@npm:^2.0.5, lilconfig@npm:^2.0.6": - version: 2.0.6 - resolution: "lilconfig@npm:2.0.6" - checksum: 40a3cd72f103b1be5975f2ac1850810b61d4053e20ab09be8d3aeddfe042187e1ba70b4651a7e70f95efa1642e7dc8b2ae395b317b7d7753b241b43cef7c0f7d + version: 2.1.0 + resolution: "lilconfig@npm:2.1.0" + checksum: 8549bb352b8192375fed4a74694cd61ad293904eee33f9d4866c2192865c44c4eb35d10782966242634e0cbc1e91fe62b1247f148dc5514918e3a966da7ea117 languageName: node linkType: hard @@ -6720,7 +6876,7 @@ __metadata: languageName: node linkType: hard -"mime-types@npm:^2.1.27": +"mime-types@npm:^2.1.12, mime-types@npm:^2.1.27": version: 2.1.35 resolution: "mime-types@npm:2.1.35" dependencies: @@ -6986,24 +7142,6 @@ __metadata: languageName: node linkType: hard -"next-contentlayer-example@workspace:examples/next-contentlayer-example": - version: 0.0.0-use.local - resolution: "next-contentlayer-example@workspace:examples/next-contentlayer-example" - dependencies: - "@types/react": 18.0.38 - autoprefixer: ^10.4.14 - contentlayer: latest - date-fns: 2.29.3 - next: 13.3.1 - next-contentlayer: latest - postcss: ^8.4.23 - react: 18.2.0 - react-dom: 18.2.0 - tailwindcss: ^3.2.7 - typescript: 5.0.4 - languageName: unknown - linkType: soft - "next-contentlayer@workspace:*, next-contentlayer@workspace:packages/next-contentlayer": version: 0.0.0-use.local resolution: "next-contentlayer@workspace:packages/next-contentlayer" @@ -7135,7 +7273,7 @@ __metadata: languageName: node linkType: hard -"node-fetch@npm:^2.6.7": +"node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7": version: 2.6.9 resolution: "node-fetch@npm:2.6.9" dependencies: @@ -7203,6 +7341,15 @@ __metadata: languageName: unknown linkType: soft +"node-script-notion-example@workspace:examples/node-script-notion": + version: 0.0.0-use.local + resolution: "node-script-notion-example@workspace:examples/node-script-notion" + dependencies: + "@notionhq/client": ^2.2.3 + contentlayer: latest + languageName: unknown + linkType: soft + "node-script-remote-content-example@workspace:examples/node-script-remote-content": version: 0.0.0-use.local resolution: "node-script-remote-content-example@workspace:examples/node-script-remote-content" @@ -7460,6 +7607,26 @@ __metadata: languageName: node linkType: hard +"p-queue@npm:^7.3.4": + version: 7.3.4 + resolution: "p-queue@npm:7.3.4" + dependencies: + eventemitter3: ^4.0.7 + p-timeout: ^5.0.2 + checksum: a21b8a4dd75f64a4988e4468cc344d1b45132506ddd2c771932d3de446d108ee68713b629e0d3f0809c227bc10eafc613edde6ae741d9f60db89b6031e40921c + languageName: node + linkType: hard + +"p-retry@npm:^5.1.2": + version: 5.1.2 + resolution: "p-retry@npm:5.1.2" + dependencies: + "@types/retry": 0.12.1 + retry: ^0.13.1 + checksum: f063c08b1adc3cf7c01de01eb2dbda841970229f9f229c5167ebf4e2080d8a38b1f4e6eccefac74bca97cfaf4436d0a0eeb0b551175b26bc8b3116195f61bba8 + languageName: node + linkType: hard + "p-throttle@npm:^4.1.1": version: 4.1.1 resolution: "p-throttle@npm:4.1.1" @@ -7467,6 +7634,13 @@ __metadata: languageName: node linkType: hard +"p-timeout@npm:^5.0.2": + version: 5.1.0 + resolution: "p-timeout@npm:5.1.0" + checksum: f5cd4e17301ff1ff1d8dbf2817df0ad88c6bba99349fc24d8d181827176ad4f8aca649190b8a5b1a428dfd6ddc091af4606835d3e0cb0656e04045da5c9e270c + languageName: node + linkType: hard + "p-try@npm:^2.0.0": version: 2.2.0 resolution: "p-try@npm:2.2.0" @@ -7511,6 +7685,13 @@ __metadata: languageName: node linkType: hard +"parse-srcset@npm:^1.0.2": + version: 1.0.2 + resolution: "parse-srcset@npm:1.0.2" + checksum: 3a0380380c6082021fcce982f0b89fb8a493ce9dfd7d308e5e6d855201e80db8b90438649b31fdd82a3d6089a8ca17dccddaa2b730a718389af4c037b8539ebf + languageName: node + linkType: hard + "parse5@npm:^6.0.0": version: 6.0.1 resolution: "parse5@npm:6.0.1" @@ -7735,7 +7916,7 @@ __metadata: languageName: node linkType: hard -"postcss@npm:^8.4.23": +"postcss@npm:^8.3.11, postcss@npm:^8.4.23": version: 8.4.23 resolution: "postcss@npm:8.4.23" dependencies: @@ -8256,6 +8437,13 @@ __metadata: languageName: node linkType: hard +"retry@npm:^0.13.1": + version: 0.13.1 + resolution: "retry@npm:0.13.1" + checksum: 47c4d5be674f7c13eee4cfe927345023972197dbbdfba5d3af7e461d13b44de1bfd663bfc80d2f601f8ef3fc8164c16dd99655a221921954a65d044a2fc1233b + languageName: node + linkType: hard + "reusify@npm:^1.0.4": version: 1.0.4 resolution: "reusify@npm:1.0.4" @@ -8382,6 +8570,20 @@ __metadata: languageName: node linkType: hard +"sanitize-html@npm:^2.10.0": + version: 2.10.0 + resolution: "sanitize-html@npm:2.10.0" + dependencies: + deepmerge: ^4.2.2 + escape-string-regexp: ^4.0.0 + htmlparser2: ^8.0.0 + is-plain-object: ^5.0.0 + parse-srcset: ^1.0.2 + postcss: ^8.3.11 + checksum: 0cb2bb330ed966a4d667b1890322dd868a67f527f87c04d7e3be1688fcfda20f7452a9a7744870751f51e255742e7264a287d9bcfcd64d4cd74a3c99f99c73d2 + languageName: node + linkType: hard + "scheduler@npm:^0.23.0": version: 0.23.0 resolution: "scheduler@npm:0.23.0" @@ -8611,6 +8813,13 @@ __metadata: languageName: node linkType: hard +"slugify@npm:^1.6.5": + version: 1.6.5 + resolution: "slugify@npm:1.6.5" + checksum: a955a1b600201030f4c1daa9bb74a17d4402a0693fc40978bbd17e44e64fd72dad3bac4037422aa8aed55b5170edd57f3f4cd8f59ba331f5cf0f10f1a7795609 + languageName: node + linkType: hard + "smart-buffer@npm:^4.2.0": version: 4.2.0 resolution: "smart-buffer@npm:4.2.0"