Updating Node-Service and Activate DuckDB

FalkWolsky · FalkWolsky · commit 86ca61e4ebb4 · 2024-11-16T00:19:34.000+01:00
diff --git a/server/node-service/src/plugins/duckdb/dataSourceConfig.ts b/server/node-service/src/plugins/duckdb/dataSourceConfig.ts
@@ -0,0 +1,27 @@
+// @server/node-service/src/plugins/duckdb/dataSourceConfig.ts
+import { ConfigToType } from "lowcoder-sdk/dataSource";
+
+const dataSourceConfig = {
+    type: "dataSource",
+    params: [
+        {
+            key: "databaseFile",
+            type: "textInput",
+            label: "Database File",
+            rules: [{ required: true, message: "Please provide a database file path 'db.duckdb' or ':memory:' for an in-memory database" }],
+            tooltip: "Please provide a database file path 'db.duckdb' or ':memory:' for an in-memory database",
+            defaultValue: ":memory:",
+        },
+        {
+            key: "options",
+            type: "textInput",
+            label: "Database Options",
+            tooltip: "Additional options to pass to the DuckDB constructor (in JSON format)",
+            defaultValue: `{"access_mode": "READ_WRITE","max_memory": "512MB","threads": "4"}`,
+        },
+    ],
+} as const;
+
+export default dataSourceConfig;
+
+export type DataSourceDataType = ConfigToType<typeof dataSourceConfig>;
diff --git a/server/node-service/src/plugins/duckdb/index.ts b/server/node-service/src/plugins/duckdb/index.ts
@@ -0,0 +1,42 @@
+import { DataSourcePlugin } from "lowcoder-sdk/dataSource";
+import dataSourceConfig, { DataSourceDataType } from "./dataSourceConfig";
+import queryConfig, { ActionDataType } from "./queryConfig";
+import { Database } from "duckdb-async";
+import { ServiceError } from "../../common/error";
+
+// Helper function to handle BigInt serialization
+function serializeBigInts(row: any): any {
+    const newRow: { [key: string]: any } = {}; // Add index signature
+    for (const [key, value] of Object.entries(row)) {
+        newRow[key] = typeof value === 'bigint' ? value.toString() : value;
+    }
+    return newRow;
+}
+
+const duckdbPlugin: DataSourcePlugin<ActionDataType, DataSourceDataType> = {
+    id: "duckdb",
+    name: "DuckDB",
+    category: "database",
+    icon: "duckdb.svg",
+    dataSourceConfig,
+    queryConfig,
+    run: async function (actionData, dataSourceConfig): Promise<any> {
+        const { databaseFile, options } = dataSourceConfig;
+        const parsedOptions = JSON.parse(options);
+        const db = await Database.create(databaseFile, parsedOptions);
+
+        if (actionData.actionName === "Query") {
+            try {
+                const result = await db.all(actionData.queryString);
+                // Apply BigInt serialization to each row
+                return result.map(serializeBigInts);
+            } catch (error) {
+                throw new ServiceError((error as Error).message);
+            } finally {
+                await db.close();
+            }
+        }
+    },
+};
+
+export default duckdbPlugin;
diff --git a/server/node-service/src/plugins/duckdb/queryConfig.ts b/server/node-service/src/plugins/duckdb/queryConfig.ts
@@ -0,0 +1,24 @@
+// @server/node-service/src/plugins/duckdb/queryConfig.ts
+import { ConfigToType } from "lowcoder-sdk/dataSource";
+
+const queryConfig = {
+    type: "query",
+    label: "Action",
+    actions: [
+        {
+            actionName: "Query",
+            label: "Query",
+            params: [
+                {
+                    label: "Query String",
+                    key: "queryString",
+                    type: "sqlInput",
+                },
+            ],
+        },
+    ],
+} as const;
+
+export type ActionDataType = ConfigToType<typeof queryConfig>;
+
+export default queryConfig;
diff --git a/server/node-service/src/plugins/index.ts b/server/node-service/src/plugins/index.ts
@@ -8,7 +8,7 @@ import couchdbPlugin from "./couchdb";
 import wooCommercePlugin from "./woocommerce";
 import openAiPlugin from "./openAi";
 import athenaPlugin from "./athena";
-// import duckdbPlugin from "./duckdb";
+import duckdbPlugin from "./duckdb";
 import lambdaPlugin from "./lambda";
 import googleCloudStorage from "./googleCloudStorage";
 import stripePlugin from "./stripe";
@@ -46,7 +46,7 @@ let plugins: (DataSourcePlugin | DataSourcePluginFactory)[] = [
   // Databases
   dynamoDBPlugin,
   couchdbPlugin,
-  // duckdbPlugin,
+  duckdbPlugin,
   faunaPlugin,
   tursoPlugin,
   firebirdsqlPlugin,
diff --git a/server/node-service/src/plugins/openAi/index.ts b/server/node-service/src/plugins/openAi/index.ts
@@ -4,10 +4,10 @@ import path from "path";
 import { OpenAPIV3, OpenAPI } from "openapi-types";
 import { ConfigToType, DataSourcePlugin } from "lowcoder-sdk/dataSource";
 import { runOpenApi } from "../openApi";
-import { parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
+import { parseOpenApi, parseMultiOpenApi, ParseOpenApiOptions } from "../openApi/parse";
 
-const spec_1_2_0 = readYaml(path.join(__dirname, "./openAI_v1.2.0.yaml"));
-const spec_2_3_0 = readYaml(path.join(__dirname, "./openAI_v2.3.0.yaml"));
+import spec_1_2_0 from "./openAI_v1.2.0.json";
+import spec_2_3_0 from "./openAI_v2.3.0.json";
 
 const specs = {
   "v1.0": spec_1_2_0,
@@ -46,8 +46,14 @@ const dataSourceConfig = {
 } as const;
 
 const parseOptions: ParseOpenApiOptions = {
-  actionLabel: (method: string, path: string, operation: OpenAPI.Operation) => {
+  /* actionLabel: (method: string, path: string, operation: OpenAPI.Operation) => {
     return _.upperFirst(operation.operationId || "");
+  }, */
+  actionLabel: (method: string, path: string, operation: OpenAPI.Operation) => {
+    return _.upperFirst(operation.operationId) || operation.summary || "";
+  },
+  actionDescription(method, path, operation) {
+    return operation.description || "";
   },
 };
 
diff --git a/server/node-service/src/plugins/openAi/openAI_v1.2.0.json b/server/node-service/src/plugins/openAi/openAI_v1.2.0.json
diff --git a/server/node-service/src/plugins/openAi/openAI_v2.3.0.json b/server/node-service/src/plugins/openAi/openAI_v2.3.0.json
