feat(privateNpmRegistry): enable package serving via internal registr…

…y proxy and package cache
lowcoder-org · FalkWolsky · Aug 21, 2024 · Jul 24, 2024 · Jul 30, 2024 · Jul 30, 2024
commit e3197367b7c76bc18cce9118a235f579ea41e698
diff --git a/client/packages/lowcoder/src/comps/utils/remote.ts b/client/packages/lowcoder/src/comps/utils/remote.ts
@@ -12,7 +12,7 @@ export function getRemoteCompType(
 }
 
 export function parseCompType(compType: string) {
-  const [type, source, packageNameAndVersion, compName] = compType.split("#");
+  let [type, source, packageNameAndVersion, compName] = compType.split("#");
   const isRemote = type === "remote";
 
   if (!isRemote) {
@@ -22,7 +22,13 @@ export function parseCompType(compType: string) {
     };
   }
 
-  const [packageName, packageVersion] = packageNameAndVersion.split("@");
+  const packageRegex = /^(?<packageName>(@[a-z0-9-~][a-z0-9-._~]*\/)?[a-z0-9-~][a-z0-9-._~]*)@(?<packageVersion>([0-9]+.[0-9]+.[0-9]+)(-[\w\d-]+)?)$/;
+  const matches = packageNameAndVersion.match(packageRegex);
+  if (!matches?.groups) {
+    throw new Error(`Invalid package name and version: ${packageNameAndVersion}`);
+  }
+
+  const {packageName, packageVersion} = matches.groups;
   return {
     compName,
     isRemote,

diff --git a/client/packages/lowcoder/src/constants/npmPlugins.ts b/client/packages/lowcoder/src/constants/npmPlugins.ts
@@ -1,2 +1,5 @@
-export const NPM_REGISTRY_URL = "https://registry.npmjs.com";
-export const NPM_PLUGIN_ASSETS_BASE_URL = "https://unpkg.com";
+import { sdkConfig } from "./sdkConfig";
+
+const baseUrl = sdkConfig.baseURL || LOWCODER_NODE_SERVICE_URL || "";
+export const NPM_REGISTRY_URL = `${baseUrl}/node-service/api/npm/registry`;
+export const NPM_PLUGIN_ASSETS_BASE_URL = `${baseUrl}/node-service/api/npm/package`;
diff --git a/server/node-service/src/controllers/npm.ts b/server/node-service/src/controllers/npm.ts
@@ -0,0 +1,195 @@
+import "../common/logger";
+import fs from "fs/promises";
+import { spawn } from "child_process";
+import { Request as ServerRequest, Response as ServerResponse } from "express";
+import { NpmRegistryService, NpmRegistryConfigEntry } from "../services/npmRegistry";
+
+
+type PackagesVersionInfo = {
+    "dist-tags": {
+        latest: string
+    },
+    versions: {
+        [version: string]: {
+            dist: {
+                tarball: string
+            }
+        }
+    }
+};
+
+
+/**
+ * Initializes npm registry cache directory
+ */
+const CACHE_DIR = process.env.NPM_CACHE_DIR || "/tmp/npm-package-cache";
+try {
+    fs.mkdir(CACHE_DIR, { recursive: true });
+} catch (error) {
+    console.error("Error creating cache directory", error);
+}
+
+
+/**
+ * Fetches package info from npm registry
+ */
+const fetchRegistryBasePath = "/npm/registry";
+export async function fetchRegistry(request: ServerRequest, response: ServerResponse) {
+    try {
+        const path = request.path.replace(fetchRegistryBasePath, "");
+        logger.info(`Fetch registry info for path: ${path}`);
+
+        const pathPackageInfo = parsePackageInfoFromPath(path);
+        if (!pathPackageInfo) {
+            return response.status(400).send(`Invalid package path: ${path}`);
+        }
+        const {organization, name} = pathPackageInfo;
+        const packageName = organization ? `@${organization}/${name}` : name;
+
+        const registryResponse = await fetchFromRegistry(packageName, path);
+        response.json(await registryResponse.json());
+    } catch (error) {
+        logger.error("Error fetching registry", error);
+        response.status(500).send("Internal server error");
+    }
+}
+
+
+/**
+ * Fetches package files from npm registry if not yet cached
+ */
+const fetchPackageFileBasePath = "/npm/package";
+export async function fetchPackageFile(request: ServerRequest, response: ServerResponse) {
+    try {
+        const path = request.path.replace(fetchPackageFileBasePath, "");
+        logger.info(`Fetch file for path: ${path}`);
+
+        const pathPackageInfo = parsePackageInfoFromPath(path);
+        if (!pathPackageInfo) {
+            return response.status(400).send(`Invalid package path: ${path}`);
+        }
+
+        logger.info(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`);
+        const {organization, name, version, file} = pathPackageInfo;
+        const packageName = organization ? `@${organization}/${name}` : name;
+        let packageVersion = version;
+
+        let packageInfo: PackagesVersionInfo | null = null;
+        if (version === "latest") {
+            const packageInfo: PackagesVersionInfo = await fetchPackageInfo(packageName);
+            packageVersion = packageInfo["dist-tags"].latest;
+        }
+
+        const packageBaseDir = `${CACHE_DIR}/${packageName}/${packageVersion}/package`;
+        const packageExists = await fileExists(`${packageBaseDir}/package.json`)
+        if (!packageExists) {
+            if (!packageInfo) {
+                packageInfo = await fetchPackageInfo(packageName);
+            }
+
+            if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) {
+                return response.status(404).send("Not found");
+            }
+
+            const tarball = packageInfo.versions[packageVersion].dist.tarball;
+            logger.info("Fetching tarball...", tarball);
+            await fetchAndUnpackTarball(tarball, packageName, packageVersion);
+        }
+
+        // Fallback to index.mjs if index.js is not present
+        if (file === "index.js" && !await fileExists(`${packageBaseDir}/${file}`)) {
+            logger.info("Fallback to index.mjs");
+            return response.sendFile(`${packageBaseDir}/index.mjs`);
+        }
+
+        return response.sendFile(`${packageBaseDir}/${file}`);
+    } catch (error) {
+        logger.error("Error fetching package file", error);
+        response.status(500).send("Internal server error");
+    }
+};
+
+
+/**
+ * Helpers
+ */
+
+function parsePackageInfoFromPath(path: string): {organization: string, name: string, version: string, file: string} | undefined { 
+    logger.info(`Parse package info from path: ${path}`);
+    //@ts-ignore - regex groups
+    const packageInfoRegex = /^\/?(?<fullName>(?:@(?<organization>[a-z0-9-~][a-z0-9-._~]*)\/)?(?<name>[a-z0-9-~][a-z0-9-._~]*))(?:@(?<version>[-a-z0-9><=_.^~]+))?\/(?<file>[^\r\n]*)?$/;
+    const matches = path.match(packageInfoRegex);
+    logger.info(`Parse package matches: ${JSON.stringify(matches)}`);
+    if (!matches?.groups) {
+        return;
+    }
+
+    let {organization, name, version, file} = matches.groups;
+    version = /^\d+\.\d+\.\d+(-[\w\d]+)?/.test(version) ? version : "latest";
+
+    return {organization, name, version, file};
+}
+
+function fetchFromRegistry(packageName: string, urlOrPath: string): Promise<Response> {
+    const config: NpmRegistryConfigEntry = NpmRegistryService.getInstance().getRegistryEntryForPackage(packageName);
+    const registryUrl = config?.registry.url;
+
+    const headers: {[key: string]: string} = {}; 
+    switch (config?.registry.auth.type) {
+        case "none":
+            break;
+        case "basic":
+            const basicUserPass = config?.registry.auth?.credentials;
+            headers["Authorization"] = `Basic ${basicUserPass}`;
+            break;
+        case "bearer":
+            const bearerToken = config?.registry.auth?.credentials;
+            headers["Authorization"] = `Bearer ${bearerToken}`;
+            break;
+    }
+
+    let url = urlOrPath;
+    if (!urlOrPath.startsWith("http")) {
+        const separator = urlOrPath.startsWith("/") ? "" : "/";
+        url = `${registryUrl}${separator}${urlOrPath}`;
+    }
+
+    logger.debug(`Fetch from registry: ${url}`);
+    return fetch(url, {headers});
+}
+
+function fetchPackageInfo(packageName: string): Promise<PackagesVersionInfo> {
+    return fetchFromRegistry(packageName, packageName).then(res => res.json());
+}
+
+async function fetchAndUnpackTarball(url: string, packageName: string, packageVersion: string) {
+    const response: Response = await fetchFromRegistry(packageName, url);
+    const arrayBuffer = await response.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+    const path = `${CACHE_DIR}/${url.split("/").pop()}`;
+    await fs.writeFile(path, buffer);
+    await unpackTarball(path, packageName, packageVersion);
+    await fs.unlink(path);
+}
+
+async function unpackTarball(path: string, packageName: string, packageVersion: string) {
+    const destinationPath = `${CACHE_DIR}/${packageName}/${packageVersion}`;
+    await fs.mkdir(destinationPath, { recursive: true });
+    await new Promise<void> ((resolve, reject) => {
+        const tar = spawn("tar", ["-xvf", path, "-C", destinationPath]);
+        tar.stdout.on("data", (data) => logger.info(data));
+        tar.stderr.on("data", (data) => console.error(data));
+        tar.on("close", (code) => {
+            code === 0 ? resolve() : reject();
+        });
+    });
+}
+
+async function fileExists(filePath: string): Promise<boolean> {
+    try {
+        await fs.access(filePath);
+        return true;
+    } catch (error) {
+        return false;
+    }
+}
diff --git a/server/node-service/src/routes/apiRouter.ts b/server/node-service/src/routes/apiRouter.ts
@@ -1,6 +1,7 @@
 import express from "express";
 import * as pluginControllers from "../controllers/plugins";
 import jsControllers from "../controllers/runJavascript";
+import * as npmControllers from "../controllers/npm";
 
 const apiRouter = express.Router();
 
@@ -12,4 +13,7 @@ apiRouter.post("/runPluginQuery", pluginControllers.runPluginQuery);
 apiRouter.post("/getPluginDynamicConfig", pluginControllers.getDynamicDef);
 apiRouter.post("/validatePluginDataSourceConfig", pluginControllers.validatePluginDataSourceConfig);
 
+apiRouter.get("/npm/registry/*", npmControllers.fetchRegistry);
+apiRouter.get("/npm/package/*", npmControllers.fetchPackageFile);
+
 export default apiRouter;
diff --git a/server/node-service/src/services/npmRegistry.ts b/server/node-service/src/services/npmRegistry.ts
@@ -0,0 +1,111 @@
+type BasicAuthType = {
+    type: "basic",
+    credentials: string,
+}
+
+type BearerAuthType = {
+    type: "bearer",
+    credentials: string,
+};
+
+type NoAuthType = {
+    type: "none"
+};
+
+type OrganizationScope = {
+    type: "organization",
+    pattern: string
+};
+
+type PackageScope = {
+    type: "package",
+    pattern: string
+};
+
+type GlobalScope = {
+    type: "global"
+};
+
+export type NpmRegistryConfigEntry = {
+    scope: OrganizationScope | PackageScope | GlobalScope,
+    registry: {
+        url: string,
+        auth: BasicAuthType | BearerAuthType | NoAuthType
+    }
+};
+
+export type NpmRegistryConfig = NpmRegistryConfigEntry[];
+
+export class NpmRegistryService {
+
+    public static DEFAULT_REGISTRY: NpmRegistryConfigEntry = {
+        scope: { type: "global" },
+        registry: {
+            url: "https://registry.npmjs.org",
+            auth: { type: "none" }
+        }
+    };
+
+    private static instance: NpmRegistryService;
+
+    private readonly registryConfig: NpmRegistryConfig = [];
+
+    private constructor() {
+        const registryConfig = this.getRegistryConfig();
+        if (registryConfig.length === 0 || !registryConfig.some(entry => entry.scope.type === "global")) {
+            registryConfig.push(NpmRegistryService.DEFAULT_REGISTRY);
+        }
+        this.registryConfig = registryConfig;
+    }
+
+    public static getInstance(): NpmRegistryService {
+        if (!NpmRegistryService.instance) {
+            NpmRegistryService.instance = new NpmRegistryService();
+        }
+        return NpmRegistryService.instance;
+    }
+
+    private getRegistryConfig(): NpmRegistryConfig {
+        const registryConfig = process.env.NPM_REGISTRY_CONFIG;
+        if (!registryConfig) {
+            return [];
+        }
+
+        try {
+            const config = JSON.parse(registryConfig);
+            return NpmRegistryService.sortRegistryConfig(config);
+        } catch (error) {
+            console.error("Error parsing registry config", error);
+            return [];
+        }
+    }
+
+    private static sortRegistryConfig(registryConfig: NpmRegistryConfig): NpmRegistryConfig {
+        const globalRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "global");
+        const orgRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "organization");
+        const packageRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "package");
+        // Order of precedence: package > organization > global
+        return [...packageRegistries, ...orgRegistries, ...globalRegistries];
+    }
+
+    public getRegistryEntryForPackage(packageName: string): NpmRegistryConfigEntry {
+        const config: NpmRegistryConfigEntry | undefined = this.registryConfig.find(entry => {
+            if (entry.scope.type === "organization") {
+                return packageName.startsWith(entry.scope.pattern);
+            } else if (entry.scope.type === "package") {
+                return packageName === entry.scope.pattern;
+            } else {
+                return true;
+            }
+        });
+
+        if (!config) {
+            logger.info(`No registry entry found for package: ${packageName}`);
+            return NpmRegistryService.DEFAULT_REGISTRY;
+        } else {
+            logger.info(`Found registry entry for package: ${packageName} -> ${config.registry.url}`);
+        }
+
+        return config;
+    }
+}