From e3197367b7c76bc18cce9118a235f579ea41e698 Mon Sep 17 00:00:00 2001 From: hulutter <> Date: Wed, 24 Jul 2024 10:30:04 +0200 Subject: [PATCH 1/5] feat(privateNpmRegistry): enable package serving via internal registry proxy and package cache --- .../lowcoder/src/comps/utils/remote.ts | 10 +- .../lowcoder/src/constants/npmPlugins.ts | 7 +- server/node-service/src/controllers/npm.ts | 195 ++++++++++++++++++ server/node-service/src/routes/apiRouter.ts | 4 + .../node-service/src/services/npmRegistry.ts | 111 ++++++++++ 5 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 server/node-service/src/controllers/npm.ts create mode 100644 server/node-service/src/services/npmRegistry.ts diff --git a/client/packages/lowcoder/src/comps/utils/remote.ts b/client/packages/lowcoder/src/comps/utils/remote.ts index 49a47033c..9daf27ef9 100644 --- a/client/packages/lowcoder/src/comps/utils/remote.ts +++ b/client/packages/lowcoder/src/comps/utils/remote.ts @@ -12,7 +12,7 @@ export function getRemoteCompType( } export function parseCompType(compType: string) { - const [type, source, packageNameAndVersion, compName] = compType.split("#"); + let [type, source, packageNameAndVersion, compName] = compType.split("#"); const isRemote = type === "remote"; if (!isRemote) { @@ -22,7 +22,13 @@ export function parseCompType(compType: string) { }; } - const [packageName, packageVersion] = packageNameAndVersion.split("@"); + const packageRegex = /^(?(@[a-z0-9-~][a-z0-9-._~]*\/)?[a-z0-9-~][a-z0-9-._~]*)@(?([0-9]+.[0-9]+.[0-9]+)(-[\w\d-]+)?)$/; + const matches = packageNameAndVersion.match(packageRegex); + if (!matches?.groups) { + throw new Error(`Invalid package name and version: ${packageNameAndVersion}`); + } + + const {packageName, packageVersion} = matches.groups; return { compName, isRemote, diff --git a/client/packages/lowcoder/src/constants/npmPlugins.ts b/client/packages/lowcoder/src/constants/npmPlugins.ts index e7154a613..0e252e938 100644 --- a/client/packages/lowcoder/src/constants/npmPlugins.ts +++ b/client/packages/lowcoder/src/constants/npmPlugins.ts @@ -1,2 +1,5 @@ -export const NPM_REGISTRY_URL = "https://registry.npmjs.com"; -export const NPM_PLUGIN_ASSETS_BASE_URL = "https://unpkg.com"; +import { sdkConfig } from "./sdkConfig"; + +const baseUrl = sdkConfig.baseURL || LOWCODER_NODE_SERVICE_URL || ""; +export const NPM_REGISTRY_URL = `${baseUrl}/node-service/api/npm/registry`; +export const NPM_PLUGIN_ASSETS_BASE_URL = `${baseUrl}/node-service/api/npm/package`; diff --git a/server/node-service/src/controllers/npm.ts b/server/node-service/src/controllers/npm.ts new file mode 100644 index 000000000..249838bcf --- /dev/null +++ b/server/node-service/src/controllers/npm.ts @@ -0,0 +1,195 @@ +import "../common/logger"; +import fs from "fs/promises"; +import { spawn } from "child_process"; +import { Request as ServerRequest, Response as ServerResponse } from "express"; +import { NpmRegistryService, NpmRegistryConfigEntry } from "../services/npmRegistry"; + + +type PackagesVersionInfo = { + "dist-tags": { + latest: string + }, + versions: { + [version: string]: { + dist: { + tarball: string + } + } + } +}; + + +/** + * Initializes npm registry cache directory + */ +const CACHE_DIR = process.env.NPM_CACHE_DIR || "/tmp/npm-package-cache"; +try { + fs.mkdir(CACHE_DIR, { recursive: true }); +} catch (error) { + console.error("Error creating cache directory", error); +} + + +/** + * Fetches package info from npm registry + */ +const fetchRegistryBasePath = "/npm/registry"; +export async function fetchRegistry(request: ServerRequest, response: ServerResponse) { + try { + const path = request.path.replace(fetchRegistryBasePath, ""); + logger.info(`Fetch registry info for path: ${path}`); + + const pathPackageInfo = parsePackageInfoFromPath(path); + if (!pathPackageInfo) { + return response.status(400).send(`Invalid package path: ${path}`); + } + const {organization, name} = pathPackageInfo; + const packageName = organization ? `@${organization}/${name}` : name; + + const registryResponse = await fetchFromRegistry(packageName, path); + response.json(await registryResponse.json()); + } catch (error) { + logger.error("Error fetching registry", error); + response.status(500).send("Internal server error"); + } +} + + +/** + * Fetches package files from npm registry if not yet cached + */ +const fetchPackageFileBasePath = "/npm/package"; +export async function fetchPackageFile(request: ServerRequest, response: ServerResponse) { + try { + const path = request.path.replace(fetchPackageFileBasePath, ""); + logger.info(`Fetch file for path: ${path}`); + + const pathPackageInfo = parsePackageInfoFromPath(path); + if (!pathPackageInfo) { + return response.status(400).send(`Invalid package path: ${path}`); + } + + logger.info(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`); + const {organization, name, version, file} = pathPackageInfo; + const packageName = organization ? `@${organization}/${name}` : name; + let packageVersion = version; + + let packageInfo: PackagesVersionInfo | null = null; + if (version === "latest") { + const packageInfo: PackagesVersionInfo = await fetchPackageInfo(packageName); + packageVersion = packageInfo["dist-tags"].latest; + } + + const packageBaseDir = `${CACHE_DIR}/${packageName}/${packageVersion}/package`; + const packageExists = await fileExists(`${packageBaseDir}/package.json`) + if (!packageExists) { + if (!packageInfo) { + packageInfo = await fetchPackageInfo(packageName); + } + + if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) { + return response.status(404).send("Not found"); + } + + const tarball = packageInfo.versions[packageVersion].dist.tarball; + logger.info("Fetching tarball...", tarball); + await fetchAndUnpackTarball(tarball, packageName, packageVersion); + } + + // Fallback to index.mjs if index.js is not present + if (file === "index.js" && !await fileExists(`${packageBaseDir}/${file}`)) { + logger.info("Fallback to index.mjs"); + return response.sendFile(`${packageBaseDir}/index.mjs`); + } + + return response.sendFile(`${packageBaseDir}/${file}`); + } catch (error) { + logger.error("Error fetching package file", error); + response.status(500).send("Internal server error"); + } +}; + + +/** + * Helpers + */ + +function parsePackageInfoFromPath(path: string): {organization: string, name: string, version: string, file: string} | undefined { + logger.info(`Parse package info from path: ${path}`); + //@ts-ignore - regex groups + const packageInfoRegex = /^\/?(?(?:@(?[a-z0-9-~][a-z0-9-._~]*)\/)?(?[a-z0-9-~][a-z0-9-._~]*))(?:@(?[-a-z0-9><=_.^~]+))?\/(?[^\r\n]*)?$/; + const matches = path.match(packageInfoRegex); + logger.info(`Parse package matches: ${JSON.stringify(matches)}`); + if (!matches?.groups) { + return; + } + + let {organization, name, version, file} = matches.groups; + version = /^\d+\.\d+\.\d+(-[\w\d]+)?/.test(version) ? version : "latest"; + + return {organization, name, version, file}; +} + +function fetchFromRegistry(packageName: string, urlOrPath: string): Promise { + const config: NpmRegistryConfigEntry = NpmRegistryService.getInstance().getRegistryEntryForPackage(packageName); + const registryUrl = config?.registry.url; + + const headers: {[key: string]: string} = {}; + switch (config?.registry.auth.type) { + case "none": + break; + case "basic": + const basicUserPass = config?.registry.auth?.credentials; + headers["Authorization"] = `Basic ${basicUserPass}`; + break; + case "bearer": + const bearerToken = config?.registry.auth?.credentials; + headers["Authorization"] = `Bearer ${bearerToken}`; + break; + } + + let url = urlOrPath; + if (!urlOrPath.startsWith("http")) { + const separator = urlOrPath.startsWith("/") ? "" : "/"; + url = `${registryUrl}${separator}${urlOrPath}`; + } + + logger.debug(`Fetch from registry: ${url}`); + return fetch(url, {headers}); +} + +function fetchPackageInfo(packageName: string): Promise { + return fetchFromRegistry(packageName, packageName).then(res => res.json()); +} + +async function fetchAndUnpackTarball(url: string, packageName: string, packageVersion: string) { + const response: Response = await fetchFromRegistry(packageName, url); + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const path = `${CACHE_DIR}/${url.split("/").pop()}`; + await fs.writeFile(path, buffer); + await unpackTarball(path, packageName, packageVersion); + await fs.unlink(path); +} + +async function unpackTarball(path: string, packageName: string, packageVersion: string) { + const destinationPath = `${CACHE_DIR}/${packageName}/${packageVersion}`; + await fs.mkdir(destinationPath, { recursive: true }); + await new Promise ((resolve, reject) => { + const tar = spawn("tar", ["-xvf", path, "-C", destinationPath]); + tar.stdout.on("data", (data) => logger.info(data)); + tar.stderr.on("data", (data) => console.error(data)); + tar.on("close", (code) => { + code === 0 ? resolve() : reject(); + }); + }); +} + +async function fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch (error) { + return false; + } +} diff --git a/server/node-service/src/routes/apiRouter.ts b/server/node-service/src/routes/apiRouter.ts index ac02c8654..083a22966 100644 --- a/server/node-service/src/routes/apiRouter.ts +++ b/server/node-service/src/routes/apiRouter.ts @@ -1,6 +1,7 @@ import express from "express"; import * as pluginControllers from "../controllers/plugins"; import jsControllers from "../controllers/runJavascript"; +import * as npmControllers from "../controllers/npm"; const apiRouter = express.Router(); @@ -12,4 +13,7 @@ apiRouter.post("/runPluginQuery", pluginControllers.runPluginQuery); apiRouter.post("/getPluginDynamicConfig", pluginControllers.getDynamicDef); apiRouter.post("/validatePluginDataSourceConfig", pluginControllers.validatePluginDataSourceConfig); +apiRouter.get("/npm/registry/*", npmControllers.fetchRegistry); +apiRouter.get("/npm/package/*", npmControllers.fetchPackageFile); + export default apiRouter; diff --git a/server/node-service/src/services/npmRegistry.ts b/server/node-service/src/services/npmRegistry.ts new file mode 100644 index 000000000..79ba4fa18 --- /dev/null +++ b/server/node-service/src/services/npmRegistry.ts @@ -0,0 +1,111 @@ +type BasicAuthType = { + type: "basic", + credentials: string, +} + +type BearerAuthType = { + type: "bearer", + credentials: string, +}; + +type NoAuthType = { + type: "none" +}; + +type OrganizationScope = { + type: "organization", + pattern: string +}; + +type PackageScope = { + type: "package", + pattern: string +}; + +type GlobalScope = { + type: "global" +}; + +export type NpmRegistryConfigEntry = { + scope: OrganizationScope | PackageScope | GlobalScope, + registry: { + url: string, + auth: BasicAuthType | BearerAuthType | NoAuthType + } +}; + +export type NpmRegistryConfig = NpmRegistryConfigEntry[]; + +export class NpmRegistryService { + + public static DEFAULT_REGISTRY: NpmRegistryConfigEntry = { + scope: { type: "global" }, + registry: { + url: "https://registry.npmjs.org", + auth: { type: "none" } + } + }; + + private static instance: NpmRegistryService; + + private readonly registryConfig: NpmRegistryConfig = []; + + private constructor() { + const registryConfig = this.getRegistryConfig(); + if (registryConfig.length === 0 || !registryConfig.some(entry => entry.scope.type === "global")) { + registryConfig.push(NpmRegistryService.DEFAULT_REGISTRY); + } + this.registryConfig = registryConfig; + } + + public static getInstance(): NpmRegistryService { + if (!NpmRegistryService.instance) { + NpmRegistryService.instance = new NpmRegistryService(); + } + return NpmRegistryService.instance; + } + + private getRegistryConfig(): NpmRegistryConfig { + const registryConfig = process.env.NPM_REGISTRY_CONFIG; + if (!registryConfig) { + return []; + } + + try { + const config = JSON.parse(registryConfig); + return NpmRegistryService.sortRegistryConfig(config); + } catch (error) { + console.error("Error parsing registry config", error); + return []; + } + } + + private static sortRegistryConfig(registryConfig: NpmRegistryConfig): NpmRegistryConfig { + const globalRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "global"); + const orgRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "organization"); + const packageRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "package"); + // Order of precedence: package > organization > global + return [...packageRegistries, ...orgRegistries, ...globalRegistries]; + } + + public getRegistryEntryForPackage(packageName: string): NpmRegistryConfigEntry { + const config: NpmRegistryConfigEntry | undefined = this.registryConfig.find(entry => { + if (entry.scope.type === "organization") { + return packageName.startsWith(entry.scope.pattern); + } else if (entry.scope.type === "package") { + return packageName === entry.scope.pattern; + } else { + return true; + } + }); + + if (!config) { + logger.info(`No registry entry found for package: ${packageName}`); + return NpmRegistryService.DEFAULT_REGISTRY; + } else { + logger.info(`Found registry entry for package: ${packageName} -> ${config.registry.url}`); + } + + return config; + } +} From 89c0d380fa4cad1d9dee95456a43e36bf5d6f752 Mon Sep 17 00:00:00 2001 From: hulutter <> Date: Tue, 30 Jul 2024 06:30:18 +0200 Subject: [PATCH 2/5] feat(privateNpmRegistry): add endpoints to fetch with config send via body and queue tar retrieval --- server/node-service/src/controllers/npm.ts | 191 +++++++++++++----- server/node-service/src/routes/apiRouter.ts | 3 + .../node-service/src/services/npmRegistry.ts | 22 ++ 3 files changed, 171 insertions(+), 45 deletions(-) diff --git a/server/node-service/src/controllers/npm.ts b/server/node-service/src/controllers/npm.ts index 249838bcf..1aec12894 100644 --- a/server/node-service/src/controllers/npm.ts +++ b/server/node-service/src/controllers/npm.ts @@ -1,8 +1,8 @@ import "../common/logger"; import fs from "fs/promises"; import { spawn } from "child_process"; -import { Request as ServerRequest, Response as ServerResponse } from "express"; -import { NpmRegistryService, NpmRegistryConfigEntry } from "../services/npmRegistry"; +import { response, Request as ServerRequest, Response as ServerResponse } from "express"; +import { NpmRegistryService, NpmRegistryConfigEntry, NpmRegistryConfig } from "../services/npmRegistry"; type PackagesVersionInfo = { @@ -19,6 +19,38 @@ type PackagesVersionInfo = { }; +class PackageProcessingQueue { + public static readonly promiseRegistry: {[packageId: string]: Promise} = {}; + public static readonly resolveRegistry: {[packageId: string]:() => void} = {}; + + public static add(packageId: string) { + PackageProcessingQueue.promiseRegistry[packageId] = new Promise((resolve) => { + PackageProcessingQueue.resolveRegistry[packageId] = resolve; + }); + } + + public static has(packageId: string) { + return !!PackageProcessingQueue.promiseRegistry[packageId]; + } + + public static wait(packageId: string) { + if (!PackageProcessingQueue.has(packageId)) { + return Promise.resolve(); + } + return PackageProcessingQueue.promiseRegistry[packageId]; + } + + public static resolve(packageId: string) { + if (!PackageProcessingQueue.has(packageId)) { + return; + } + PackageProcessingQueue.resolveRegistry[packageId](); + delete PackageProcessingQueue.promiseRegistry[packageId]; + delete PackageProcessingQueue.resolveRegistry[packageId]; + } +} + + /** * Initializes npm registry cache directory */ @@ -26,14 +58,37 @@ const CACHE_DIR = process.env.NPM_CACHE_DIR || "/tmp/npm-package-cache"; try { fs.mkdir(CACHE_DIR, { recursive: true }); } catch (error) { - console.error("Error creating cache directory", error); + logger.error("Error creating cache directory", error); } /** * Fetches package info from npm registry */ + const fetchRegistryBasePath = "/npm/registry"; + +export async function fetchRegistryWithConfig(request: ServerRequest, response: ServerResponse) { + try { + const path = request.path.replace(fetchRegistryBasePath, ""); + logger.info(`Fetch registry info for path: ${path}`); + + const pathPackageInfo = parsePackageInfoFromPath(path); + if (!pathPackageInfo) { + return response.status(400).send(`Invalid package path: ${path}`); + } + + const registryConfig: NpmRegistryConfig = request.body; + const config = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig); + + const registryResponse = await fetchFromRegistry(path, config); + response.json(await registryResponse.json()); + } catch (error) { + logger.error("Error fetching registry", error); + response.status(500).send("Internal server error"); + } +} + export async function fetchRegistry(request: ServerRequest, response: ServerResponse) { try { const path = request.path.replace(fetchRegistryBasePath, ""); @@ -43,10 +98,9 @@ export async function fetchRegistry(request: ServerRequest, response: ServerResp if (!pathPackageInfo) { return response.status(400).send(`Invalid package path: ${path}`); } - const {organization, name} = pathPackageInfo; - const packageName = organization ? `@${organization}/${name}` : name; - const registryResponse = await fetchFromRegistry(packageName, path); + const config = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId); + const registryResponse = await fetchFromRegistry(path, config); response.json(await registryResponse.json()); } catch (error) { logger.error("Error fetching registry", error); @@ -58,53 +112,100 @@ export async function fetchRegistry(request: ServerRequest, response: ServerResp /** * Fetches package files from npm registry if not yet cached */ + const fetchPackageFileBasePath = "/npm/package"; + +export async function fetchPackageFileWithConfig(request: ServerRequest, response: ServerResponse) { + const path = request.path.replace(fetchPackageFileBasePath, ""); + logger.info(`Fetch file for path with config: ${path}`); + + const pathPackageInfo = parsePackageInfoFromPath(path); + if (!pathPackageInfo) { + return response.status(400).send(`Invalid package path: ${path}`); + } + + const registryConfig: NpmRegistryConfig = request.body; + const config = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig); + + fetchPackageFileInner(request, response, config); +} + export async function fetchPackageFile(request: ServerRequest, response: ServerResponse) { + const path = request.path.replace(fetchPackageFileBasePath, ""); + logger.info(`Fetch file for path: ${path}`); + + const pathPackageInfo = parsePackageInfoFromPath(path); + if (!pathPackageInfo) { + return response.status(400).send(`Invalid package path: ${path}`); + } + + const config = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId); + fetchPackageFileInner(request, response, config); +} + +async function fetchPackageFileInner(request: ServerRequest, response: ServerResponse, config: NpmRegistryConfigEntry) { try { - const path = request.path.replace(fetchPackageFileBasePath, ""); - logger.info(`Fetch file for path: ${path}`); - + const path = request.path.replace(fetchPackageFileBasePath, ""); const pathPackageInfo = parsePackageInfoFromPath(path); if (!pathPackageInfo) { return response.status(400).send(`Invalid package path: ${path}`); } - logger.info(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`); - const {organization, name, version, file} = pathPackageInfo; - const packageName = organization ? `@${organization}/${name}` : name; + logger.debug(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`); + const {packageId, version, file} = pathPackageInfo; let packageVersion = version; let packageInfo: PackagesVersionInfo | null = null; if (version === "latest") { - const packageInfo: PackagesVersionInfo = await fetchPackageInfo(packageName); + const packageInfo: PackagesVersionInfo|null = await fetchPackageInfo(packageId, config); + if (packageInfo === null) { + return response.status(404).send("Not found"); + } packageVersion = packageInfo["dist-tags"].latest; } + + // Wait for package to be processed if it's already being processed + if (PackageProcessingQueue.has(packageId)) { + logger.info("Waiting for package to be processed", packageId); + await PackageProcessingQueue.wait(packageId); + } - const packageBaseDir = `${CACHE_DIR}/${packageName}/${packageVersion}/package`; + const packageBaseDir = `${CACHE_DIR}/${packageId}/${packageVersion}/package`; const packageExists = await fileExists(`${packageBaseDir}/package.json`) if (!packageExists) { - if (!packageInfo) { - packageInfo = await fetchPackageInfo(packageName); - } - - if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) { - return response.status(404).send("Not found"); + try { + logger.info(`Package does not exist, fetch from registy: ${packageId}@${packageVersion}`); + PackageProcessingQueue.add(packageId); + if (!packageInfo) { + packageInfo = await fetchPackageInfo(packageId, config); + } + + if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) { + return response.status(404).send("Not found"); + } + + const tarball = packageInfo.versions[packageVersion].dist.tarball; + logger.info(`Fetching tarball: ${tarball}`); + await fetchAndUnpackTarball(tarball, packageId, packageVersion, config); + } catch (error) { + logger.error("Error fetching package tarball", error); + return response.status(500).send("Internal server error"); + } finally { + PackageProcessingQueue.resolve(packageId); } - - const tarball = packageInfo.versions[packageVersion].dist.tarball; - logger.info("Fetching tarball...", tarball); - await fetchAndUnpackTarball(tarball, packageName, packageVersion); + } else { + logger.info(`Package already exists, serve from cache: ${packageBaseDir}/${file}`) } // Fallback to index.mjs if index.js is not present if (file === "index.js" && !await fileExists(`${packageBaseDir}/${file}`)) { - logger.info("Fallback to index.mjs"); + logger.debug("Fallback to index.mjs"); return response.sendFile(`${packageBaseDir}/index.mjs`); } return response.sendFile(`${packageBaseDir}/${file}`); } catch (error) { - logger.error("Error fetching package file", error); + logger.error(`Error fetching package file: ${error} ${(error as {stack: string})?.stack?.toString()}`); response.status(500).send("Internal server error"); } }; @@ -114,26 +215,22 @@ export async function fetchPackageFile(request: ServerRequest, response: ServerR * Helpers */ -function parsePackageInfoFromPath(path: string): {organization: string, name: string, version: string, file: string} | undefined { - logger.info(`Parse package info from path: ${path}`); +function parsePackageInfoFromPath(path: string): {packageId: string, organization: string, name: string, version: string, file: string} | undefined { //@ts-ignore - regex groups - const packageInfoRegex = /^\/?(?(?:@(?[a-z0-9-~][a-z0-9-._~]*)\/)?(?[a-z0-9-~][a-z0-9-._~]*))(?:@(?[-a-z0-9><=_.^~]+))?\/(?[^\r\n]*)?$/; + const packageInfoRegex = /^\/?(?(?:@(?[a-z0-9-~][a-z0-9-._~]*)\/)?(?[a-z0-9-~][a-z0-9-._~]*))(?:@(?[-a-z0-9><=_.^~]+))?\/(?[^\r\n]*)?$/; const matches = path.match(packageInfoRegex); - logger.info(`Parse package matches: ${JSON.stringify(matches)}`); if (!matches?.groups) { return; } - let {organization, name, version, file} = matches.groups; + let {packageId, organization, name, version, file} = matches.groups; version = /^\d+\.\d+\.\d+(-[\w\d]+)?/.test(version) ? version : "latest"; - return {organization, name, version, file}; + return {packageId, organization, name, version, file}; } -function fetchFromRegistry(packageName: string, urlOrPath: string): Promise { - const config: NpmRegistryConfigEntry = NpmRegistryService.getInstance().getRegistryEntryForPackage(packageName); +function fetchFromRegistry(urlOrPath: string, config: NpmRegistryConfigEntry): Promise { const registryUrl = config?.registry.url; - const headers: {[key: string]: string} = {}; switch (config?.registry.auth.type) { case "none": @@ -154,31 +251,35 @@ function fetchFromRegistry(packageName: string, urlOrPath: string): Promise { - return fetchFromRegistry(packageName, packageName).then(res => res.json()); +function fetchPackageInfo(packageName: string, config: NpmRegistryConfigEntry): Promise { + return fetchFromRegistry(`/${packageName}`, config).then(res => { + if (!res.ok) { + logger.error(`Failed to fetch package info for package ${packageName}: ${res.statusText}`); + return null; + } + return res.json(); + }); } -async function fetchAndUnpackTarball(url: string, packageName: string, packageVersion: string) { - const response: Response = await fetchFromRegistry(packageName, url); +async function fetchAndUnpackTarball(url: string, packageId: string, packageVersion: string, config: NpmRegistryConfigEntry) { + const response: Response = await fetchFromRegistry(url, config); const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const path = `${CACHE_DIR}/${url.split("/").pop()}`; await fs.writeFile(path, buffer); - await unpackTarball(path, packageName, packageVersion); + await unpackTarball(path, packageId, packageVersion); await fs.unlink(path); } -async function unpackTarball(path: string, packageName: string, packageVersion: string) { - const destinationPath = `${CACHE_DIR}/${packageName}/${packageVersion}`; +async function unpackTarball(path: string, packageId: string, packageVersion: string) { + const destinationPath = `${CACHE_DIR}/${packageId}/${packageVersion}`; await fs.mkdir(destinationPath, { recursive: true }); await new Promise ((resolve, reject) => { const tar = spawn("tar", ["-xvf", path, "-C", destinationPath]); - tar.stdout.on("data", (data) => logger.info(data)); - tar.stderr.on("data", (data) => console.error(data)); tar.on("close", (code) => { code === 0 ? resolve() : reject(); }); diff --git a/server/node-service/src/routes/apiRouter.ts b/server/node-service/src/routes/apiRouter.ts index 083a22966..32fadcaee 100644 --- a/server/node-service/src/routes/apiRouter.ts +++ b/server/node-service/src/routes/apiRouter.ts @@ -16,4 +16,7 @@ apiRouter.post("/validatePluginDataSourceConfig", pluginControllers.validatePlug apiRouter.get("/npm/registry/*", npmControllers.fetchRegistry); apiRouter.get("/npm/package/*", npmControllers.fetchPackageFile); +apiRouter.post("/npm/registry/*", npmControllers.fetchRegistryWithConfig); +apiRouter.post("/npm/package/*", npmControllers.fetchPackageFileWithConfig); + export default apiRouter; diff --git a/server/node-service/src/services/npmRegistry.ts b/server/node-service/src/services/npmRegistry.ts index 79ba4fa18..e5cc6205f 100644 --- a/server/node-service/src/services/npmRegistry.ts +++ b/server/node-service/src/services/npmRegistry.ts @@ -108,4 +108,26 @@ export class NpmRegistryService { return config; } + + public static getRegistryEntryForPackageWithConfig(packageName: string, registryConfig: NpmRegistryConfig): NpmRegistryConfigEntry { + registryConfig = NpmRegistryService.sortRegistryConfig(registryConfig); + const config: NpmRegistryConfigEntry | undefined = registryConfig.find(entry => { + if (entry.scope.type === "organization") { + return packageName.startsWith(entry.scope.pattern); + } else if (entry.scope.type === "package") { + return packageName === entry.scope.pattern; + } else { + return true; + } + }); + + if (!config) { + logger.info(`No registry entry found for package: ${packageName}`); + return NpmRegistryService.DEFAULT_REGISTRY; + } else { + logger.info(`Found registry entry for package: ${packageName} -> ${config.registry.url}`); + } + + return config; + } } From 3ad71b4ecd8d760e0e285c52abf3b4a79301ddfb Mon Sep 17 00:00:00 2001 From: hulutter <> Date: Tue, 30 Jul 2024 07:48:18 +0200 Subject: [PATCH 3/5] feat(privateNpmRegistry): add workspaceId support for cache directory structure --- server/node-service/src/controllers/npm.ts | 97 +++++++++++++++------- 1 file changed, 65 insertions(+), 32 deletions(-) diff --git a/server/node-service/src/controllers/npm.ts b/server/node-service/src/controllers/npm.ts index 1aec12894..535c95a28 100644 --- a/server/node-service/src/controllers/npm.ts +++ b/server/node-service/src/controllers/npm.ts @@ -1,10 +1,20 @@ import "../common/logger"; import fs from "fs/promises"; import { spawn } from "child_process"; -import { response, Request as ServerRequest, Response as ServerResponse } from "express"; +import { Request as ServerRequest, Response as ServerResponse } from "express"; import { NpmRegistryService, NpmRegistryConfigEntry, NpmRegistryConfig } from "../services/npmRegistry"; +type RequestConfig = { + workspaceId: string; + npmRegistryConfig: NpmRegistryConfig; +} + +type InnerRequestConfig = { + workspaceId?: string; + registry: NpmRegistryConfigEntry; +} + type PackagesVersionInfo = { "dist-tags": { latest: string @@ -23,24 +33,24 @@ class PackageProcessingQueue { public static readonly promiseRegistry: {[packageId: string]: Promise} = {}; public static readonly resolveRegistry: {[packageId: string]:() => void} = {}; - public static add(packageId: string) { + public static add(packageId: string): void { PackageProcessingQueue.promiseRegistry[packageId] = new Promise((resolve) => { PackageProcessingQueue.resolveRegistry[packageId] = resolve; }); } - public static has(packageId: string) { + public static has(packageId: string): boolean { return !!PackageProcessingQueue.promiseRegistry[packageId]; } - public static wait(packageId: string) { + public static wait(packageId: string): Promise { if (!PackageProcessingQueue.has(packageId)) { return Promise.resolve(); } return PackageProcessingQueue.promiseRegistry[packageId]; } - public static resolve(packageId: string) { + public static resolve(packageId: string): void { if (!PackageProcessingQueue.has(packageId)) { return; } @@ -78,10 +88,18 @@ export async function fetchRegistryWithConfig(request: ServerRequest, response: return response.status(400).send(`Invalid package path: ${path}`); } - const registryConfig: NpmRegistryConfig = request.body; - const config = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig); + if (!request.body.workspaceId && !request.body.npmRegistryConfig) { + return response.status(400).send("Missing workspaceId and/or npmRegistryConfig"); + } + + const {npmRegistryConfig}: RequestConfig = request.body; + + const registry = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, npmRegistryConfig); - const registryResponse = await fetchFromRegistry(path, config); + const registryResponse = await fetchFromRegistry(path, registry); + if (!registryResponse.ok) { + return response.status(registryResponse.status).send(await registryResponse.text()); + } response.json(await registryResponse.json()); } catch (error) { logger.error("Error fetching registry", error); @@ -99,8 +117,11 @@ export async function fetchRegistry(request: ServerRequest, response: ServerResp return response.status(400).send(`Invalid package path: ${path}`); } - const config = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId); - const registryResponse = await fetchFromRegistry(path, config); + const registry = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId); + const registryResponse = await fetchFromRegistry(path, registry); + if (!registryResponse.ok) { + return response.status(registryResponse.status).send(await registryResponse.text()); + } response.json(await registryResponse.json()); } catch (error) { logger.error("Error fetching registry", error); @@ -124,10 +145,15 @@ export async function fetchPackageFileWithConfig(request: ServerRequest, respons return response.status(400).send(`Invalid package path: ${path}`); } - const registryConfig: NpmRegistryConfig = request.body; - const config = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig); + if (!request.body.workspaceId && !request.body.npmRegistryConfig) { + return response.status(400).send("Missing workspaceId and/or npmRegistryConfig"); + } - fetchPackageFileInner(request, response, config); + const {workspaceId, npmRegistryConfig}: RequestConfig = request.body; + const registryConfig: NpmRegistryConfig = npmRegistryConfig; + const registry = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig); + + fetchPackageFileInner(request, response, {workspaceId, registry}); } export async function fetchPackageFile(request: ServerRequest, response: ServerResponse) { @@ -139,12 +165,14 @@ export async function fetchPackageFile(request: ServerRequest, response: ServerR return response.status(400).send(`Invalid package path: ${path}`); } - const config = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId); - fetchPackageFileInner(request, response, config); + const registry = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId); + fetchPackageFileInner(request, response, {registry}); } -async function fetchPackageFileInner(request: ServerRequest, response: ServerResponse, config: NpmRegistryConfigEntry) { +async function fetchPackageFileInner(request: ServerRequest, response: ServerResponse, config: InnerRequestConfig) { try { + const {workspaceId, registry} = config + logger.info(`Fetch file for workspaceId: ${workspaceId}`); const path = request.path.replace(fetchPackageFileBasePath, ""); const pathPackageInfo = parsePackageInfoFromPath(path); if (!pathPackageInfo) { @@ -157,7 +185,7 @@ async function fetchPackageFileInner(request: ServerRequest, response: ServerRes let packageInfo: PackagesVersionInfo | null = null; if (version === "latest") { - const packageInfo: PackagesVersionInfo|null = await fetchPackageInfo(packageId, config); + const packageInfo: PackagesVersionInfo|null = await fetchPackageInfo(packageId, registry); if (packageInfo === null) { return response.status(404).send("Not found"); } @@ -170,14 +198,15 @@ async function fetchPackageFileInner(request: ServerRequest, response: ServerRes await PackageProcessingQueue.wait(packageId); } - const packageBaseDir = `${CACHE_DIR}/${packageId}/${packageVersion}/package`; + const baseDir = `${CACHE_DIR}/${workspaceId ?? "default"}`; + const packageBaseDir = `${baseDir}/${packageId}/${packageVersion}/package`; const packageExists = await fileExists(`${packageBaseDir}/package.json`) if (!packageExists) { try { logger.info(`Package does not exist, fetch from registy: ${packageId}@${packageVersion}`); PackageProcessingQueue.add(packageId); if (!packageInfo) { - packageInfo = await fetchPackageInfo(packageId, config); + packageInfo = await fetchPackageInfo(packageId, registry); } if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) { @@ -186,9 +215,9 @@ async function fetchPackageFileInner(request: ServerRequest, response: ServerRes const tarball = packageInfo.versions[packageVersion].dist.tarball; logger.info(`Fetching tarball: ${tarball}`); - await fetchAndUnpackTarball(tarball, packageId, packageVersion, config); + await fetchAndUnpackTarball(tarball, packageId, packageVersion, registry, baseDir); } catch (error) { - logger.error("Error fetching package tarball", error); + logger.error(`Error fetching package: ${error} ${(error as {stack: string}).stack}`); return response.status(500).send("Internal server error"); } finally { PackageProcessingQueue.resolve(packageId); @@ -224,6 +253,7 @@ function parsePackageInfoFromPath(path: string): {packageId: string, organizatio } let {packageId, organization, name, version, file} = matches.groups; + // also test for alpha and beta versions like 0.0.1-beta1 version = /^\d+\.\d+\.\d+(-[\w\d]+)?/.test(version) ? version : "latest"; return {packageId, organization, name, version, file}; @@ -265,25 +295,28 @@ function fetchPackageInfo(packageName: string, config: NpmRegistryConfigEntry): }); } -async function fetchAndUnpackTarball(url: string, packageId: string, packageVersion: string, config: NpmRegistryConfigEntry) { +async function fetchAndUnpackTarball(url: string, packageId: string, packageVersion: string, config: NpmRegistryConfigEntry, baseDir: string) { + if (!await fileExists(baseDir)) { + await fs.mkdir(baseDir, { recursive: true }); + } + + // Fetch tarball const response: Response = await fetchFromRegistry(url, config); const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); - const path = `${CACHE_DIR}/${url.split("/").pop()}`; + const path = `${baseDir}/${url.split("/").pop()}`; await fs.writeFile(path, buffer); - await unpackTarball(path, packageId, packageVersion); - await fs.unlink(path); -} - -async function unpackTarball(path: string, packageId: string, packageVersion: string) { - const destinationPath = `${CACHE_DIR}/${packageId}/${packageVersion}`; + + // Unpack tarball + const destinationPath = `${baseDir}/${packageId}/${packageVersion}`; await fs.mkdir(destinationPath, { recursive: true }); await new Promise ((resolve, reject) => { const tar = spawn("tar", ["-xvf", path, "-C", destinationPath]); - tar.on("close", (code) => { - code === 0 ? resolve() : reject(); - }); + tar.on("close", (code) => code === 0 ? resolve() : reject()); }); + + // Cleanup + await fs.unlink(path); } async function fileExists(filePath: string): Promise { From aec795b675cc3b95a68706ede87d1093dcf175ad Mon Sep 17 00:00:00 2001 From: hulutter <> Date: Wed, 31 Jul 2024 11:53:04 +0200 Subject: [PATCH 4/5] feat(privateNpmRegistry): add workspace config --- .../src/components/NpmRegistryConfig.tsx | 299 ++++++++++++++++++ .../packages/lowcoder/src/i18n/locales/en.ts | 19 +- .../setting/advanced/AdvancedSetting.tsx | 18 +- .../uiReducers/commonSettingsReducer.ts | 15 + 4 files changed, 348 insertions(+), 3 deletions(-) create mode 100644 client/packages/lowcoder/src/components/NpmRegistryConfig.tsx diff --git a/client/packages/lowcoder/src/components/NpmRegistryConfig.tsx b/client/packages/lowcoder/src/components/NpmRegistryConfig.tsx new file mode 100644 index 000000000..6dadfed36 --- /dev/null +++ b/client/packages/lowcoder/src/components/NpmRegistryConfig.tsx @@ -0,0 +1,299 @@ +import { useEffect, useState } from "react"; +import { HelpText } from "./HelpText"; +import { FormInputItem, FormSelectItem, TacoSwitch } from "lowcoder-design"; +import { Form } from "antd"; +import { trans } from "@lowcoder-ee/i18n"; +import { FormStyled } from "@lowcoder-ee/pages/setting/idSource/styledComponents"; +import { SaveButton } from "@lowcoder-ee/pages/setting/styled"; +import { NpmRegistryConfigEntry } from "@lowcoder-ee/redux/reducers/uiReducers/commonSettingsReducer"; + +type NpmRegistryConfigEntryInput = { + url: string; + scope: "global" | "organization" | "package"; + pattern: string; + authType: "none" | "basic" | "bearer"; + credentials: string; +}; + +const initialRegistryConfig: NpmRegistryConfigEntryInput = { + scope: "global", + pattern: "", + url: "", + authType: "none", + credentials: "", +}; + +interface NpmRegistryConfigProps { + initialData?: NpmRegistryConfigEntry; + onSave: (registryConfig: NpmRegistryConfigEntry|null) => void; +} + +export function NpmRegistryConfig(props: NpmRegistryConfigProps) { + const [initialConfigSet, setItialConfigSet] = useState(false); + const [enableRegistry, setEnableRegistry] = useState(!!props.initialData); + const [registryConfig, setRegistryConfig] = useState(initialRegistryConfig); + + useEffect(() => { + if (props.initialData && !initialConfigSet) { + let initConfig: NpmRegistryConfigEntryInput = {...initialRegistryConfig}; + if (props.initialData) { + const {scope} = props.initialData; + const {type: scopeTye, pattern} = scope; + const {url, auth} = props.initialData.registry; + const {type: authType, credentials} = props.initialData.registry.auth; + initConfig.scope = scopeTye; + initConfig.pattern = pattern || ""; + initConfig.url = url; + initConfig.authType = authType; + initConfig.credentials = credentials || ""; + } + + form.setFieldsValue(initConfig); + setRegistryConfig(initConfig); + setEnableRegistry(true); + setItialConfigSet(true); + } + }, [props.initialData, initialConfigSet]); + + useEffect(() => { + if (!enableRegistry) { + form.resetFields(); + setRegistryConfig(initialRegistryConfig); + } + }, [enableRegistry]); + + const [form] = Form.useForm(); + + const handleRegistryConfigChange = async (key: string, value: string) => { + let keyConfg = { [key]: value }; + form.validateFields([key]); + + // Reset the pattern field if the scope is global + if (key === "scope") { + if (value !== "global") { + registryConfig.scope !== "global" && form.validateFields(["pattern"]); + } else { + form.resetFields(["pattern"]); + keyConfg = { + ...keyConfg, + pattern: "" + }; + } + } + + // Reset the credentials field if the auth type is none + if (key === "authType") { + if (value !== "none") { + registryConfig.authType !== "none" && form.validateFields(["credentials"]); + } else { + form.resetFields(["credentials"]); + keyConfg = { + ...keyConfg, + credentials: "" + }; + } + } + + // Update the registry config + setRegistryConfig((prevConfig) => ({ + ...prevConfig, + ...keyConfg, + })); + }; + + const scopeOptions = [ + { + value: "global", + label: "Global", + }, + { + value: "organization", + label: "Organization", + }, + { + value: "package", + label: "Package", + }, + ]; + + const authOptions = [ + { + value: "none", + label: "None", + }, + { + value: "basic", + label: "Basic", + }, + { + value: "bearer", + label: "Token", + }, + ]; + + const onFinsish = () => { + const registryConfigEntry: NpmRegistryConfigEntry = { + scope: { + type: registryConfig.scope, + pattern: registryConfig.pattern, + }, + registry: { + url: registryConfig.url, + auth: { + type: registryConfig.authType, + credentials: registryConfig.credentials, + }, + }, + }; + props.onSave(registryConfigEntry); + } + + return ( + { + for (const key in changedValues) { + handleRegistryConfigChange(key, changedValues[key]); + } + }} + onFinish={onFinsish} + > +
+ +
+ + + { + if (!enableRegistry) { + return props.onSave(null); + } + } + }> + {trans("advanced.saveBtn")} + + + + ); +} diff --git a/client/packages/lowcoder/src/i18n/locales/en.ts b/client/packages/lowcoder/src/i18n/locales/en.ts index ffccf2aca..4c52c1910 100644 --- a/client/packages/lowcoder/src/i18n/locales/en.ts +++ b/client/packages/lowcoder/src/i18n/locales/en.ts @@ -2600,8 +2600,9 @@ export const en = { "APIConsumption": "API Consumption", "APIConsumptionDescription": "Here you can see the API Consumption for All Apps in the Current Workspace.", "overallAPIConsumption": "Overall API Consumption in this Workspace till now", - "lastMonthAPIConsumption": "Last Month API Consumption, in this Workspace" - + "lastMonthAPIConsumption": "Last Month API Consumption, in this Workspace", + "npmRegistryTitle": "Custom NPM Registry", + "npmRegistryHelp": "Setup a custom NPM Registry to enable fetching of plugins from a private NPM registry.", }, @@ -2949,6 +2950,20 @@ export const en = { "createAppContent": "Welcome! Click 'App' and Start to Create Your First Application.", "createAppTitle": "Create App" }, + "npmRegistry": { + "npmRegistryEnable": "Enable custom NPM Registry", + "npmRegistryUrl": "NPM Registry Url", + "npmRegistryUrlRequired": "Please input the registry URL", + "npmRegistryUrlInvalid": "Please input a valid URL", + "npmRegistryScope": "Package Scope", + "npmRegistryPattern": "Pattern", + "npmRegistryPatternInvalid": "Please input a valid pattern (starting with @ for oragnizations).", + "npmRegistryAuth": "Authentication", + "npmRegistryAuthType": "Authentication Type", + "npmRegistryAuthCredentials": "Authentication Credentials", + "npmRegistryAuthCredentialsRequired": "Please input the registry credentials", + "npmRegistryAuthCredentialsHelp": "For basic auth provide the base64 encoded username and password in the format 'base64(username:password)', for token auth provide the token.", + }, // nineteenth part diff --git a/client/packages/lowcoder/src/pages/setting/advanced/AdvancedSetting.tsx b/client/packages/lowcoder/src/pages/setting/advanced/AdvancedSetting.tsx index 2a503fae6..22ab6ba65 100644 --- a/client/packages/lowcoder/src/pages/setting/advanced/AdvancedSetting.tsx +++ b/client/packages/lowcoder/src/pages/setting/advanced/AdvancedSetting.tsx @@ -2,7 +2,7 @@ import { EmptyContent } from "components/EmptyContent"; import { HelpText } from "components/HelpText"; import { GreyTextColor } from "constants/style"; import { CustomModal, CustomSelect, TacoButton } from "lowcoder-design"; -import React, { lazy, useEffect, useState } from "react"; +import { lazy, useEffect, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; import { fetchCommonSettings, setCommonSettings } from "redux/reduxActions/commonSettingsActions"; import { getCommonSettings } from "redux/selectors/commonSettingSelectors"; @@ -25,6 +25,8 @@ import { getGlobalSettings } from "comps/utils/globalSettings"; import { fetchJSLibrary } from "util/jsLibraryUtils"; import { evalFunc } from "lowcoder-core"; import { messageInstance } from "lowcoder-design/src/components/GlobalInstances"; +import { NpmRegistryConfig } from "@lowcoder-ee/components/NpmRegistryConfig"; +import { NpmRegistryConfigEntry } from "@lowcoder-ee/redux/reducers/uiReducers/commonSettingsReducer"; const CodeEditor = lazy( () => import("base/codeEditor/codeEditor") @@ -277,6 +279,20 @@ export function AdvancedSetting() { /> )} +
{trans("advanced.npmRegistryTitle")}
+ {trans("advanced.npmRegistryHelp")} +
+
+ { + // Wrap in array to enable future option for multiple registries + if (config === null) { + handleSave("npmRegistries")([]); + } else { + handleSave("npmRegistries")([config]); + } + }} /> +
+
{extraAdvanceSettings}
{trans("advanced.APIConsumption")}
{trans("advanced.APIConsumptionDescription")} diff --git a/client/packages/lowcoder/src/redux/reducers/uiReducers/commonSettingsReducer.ts b/client/packages/lowcoder/src/redux/reducers/uiReducers/commonSettingsReducer.ts index e950dc2b3..e423ceee1 100644 --- a/client/packages/lowcoder/src/redux/reducers/uiReducers/commonSettingsReducer.ts +++ b/client/packages/lowcoder/src/redux/reducers/uiReducers/commonSettingsReducer.ts @@ -3,9 +3,24 @@ import { ReduxAction, ReduxActionTypes } from "constants/reduxActionConstants"; import { CommonSettingResponseData, ThemeType } from "api/commonSettingApi"; import { GenericApiResponse } from "api/apiResponses"; +export interface NpmRegistryConfigEntry { + scope: { + type: "organization" | "package" | "global"; + pattern?: string; + }; + registry: { + url: string; + auth: { + type: "none" | "basic" | "bearer"; + credentials?: string; + }; + }; +} + export interface CommonSettingsState { settings: { npmPlugins?: string[] | null; + npmRegistries?: NpmRegistryConfigEntry[] | null themeList?: ThemeType[] | null; defaultTheme?: string | null; defaultHomePage?: string | null; From 9857ef2b38aa0b9b7f8363218a238fba8ef1eec6 Mon Sep 17 00:00:00 2001 From: hulutter <> Date: Wed, 31 Jul 2024 11:56:43 +0200 Subject: [PATCH 5/5] feat(privateNpmRegistry): cleanup and comments --- server/node-service/src/routes/apiRouter.ts | 8 +++++--- server/node-service/src/services/npmRegistry.ts | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/node-service/src/routes/apiRouter.ts b/server/node-service/src/routes/apiRouter.ts index 32fadcaee..bdf68a00f 100644 --- a/server/node-service/src/routes/apiRouter.ts +++ b/server/node-service/src/routes/apiRouter.ts @@ -13,10 +13,12 @@ apiRouter.post("/runPluginQuery", pluginControllers.runPluginQuery); apiRouter.post("/getPluginDynamicConfig", pluginControllers.getDynamicDef); apiRouter.post("/validatePluginDataSourceConfig", pluginControllers.validatePluginDataSourceConfig); -apiRouter.get("/npm/registry/*", npmControllers.fetchRegistry); -apiRouter.get("/npm/package/*", npmControllers.fetchPackageFile); - +// routes for npm registry and package fetching with config called by the api-service apiRouter.post("/npm/registry/*", npmControllers.fetchRegistryWithConfig); apiRouter.post("/npm/package/*", npmControllers.fetchPackageFileWithConfig); +// temporary routes for testing npm registry and package routes by directly calling the node-service +apiRouter.get("/npm/registry/*", npmControllers.fetchRegistry); +apiRouter.get("/npm/package/*", npmControllers.fetchPackageFile); + export default apiRouter; diff --git a/server/node-service/src/services/npmRegistry.ts b/server/node-service/src/services/npmRegistry.ts index e5cc6205f..4c69314a2 100644 --- a/server/node-service/src/services/npmRegistry.ts +++ b/server/node-service/src/services/npmRegistry.ts @@ -75,7 +75,7 @@ export class NpmRegistryService { const config = JSON.parse(registryConfig); return NpmRegistryService.sortRegistryConfig(config); } catch (error) { - console.error("Error parsing registry config", error); + logger.error(`Error parsing NPM registry config: ${error}`); return []; } }