Skip to content

Commit 89c0d38

Browse files
author
hulutter
committed
feat(privateNpmRegistry): add endpoints to fetch with config send via body and queue tar retrieval
1 parent e319736 commit 89c0d38

File tree

3 files changed

+171
-45
lines changed

3 files changed

+171
-45
lines changed

‎server/node-service/src/controllers/npm.ts

Lines changed: 146 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import "../common/logger";
22
import fs from "fs/promises";
33
import { spawn } from "child_process";
4-
import { Request as ServerRequest, Response as ServerResponse } from "express";
5-
import { NpmRegistryService, NpmRegistryConfigEntry } from "../services/npmRegistry";
4+
import { response, Request as ServerRequest, Response as ServerResponse } from "express";
5+
import { NpmRegistryService, NpmRegistryConfigEntry, NpmRegistryConfig } from "../services/npmRegistry";
66

77

88
type PackagesVersionInfo = {
@@ -19,21 +19,76 @@ type PackagesVersionInfo = {
1919
};
2020

2121

22+
class PackageProcessingQueue {
23+
public static readonly promiseRegistry: {[packageId: string]: Promise<void>} = {};
24+
public static readonly resolveRegistry: {[packageId: string]:() => void} = {};
25+
26+
public static add(packageId: string) {
27+
PackageProcessingQueue.promiseRegistry[packageId] = new Promise<void>((resolve) => {
28+
PackageProcessingQueue.resolveRegistry[packageId] = resolve;
29+
});
30+
}
31+
32+
public static has(packageId: string) {
33+
return !!PackageProcessingQueue.promiseRegistry[packageId];
34+
}
35+
36+
public static wait(packageId: string) {
37+
if (!PackageProcessingQueue.has(packageId)) {
38+
return Promise.resolve();
39+
}
40+
return PackageProcessingQueue.promiseRegistry[packageId];
41+
}
42+
43+
public static resolve(packageId: string) {
44+
if (!PackageProcessingQueue.has(packageId)) {
45+
return;
46+
}
47+
PackageProcessingQueue.resolveRegistry[packageId]();
48+
delete PackageProcessingQueue.promiseRegistry[packageId];
49+
delete PackageProcessingQueue.resolveRegistry[packageId];
50+
}
51+
}
52+
53+
2254
/**
2355
* Initializes npm registry cache directory
2456
*/
2557
const CACHE_DIR = process.env.NPM_CACHE_DIR || "/tmp/npm-package-cache";
2658
try {
2759
fs.mkdir(CACHE_DIR, { recursive: true });
2860
} catch (error) {
29-
console.error("Error creating cache directory", error);
61+
logger.error("Error creating cache directory", error);
3062
}
3163

3264

3365
/**
3466
* Fetches package info from npm registry
3567
*/
68+
3669
const fetchRegistryBasePath = "/npm/registry";
70+
71+
export async function fetchRegistryWithConfig(request: ServerRequest, response: ServerResponse) {
72+
try {
73+
const path = request.path.replace(fetchRegistryBasePath, "");
74+
logger.info(`Fetch registry info for path: ${path}`);
75+
76+
const pathPackageInfo = parsePackageInfoFromPath(path);
77+
if (!pathPackageInfo) {
78+
return response.status(400).send(`Invalid package path: ${path}`);
79+
}
80+
81+
const registryConfig: NpmRegistryConfig = request.body;
82+
const config = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig);
83+
84+
const registryResponse = await fetchFromRegistry(path, config);
85+
response.json(await registryResponse.json());
86+
} catch (error) {
87+
logger.error("Error fetching registry", error);
88+
response.status(500).send("Internal server error");
89+
}
90+
}
91+
3792
export async function fetchRegistry(request: ServerRequest, response: ServerResponse) {
3893
try {
3994
const path = request.path.replace(fetchRegistryBasePath, "");
@@ -43,10 +98,9 @@ export async function fetchRegistry(request: ServerRequest, response: ServerResp
4398
if (!pathPackageInfo) {
4499
return response.status(400).send(`Invalid package path: ${path}`);
45100
}
46-
const {organization, name} = pathPackageInfo;
47-
const packageName = organization ? `@${organization}/${name}` : name;
48101

49-
const registryResponse = await fetchFromRegistry(packageName, path);
102+
const config = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId);
103+
const registryResponse = await fetchFromRegistry(path, config);
50104
response.json(await registryResponse.json());
51105
} catch (error) {
52106
logger.error("Error fetching registry", error);
@@ -58,53 +112,100 @@ export async function fetchRegistry(request: ServerRequest, response: ServerResp
58112
/**
59113
* Fetches package files from npm registry if not yet cached
60114
*/
115+
61116
const fetchPackageFileBasePath = "/npm/package";
117+
118+
export async function fetchPackageFileWithConfig(request: ServerRequest, response: ServerResponse) {
119+
const path = request.path.replace(fetchPackageFileBasePath, "");
120+
logger.info(`Fetch file for path with config: ${path}`);
121+
122+
const pathPackageInfo = parsePackageInfoFromPath(path);
123+
if (!pathPackageInfo) {
124+
return response.status(400).send(`Invalid package path: ${path}`);
125+
}
126+
127+
const registryConfig: NpmRegistryConfig = request.body;
128+
const config = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig);
129+
130+
fetchPackageFileInner(request, response, config);
131+
}
132+
62133
export async function fetchPackageFile(request: ServerRequest, response: ServerResponse) {
134+
const path = request.path.replace(fetchPackageFileBasePath, "");
135+
logger.info(`Fetch file for path: ${path}`);
136+
137+
const pathPackageInfo = parsePackageInfoFromPath(path);
138+
if (!pathPackageInfo) {
139+
return response.status(400).send(`Invalid package path: ${path}`);
140+
}
141+
142+
const config = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId);
143+
fetchPackageFileInner(request, response, config);
144+
}
145+
146+
async function fetchPackageFileInner(request: ServerRequest, response: ServerResponse, config: NpmRegistryConfigEntry) {
63147
try {
64-
const path = request.path.replace(fetchPackageFileBasePath, "");
65-
logger.info(`Fetch file for path: ${path}`);
66-
148+
const path = request.path.replace(fetchPackageFileBasePath, "");
67149
const pathPackageInfo = parsePackageInfoFromPath(path);
68150
if (!pathPackageInfo) {
69151
return response.status(400).send(`Invalid package path: ${path}`);
70152
}
71153

72-
logger.info(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`);
73-
const {organization, name, version, file} = pathPackageInfo;
74-
const packageName = organization ? `@${organization}/${name}` : name;
154+
logger.debug(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`);
155+
const {packageId, version, file} = pathPackageInfo;
75156
let packageVersion = version;
76157

77158
let packageInfo: PackagesVersionInfo | null = null;
78159
if (version === "latest") {
79-
const packageInfo: PackagesVersionInfo = await fetchPackageInfo(packageName);
160+
const packageInfo: PackagesVersionInfo|null = await fetchPackageInfo(packageId, config);
161+
if (packageInfo === null) {
162+
return response.status(404).send("Not found");
163+
}
80164
packageVersion = packageInfo["dist-tags"].latest;
81165
}
166+
167+
// Wait for package to be processed if it's already being processed
168+
if (PackageProcessingQueue.has(packageId)) {
169+
logger.info("Waiting for package to be processed", packageId);
170+
await PackageProcessingQueue.wait(packageId);
171+
}
82172

83-
const packageBaseDir = `${CACHE_DIR}/${packageName}/${packageVersion}/package`;
173+
const packageBaseDir = `${CACHE_DIR}/${packageId}/${packageVersion}/package`;
84174
const packageExists = await fileExists(`${packageBaseDir}/package.json`)
85175
if (!packageExists) {
86-
if (!packageInfo) {
87-
packageInfo = await fetchPackageInfo(packageName);
88-
}
89-
90-
if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) {
91-
return response.status(404).send("Not found");
176+
try {
177+
logger.info(`Package does not exist, fetch from registy: ${packageId}@${packageVersion}`);
178+
PackageProcessingQueue.add(packageId);
179+
if (!packageInfo) {
180+
packageInfo = await fetchPackageInfo(packageId, config);
181+
}
182+
183+
if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) {
184+
return response.status(404).send("Not found");
185+
}
186+
187+
const tarball = packageInfo.versions[packageVersion].dist.tarball;
188+
logger.info(`Fetching tarball: ${tarball}`);
189+
await fetchAndUnpackTarball(tarball, packageId, packageVersion, config);
190+
} catch (error) {
191+
logger.error("Error fetching package tarball", error);
192+
return response.status(500).send("Internal server error");
193+
} finally {
194+
PackageProcessingQueue.resolve(packageId);
92195
}
93-
94-
const tarball = packageInfo.versions[packageVersion].dist.tarball;
95-
logger.info("Fetching tarball...", tarball);
96-
await fetchAndUnpackTarball(tarball, packageName, packageVersion);
196+
} else {
197+
logger.info(`Package already exists, serve from cache: ${packageBaseDir}/${file}`)
97198
}
98199

99200
// Fallback to index.mjs if index.js is not present
100201
if (file === "index.js" && !await fileExists(`${packageBaseDir}/${file}`)) {
101-
logger.info("Fallback to index.mjs");
202+
logger.debug("Fallback to index.mjs");
102203
return response.sendFile(`${packageBaseDir}/index.mjs`);
103204
}
104205

105206
return response.sendFile(`${packageBaseDir}/${file}`);
106207
} catch (error) {
107-
logger.error("Error fetching package file", error);
208+
logger.error(`Error fetching package file: ${error} ${(error as {stack: string})?.stack?.toString()}`);
108209
response.status(500).send("Internal server error");
109210
}
110211
};
@@ -114,26 +215,22 @@ export async function fetchPackageFile(request: ServerRequest, response: ServerR
114215
* Helpers
115216
*/
116217

117-
function parsePackageInfoFromPath(path: string): {organization: string, name: string, version: string, file: string} | undefined {
118-
logger.info(`Parse package info from path: ${path}`);
218+
function parsePackageInfoFromPath(path: string): {packageId: string, organization: string, name: string, version: string, file: string} | undefined {
119219
//@ts-ignore - regex groups
120-
const packageInfoRegex = /^\/?(?<fullName>(?:@(?<organization>[a-z0-9-~][a-z0-9-._~]*)\/)?(?<name>[a-z0-9-~][a-z0-9-._~]*))(?:@(?<version>[-a-z0-9><=_.^~]+))?\/(?<file>[^\r\n]*)?$/;
220+
const packageInfoRegex = /^\/?(?<packageId>(?:@(?<organization>[a-z0-9-~][a-z0-9-._~]*)\/)?(?<name>[a-z0-9-~][a-z0-9-._~]*))(?:@(?<version>[-a-z0-9><=_.^~]+))?\/(?<file>[^\r\n]*)?$/;
121221
const matches = path.match(packageInfoRegex);
122-
logger.info(`Parse package matches: ${JSON.stringify(matches)}`);
123222
if (!matches?.groups) {
124223
return;
125224
}
126225

127-
let {organization, name, version, file} = matches.groups;
226+
let {packageId, organization, name, version, file} = matches.groups;
128227
version = /^\d+\.\d+\.\d+(-[\w\d]+)?/.test(version) ? version : "latest";
129228

130-
return {organization, name, version, file};
229+
return {packageId, organization, name, version, file};
131230
}
132231

133-
function fetchFromRegistry(packageName: string, urlOrPath: string): Promise<Response> {
134-
const config: NpmRegistryConfigEntry = NpmRegistryService.getInstance().getRegistryEntryForPackage(packageName);
232+
function fetchFromRegistry(urlOrPath: string, config: NpmRegistryConfigEntry): Promise<Response> {
135233
const registryUrl = config?.registry.url;
136-
137234
const headers: {[key: string]: string} = {};
138235
switch (config?.registry.auth.type) {
139236
case "none":
@@ -154,31 +251,35 @@ function fetchFromRegistry(packageName: string, urlOrPath: string): Promise<Resp
154251
url = `${registryUrl}${separator}${urlOrPath}`;
155252
}
156253

157-
logger.debug(`Fetch from registry: ${url}`);
254+
logger.debug(`Fetch from registry: ${url}, ${JSON.stringify(headers)}`);
158255
return fetch(url, {headers});
159256
}
160257

161-
function fetchPackageInfo(packageName: string): Promise<PackagesVersionInfo> {
162-
return fetchFromRegistry(packageName, packageName).then(res => res.json());
258+
function fetchPackageInfo(packageName: string, config: NpmRegistryConfigEntry): Promise<PackagesVersionInfo|null> {
259+
return fetchFromRegistry(`/${packageName}`, config).then(res => {
260+
if (!res.ok) {
261+
logger.error(`Failed to fetch package info for package ${packageName}: ${res.statusText}`);
262+
return null;
263+
}
264+
return res.json();
265+
});
163266
}
164267

165-
async function fetchAndUnpackTarball(url: string, packageName: string, packageVersion: string) {
166-
const response: Response = await fetchFromRegistry(packageName, url);
268+
async function fetchAndUnpackTarball(url: string, packageId: string, packageVersion: string, config: NpmRegistryConfigEntry) {
269+
const response: Response = await fetchFromRegistry(url, config);
167270
const arrayBuffer = await response.arrayBuffer();
168271
const buffer = Buffer.from(arrayBuffer);
169272
const path = `${CACHE_DIR}/${url.split("/").pop()}`;
170273
await fs.writeFile(path, buffer);
171-
await unpackTarball(path, packageName, packageVersion);
274+
await unpackTarball(path, packageId, packageVersion);
172275
await fs.unlink(path);
173276
}
174277

175-
async function unpackTarball(path: string, packageName: string, packageVersion: string) {
176-
const destinationPath = `${CACHE_DIR}/${packageName}/${packageVersion}`;
278+
async function unpackTarball(path: string, packageId: string, packageVersion: string) {
279+
const destinationPath = `${CACHE_DIR}/${packageId}/${packageVersion}`;
177280
await fs.mkdir(destinationPath, { recursive: true });
178281
await new Promise<void> ((resolve, reject) => {
179282
const tar = spawn("tar", ["-xvf", path, "-C", destinationPath]);
180-
tar.stdout.on("data", (data) => logger.info(data));
181-
tar.stderr.on("data", (data) => console.error(data));
182283
tar.on("close", (code) => {
183284
code === 0 ? resolve() : reject();
184285
});

‎server/node-service/src/routes/apiRouter.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,7 @@ apiRouter.post("/validatePluginDataSourceConfig", pluginControllers.validatePlug
1616
apiRouter.get("/npm/registry/*", npmControllers.fetchRegistry);
1717
apiRouter.get("/npm/package/*", npmControllers.fetchPackageFile);
1818

19+
apiRouter.post("/npm/registry/*", npmControllers.fetchRegistryWithConfig);
20+
apiRouter.post("/npm/package/*", npmControllers.fetchPackageFileWithConfig);
21+
1922
export default apiRouter;

‎server/node-service/src/services/npmRegistry.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,4 +108,26 @@ export class NpmRegistryService {
108108

109109
return config;
110110
}
111+
112+
public static getRegistryEntryForPackageWithConfig(packageName: string, registryConfig: NpmRegistryConfig): NpmRegistryConfigEntry {
113+
registryConfig = NpmRegistryService.sortRegistryConfig(registryConfig);
114+
const config: NpmRegistryConfigEntry | undefined = registryConfig.find(entry => {
115+
if (entry.scope.type === "organization") {
116+
return packageName.startsWith(entry.scope.pattern);
117+
} else if (entry.scope.type === "package") {
118+
return packageName === entry.scope.pattern;
119+
} else {
120+
return true;
121+
}
122+
});
123+
124+
if (!config) {
125+
logger.info(`No registry entry found for package: ${packageName}`);
126+
return NpmRegistryService.DEFAULT_REGISTRY;
127+
} else {
128+
logger.info(`Found registry entry for package: ${packageName} -> ${config.registry.url}`);
129+
}
130+
131+
return config;
132+
}
111133
}

0 commit comments

Comments
 (0)