From fc5c9b5a16e4f21b688806a97e24a1cd7b054d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AC=A7=E5=A6=82=E6=A0=8B?= Date: Mon, 3 Jul 2023 15:51:53 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/index.js | 23 ++++++++++++++++++----- package.json | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/bin/index.js b/bin/index.js index 0e1744f..e8811d6 100755 --- a/bin/index.js +++ b/bin/index.js @@ -74,10 +74,12 @@ const upload = async (filePath, parts = [], requestUrl) => { } else { console.log(chalk.red('网络连接异常,请重新执行命令继续上传')); logger.error(`分片(${currentChunkIndex})上传时网络连接异常 (path: ${filePath}) , url: ${requestUrl})`); + await logger.close(); process.exit(1); } } else { console.log(chalk.red((error.response && error.response.data) || error.message)); + await logger.close(); process.exit(1); } } @@ -119,6 +121,7 @@ const upload = async (filePath, parts = [], requestUrl) => { logger.error(error.message); logger.error(error.stack); console.log(chalk(error.message)); + await logger.close(); process.exit(1); } @@ -150,6 +153,7 @@ const upload = async (filePath, parts = [], requestUrl) => { logger.error(error.message); logger.error(error.stack); console.log(chalk.red((error.response && error.response.data) || error.message)); + await logger.close(); process.exit(1); } @@ -169,6 +173,7 @@ const getFileMD5Success = async (filePath, requestUrl) => { Authorization }); if (res.code) { + logger.info(`获取已上传信息错误(1): ${JSON.stringify(res)} (path: ${filePath} , url: ${requestUrl})`); throw (res.message); } uploadId = res.data.uploadId; @@ -181,10 +186,11 @@ const getFileMD5Success = async (filePath, requestUrl) => { uploadedParts = [] } } catch (error) { - logger.error(`获取已上传信息错误 (path: ${filePath} , url: ${requestUrl})`); + logger.error(`获取已上传信息错误(2) (path: ${filePath} , url: ${requestUrl})`); logger.error(error.message); logger.error(error.stack); console.log(chalk.red((error.response && error.response.data) || error.message)); + await logger.close(); process.exit(1); } @@ -227,13 +233,14 @@ const getFileMD5 = async (filePath, requestUrl) => { console.log(chalk.red((error.response && error.response.data) || error.message)); logger.error(error.message); logger.error(error.stack); + await logger.close(); process.exit(1); } } const uploadFile = async (filePath, size, requestUrl) => { fileSize = size; - logger.info(`('************************ 开始上传 (${filePath}) ('************************`); + logger.info(`************************ 开始上传 (${filePath}) ************************`); await getFileMD5(filePath, requestUrl); md5 = ''; uploadId = ''; @@ -262,6 +269,7 @@ const uploadDir = async (dir) => { console.log(chalk.red((error.response && error.response.data) || error.message)); logger.error(error.message); logger.error(error.stack); + await logger.close(); process.exit(1); } else { return files; @@ -290,9 +298,11 @@ const beforeUpload = async (filePath) => { const isDirectory = stat.isDirectory(); if (isDirectory && !isUploadDir) { console.log(chalk.red(`\n${filePath}不合法,需指定一个文件\n`)) + await logger.close(); process.exit(1); } else if (!isDirectory && isUploadDir) { console.log(chalk.red(`\n${filePath}不合法,需指定一个文件夹\n`)) + await logger.close(); process.exit(1); } fSize = stat.size; @@ -304,6 +314,7 @@ const beforeUpload = async (filePath) => { logger.error(error.stack); console.log(chalk.red((error.response && error.response.data) || error.message)); } + await logger.close(); process.exit(1); } if (isUploadDir) { @@ -313,16 +324,18 @@ const beforeUpload = async (filePath) => { } } -const onUpload = (_username, _password) => { +const onUpload = async (_username, _password) => { Authorization = generateAuthorization(_username, _password); logger.info('************************ 准备上传 ************************') if (path.isAbsolute(argv.path)) { - beforeUpload(argv.path); + await beforeUpload(argv.path); } else { - beforeUpload(path.join(process.cwd(), argv.path)) + await beforeUpload(path.join(process.cwd(), argv.path)) } + + await logger.close(); } const [username, password] = argv.username.split(':'); diff --git a/package.json b/package.json index c43e1a5..5a5ba6e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "coding-generic", - "version": "1.2.9", + "version": "1.2.10", "description": "", "main": "index.js", "bin": { From b776111dd62c81b8b2082941eba5158b2cd21d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AC=A7=E5=A6=82=E6=A0=8B?= Date: Tue, 4 Jul 2023 18:14:11 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=97=A5=E5=BF=97=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/index.js | 40 +++++++++++++++++++-------------------- lib/log.js | 5 ++++- lib/request.js | 51 ++++++++++++++++++++++++++++++++++++++------------ package.json | 2 +- 4 files changed, 63 insertions(+), 35 deletions(-) diff --git a/bin/index.js b/bin/index.js index e8811d6..aadf408 100755 --- a/bin/index.js +++ b/bin/index.js @@ -74,13 +74,13 @@ const upload = async (filePath, parts = [], requestUrl) => { } else { console.log(chalk.red('网络连接异常,请重新执行命令继续上传')); logger.error(`分片(${currentChunkIndex})上传时网络连接异常 (path: ${filePath}) , url: ${requestUrl})`); - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); + throw error; } } else { console.log(chalk.red((error.response && error.response.data) || error.message)); - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); + throw error; } } } @@ -121,8 +121,8 @@ const upload = async (filePath, parts = [], requestUrl) => { logger.error(error.message); logger.error(error.stack); console.log(chalk(error.message)); - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); + throw error; } @@ -153,8 +153,8 @@ const upload = async (filePath, parts = [], requestUrl) => { logger.error(error.message); logger.error(error.stack); console.log(chalk.red((error.response && error.response.data) || error.message)); - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); + throw error; } console.log(chalk.green(`\n上传完毕 (${filePath})\n`)) @@ -189,9 +189,9 @@ const getFileMD5Success = async (filePath, requestUrl) => { logger.error(`获取已上传信息错误(2) (path: ${filePath} , url: ${requestUrl})`); logger.error(error.message); logger.error(error.stack); - console.log(chalk.red((error.response && error.response.data) || error.message)); - await logger.close(); - process.exit(1); + console.log(chalk.red((error.response && error.response.data) || error.message), `(path: ${filePath} , url: ${requestUrl}`); + await logger.close(() => process.exit(1)); + throw error; } await upload(filePath, uploadedParts, requestUrl); @@ -233,8 +233,8 @@ const getFileMD5 = async (filePath, requestUrl) => { console.log(chalk.red((error.response && error.response.data) || error.message)); logger.error(error.message); logger.error(error.stack); - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); + throw error; } } @@ -269,8 +269,8 @@ const uploadDir = async (dir) => { console.log(chalk.red((error.response && error.response.data) || error.message)); logger.error(error.message); logger.error(error.stack); - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); + throw error; } else { return files; } @@ -298,12 +298,10 @@ const beforeUpload = async (filePath) => { const isDirectory = stat.isDirectory(); if (isDirectory && !isUploadDir) { console.log(chalk.red(`\n${filePath}不合法,需指定一个文件\n`)) - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); } else if (!isDirectory && isUploadDir) { console.log(chalk.red(`\n${filePath}不合法,需指定一个文件夹\n`)) - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); } fSize = stat.size; } catch (error) { @@ -314,8 +312,8 @@ const beforeUpload = async (filePath) => { logger.error(error.stack); console.log(chalk.red((error.response && error.response.data) || error.message)); } - await logger.close(); - process.exit(1); + await logger.close(() => process.exit(1)); + throw error; } if (isUploadDir) { await uploadDir(filePath); diff --git a/lib/log.js b/lib/log.js index 08ed0fd..c8bf8f8 100644 --- a/lib/log.js +++ b/lib/log.js @@ -1,9 +1,12 @@ const { createLogger, format, transports } = require('winston'); const { combine, timestamp, printf } = format; +const util = require('util') const userHome = process.env.HOME || process.env.USERPROFILE; -const formatLog = printf(({ level, message, timestamp }) => `${timestamp} ${level}: ${JSON.stringify(message)}`); +const formatLog = printf(({ level, message, timestamp }) => { + return `${timestamp} ${level}: ${JSON.stringify(util.inspect(message))}` +}); const transport = new (transports.DailyRotateFile)({ filename: `${userHome}/.coding/log/coding-generic/%DATE%.log`, zippedArchive: true, diff --git a/lib/request.js b/lib/request.js index abfd037..fd1226e 100644 --- a/lib/request.js +++ b/lib/request.js @@ -1,4 +1,35 @@ const axios = require('axios'); +const util = require('util') +const logger = require('./log'); + + +const http = axios.create({ + withCredentials: true, +}) + + +// 响应拦截器 +const responseSuccess = response => { + return Promise.resolve(response) +} + +const responseFailed = error => { + const url = error.config?.url + console.error('网络请求错误', `(${url})`); + logger.error(`网络请求错误 (${url})`); + logger.error(JSON.stringify(util.inspect(error))); + const { response } = error + if (response) { + console.error('网络请求错误', response.data); + logger.error(response.data); + logger.error(response); + + } + return Promise.reject(error) +} +http.interceptors.response.use(responseSuccess, responseFailed) + + /** * 获取已经上传完成的分片信息 @@ -12,10 +43,10 @@ const getExistChunks = (requestUrl, { fileSize, version, fileTag -}, { +}, { Authorization }) => { - return axios.post(`${requestUrl}?version=${version}&fileTag=${fileTag}&fileSize=${fileSize}&action=part-init`, {}, { + return http.post(`${requestUrl}?version=${version}&fileTag=${fileTag}&fileSize=${fileSize}&action=part-init`, {}, { headers: { Authorization } }) } @@ -32,22 +63,18 @@ const getExistChunks = (requestUrl, { * @param {string} Authorization */ const uploadChunk = (requestUrl, { - uploadId, + uploadId, version, - partNumber, - size, + partNumber, + size, currentChunk, }, { headers, Authorization }) => { - return axios({ + return http.post(`${requestUrl}?version=${version}&uploadId=${uploadId}&partNumber=${partNumber}&size=${size}&action=part-upload`, currentChunk, { maxContentLength: Infinity, - maxBodyLength: Infinity, - method: 'post', - url: `${requestUrl}?version=${version}&uploadId=${uploadId}&partNumber=${partNumber}&size=${size}&action=part-upload`, - data: currentChunk, - headers: { Authorization, ...headers } + maxBodyLength: Infinity, headers: { Authorization, ...headers } }) } @@ -68,7 +95,7 @@ const mergeAllChunks = (requestUrl, { }, { Authorization }) => { - return axios.post(`${requestUrl}?version=${version}&uploadId=${uploadId}&fileTag=${fileTag}&size=${fileSize}&action=part-complete`, {}, { + return http.post(`${requestUrl}?version=${version}&uploadId=${uploadId}&fileTag=${fileTag}&size=${fileSize}&action=part-complete`, {}, { headers: { Authorization } }) } diff --git a/package.json b/package.json index 5a5ba6e..3042290 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "coding-generic", - "version": "1.2.10", + "version": "1.2.11", "description": "", "main": "index.js", "bin": { From 2e7ba931ea6b577cf4f7bb0efa565890018fa33f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AC=A7=E5=A6=82=E6=A0=8B?= Date: Wed, 5 Jul 2023 10:30:45 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E8=AF=AD=E6=B3=95=E5=85=BC=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/request.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/request.js b/lib/request.js index fd1226e..30a3681 100644 --- a/lib/request.js +++ b/lib/request.js @@ -14,7 +14,7 @@ const responseSuccess = response => { } const responseFailed = error => { - const url = error.config?.url + const url = error && error.config && error.config.url console.error('网络请求错误', `(${url})`); logger.error(`网络请求错误 (${url})`); logger.error(JSON.stringify(util.inspect(error))); diff --git a/package.json b/package.json index 3042290..6686fb2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "coding-generic", - "version": "1.2.11", + "version": "1.2.12", "description": "", "main": "index.js", "bin": { From 7673b7c7696b98f3a2fc0ae7f7d18a5c3ce589fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AC=A7=E5=A6=82=E6=A0=8B?= Date: Fri, 20 Oct 2023 15:59:32 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +++++ bin/index.js | 12 +++++++++-- lib/argv.js | 7 ++++++- lib/download.js | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/request.js | 33 ++++++++++++++++++++++++++++-- lib/utils.js | 3 ++- package.json | 2 +- 7 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 lib/download.js diff --git a/README.md b/README.md index 47ac96c..f38d2bf 100644 --- a/README.md +++ b/README.md @@ -16,4 +16,9 @@ coding-generic -u=[:password] --path= --registry=[:password] --dir --path= --registry= +``` + +- 下载文件夹(仅 1.2.13 及以上版本支持) +```shell +coding-generic --pull -u=[:password] --registry=/list/?version= ``` \ No newline at end of file diff --git a/bin/index.js b/bin/index.js index aadf408..758580c 100755 --- a/bin/index.js +++ b/bin/index.js @@ -18,6 +18,7 @@ const { getExistChunks: _getExistChunks, uploadChunk: _uploadChunk, mergeAllChun const { withRetry } = require('../lib/withRetry'); const argv = require('../lib/argv'); +const { onDownload } = require('../lib/download'); const { requestUrl, version } = getRegistryInfo(argv.registry); @@ -339,7 +340,11 @@ const onUpload = async (_username, _password) => { const [username, password] = argv.username.split(':'); if (username && password) { - onUpload(username, password); + if (argv.pull) { + onDownload() + } else { + onUpload(username, password); + } } else { prompts([ { @@ -353,7 +358,10 @@ if (username && password) { ).then(async (answers) => { if (!answers.password) { return; + } if (argv.pull) { + onDownload() + } else { + onUpload(argv.username, answers.password); } - onUpload(argv.username, answers.password); }) } diff --git a/lib/argv.js b/lib/argv.js index 3f4939c..e76b8ea 100644 --- a/lib/argv.js +++ b/lib/argv.js @@ -1,6 +1,7 @@ const argv = require('yargs') .usage('上传文件: coding-generic --username=[:PASSWORD] --path= --registry=') .usage('上传文件夹: coding-generic --username=[:PASSWORD] --dir --path= --registry=') + .usage('下载文件夹: coding-generic --pull --username=[:PASSWORD] --registry=/list/?version=') .options({ username: { alias: 'u', @@ -10,7 +11,7 @@ const argv = require('yargs') path: { alias: 'p', describe: '需要上传的文件路径', - demandOption: true + // demandOption: true }, registry: { alias: 'r', @@ -27,6 +28,10 @@ const argv = require('yargs') alias: 'd', describe: '上传文件夹', boolean: true, + }, + pull: { + describe: '下载', + boolean: true, } }) .alias('version', 'v') diff --git a/lib/download.js b/lib/download.js new file mode 100644 index 0000000..a2a6f47 --- /dev/null +++ b/lib/download.js @@ -0,0 +1,54 @@ +const fs = require('fs'); +const path = require('path'); +const logger = require('./log'); +const { generateAuthorization, getRegistryInfo } = require('./utils'); +const { fetchDownloadList, downloadFile } = require('../lib/request'); +const argv = require('./argv'); + +const { version, host, protocol, pathname } = getRegistryInfo(argv.registry); + +let Authorization = ''; + +const onDownload = async () => { + console.log('************************ 准备下载 ************************'); + logger.info('************************ 准备下载 ************************'); + Authorization = generateAuthorization(argv.username, argv.password); + const res = await fetchDownloadList(argv.registry, Authorization) + const { status, fileInfos = [] } = res.data + if (status === 200) { + await downloadFiles(fileInfos) + console.log('************************ 下载完毕 ************************'); + logger.info('************************ 下载完毕 ************************'); + } +} + +const downloadFiles = async (fileInfos = []) => { + try { + return await Promise.all(fileInfos.map(async info => { + console.log(`正在下载 ${info.fileName} ...`); + logger.info(`正在下载 ${info.fileName} ...`); + const p = path.join(process.cwd(), info.fileName); + const dir = p.split('/').slice(0, -1).join('/'); + if (dir && !fs.existsSync(dir)) { + fs.mkdirSync(dir); + } + const writer = fs.createWriteStream(p); + const url = `${protocol}//${path.join(host, path.join(pathname.split('/').slice(0, -2).join('/'), info.fileName))}` + const res = await downloadFile(url, { version }, Authorization); + await res.data.pipe(writer) + await writer.end(); + await writer.close(); + console.log(`下载 ${info.fileName} 完成`); + logger.info(`下载 ${info.fileName} 完成`); + })); + } catch (error) { + console.log(error); + logger.error(error); + throw error; + } + +} + +module.exports = { + onDownload +} \ No newline at end of file diff --git a/lib/request.js b/lib/request.js index 30a3681..8f8a4f0 100644 --- a/lib/request.js +++ b/lib/request.js @@ -14,6 +14,7 @@ const responseSuccess = response => { } const responseFailed = error => { + console.log('eeee=>', error) const url = error && error.config && error.config.url console.error('网络请求错误', `(${url})`); logger.error(`网络请求错误 (${url})`); @@ -100,8 +101,36 @@ const mergeAllChunks = (requestUrl, { }) } + +const fetchDownloadList = async (registry, Authorization) => { + return http.post(registry, { + }, { + headers: { Authorization } + }) + +} + +//http:/codingcorp-generic.pkg.coding-artifacts.test-codingcorp.woa.com/coding-xxx-567023e/generic-public/test/coding-coding +//http://codingcorp-generic.pkg.coding-artifacts.test-codingcorp.woa.com/coding-xxx-567023e/generic-public/test/coding-coding + +const downloadFile = async (url, params, Authorization) => { + return axios.get(url, { + params, + headers: { + Authorization + }, + responseType: 'stream' + }); + +} + + module.exports = { getExistChunks, uploadChunk, - mergeAllChunks -} \ No newline at end of file + mergeAllChunks, + fetchDownloadList, + downloadFile +} + + diff --git a/lib/utils.js b/lib/utils.js index 91ffcae..eea611d 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -23,7 +23,8 @@ const getRegistryInfo = (registry) => { const { version } = querystring.parse(query) return { requestUrl: `${protocol}//${path.join(host, pathname)}`, - version: !version || version === '' ? 'latest' : version + version: !version || version === '' ? 'latest' : version, + host, protocol, pathname } } diff --git a/package.json b/package.json index 6686fb2..2082296 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "coding-generic", - "version": "1.2.12", + "version": "1.2.13", "description": "", "main": "index.js", "bin": {