diff --git a/package-lock.json b/package-lock.json index 4e33da7..7abeb93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@nestjs/websockets": "^8.4.4", "axios": "^0.27.2", "cache-manager": "^4.1.0", + "csv": "^6.3.6", "handlebars": "^4.7.7", "hbs": "^4.2.0", "imap-simple": "^5.1.0", @@ -3741,6 +3742,35 @@ "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", "dev": true }, + "node_modules/csv": { + "version": "6.3.6", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz", + "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==", + "dependencies": { + "csv-generate": "^4.3.1", + "csv-parse": "^5.5.3", + "csv-stringify": "^6.4.5", + "stream-transform": "^3.3.0" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz", + "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w==" + }, + "node_modules/csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==" + }, + "node_modules/csv-stringify": { + "version": "6.4.5", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz", + "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ==" + }, "node_modules/dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -9067,6 +9097,11 @@ "node": ">=0.10.0" } }, + "node_modules/stream-transform": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz", + "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA==" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -13277,6 +13312,32 @@ } } }, + "csv": { + "version": "6.3.6", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz", + "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==", + "requires": { + "csv-generate": "^4.3.1", + "csv-parse": "^5.5.3", + "csv-stringify": "^6.4.5", + "stream-transform": "^3.3.0" + } + }, + "csv-generate": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz", + "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w==" + }, + "csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==" + }, + "csv-stringify": { + "version": "6.4.5", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz", + "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ==" + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -17260,6 +17321,11 @@ "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", "integrity": "sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g==" }, + "stream-transform": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz", + "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA==" + }, "streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", diff --git a/package.json b/package.json index ad4a2a9..eacd683 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "@nestjs/websockets": "^8.4.4", "axios": "^0.27.2", "cache-manager": "^4.1.0", + "csv": "^6.3.6", "handlebars": "^4.7.7", "hbs": "^4.2.0", "imap-simple": "^5.1.0", diff --git a/tools/sync-issues.mjs b/tools/sync-issues.mjs new file mode 100644 index 0000000..4d4db5e --- /dev/null +++ b/tools/sync-issues.mjs @@ -0,0 +1,240 @@ +import { parse } from 'jsonc-parser'; +import { readFileSync } from 'fs'; +import { parseArgs } from 'util'; +import { parse as csvParse } from 'csv/sync'; +import { DateTime } from 'luxon'; +import { createInterface } from 'readline/promises'; + +const helpMsg = ` +Синхронизация задач из Redmine с Redmine Issue Event Emitter + +* --config=sync-issues.jsonc - конфигурационный файл для постановки задачи на выполнение синхронизации +* --test - тестовый прогон без запуска задачи на синхронизацию +* --yes - автоматически отвечать 'y' + +Пример конфигурационного файла: + +{ + "redmineUrl": "http://redmine.my-company.local", + "redmineToken": "... ... ...", + "eventEmitterUrl": "http://localhost:3000" + "forceUpdate": true, // true - принудительное обновление, + // false - сравнение даты обновления в redmine и в event-emitter-е + // и обновление только просроченных задач + "updatedAtFieldName": "Обновлено", + "dateTimeFormat": "dd.MM.yyyy HH:mm", + + "csvLinks": [ + "http://redmine.my-company.local/projects/test-project/issues.csv?query_id=2011", // закрытые за сутки + "http://redmine.my-company.local/projects/test-project/issues.csv?query_id=2012" // обновленые за сутки + ], + "csvFiles": [ + "/tmp/file1.csv", + "/tmp/file2.csv" + ], +} +`; + +var args = parseArgs({ + options: { + config: { + type: 'string', + }, + test: { + type: 'boolean', + }, + yes: { + type: 'boolean', + }, + }, +}); + +if (!args?.values?.config) { + console.log(helpMsg); + process.exit(0); +} + +function readConfig(fileName) { + const rawData = readFileSync(fileName, { encoding: 'utf8' }); + return parse(rawData); +} + +var config = readConfig(args.values.config); + +console.log(`Конфигурационный файл:\n${JSON.stringify(config, null, ' ')}\n`); // DEBUG + +const UNLIMIT = 999999; + +async function getIssuesByQuery(selector) { + const resp = await fetch(`${config.eventEmitterUrl}/issues/find`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + selector: selector, + limit: UNLIMIT, + }), + }); + return await resp.json(); +} + +async function getIssuesByList(issueIds) { + return await getIssuesByQuery({ + id: { + $in: issueIds, + }, + }); +} + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +async function getCsvFromRedmine(url, token) { + const resp = await fetch(url, { + headers: { + 'X-Redmine-API-Key': token, + }, + }); + const rawData = await resp.text(); + return csvParse(rawData, { + delimiter: ';', + quote: '"', + columns: true, + skip_empty_lines: true, + cast: (value, context) => { + if (context.column == '#') { + return Number(value); + } + return value; + }, + }); +} + +async function getCsvFromLocalFile(fileName) { + const rawData = readFileSync(fileName, { encoding: 'utf8' }); + const data = csvParse(rawData, { + delimiter: ';', + quote: '"', + columns: true, + skip_empty_lines: true, + cast: (value, context) => { + if (context.column == '#') { + return Number(value); + } + return value; + }, + }); + return data; +} + +async function loadAllCsv() { + const res = {}; + const csvLinks = config?.csvLinks || []; + const csvFiles = config?.csvFiles || []; + const token = config?.redmineToken || ''; + function appendIssuesToRes(issues) { + issues.forEach((issue) => { + const issueId = Object.values(issue)[0]; + issue.id = Number(issueId); + const updatedAt = DateTime.fromFormat( + issue[config.updatedAtFieldName], + config.dateTimeFormat, + ); + if (!res[issueId]) { + res[issueId] = issue; + } else { + const existsIssue = res[issueId]; + const existsUpdatedAt = DateTime.fromFormat( + existsIssue[config.updatedAtFieldName], + config.dateTimeFormat, + ); + if (updatedAt.diff(existsUpdatedAt).as('seconds') > 0) { + res[issueId] = issue; + } + } + }); + } + for (let i = 0; i < csvLinks.length; i++) { + const link = csvLinks[i]; + const issues = await getCsvFromRedmine(link, token); + appendIssuesToRes(issues); + } + for (let i = 0; i < csvFiles.length; i++) { + const file = csvFiles[i]; + const issues = await getCsvFromLocalFile(file); + appendIssuesToRes(issues); + } + return Object.values(res); +} + +async function loadFromEventEmitter(issueIds) { + return await getIssuesByList(issueIds); +} + +async function getIssueIdsForUpdate(newIssues) { + if (config.forceUpdate) { + return newIssues.map((i) => i.id); + } + const existsIssue = await loadFromEventEmitter(newIssues.map((i) => i.id)); + const existsIssueMap = existsIssue.reduce((acc, issue) => { + acc[issue.id] = issue; + return acc; + }, {}); + const newIds = newIssues + .filter((newIssue) => { + const existsIssue = existsIssueMap[newIssue.id]; + if (!existsIssue) return true; + const newUpdatedAt = DateTime.fromFormat( + newIssue[config.updatedAtFieldName], + config.dateTimeFormat, + ); + const existsUpdatedTimestamp = DateTime.fromMillis( + existsIssue.timestamp__, + ); + if (newUpdatedAt.diff(existsUpdatedTimestamp).as('seconds') > 0) { + return true; + } + return false; + }) + .map((i) => i.id); + return newIds; +} + +const issues = await loadAllCsv(); +const idsForUpdate = await getIssueIdsForUpdate(issues); + +console.log(`Из csv получено задач - ${issues.length} шт.`); +console.log(`Нужно обновить задач - ${idsForUpdate.length} шт.`); +console.log(`Задачи для обновления - ` + JSON.stringify(idsForUpdate)); + +async function continueQuestion() { + if (args.values.yes) { + return true; + } + const readline = createInterface({ + input: process.stdin, + output: process.stdout, + }); + const answer = await readline.question(`\nПродолжить? (y/n)\n> `); + const res = answer == 'y'; + readline.close(); + console.log(res ? `\nПродолжаем...` : '\nНу и мне тогда это не надо'); + return res; +} + +if (!args.values.test) { + const answer = await continueQuestion(); + if (answer) { + console.log(`Отправка задачи на синхронизацию...`); + await fetch( + `${config.eventEmitterUrl}/redmine-event-emitter/append-issues`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(idsForUpdate), + }, + ); + console.log(`Синхронизация запущена`); + } +}