Добавлен cli скрипт для синхронизации закешированных задач с задачами в redmine
This commit is contained in:
Pavel Gnedov
parent
f5b547e8ce
commit
afad88c057
3 changed files with 307 additions and 0 deletions
66
package-lock.json
generated
66
package-lock.json
generated
|
|
@ -19,6 +19,7 @@
|
|||
"@nestjs/websockets": "^8.4.4",
|
||||
"axios": "^0.27.2",
|
||||
"cache-manager": "^4.1.0",
|
||||
"csv": "^6.3.6",
|
||||
"handlebars": "^4.7.7",
|
||||
"hbs": "^4.2.0",
|
||||
"imap-simple": "^5.1.0",
|
||||
|
|
@ -3741,6 +3742,35 @@
|
|||
"integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/csv": {
|
||||
"version": "6.3.6",
|
||||
"resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz",
|
||||
"integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==",
|
||||
"dependencies": {
|
||||
"csv-generate": "^4.3.1",
|
||||
"csv-parse": "^5.5.3",
|
||||
"csv-stringify": "^6.4.5",
|
||||
"stream-transform": "^3.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.1.90"
|
||||
}
|
||||
},
|
||||
"node_modules/csv-generate": {
|
||||
"version": "4.3.1",
|
||||
"resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz",
|
||||
"integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w=="
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "5.5.3",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz",
|
||||
"integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A=="
|
||||
},
|
||||
"node_modules/csv-stringify": {
|
||||
"version": "6.4.5",
|
||||
"resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz",
|
||||
"integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ=="
|
||||
},
|
||||
"node_modules/dashdash": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
|
||||
|
|
@ -9067,6 +9097,11 @@
|
|||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/stream-transform": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz",
|
||||
"integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA=="
|
||||
},
|
||||
"node_modules/streamsearch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
|
||||
|
|
@ -13277,6 +13312,32 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"csv": {
|
||||
"version": "6.3.6",
|
||||
"resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz",
|
||||
"integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==",
|
||||
"requires": {
|
||||
"csv-generate": "^4.3.1",
|
||||
"csv-parse": "^5.5.3",
|
||||
"csv-stringify": "^6.4.5",
|
||||
"stream-transform": "^3.3.0"
|
||||
}
|
||||
},
|
||||
"csv-generate": {
|
||||
"version": "4.3.1",
|
||||
"resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz",
|
||||
"integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w=="
|
||||
},
|
||||
"csv-parse": {
|
||||
"version": "5.5.3",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz",
|
||||
"integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A=="
|
||||
},
|
||||
"csv-stringify": {
|
||||
"version": "6.4.5",
|
||||
"resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz",
|
||||
"integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ=="
|
||||
},
|
||||
"dashdash": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
|
||||
|
|
@ -17260,6 +17321,11 @@
|
|||
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
|
||||
"integrity": "sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g=="
|
||||
},
|
||||
"stream-transform": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz",
|
||||
"integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA=="
|
||||
},
|
||||
"streamsearch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@
|
|||
"@nestjs/websockets": "^8.4.4",
|
||||
"axios": "^0.27.2",
|
||||
"cache-manager": "^4.1.0",
|
||||
"csv": "^6.3.6",
|
||||
"handlebars": "^4.7.7",
|
||||
"hbs": "^4.2.0",
|
||||
"imap-simple": "^5.1.0",
|
||||
|
|
|
|||
240
tools/sync-issues.mjs
Normal file
240
tools/sync-issues.mjs
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
import { parse } from 'jsonc-parser';
|
||||
import { readFileSync } from 'fs';
|
||||
import { parseArgs } from 'util';
|
||||
import { parse as csvParse } from 'csv/sync';
|
||||
import { DateTime } from 'luxon';
|
||||
import { createInterface } from 'readline/promises';
|
||||
|
||||
const helpMsg = `
|
||||
Синхронизация задач из Redmine с Redmine Issue Event Emitter
|
||||
|
||||
* --config=sync-issues.jsonc - конфигурационный файл для постановки задачи на выполнение синхронизации
|
||||
* --test - тестовый прогон без запуска задачи на синхронизацию
|
||||
* --yes - автоматически отвечать 'y'
|
||||
|
||||
Пример конфигурационного файла:
|
||||
|
||||
{
|
||||
"redmineUrl": "http://redmine.my-company.local",
|
||||
"redmineToken": "... ... ...",
|
||||
"eventEmitterUrl": "http://localhost:3000"
|
||||
"forceUpdate": true, // true - принудительное обновление,
|
||||
// false - сравнение даты обновления в redmine и в event-emitter-е
|
||||
// и обновление только просроченных задач
|
||||
"updatedAtFieldName": "Обновлено",
|
||||
"dateTimeFormat": "dd.MM.yyyy HH:mm",
|
||||
|
||||
"csvLinks": [
|
||||
"http://redmine.my-company.local/projects/test-project/issues.csv?query_id=2011", // закрытые за сутки
|
||||
"http://redmine.my-company.local/projects/test-project/issues.csv?query_id=2012" // обновленые за сутки
|
||||
],
|
||||
"csvFiles": [
|
||||
"/tmp/file1.csv",
|
||||
"/tmp/file2.csv"
|
||||
],
|
||||
}
|
||||
`;
|
||||
|
||||
var args = parseArgs({
|
||||
options: {
|
||||
config: {
|
||||
type: 'string',
|
||||
},
|
||||
test: {
|
||||
type: 'boolean',
|
||||
},
|
||||
yes: {
|
||||
type: 'boolean',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!args?.values?.config) {
|
||||
console.log(helpMsg);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
function readConfig(fileName) {
|
||||
const rawData = readFileSync(fileName, { encoding: 'utf8' });
|
||||
return parse(rawData);
|
||||
}
|
||||
|
||||
var config = readConfig(args.values.config);
|
||||
|
||||
console.log(`Конфигурационный файл:\n${JSON.stringify(config, null, ' ')}\n`); // DEBUG
|
||||
|
||||
const UNLIMIT = 999999;
|
||||
|
||||
async function getIssuesByQuery(selector) {
|
||||
const resp = await fetch(`${config.eventEmitterUrl}/issues/find`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
selector: selector,
|
||||
limit: UNLIMIT,
|
||||
}),
|
||||
});
|
||||
return await resp.json();
|
||||
}
|
||||
|
||||
async function getIssuesByList(issueIds) {
|
||||
return await getIssuesByQuery({
|
||||
id: {
|
||||
$in: issueIds,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
async function getCsvFromRedmine(url, token) {
|
||||
const resp = await fetch(url, {
|
||||
headers: {
|
||||
'X-Redmine-API-Key': token,
|
||||
},
|
||||
});
|
||||
const rawData = await resp.text();
|
||||
return csvParse(rawData, {
|
||||
delimiter: ';',
|
||||
quote: '"',
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
cast: (value, context) => {
|
||||
if (context.column == '#') {
|
||||
return Number(value);
|
||||
}
|
||||
return value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function getCsvFromLocalFile(fileName) {
|
||||
const rawData = readFileSync(fileName, { encoding: 'utf8' });
|
||||
const data = csvParse(rawData, {
|
||||
delimiter: ';',
|
||||
quote: '"',
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
cast: (value, context) => {
|
||||
if (context.column == '#') {
|
||||
return Number(value);
|
||||
}
|
||||
return value;
|
||||
},
|
||||
});
|
||||
return data;
|
||||
}
|
||||
|
||||
async function loadAllCsv() {
|
||||
const res = {};
|
||||
const csvLinks = config?.csvLinks || [];
|
||||
const csvFiles = config?.csvFiles || [];
|
||||
const token = config?.redmineToken || '';
|
||||
function appendIssuesToRes(issues) {
|
||||
issues.forEach((issue) => {
|
||||
const issueId = Object.values(issue)[0];
|
||||
issue.id = Number(issueId);
|
||||
const updatedAt = DateTime.fromFormat(
|
||||
issue[config.updatedAtFieldName],
|
||||
config.dateTimeFormat,
|
||||
);
|
||||
if (!res[issueId]) {
|
||||
res[issueId] = issue;
|
||||
} else {
|
||||
const existsIssue = res[issueId];
|
||||
const existsUpdatedAt = DateTime.fromFormat(
|
||||
existsIssue[config.updatedAtFieldName],
|
||||
config.dateTimeFormat,
|
||||
);
|
||||
if (updatedAt.diff(existsUpdatedAt).as('seconds') > 0) {
|
||||
res[issueId] = issue;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
for (let i = 0; i < csvLinks.length; i++) {
|
||||
const link = csvLinks[i];
|
||||
const issues = await getCsvFromRedmine(link, token);
|
||||
appendIssuesToRes(issues);
|
||||
}
|
||||
for (let i = 0; i < csvFiles.length; i++) {
|
||||
const file = csvFiles[i];
|
||||
const issues = await getCsvFromLocalFile(file);
|
||||
appendIssuesToRes(issues);
|
||||
}
|
||||
return Object.values(res);
|
||||
}
|
||||
|
||||
async function loadFromEventEmitter(issueIds) {
|
||||
return await getIssuesByList(issueIds);
|
||||
}
|
||||
|
||||
async function getIssueIdsForUpdate(newIssues) {
|
||||
if (config.forceUpdate) {
|
||||
return newIssues.map((i) => i.id);
|
||||
}
|
||||
const existsIssue = await loadFromEventEmitter(newIssues.map((i) => i.id));
|
||||
const existsIssueMap = existsIssue.reduce((acc, issue) => {
|
||||
acc[issue.id] = issue;
|
||||
return acc;
|
||||
}, {});
|
||||
const newIds = newIssues
|
||||
.filter((newIssue) => {
|
||||
const existsIssue = existsIssueMap[newIssue.id];
|
||||
if (!existsIssue) return true;
|
||||
const newUpdatedAt = DateTime.fromFormat(
|
||||
newIssue[config.updatedAtFieldName],
|
||||
config.dateTimeFormat,
|
||||
);
|
||||
const existsUpdatedTimestamp = DateTime.fromMillis(
|
||||
existsIssue.timestamp__,
|
||||
);
|
||||
if (newUpdatedAt.diff(existsUpdatedTimestamp).as('seconds') > 0) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
})
|
||||
.map((i) => i.id);
|
||||
return newIds;
|
||||
}
|
||||
|
||||
const issues = await loadAllCsv();
|
||||
const idsForUpdate = await getIssueIdsForUpdate(issues);
|
||||
|
||||
console.log(`Из csv получено задач - ${issues.length} шт.`);
|
||||
console.log(`Нужно обновить задач - ${idsForUpdate.length} шт.`);
|
||||
console.log(`Задачи для обновления - ` + JSON.stringify(idsForUpdate));
|
||||
|
||||
async function continueQuestion() {
|
||||
if (args.values.yes) {
|
||||
return true;
|
||||
}
|
||||
const readline = createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
const answer = await readline.question(`\nПродолжить? (y/n)\n> `);
|
||||
const res = answer == 'y';
|
||||
readline.close();
|
||||
console.log(res ? `\nПродолжаем...` : '\nНу и мне тогда это не надо');
|
||||
return res;
|
||||
}
|
||||
|
||||
if (!args.values.test) {
|
||||
const answer = await continueQuestion();
|
||||
if (answer) {
|
||||
console.log(`Отправка задачи на синхронизацию...`);
|
||||
await fetch(
|
||||
`${config.eventEmitterUrl}/redmine-event-emitter/append-issues`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(idsForUpdate),
|
||||
},
|
||||
);
|
||||
console.log(`Синхронизация запущена`);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue