Добавлен cli скрипт для синхронизации закешированных задач с задачами в redmine

package-lock.json

@@ -19,6 +19,7 @@
         "@nestjs/websockets": "^8.4.4",
         "axios": "^0.27.2",
         "cache-manager": "^4.1.0",
+        "csv": "^6.3.6",
         "handlebars": "^4.7.7",
         "hbs": "^4.2.0",
         "imap-simple": "^5.1.0",
@@ -3741,6 +3742,35 @@
       "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==",
       "dev": true
     },
+    "node_modules/csv": {
+      "version": "6.3.6",
+      "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz",
+      "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==",
+      "dependencies": {
+        "csv-generate": "^4.3.1",
+        "csv-parse": "^5.5.3",
+        "csv-stringify": "^6.4.5",
+        "stream-transform": "^3.3.0"
+      },
+      "engines": {
+        "node": ">= 0.1.90"
+      }
+    },
+    "node_modules/csv-generate": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz",
+      "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w=="
+    },
+    "node_modules/csv-parse": {
+      "version": "5.5.3",
+      "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz",
+      "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A=="
+    },
+    "node_modules/csv-stringify": {
+      "version": "6.4.5",
+      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz",
+      "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ=="
+    },
     "node_modules/dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@@ -9067,6 +9097,11 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/stream-transform": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz",
+      "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA=="
+    },
     "node_modules/streamsearch": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
@@ -13277,6 +13312,32 @@
         }
       }
     },
+    "csv": {
+      "version": "6.3.6",
+      "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz",
+      "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==",
+      "requires": {
+        "csv-generate": "^4.3.1",
+        "csv-parse": "^5.5.3",
+        "csv-stringify": "^6.4.5",
+        "stream-transform": "^3.3.0"
+      }
+    },
+    "csv-generate": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz",
+      "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w=="
+    },
+    "csv-parse": {
+      "version": "5.5.3",
+      "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz",
+      "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A=="
+    },
+    "csv-stringify": {
+      "version": "6.4.5",
+      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz",
+      "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ=="
+    },
     "dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@@ -17260,6 +17321,11 @@
       "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
       "integrity": "sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g=="
     },
+    "stream-transform": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz",
+      "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA=="
+    },
     "streamsearch": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",

package.json

@@ -31,6 +31,7 @@
     "@nestjs/websockets": "^8.4.4",
     "axios": "^0.27.2",
     "cache-manager": "^4.1.0",
+    "csv": "^6.3.6",
     "handlebars": "^4.7.7",
     "hbs": "^4.2.0",
     "imap-simple": "^5.1.0",

tools/sync-issues.mjs

@@ -0,0 +1,240 @@
+import { parse } from 'jsonc-parser';
+import { readFileSync } from 'fs';
+import { parseArgs } from 'util';
+import { parse as csvParse } from 'csv/sync';
+import { DateTime } from 'luxon';
+import { createInterface } from 'readline/promises';
+
+const helpMsg = `
+Синхронизация задач из Redmine с Redmine Issue Event Emitter
+
+* --config=sync-issues.jsonc - конфигурационный файл для постановки задачи на выполнение синхронизации
+* --test - тестовый прогон без запуска задачи на синхронизацию
+* --yes - автоматически отвечать 'y'
+
+Пример конфигурационного файла:
+
+{
+  "redmineUrl": "http://redmine.my-company.local",
+  "redmineToken": "... ... ...",
+  "eventEmitterUrl": "http://localhost:3000"
+  "forceUpdate": true, // true - принудительное обновление,
+                       // false - сравнение даты обновления в redmine и в event-emitter-е
+                       // и обновление только просроченных задач
+  "updatedAtFieldName": "Обновлено",
+  "dateTimeFormat": "dd.MM.yyyy HH:mm",
+  
+  "csvLinks": [
+    "http://redmine.my-company.local/projects/test-project/issues.csv?query_id=2011", // закрытые за сутки
+    "http://redmine.my-company.local/projects/test-project/issues.csv?query_id=2012"  // обновленые за сутки
+  ],
+  "csvFiles": [
+    "/tmp/file1.csv",
+    "/tmp/file2.csv"
+  ],
+}
+`;
+
+var args = parseArgs({
+  options: {
+    config: {
+      type: 'string',
+    },
+    test: {
+      type: 'boolean',
+    },
+    yes: {
+      type: 'boolean',
+    },
+  },
+});
+
+if (!args?.values?.config) {
+  console.log(helpMsg);
+  process.exit(0);
+}
+
+function readConfig(fileName) {
+  const rawData = readFileSync(fileName, { encoding: 'utf8' });
+  return parse(rawData);
+}
+
+var config = readConfig(args.values.config);
+
+console.log(`Конфигурационный файл:\n${JSON.stringify(config, null, '  ')}\n`); // DEBUG
+
+const UNLIMIT = 999999;
+
+async function getIssuesByQuery(selector) {
+  const resp = await fetch(`${config.eventEmitterUrl}/issues/find`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      selector: selector,
+      limit: UNLIMIT,
+    }),
+  });
+  return await resp.json();
+}
+
+async function getIssuesByList(issueIds) {
+  return await getIssuesByQuery({
+    id: {
+      $in: issueIds,
+    },
+  });
+}
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+async function getCsvFromRedmine(url, token) {
+  const resp = await fetch(url, {
+    headers: {
+      'X-Redmine-API-Key': token,
+    },
+  });
+  const rawData = await resp.text();
+  return csvParse(rawData, {
+    delimiter: ';',
+    quote: '"',
+    columns: true,
+    skip_empty_lines: true,
+    cast: (value, context) => {
+      if (context.column == '#') {
+        return Number(value);
+      }
+      return value;
+    },
+  });
+}
+
+async function getCsvFromLocalFile(fileName) {
+  const rawData = readFileSync(fileName, { encoding: 'utf8' });
+  const data = csvParse(rawData, {
+    delimiter: ';',
+    quote: '"',
+    columns: true,
+    skip_empty_lines: true,
+    cast: (value, context) => {
+      if (context.column == '#') {
+        return Number(value);
+      }
+      return value;
+    },
+  });
+  return data;
+}
+
+async function loadAllCsv() {
+  const res = {};
+  const csvLinks = config?.csvLinks || [];
+  const csvFiles = config?.csvFiles || [];
+  const token = config?.redmineToken || '';
+  function appendIssuesToRes(issues) {
+    issues.forEach((issue) => {
+      const issueId = Object.values(issue)[0];
+      issue.id = Number(issueId);
+      const updatedAt = DateTime.fromFormat(
+        issue[config.updatedAtFieldName],
+        config.dateTimeFormat,
+      );
+      if (!res[issueId]) {
+        res[issueId] = issue;
+      } else {
+        const existsIssue = res[issueId];
+        const existsUpdatedAt = DateTime.fromFormat(
+          existsIssue[config.updatedAtFieldName],
+          config.dateTimeFormat,
+        );
+        if (updatedAt.diff(existsUpdatedAt).as('seconds') > 0) {
+          res[issueId] = issue;
+        }
+      }
+    });
+  }
+  for (let i = 0; i < csvLinks.length; i++) {
+    const link = csvLinks[i];
+    const issues = await getCsvFromRedmine(link, token);
+    appendIssuesToRes(issues);
+  }
+  for (let i = 0; i < csvFiles.length; i++) {
+    const file = csvFiles[i];
+    const issues = await getCsvFromLocalFile(file);
+    appendIssuesToRes(issues);
+  }
+  return Object.values(res);
+}
+
+async function loadFromEventEmitter(issueIds) {
+  return await getIssuesByList(issueIds);
+}
+
+async function getIssueIdsForUpdate(newIssues) {
+  if (config.forceUpdate) {
+    return newIssues.map((i) => i.id);
+  }
+  const existsIssue = await loadFromEventEmitter(newIssues.map((i) => i.id));
+  const existsIssueMap = existsIssue.reduce((acc, issue) => {
+    acc[issue.id] = issue;
+    return acc;
+  }, {});
+  const newIds = newIssues
+    .filter((newIssue) => {
+      const existsIssue = existsIssueMap[newIssue.id];
+      if (!existsIssue) return true;
+      const newUpdatedAt = DateTime.fromFormat(
+        newIssue[config.updatedAtFieldName],
+        config.dateTimeFormat,
+      );
+      const existsUpdatedTimestamp = DateTime.fromMillis(
+        existsIssue.timestamp__,
+      );
+      if (newUpdatedAt.diff(existsUpdatedTimestamp).as('seconds') > 0) {
+        return true;
+      }
+      return false;
+    })
+    .map((i) => i.id);
+  return newIds;
+}
+
+const issues = await loadAllCsv();
+const idsForUpdate = await getIssueIdsForUpdate(issues);
+
+console.log(`Из csv получено задач - ${issues.length} шт.`);
+console.log(`Нужно обновить задач - ${idsForUpdate.length} шт.`);
+console.log(`Задачи для обновления - ` + JSON.stringify(idsForUpdate));
+
+async function continueQuestion() {
+  if (args.values.yes) {
+    return true;
+  }
+  const readline = createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+  const answer = await readline.question(`\nПродолжить? (y/n)\n> `);
+  const res = answer == 'y';
+  readline.close();
+  console.log(res ? `\nПродолжаем...` : '\nНу и мне тогда это не надо');
+  return res;
+}
+
+if (!args.values.test) {
+  const answer = await continueQuestion();
+  if (answer) {
+    console.log(`Отправка задачи на синхронизацию...`);
+    await fetch(
+      `${config.eventEmitterUrl}/redmine-event-emitter/append-issues`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(idsForUpdate),
+      },
+    );
+    console.log(`Синхронизация запущена`);
+  }
+}