From a2131e9b03c2760e3d79e24b672dba22ec6c225e Mon Sep 17 00:00:00 2001 From: Pavel Gnedov Date: Tue, 10 Dec 2024 09:20:45 +0700 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=BE=20=D0=B0=D0=B2=D1=82=D0=BE=D0=BE=D0=B1=D0=BD=D0=B0?= =?UTF-8?q?=D1=80=D1=83=D0=B6=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B2=20=D0=B2?= =?UTF-8?q?=D0=B8=D0=B4=D0=B6=D0=B5=D1=82=D0=B0=D1=85=20cron-=D0=B7=D0=B0?= =?UTF-8?q?=D0=B4=D0=B0=D1=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Поиск обновлённых виджетов daily-eccm-v2 * Автоматическая инициализация поиска со старта приложения --- .../src/dashboards/dashboards.service.ts | 37 +++++++--- src/app.module.ts | 33 +++++++-- .../daily-eccm-reports-v2.datasource.ts | 5 +- src/reports/daily-eccm-v2.report.service.ts | 72 ++++++++++++++----- 4 files changed, 107 insertions(+), 40 deletions(-) diff --git a/libs/event-emitter/src/dashboards/dashboards.service.ts b/libs/event-emitter/src/dashboards/dashboards.service.ts index 0a4ebcd..9de17cf 100644 --- a/libs/event-emitter/src/dashboards/dashboards.service.ts +++ b/libs/event-emitter/src/dashboards/dashboards.service.ts @@ -4,6 +4,7 @@ import * as DashboardModel from '../models/dashboard'; import nano from 'nano'; import { randomUUID } from 'crypto'; import { createAppError } from '../utils/result'; +import { TimestampNowFill } from '../utils/timestamp-now-fill'; @Injectable() export class DashboardsService { @@ -56,18 +57,18 @@ export class DashboardsService { } async save(id: string, data: DashboardModel.Data): Promise { - this.logger.debug( - `Save dashboard id - ${id}, data - ${JSON.stringify(data)}`, + this.logger.log( + `Save dashboard id - ${id}, title - ${JSON.stringify(data.title)}`, ); const ds = await this.db.getDatasource(); const prevValue = await this.loadRawData(id); - const newValue = { + const newValue = TimestampNowFill({ _id: prevValue._id, _rev: prevValue._rev, id: prevValue.id, data: data, - }; + }); await ds.insert(newValue); return; } @@ -89,17 +90,33 @@ export class DashboardsService { async findDashboardsByWidgetType( widgetType: string, + updatedAfter?: number, ): Promise { const ds = await this.db.getDatasource(); - const data = await ds.find({ - selector: { - 'data.widgets': { - $elemMatch: { - type: widgetType, - }, + const selector = { + 'data.widgets': { + $elemMatch: { + type: widgetType, }, }, + }; + if (updatedAfter > 0) { + selector['timestamp__'] = { + $gte: updatedAfter, + }; + } + this.logger.debug( + `Find dashboards by widget type - ${widgetType} ` + + `and selector - ${JSON.stringify(selector)}`, + ); + const data = await ds.find({ + selector: selector, }); + this.logger.debug( + `Found dashboards by widget type - ${widgetType} ` + + `, selector - ${JSON.stringify(selector)}` + + `, result - ${JSON.stringify(data)}`, + ); if (!data.docs) throw createAppError('DASHBOARDS_NOT_FOUND'); return data.docs; } diff --git a/src/app.module.ts b/src/app.module.ts index 13cefab..1a7659d 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -54,6 +54,9 @@ import { CalendarEnhancer } from '@app/event-emitter/issue-enhancers/calendar-en import { Dashboards as DashboardsDs } from '@app/event-emitter/couchdb-datasources/dashboards'; import { DashboardInitService } from './dashboards/dashboard-init.service'; import { TelegramBotController } from './telegram-bot/telegram-bot.controller'; +import { DailyEccmV2ReportService } from './reports/daily-eccm-v2.report.service'; +import { DashboardsService } from '@app/event-emitter/dashboards/dashboards.service'; +import { DailyEccmReportsV2Datasource } from './couchdb-datasources/daily-eccm-reports-v2.datasource'; @Module({ imports: [ @@ -115,6 +118,9 @@ import { TelegramBotController } from './telegram-bot/telegram-bot.controller'; }, CreateTagManagerServiceProvider('TAG_MANAGER_SERVICE'), DashboardInitService, + DashboardsService, + DailyEccmReportsV2Datasource, + DailyEccmV2ReportService, ], }) export class AppModule implements OnModuleInit { @@ -143,16 +149,22 @@ export class AppModule implements OnModuleInit { private calendarEnhancer: CalendarEnhancer, private dashboardInitService: DashboardInitService, + + private dashboardsService: DashboardsService, + private dailyEccmV2ReportService: DailyEccmV2ReportService, ) {} onModuleInit() { - Issues.getDatasource(); - Users.getDatasource(); - Changes.getDatasource(); - UserMetaInfo.getDatasource(); - DailyEccmReportsDatasource.getDatasource(); - DailyEccmReportsUserCommentsDatasource.getDatasource(); - DashboardsDs.getDatasource(); + const datasources = [ + Issues.getDatasource(), + Users.getDatasource(), + Changes.getDatasource(), + UserMetaInfo.getDatasource(), + DailyEccmReportsDatasource.getDatasource(), + DailyEccmReportsUserCommentsDatasource.getDatasource(), + DashboardsDs.getDatasource(), + DailyEccmReportsV2Datasource.getDatasource(), + ]; this.enhancerService.addEnhancer([ this.timestampEnhancer, @@ -222,6 +234,13 @@ export class AppModule implements OnModuleInit { this.initDailyEccmUserCommentsPipeline(); this.initDashbordProviders(); + + Promise.all(datasources).then(() => { + this.dailyEccmV2ReportService.setDashboardsService( + this.dashboardsService, + ); + this.dailyEccmV2ReportService.initAutoScanJobs(); + }); } private initDailyEccmUserCommentsPipeline(): void { diff --git a/src/couchdb-datasources/daily-eccm-reports-v2.datasource.ts b/src/couchdb-datasources/daily-eccm-reports-v2.datasource.ts index cf2210b..3685814 100644 --- a/src/couchdb-datasources/daily-eccm-reports-v2.datasource.ts +++ b/src/couchdb-datasources/daily-eccm-reports-v2.datasource.ts @@ -1,10 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { CouchDb } from '@app/event-emitter/couchdb-datasources/couchdb'; import nano from 'nano'; -import { - DailyEccmV2ReportService, - Report, -} from 'src/reports/daily-eccm-v2.report.service'; +import { Report } from 'src/reports/daily-eccm-v2.report.service'; @Injectable() export class DailyEccmReportsV2Datasource { diff --git a/src/reports/daily-eccm-v2.report.service.ts b/src/reports/daily-eccm-v2.report.service.ts index e00622e..75c2cc0 100644 --- a/src/reports/daily-eccm-v2.report.service.ts +++ b/src/reports/daily-eccm-v2.report.service.ts @@ -1,7 +1,7 @@ import { DashboardsService } from '@app/event-emitter/dashboards/dashboards.service'; import { Dashboard, Widget } from '@app/event-emitter/models/dashboard'; import { Injectable, Logger } from '@nestjs/common'; -import { Cron, SchedulerRegistry } from '@nestjs/schedule'; +import { SchedulerRegistry } from '@nestjs/schedule'; import { CronJob } from 'cron'; export type Params = { @@ -23,19 +23,38 @@ export type Job = { params: Params; }; -export const WIDGET_TYPE = 'daily-eccm-v2'; +export const WIDGET_TYPE = 'daily_eccm_v2'; export const JOB_PREFIX = 'daily_eccm_v2'; +export const UPDATE_RATE = 60 * 1000; + @Injectable() export class DailyEccmV2ReportService { private logger = new Logger(DailyEccmV2ReportService.name); private dashboardsService: DashboardsService; + private previousAutoScanTime = 0; + + private cronJobs: Record = {}; + constructor(private schedulerRegistry: SchedulerRegistry) {} - @Cron('* * * * *') + /** + * Auto scan jobs every UPDATE_RATE seconds. + * First call to autoScanJobs is done immediately. + * Each subsequent call is done after the timeout. + * The timeout is reset after each call to autoScanJobs. + */ + initAutoScanJobs() { + const tick = () => { + setTimeout(tick, UPDATE_RATE); + this.autoScanJobs(); + }; + tick(); + } + async autoScanJobs() { this.logger.debug('Auto scan jobs started'); const dbs = this.getDashboardsService(); @@ -44,32 +63,43 @@ export class DailyEccmV2ReportService { this.logger.debug('Auto scan jobs finished'); return; } - const dashboards = await dbs.findDashboardsByWidgetType(WIDGET_TYPE); + const nowTime = new Date().getTime(); + const dashboards = await dbs.findDashboardsByWidgetType( + WIDGET_TYPE, + this.previousAutoScanTime, + ); + this.previousAutoScanTime = nowTime; for (let i = 0; i < dashboards.length; i++) { const dashboard: Dashboard = dashboards[i]; for (let j = 0; j < dashboard.data.widgets.length; j++) { const widget = dashboard.data.widgets[j]; if (widget.type === WIDGET_TYPE) { const jobId = `${JOB_PREFIX}_${dashboard.id}_${widget.id}`; - let cronJob = this.schedulerRegistry.getCronJob(jobId); - if (!cronJob) { - cronJob = new CronJob( - widget.dataLoaderParams.schedule, - async () => { - this.logger.debug(`Cron job ${jobId} started`); - // const report = await dbs.getReport(dashboard.id, widget.id); - this.logger.debug(`Cron job ${jobId} finished`); - // return report; - }, - ); - this.schedulerRegistry.addCronJob(jobId, cronJob); - } + this.updateCronJob(jobId, widget, dashboard); } } } this.logger.debug('Auto scan jobs finished'); } + private updateCronJob( + jobId: string, + widget: Widget, + dashboard: Dashboard, + ): void { + if (this.cronJobs[jobId]) { + this.cronJobs[jobId].stop(); + this.schedulerRegistry.deleteCronJob(jobId); + } + const job = new CronJob( + widget.dataLoaderParams?.schedule || '* * * * *', + this.createJobHandler(jobId, dashboard, widget), + ); + this.cronJobs[jobId] = job; + this.schedulerRegistry.addCronJob(jobId, job); + job.start(); + } + getDashboardsService(): DashboardsService | null { if (!this.dashboardsService) { this.logger.warn('Dashboards service not initialized'); @@ -88,8 +118,12 @@ export class DailyEccmV2ReportService { widget: Widget, ): () => void { return async () => { - this.logger.debug(`Cron job ${jobId} started`); - this.logger.debug(`Cron job ${jobId} finished`); + this.logger.debug( + `Cron job ${jobId} started - dashboard ${dashboard.id}, widget ${widget.id}`, + ); + this.logger.debug( + `Cron job ${jobId} finished - dashboard ${dashboard.id}, widget ${widget.id}`, + ); }; } }