[core] API - messages filters

fix #449, fix #140, fix #458

src/api/chats.controller.ts

@@ -18,7 +18,6 @@ import {
   SessionApiParam,
   WorkingSessionParam,
 } from '@waha/nestjs/params/SessionApiParam';
-import { PaginationParams } from '@waha/structures/pagination.dto';
 
 import { SessionManager } from '../core/abc/manager.abc';
 import { WhatsappSession } from '../core/abc/session.abc';
@@ -26,6 +25,7 @@ import { parseBool } from '../helpers';
 import {
   ChatsPaginationParams,
   GetChatMessageQuery,
+  GetChatMessagesFilter,
   GetChatMessagesQuery,
 } from '../structures/chats.dto';
 import { EditMessageRequest } from '../structures/chatting.dto';
@@ -62,13 +62,14 @@ class ChatsController {
   @SessionApiParam
   @ApiOperation({ summary: 'Gets messages in the chat' })
   @ChatIdApiParam
+  @UsePipes(new ValidationPipe({ transform: true, whitelist: true }))
   getChatMessages(
     @Query() query: GetChatMessagesQuery,
+    @Query() filter: GetChatMessagesFilter,
     @WorkingSessionParam session: WhatsappSession,
     @Param('chatId') chatId: string,
   ) {
-    const downloadMedia = parseBool(query.downloadMedia);
-    return session.getChatMessages(chatId, query.limit, downloadMedia);
+    return session.getChatMessages(chatId, query, filter);
   }
 
   @Get(':chatId/messages/:messageId')

src/api/chatting.controller.ts

@@ -9,6 +9,7 @@ import {
 } from '@nestjs/common';
 import { ApiOperation, ApiSecurity, ApiTags } from '@nestjs/swagger';
 import { WAHAValidationPipe } from '@waha/nestjs/pipes/WAHAValidationPipe';
+import { GetChatMessagesFilter } from '@waha/structures/chats.dto';
 import { SendButtonsRequest } from '@waha/structures/chatting.buttons.dto';
 
 import { SessionManager } from '../core/abc/manager.abc';
@@ -171,9 +172,12 @@ export class ChattingController {
 
   @Get('/messages')
   @ApiOperation({ summary: 'Get messages in a chat' })
-  async getMessages(@Query() query: GetMessageQuery) {
+  async getMessages(
+    @Query() query: GetMessageQuery,
+    @Query() filter: GetChatMessagesFilter,
+  ) {
     const whatsapp = await this.manager.getWorkingSession(query.session);
-    return whatsapp.getMessages(query);
+    return whatsapp.getChatMessages(query.chatId, query, filter);
   }
 
   @Get('/sendText')

src/core/abc/session.abc.ts

@@ -3,7 +3,11 @@ import {
   CreateChannelRequest,
   ListChannelsQuery,
 } from '@waha/structures/channels.dto';
-import { GetChatMessageQuery } from '@waha/structures/chats.dto';
+import {
+  GetChatMessageQuery,
+  GetChatMessagesFilter,
+  GetChatMessagesQuery,
+} from '@waha/structures/chats.dto';
 import { SendButtonsRequest } from '@waha/structures/chatting.buttons.dto';
 import { Label, LabelID } from '@waha/structures/labels.dto';
 import { PaginationParams } from '@waha/structures/pagination.dto';
@@ -308,10 +312,6 @@ export abstract class WhatsappSession {
 
   abstract stopTyping(chat: ChatRequest);
 
-  getMessages(query: GetMessageQuery) {
-    throw new NotImplementedByEngineError();
-  }
-
   abstract setReaction(request: MessageReactionRequest);
 
   setStar(request: MessageStarRequest): Promise<void> {
@@ -331,8 +331,8 @@ export abstract class WhatsappSession {
 
   public getChatMessages(
     chatId: string,
-    limit: number,
-    downloadMedia: boolean,
+    query: GetChatMessagesQuery,
+    filter: GetChatMessagesFilter,
   ) {
     throw new NotImplementedByEngineError();
   }

src/core/engines/noweb/session.noweb.core.ts

@@ -38,7 +38,11 @@ import {
   CreateChannelRequest,
   ListChannelsQuery,
 } from '@waha/structures/channels.dto';
-import { GetChatMessageQuery } from '@waha/structures/chats.dto';
+import {
+  GetChatMessageQuery,
+  GetChatMessagesFilter,
+  GetChatMessagesQuery,
+} from '@waha/structures/chats.dto';
 import { SendButtonsRequest } from '@waha/structures/chatting.buttons.dto';
 import { ContactQuery, ContactRequest } from '@waha/structures/contacts.dto';
 import { BinaryFile, RemoteFile } from '@waha/structures/files.dto';
@@ -68,7 +72,6 @@ import {
   ChatRequest,
   CheckNumberStatusQuery,
   EditMessageRequest,
-  GetMessageQuery,
   MessageContactVcardRequest,
   MessageDestination,
   MessageFileRequest,
@@ -755,19 +758,17 @@ export class WhatsappSessionNoWebCore extends WhatsappSession {
     return this.sock.sendPresenceUpdate('paused', chatId);
   }
 
-  async getMessages(query: GetMessageQuery) {
-    return this.getChatMessages(query.chatId, query.limit, query.downloadMedia);
-  }
-
   public async getChatMessages(
     chatId: string,
-    limit: number,
-    downloadMedia: boolean,
+    query: GetChatMessagesQuery,
+    filter: GetChatMessagesFilter,
   ) {
-    downloadMedia = parseBool(downloadMedia);
+    const downloadMedia = query.downloadMedia;
+    const pagination = query as PaginationParams;
     const messages = await this.store.getMessagesByJid(
       toJID(chatId),
-      toNumber(limit),
+      filter,
+      pagination,
     );
 
     const promises = [];
@@ -830,7 +831,7 @@ export class WhatsappSessionNoWebCore extends WhatsappSession {
     archive: boolean,
   ): Promise<any> {
     const jid = toJID(chatId);
-    const messages = await this.store.getMessagesByJid(jid, 1);
+    const messages = await this.store.getMessagesByJid(jid, {}, { limit: 1 });
     return await this.sock.chatModify(
       { archive: archive, lastMessages: messages },
       jid,
@@ -847,7 +848,7 @@ export class WhatsappSessionNoWebCore extends WhatsappSession {
 
   public async chatsUnreadChat(chatId: string): Promise<any> {
     const jid = toJID(chatId);
-    const messages = await this.store.getMessagesByJid(jid, 1);
+    const messages = await this.store.getMessagesByJid(jid, {}, { limit: 1 });
     return await this.sock.chatModify(
       { markRead: false, lastMessages: messages },
       jid,
@@ -1723,6 +1724,8 @@ function toCusFormat(remoteJid) {
   return ensureSuffix(number);
 }
 
+export const ALL_JID = 'all@s.whatsapp.net';
+
 /**
  * Convert from 11111111111@c.us to 11111111111@s.whatsapp.net
  * @param chatId

src/core/engines/noweb/store/IMessagesRepository.ts

@@ -1,3 +1,6 @@
+import { GetChatMessagesFilter } from '@waha/structures/chats.dto';
+import { PaginationParams } from '@waha/structures/pagination.dto';
+
 export interface IMessagesRepository {
   deleteAll(): Promise<void>;
 
@@ -5,7 +8,11 @@ export interface IMessagesRepository {
 
   upsertOne(message: any): Promise<void>;
 
-  getAllByJid(jid: string, limit: number): Promise<any[]>;
+  getAllByJid(
+    jid: string,
+    filter: GetChatMessagesFilter,
+    pagination: PaginationParams,
+  ): Promise<any[]>;
 
   getByJidById(jid: string, id: string): Promise<any | null>;
 

src/core/engines/noweb/store/INowebStore.ts

@@ -5,6 +5,7 @@ import {
   proto,
 } from '@adiwajshing/baileys';
 import { Label } from '@adiwajshing/baileys/lib/Types/Label';
+import { GetChatMessagesFilter } from '@waha/structures/chats.dto';
 import { PaginationParams } from '@waha/structures/pagination.dto';
 
 export interface INowebStore {
@@ -18,7 +19,11 @@ export interface INowebStore {
 
   loadMessage(jid: string, id: string): Promise<proto.IWebMessageInfo>;
 
-  getMessagesByJid(chatId: string, limit: number): Promise<any>;
+  getMessagesByJid(
+    chatId: string,
+    filter: GetChatMessagesFilter,
+    pagination: PaginationParams,
+  ): Promise<any>;
 
   getMessageById(chatId: string, messageId: string): Promise<any>;
 

src/core/engines/noweb/store/NowebInMemoryStore.ts

@@ -1,6 +1,7 @@
 import { Chat, Contact, makeInMemoryStore, proto } from '@adiwajshing/baileys';
 import { Label } from '@adiwajshing/baileys/lib/Types/Label';
 import { BadRequestException } from '@nestjs/common';
+import { GetChatMessagesFilter } from '@waha/structures/chats.dto';
 import { PaginationParams } from '@waha/structures/pagination.dto';
 
 import { INowebStore } from './INowebStore';
@@ -38,7 +39,11 @@ export class NowebInMemoryStore implements INowebStore {
     return this.store.loadMessage(jid, id);
   }
 
-  getMessagesByJid(chatId: string, limit: number): Promise<any> {
+  getMessagesByJid(
+    chatId: string,
+    filter: GetChatMessagesFilter,
+    pagination: PaginationParams,
+  ): Promise<any> {
     throw new BadRequestException(this.errorMessage);
   }
 

src/core/engines/noweb/store/NowebPersistentStore.ts

@@ -16,6 +16,7 @@ import {
 } from '@adiwajshing/baileys/lib/Types/LabelAssociation';
 import { ILabelAssociationRepository } from '@waha/core/engines/noweb/store/ILabelAssociationsRepository';
 import { ILabelsRepository } from '@waha/core/engines/noweb/store/ILabelsRepository';
+import { GetChatMessagesFilter } from '@waha/structures/chats.dto';
 import { PaginationParams, SortOrder } from '@waha/structures/pagination.dto';
 import { toNumber } from 'lodash';
 import { Logger } from 'pino';
@@ -323,8 +324,14 @@ export class NowebPersistentStore implements INowebStore {
     return proto.WebMessageInfo.fromObject(data);
   }
 
-  getMessagesByJid(chatId: string, limit: number) {
-    return this.messagesRepo.getAllByJid(toJID(chatId), toNumber(limit));
+  getMessagesByJid(
+    chatId: string,
+    filter: GetChatMessagesFilter,
+    pagination: PaginationParams,
+  ): Promise<any> {
+    pagination.sortBy = 'messageTimestamp';
+    pagination.sortOrder = SortOrder.DESC;
+    return this.messagesRepo.getAllByJid(chatId, filter, pagination);
   }
 
   getMessageById(chatId: string, messageId: string): Promise<any> {

src/core/engines/noweb/store/sqlite3/Sqlite3MessagesRepository.ts

@@ -1,3 +1,7 @@
+import { ALL_JID } from '@waha/core/engines/noweb/session.noweb.core';
+import { GetChatMessagesFilter } from '@waha/structures/chats.dto';
+import { PaginationParams } from '@waha/structures/pagination.dto';
+
 import { IMessagesRepository } from '../IMessagesRepository';
 import { NOWEBSqlite3KVRepository } from './NOWEBSqlite3KVRepository';
 
@@ -9,11 +13,36 @@ export class Sqlite3MessagesRepository
     return this.upsertMany(messages);
   }
 
-  getAllByJid(jid: string, limit: number): Promise<any[]> {
-    const query = this.select()
-      .where({ jid: jid })
-      .limit(limit)
-      .orderBy('messageTimestamp', 'DESC');
+  async getAllByJid(
+    jid: string,
+    filter: GetChatMessagesFilter,
+    pagination: PaginationParams,
+  ): Promise<any[]> {
+    let query = this.select();
+    if (jid !== ALL_JID) {
+      query = this.select().where({ jid: jid });
+    }
+    if (filter['filter.timestamp.lte'] != null) {
+      query = query.where(
+        'messageTimestamp',
+        '<=',
+        filter['filter.timestamp.lte'],
+      );
+    }
+    if (filter['filter.timestamp.gte'] != null) {
+      query = query.where(
+        'messageTimestamp',
+        '>=',
+        filter['filter.timestamp.gte'],
+      );
+    }
+    if (filter['filter.fromMe'] != null) {
+      // filter by data->"$.key.fromMe"
+      query = query.whereRaw("data->'$.key.fromMe' = ?", [
+        filter['filter.fromMe'] ? 'true' : 'false',
+      ]);
+    }
+    query = this.pagination(query, pagination);
     return this.all(query);
   }
 

src/core/engines/webjs/WebjsClient.ts

@@ -1,6 +1,8 @@
+import { GetChatMessagesFilter } from '@waha/structures/chats.dto';
 import { PaginationParams } from '@waha/structures/pagination.dto';
 import * as lodash from 'lodash';
 import { Client } from 'whatsapp-web.js';
+import { Message } from 'whatsapp-web.js/src/structures';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { LoadWAHA } = require('./_WAHA.js');
@@ -43,4 +45,71 @@ export class WebjsClient extends Client {
 
     return chats.map((chat) => ChatFactory.create(this, chat));
   }
+
+  async getMessages(
+    chatId: string,
+    filter: GetChatMessagesFilter,
+    pagination: PaginationParams,
+  ) {
+    const messages = await this.pupPage.evaluate(
+      async (chatId, filter, pagination) => {
+        pagination.limit ||= Infinity;
+        pagination.offset ||= 0;
+
+        const msgFilter = (m) => {
+          if (m.isNotification) {
+            return false;
+          }
+          if (
+            filter['filter.fromMe'] != null &&
+            m.id.fromMe !== filter['filter.fromMe']
+          ) {
+            return false;
+          }
+          if (
+            filter['filter.timestamp.gte'] != null &&
+            m.t < filter['filter.timestamp.gte']
+          ) {
+            return false;
+          }
+          if (
+            filter['filter.timestamp.lte'] != null &&
+            m.t > filter['filter.timestamp.lte']
+          ) {
+            return false;
+          }
+          return true;
+        };
+
+        // @ts-ignore
+        const chat = await window.WWebJS.getChat(chatId, { getAsModel: false });
+        let msgs = chat.msgs.getModelsArray().filter(msgFilter);
+
+        while (msgs.length < pagination.limit + pagination.offset) {
+          const loadedMessages =
+            // @ts-ignore
+            await window.Store.ConversationMsgs.loadEarlierMsgs(chat);
+          if (!loadedMessages || !loadedMessages.length) break;
+          msgs = [...loadedMessages.filter(msgFilter), ...msgs];
+        }
+
+        if (msgs.length > pagination.limit + pagination.offset) {
+          // sort by t - new first
+          msgs = msgs.sort((a, b) => b.t - a.t);
+          msgs = msgs.slice(
+            pagination.offset,
+            pagination.limit + pagination.offset,
+          );
+        }
+
+        // @ts-ignore
+        return msgs.map((m) => window.WWebJS.getMessageModel(m));
+      },
+      chatId,
+      filter,
+      pagination,
+    );
+
+    return messages.map((m) => new Message(this, m));
+  }
 }

src/core/engines/webjs/session.webjs.core.ts

@@ -10,7 +10,7 @@ import {
 } from '@waha/core/exceptions';
 import { IMediaEngineProcessor } from '@waha/core/media/IMediaEngineProcessor';
 import { QR } from '@waha/core/QR';
-import { parseBool, splitAt } from '@waha/helpers';
+import { splitAt } from '@waha/helpers';
 import { PairingCodeResponse } from '@waha/structures/auth.dto';
 import { CallData } from '@waha/structures/calls.dto';
 import {
@@ -23,12 +23,13 @@ import {
   ChatArchiveEvent,
   ChatSortField,
   GetChatMessageQuery,
+  GetChatMessagesFilter,
+  GetChatMessagesQuery,
 } from '@waha/structures/chats.dto';
 import {
   ChatRequest,
   CheckNumberStatusQuery,
   EditMessageRequest,
-  GetMessageQuery,
   MessageFileRequest,
   MessageForwardRequest,
   MessageImageRequest,
@@ -499,10 +500,6 @@ export class WhatsappSessionWebJSCore extends WhatsappSession {
     await chat.clearState();
   }
 
-  async getMessages(query: GetMessageQuery) {
-    return this.getChatMessages(query.chatId, query.limit, query.downloadMedia);
-  }
-
   async setReaction(request: MessageReactionRequest) {
     const message = this.recreateMessage(request.messageId);
     return message.react(request.reaction);
@@ -543,14 +540,27 @@ export class WhatsappSessionWebJSCore extends WhatsappSession {
     return this.whatsapp.getChats(pagination);
   }
 
-  async getChatMessages(chatId: string, limit: number, downloadMedia: boolean) {
-    downloadMedia = parseBool(downloadMedia);
-    const chat: Chat = await this.whatsapp.getChatById(
+  public async getChatMessages(
+    chatId: string,
+    query: GetChatMessagesQuery,
+    filter: GetChatMessagesFilter,
+  ) {
+    if (chatId == 'all') {
+      throw new NotImplementedByEngineError(
+        "Can not get messages from 'all' in WEBJS",
+      );
+    }
+
+    const limit = query.limit;
+    const downloadMedia = query.downloadMedia;
+    // Test there's chat with id
+    await this.whatsapp.getChatById(this.ensureSuffix(chatId));
+    const pagination: PaginationParams = query;
+    const messages = await this.whatsapp.getMessages(
       this.ensureSuffix(chatId),
+      filter,
+      pagination,
     );
-    const messages = await chat.fetchMessages({
-      limit: limit,
-    });
     const promises = [];
     for (const msg of messages) {
       promises.push(this.processIncomingMessage(msg, downloadMedia));

src/core/exceptions.ts

@@ -6,7 +6,7 @@ export class NotImplementedByEngineError extends UnprocessableEntityException {
   constructor(msg = '') {
     let error = 'The method is not implemented by the engine.';
     if (msg) {
-      error += ` ${msg}`;
+      error = `${msg} ${error}`;
     }
     super(error);
   }

src/structures/chats.dto.ts

@@ -2,23 +2,61 @@ import { ApiProperty } from '@nestjs/swagger';
 import { BooleanString } from '@waha/nestjs/validation/BooleanString';
 import { PaginationParams } from '@waha/structures/pagination.dto';
 import { ChatIdProperty } from '@waha/structures/properties.dto';
-import { Transform } from 'class-transformer';
-import { IsBoolean, IsEnum, IsNotEmpty, IsOptional } from 'class-validator';
-
-import { SessionQuery, WHATSAPP_DEFAULT_SESSION_NAME } from './base.dto';
+import { Transform, Type } from 'class-transformer';
+import { IsBoolean, IsEnum, IsNumber, IsOptional } from 'class-validator';
 
 /**
  * Queries
  */
 
-export class GetChatMessagesQuery extends SessionQuery {
+export class GetChatMessagesFilter {
+  @ApiProperty({
+    required: false,
+    description: 'Filter messages before this timestamp (inclusive)',
+  })
+  @IsNumber()
+  @IsOptional()
+  @Type(() => Number)
+  'filter.timestamp.lte'?: number;
+
+  @ApiProperty({
+    required: false,
+    description: 'Filter messages after this timestamp (inclusive)',
+  })
+  @IsNumber()
+  @IsOptional()
+  @Type(() => Number)
+  'filter.timestamp.gte'?: number;
+
+  @ApiProperty({
+    required: false,
+    description: 'From me filter (by default shows all messages)',
+  })
+  @Transform(BooleanString)
+  @IsBoolean()
+  @IsOptional()
+  'filter.fromMe'?: boolean;
+}
+
+export class GetChatMessagesQuery {
+  @IsNumber()
+  @IsOptional()
+  @Type(() => Number)
   limit: number = 100;
 
+  @IsNumber()
+  @IsOptional()
+  @Type(() => Number)
+  offset?: number;
+
   @ApiProperty({
     example: true,
     required: false,
     description: 'Download media for messages',
   })
+  @Transform(BooleanString)
+  @IsBoolean()
+  @IsOptional()
   downloadMedia: boolean = true;
 }
 

src/structures/chatting.dto.ts

@@ -4,9 +4,14 @@ import {
   ApiProperty,
   getSchemaPath,
 } from '@nestjs/swagger';
-import { IsNumber, IsString } from 'class-validator';
+import { GetChatMessagesQuery } from '@waha/structures/chats.dto';
+import { IsNotEmpty, IsNumber, IsString } from 'class-validator';
 
-import { SessionBaseRequest, SessionQuery } from './base.dto';
+import {
+  SessionBaseRequest,
+  SessionQuery,
+  WHATSAPP_DEFAULT_SESSION_NAME,
+} from './base.dto';
 import {
   BinaryFile,
   RemoteFile,
@@ -42,16 +47,12 @@ export class ChatQuery extends SessionQuery {
   chatId: string;
 }
 
-export class GetMessageQuery extends ChatQuery {
-  @IsNumber()
-  limit: number;
+export class GetMessageQuery extends GetChatMessagesQuery {
+  @IsNotEmpty()
+  session: string = WHATSAPP_DEFAULT_SESSION_NAME;
 
-  @ApiProperty({
-    example: true,
-    required: false,
-    description: 'Download media for messages',
-  })
-  downloadMedia: true;
+  @ChatIdProperty()
+  chatId: string;
 }
 
 export class GetPresenceQuery extends ChatQuery {}