From 809b3eecec92d96fcfbe833370997a51483b74eb Mon Sep 17 00:00:00 2001 From: Mohammed S Date: Thu, 9 Nov 2023 19:05:44 +0530 Subject: [PATCH 1/2] fix: optimization fixes --- .../lib/chat/getAllGroupMembersPublicKeys.ts | 30 +++++ .../src/lib/chat/getChatMemberCount.ts | 2 +- .../restapi/src/lib/chat/getGroupMembers.ts | 2 +- .../src/lib/chat/getGroupMembersPublicKeys.ts | 38 ++++++ .../restapi/src/lib/chat/helpers/validator.ts | 4 +- packages/restapi/src/lib/chat/index.ts | 1 + packages/restapi/src/lib/types/index.ts | 1 + .../tests/lib/benchmark/privateGroup.test.ts | 109 ++++++++++++++++-- 8 files changed, 172 insertions(+), 15 deletions(-) create mode 100644 packages/restapi/src/lib/chat/getAllGroupMembersPublicKeys.ts create mode 100644 packages/restapi/src/lib/chat/getGroupMembersPublicKeys.ts diff --git a/packages/restapi/src/lib/chat/getAllGroupMembersPublicKeys.ts b/packages/restapi/src/lib/chat/getAllGroupMembersPublicKeys.ts new file mode 100644 index 000000000..9ccf44db3 --- /dev/null +++ b/packages/restapi/src/lib/chat/getAllGroupMembersPublicKeys.ts @@ -0,0 +1,30 @@ +import { getGroupMembers } from './getGroupMembers'; +import { getChatMemberCount } from './getChatMemberCount'; +import { ChatMemberProfile, EnvOptionsType } from '../types'; +import { getGroupMembersPublicKeys } from './getGroupMembersPublicKeys'; + +export const getAllGroupMembersPublicKeys = async (options: { + chatId: string; + env: EnvOptionsType['env']; +}): Promise => { + const { chatId, env } = options; + const count = await getChatMemberCount({ chatId, env }); + const overallCount = count.approvedCount; + const limit = 5000; + const totalPages = Math.ceil(overallCount / limit); + const pageNumbers = Array.from({ length: totalPages }, (_, i) => i + 1); + const groupMembers: ChatMemberProfile[] = []; + + const memberFetchPromises = pageNumbers.map((page) => + getGroupMembersPublicKeys({ chatId, env, page, limit }) + ); + + const membersResults = await Promise.all(memberFetchPromises); + membersResults.forEach((result) => { + if (result.members.length > 0) { + groupMembers.push(...result.members); + } + }); + + return groupMembers; +}; diff --git a/packages/restapi/src/lib/chat/getChatMemberCount.ts b/packages/restapi/src/lib/chat/getChatMemberCount.ts index 13ea9c82e..b7902033c 100644 --- a/packages/restapi/src/lib/chat/getChatMemberCount.ts +++ b/packages/restapi/src/lib/chat/getChatMemberCount.ts @@ -23,7 +23,7 @@ export const getChatMemberCount = async ( } const API_BASE_URL = getAPIBaseUrls(env); - const requestUrl = `${API_BASE_URL}/v1/chat/${chatId}/members/count`; + const requestUrl = `${API_BASE_URL}/v1/chat/groups/${chatId}/members/count`; const response = await axios.get(requestUrl); const { totalMembersCount } = response.data; diff --git a/packages/restapi/src/lib/chat/getGroupMembers.ts b/packages/restapi/src/lib/chat/getGroupMembers.ts index 3daa7e556..2f245d203 100644 --- a/packages/restapi/src/lib/chat/getGroupMembers.ts +++ b/packages/restapi/src/lib/chat/getGroupMembers.ts @@ -28,7 +28,7 @@ export const getGroupMembers = async ( } const API_BASE_URL = getAPIBaseUrls(env); - const requestUrl = `${API_BASE_URL}/v1/chat/${chatId}/members?pageNumber=${page}&pageSize=${limit}`; + const requestUrl = `${API_BASE_URL}/v1/chat/groups/${chatId}/members?pageNumber=${page}&pageSize=${limit}`; const response = await axios.get(requestUrl); return response.data; diff --git a/packages/restapi/src/lib/chat/getGroupMembersPublicKeys.ts b/packages/restapi/src/lib/chat/getGroupMembersPublicKeys.ts new file mode 100644 index 000000000..c887e26f8 --- /dev/null +++ b/packages/restapi/src/lib/chat/getGroupMembersPublicKeys.ts @@ -0,0 +1,38 @@ +import axios from 'axios'; +import { getAPIBaseUrls } from '../helpers'; +import Constants, { ENV } from '../constants'; +import { ChatMemberCounts, ChatMemberProfile } from '../types'; +import { FetchChatGroupInfoType } from './getGroupMembers'; + +/** + * GET /v1/chat/:chatId/members/public/keys + */ + +export const getGroupMembersPublicKeys = async ( + options: FetchChatGroupInfoType +): Promise<{ + totalMembersCount: ChatMemberCounts; + members: ChatMemberProfile[]; +}> => { + const { chatId, page = 1, limit = 20, env = Constants.ENV.PROD } = options; + + try { + if (!chatId) { + throw new Error('Chat ID is required.'); + } + + const API_BASE_URL = getAPIBaseUrls(env); + const requestUrl = `${API_BASE_URL}/v1/chat/groups/${chatId}/members/public/keys?pageNumber=${page}&pageSize=${limit}`; + + const response = await axios.get(requestUrl); + return response.data; + } catch (error) { + console.error( + `[Push SDK] - API - Error - API ${getGroupMembersPublicKeys.name} -: `, + error + ); + throw new Error( + `[Push SDK] - API - Error - API ${getGroupMembersPublicKeys.name} -: ${error}` + ); + } +}; diff --git a/packages/restapi/src/lib/chat/helpers/validator.ts b/packages/restapi/src/lib/chat/helpers/validator.ts index b49f1220a..3f60df430 100644 --- a/packages/restapi/src/lib/chat/helpers/validator.ts +++ b/packages/restapi/src/lib/chat/helpers/validator.ts @@ -227,9 +227,7 @@ export const validateGroupMemberUpdateOptions = ( `Invalid role: ${role}. Allowed roles are ${allowedRoles.join(', ')}.` ); } - if (upsert[role] && upsert[role].length > 100) { - throw new Error(`${role} array cannot have more than 100 addresses.`); - } + // Assuming you have a function `isValidETHAddress` to validate Ethereum addresses upsert[role].forEach((address) => { if (!isValidETHAddress(address)) { diff --git a/packages/restapi/src/lib/chat/index.ts b/packages/restapi/src/lib/chat/index.ts index ab6a4580e..8090dad2d 100644 --- a/packages/restapi/src/lib/chat/index.ts +++ b/packages/restapi/src/lib/chat/index.ts @@ -26,3 +26,4 @@ export * from './getGroupMemberStatus'; export * from './getGroupMembers'; export * from './getGroupInfo'; export * from './getChatMemberCount'; +export * from './getGroupMembersPublicKeys'; diff --git a/packages/restapi/src/lib/types/index.ts b/packages/restapi/src/lib/types/index.ts index c6d032f3c..3438d8292 100644 --- a/packages/restapi/src/lib/types/index.ts +++ b/packages/restapi/src/lib/types/index.ts @@ -373,6 +373,7 @@ export interface ChatMemberCounts { adminsCount: number; membersCount: number; pendingCount: number; + approvedCount: number; } export interface ChatMemberProfile { diff --git a/packages/restapi/tests/lib/benchmark/privateGroup.test.ts b/packages/restapi/tests/lib/benchmark/privateGroup.test.ts index c7b6bdfd8..2b8060559 100644 --- a/packages/restapi/tests/lib/benchmark/privateGroup.test.ts +++ b/packages/restapi/tests/lib/benchmark/privateGroup.test.ts @@ -9,7 +9,7 @@ const _env = Constants.ENV.LOCAL; * THIS TEST GROUP IS FOR BENCHMARKING SEND MESSAGE FOR PRIVATE GROUP * These tests will be skipped */ -describe.only('Private Groups', () => { +describe('Private Groups', () => { let account: string; let account2: string; let userAlice: PushAPI; @@ -341,16 +341,21 @@ describe.only('Private Groups', () => { console.log('Duration in ms : ', duration); }); it('5000 Members', async () => { + await createGroupAndSendMessages(userAlice, 5000); + }); + it('10000 Members', async () => { + await createGroupAndSendMessages(userAlice, 10000); + }); + it('15000 Members', async () => { + await createGroupAndSendMessages(userAlice, 15000); + }); + }); + + describe('Update Group with Pending members', () => { + it.only('10 Members', async () => { const chatId = - '33c9295913786a8c446ceca46af8ee29a3a7144ba63071c24e5f05a5407bccdf'; - const startTime = new Date(); - await userAlice.chat.send(chatId, { - content: 'Sending Message to Private Grp', - type: 'Text', - }); - const endTime = new Date(); - const duration = endTime.getTime() - startTime.getTime(); - console.log('Duration in ms : ', duration); + 'd8892a41ccbb7d0c627d1e3976f3a0bd64540d1d535b1a339680f2ce5b0fbcf0'; + await updateGroupWithPendingMembers(userAlice, chatId, 500); }); }); @@ -465,6 +470,90 @@ const createGroupWithPendingMembers = async ( return createdGroup.chatId; }; + +/** + * CREATE GROUP WITH GIVEN MEMBERS COUNT PENDING MEMBERS + * @dev - Added members are pending members + */ +const updateGroupWithPendingMembers = async ( + user: PushAPI, + chatId: string, + memberCount: number +): Promise => { + /** + * STEP 1: Generate ENOUGH USERS + */ + console.log('Generating Users'); + const users = await generateUsers(memberCount); + + /** + * STEP 2: Add Members to Group + * Note - At max 100 members can be added at once + */ + console.log('Adding Members to Group'); + let currentMemberCount = 1; + while (currentMemberCount < memberCount) { + const currentUsersIndex = currentMemberCount - 1; + if (currentMemberCount + 100 > memberCount) { + currentMemberCount = memberCount; + } else { + currentMemberCount += 100; + } + const nextUsersIndex = currentMemberCount - 1; + + const membersToBeAdded = []; + for (let i = currentUsersIndex; i <= nextUsersIndex; i++) { + membersToBeAdded.push(users[i]); + } + await user.chat.group.add(chatId, { + role: 'MEMBER', + accounts: membersToBeAdded, + }); + } + console.log('Added Members to Group : ', currentMemberCount); + return chatId; +}; + +const generateUsers = async (memberCount: number): Promise => { + let users: string[] = []; // Now 'users' is explicitly typed as an array of strings + let generationCount = 0; + const batchSize = 20; + + while (generationCount < memberCount) { + const userPromises: Promise[] = []; // An array to hold the promises which will resolve to strings + for (let i = 0; i < batchSize && generationCount < memberCount; i++) { + userPromises.push( + (async () => { + const WALLET = ethers.Wallet.createRandom(); + const signer = new ethers.Wallet(WALLET.privateKey); + const account = `eip155:${signer.address}`; + // Assume that PushAPI.initialize resolves successfully and you don't need anything from the resolved value + await PushAPI.initialize(signer, { + env: _env, + streamOptions: { enabled: false }, + }); + return account; // This resolves to a string + })() + ); + generationCount++; + } + + // Wait for all promises in the batch to resolve, and then spread their results into the 'users' array + const batchResults = await Promise.all(userPromises); + users = [...users, ...batchResults]; + + if (generationCount % 100 == 0) { + console.log('Generated Users : ', generationCount); + } + } + + console.log( + `User Generation Completed, users generated : ${generationCount}` + ); + return users; // 'users' is an array of strings representing accounts +}; + + /** * CREATE GROUP WITH GIVEN MEMBERS COUNT NON-PENDING MEMBERS * @dev - Added members are pending members From f3c3d0cee160cda3316e1286c276a4caed4b5505 Mon Sep 17 00:00:00 2001 From: aman035 Date: Thu, 9 Nov 2023 20:12:53 +0530 Subject: [PATCH 2/2] fix: minor issues --- .../restapi/src/lib/chat/approveRequest.ts | 11 +++++-- .../lib/chat/getAllGroupMembersPublicKeys.ts | 7 ++--- .../src/lib/chat/getGroupMembersPublicKeys.ts | 5 +--- .../restapi/src/lib/chat/helpers/validator.ts | 10 +++++-- .../src/lib/chat/updateGroupMembers.ts | 16 ++++------ .../tests/lib/benchmark/privateGroup.test.ts | 29 ++++++++++--------- 6 files changed, 40 insertions(+), 38 deletions(-) diff --git a/packages/restapi/src/lib/chat/approveRequest.ts b/packages/restapi/src/lib/chat/approveRequest.ts index 4fe9958f5..b8a59fb9e 100644 --- a/packages/restapi/src/lib/chat/approveRequest.ts +++ b/packages/restapi/src/lib/chat/approveRequest.ts @@ -13,6 +13,8 @@ import { import * as CryptoJS from 'crypto-js'; import { getGroup } from './getGroup'; import * as AES from '../chat/helpers/aes'; +import { getGroupInfo } from './getGroupInfo'; +import { getAllGroupMembersPublicKeys } from './getAllGroupMembersPublicKeys'; export interface ApproveRequestOptionsType extends EnvOptionsType { /** @@ -88,13 +90,18 @@ export const approveCore = async ( // pgpv2 is used for private grps let sigType: 'pgp' | 'pgpv2' = 'pgp'; if (isGroup) { - const group = await getGroup({ chatId: senderAddress, env }); + const group = await getGroupInfo({ chatId: senderAddress, env }); if (group && !group.isPublic) { sigType = 'pgpv2'; const secretKey = AES.generateRandomSecret(15); + + const groupMembers = await getAllGroupMembersPublicKeys({ + chatId: group.chatId, + env, + }); // Encrypt secret key with group members public keys - const publicKeys: string[] = group.members.map( + const publicKeys: string[] = groupMembers.map( (member) => member.publicKey ); publicKeys.push(connectedUser.publicKey); diff --git a/packages/restapi/src/lib/chat/getAllGroupMembersPublicKeys.ts b/packages/restapi/src/lib/chat/getAllGroupMembersPublicKeys.ts index 9ccf44db3..a4a66f32b 100644 --- a/packages/restapi/src/lib/chat/getAllGroupMembersPublicKeys.ts +++ b/packages/restapi/src/lib/chat/getAllGroupMembersPublicKeys.ts @@ -1,19 +1,18 @@ -import { getGroupMembers } from './getGroupMembers'; import { getChatMemberCount } from './getChatMemberCount'; -import { ChatMemberProfile, EnvOptionsType } from '../types'; +import { EnvOptionsType } from '../types'; import { getGroupMembersPublicKeys } from './getGroupMembersPublicKeys'; export const getAllGroupMembersPublicKeys = async (options: { chatId: string; env: EnvOptionsType['env']; -}): Promise => { +}): Promise<{ did: string; publicKey: string }[]> => { const { chatId, env } = options; const count = await getChatMemberCount({ chatId, env }); const overallCount = count.approvedCount; const limit = 5000; const totalPages = Math.ceil(overallCount / limit); const pageNumbers = Array.from({ length: totalPages }, (_, i) => i + 1); - const groupMembers: ChatMemberProfile[] = []; + const groupMembers: { did: string; publicKey: string }[] = []; const memberFetchPromises = pageNumbers.map((page) => getGroupMembersPublicKeys({ chatId, env, page, limit }) diff --git a/packages/restapi/src/lib/chat/getGroupMembersPublicKeys.ts b/packages/restapi/src/lib/chat/getGroupMembersPublicKeys.ts index c887e26f8..6053f331e 100644 --- a/packages/restapi/src/lib/chat/getGroupMembersPublicKeys.ts +++ b/packages/restapi/src/lib/chat/getGroupMembersPublicKeys.ts @@ -10,10 +10,7 @@ import { FetchChatGroupInfoType } from './getGroupMembers'; export const getGroupMembersPublicKeys = async ( options: FetchChatGroupInfoType -): Promise<{ - totalMembersCount: ChatMemberCounts; - members: ChatMemberProfile[]; -}> => { +): Promise<{ members: [{ did: string; publicKey: string }] }> => { const { chatId, page = 1, limit = 20, env = Constants.ENV.PROD } = options; try { diff --git a/packages/restapi/src/lib/chat/helpers/validator.ts b/packages/restapi/src/lib/chat/helpers/validator.ts index 3f60df430..78e4053eb 100644 --- a/packages/restapi/src/lib/chat/helpers/validator.ts +++ b/packages/restapi/src/lib/chat/helpers/validator.ts @@ -227,7 +227,11 @@ export const validateGroupMemberUpdateOptions = ( `Invalid role: ${role}. Allowed roles are ${allowedRoles.join(', ')}.` ); } - + + if (upsert[role] && upsert[role].length > 1000) { + throw new Error(`${role} array cannot have more than 1000 addresses.`); + } + // Assuming you have a function `isValidETHAddress` to validate Ethereum addresses upsert[role].forEach((address) => { if (!isValidETHAddress(address)) { @@ -237,8 +241,8 @@ export const validateGroupMemberUpdateOptions = ( }); // Validating remove array - if (remove && remove.length > 100) { - throw new Error('Remove array cannot have more than 100 addresses.'); + if (remove && remove.length > 1000) { + throw new Error('Remove array cannot have more than 1000 addresses.'); } remove.forEach((address) => { if (!isValidETHAddress(address)) { diff --git a/packages/restapi/src/lib/chat/updateGroupMembers.ts b/packages/restapi/src/lib/chat/updateGroupMembers.ts index e1d97b31c..49ef6e131 100644 --- a/packages/restapi/src/lib/chat/updateGroupMembers.ts +++ b/packages/restapi/src/lib/chat/updateGroupMembers.ts @@ -14,7 +14,7 @@ import { EnvOptionsType, GroupInfoDTO, SignerType } from '../types'; import { getGroupInfo } from './getGroupInfo'; import { getGroupMemberStatus } from './getGroupMemberStatus'; import * as AES from '../chat/helpers/aes'; -import { getAllGroupMembers } from './getAllGroupMembers'; +import { getAllGroupMembersPublicKeys } from './getAllGroupMembersPublicKeys'; export interface GroupMemberUpdateOptions extends EnvOptionsType { chatId: string; @@ -79,7 +79,7 @@ export const updateGroupMembers = async ( env, }); - const groupMembers = await getAllGroupMembers({ chatId, env }); + const groupMembers = await getAllGroupMembersPublicKeys({ chatId, env }); const removeParticipantSet = new Set( convertedRemove.map((participant) => participant.toLowerCase()) @@ -87,10 +87,7 @@ export const updateGroupMembers = async ( let sameMembers = true; groupMembers.map((element) => { - if ( - element.intent && - removeParticipantSet.has(element.address.toLowerCase()) - ) { + if (removeParticipantSet.has(element.did.toLowerCase())) { sameMembers = false; } }); @@ -101,11 +98,8 @@ export const updateGroupMembers = async ( const publicKeys: string[] = []; // This will now only take keys of non-removed members groupMembers.map((element) => { - if ( - element.intent && - !removeParticipantSet.has(element.address.toLowerCase()) - ) { - publicKeys.push(element.userInfo.publicKey as string); + if (!removeParticipantSet.has(element.did.toLowerCase())) { + publicKeys.push(element.publicKey as string); } }); diff --git a/packages/restapi/tests/lib/benchmark/privateGroup.test.ts b/packages/restapi/tests/lib/benchmark/privateGroup.test.ts index 2b8060559..d83b86d4d 100644 --- a/packages/restapi/tests/lib/benchmark/privateGroup.test.ts +++ b/packages/restapi/tests/lib/benchmark/privateGroup.test.ts @@ -9,7 +9,7 @@ const _env = Constants.ENV.LOCAL; * THIS TEST GROUP IS FOR BENCHMARKING SEND MESSAGE FOR PRIVATE GROUP * These tests will be skipped */ -describe('Private Groups', () => { +describe.skip('Private Groups', () => { let account: string; let account2: string; let userAlice: PushAPI; @@ -180,7 +180,7 @@ describe('Private Groups', () => { * STEP 3 - AUTOJOIN * This is imp for generating session keys so , do skip this test */ - describe.skip('Private Group AutoJoin', () => { + describe('Private Group AutoJoin', () => { it('10 Members', async () => { const chatId = '9e8bea378b4e4860956c177146786c2e96a0db8aa7c4156299181b3e56290a57'; @@ -341,18 +341,21 @@ describe('Private Groups', () => { console.log('Duration in ms : ', duration); }); it('5000 Members', async () => { - await createGroupAndSendMessages(userAlice, 5000); - }); - it('10000 Members', async () => { - await createGroupAndSendMessages(userAlice, 10000); - }); - it('15000 Members', async () => { - await createGroupAndSendMessages(userAlice, 15000); + const chatId = + '33c9295913786a8c446ceca46af8ee29a3a7144ba63071c24e5f05a5407bccdf'; + const startTime = new Date(); + await userAlice.chat.send(chatId, { + content: 'Sending Message to Private Grp', + type: 'Text', + }); + const endTime = new Date(); + const duration = endTime.getTime() - startTime.getTime(); + console.log('Duration in ms : ', duration); }); }); - describe('Update Group with Pending members', () => { - it.only('10 Members', async () => { + describe.skip('Update Group with Pending members', () => { + it('10 Members', async () => { const chatId = 'd8892a41ccbb7d0c627d1e3976f3a0bd64540d1d535b1a339680f2ce5b0fbcf0'; await updateGroupWithPendingMembers(userAlice, chatId, 500); @@ -470,7 +473,6 @@ const createGroupWithPendingMembers = async ( return createdGroup.chatId; }; - /** * CREATE GROUP WITH GIVEN MEMBERS COUNT PENDING MEMBERS * @dev - Added members are pending members @@ -484,7 +486,7 @@ const updateGroupWithPendingMembers = async ( * STEP 1: Generate ENOUGH USERS */ console.log('Generating Users'); - const users = await generateUsers(memberCount); + const users = await generateUsers(memberCount); /** * STEP 2: Add Members to Group @@ -553,7 +555,6 @@ const generateUsers = async (memberCount: number): Promise => { return users; // 'users' is an array of strings representing accounts }; - /** * CREATE GROUP WITH GIVEN MEMBERS COUNT NON-PENDING MEMBERS * @dev - Added members are pending members