From 717fbc3383d71adb69ecc9200cd1f4f84aff5083 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Tue, 17 Dec 2024 22:41:38 +0800 Subject: [PATCH 01/45] feat: cio optimization --- package-lock.json | 79 ++++++++- package.json | 1 + src/cio.ts | 78 ++++++--- src/common/async.ts | 23 +++ src/cron/validateActiveUsers.ts | 159 ++++++++++++++++++ src/entity/user/User.ts | 3 + .../1734294820179-UserCioRegistered.ts | 19 +++ 7 files changed, 333 insertions(+), 29 deletions(-) create mode 100644 src/common/async.ts create mode 100644 src/cron/validateActiveUsers.ts create mode 100644 src/migration/1734294820179-UserCioRegistered.ts diff --git a/package-lock.json b/package-lock.json index 912101b9a..545f1cc59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@fastify/http-proxy": "^10.0.1", "@fastify/rate-limit": "^10.1.1", "@fastify/websocket": "^11.0.1", + "@google-cloud/bigquery": "^7.9.1", "@google-cloud/opentelemetry-resource-util": "^2.4.0", "@google-cloud/pubsub": "^4.8.0", "@google-cloud/storage": "^7.13.0", @@ -1328,6 +1329,58 @@ "ws": "^8.16.0" } }, + "node_modules/@google-cloud/bigquery": { + "version": "7.9.1", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.9.1.tgz", + "integrity": "sha512-ZkcRMpBoFLxIh6TiQBywA22yT3c2j0f07AHWEMjtYqMQzZQbFrpxuJU2COp3tyjZ91ZIGHe4gY7/dGZL88cltg==", + "dependencies": { + "@google-cloud/common": "^5.0.0", + "@google-cloud/paginator": "^5.0.2", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/bigquery/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@google-cloud/common": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", + "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", + "dependencies": { + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "extend": "^3.0.2", + "google-auth-library": "^9.0.0", + "html-entities": "^2.5.2", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@google-cloud/opentelemetry-resource-util": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@google-cloud/opentelemetry-resource-util/-/opentelemetry-resource-util-2.4.0.tgz", @@ -1344,9 +1397,9 @@ } }, "node_modules/@google-cloud/paginator": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.0.tgz", - "integrity": "sha512-87aeg6QQcEPxGCOthnpUjvw4xAZ57G7pL8FS0C4e/81fr3FjkpUpibf1s2v5XGyGhUVGF4Jfg7yEcxqn2iUw1w==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", + "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", "dependencies": { "arrify": "^2.0.0", "extend": "^3.0.2" @@ -4769,6 +4822,18 @@ "node": ">=0.6" } }, + "node_modules/big.js": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", + "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" + } + }, "node_modules/bignumber.js": { "version": "9.1.2", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", @@ -7714,6 +7779,14 @@ "node": ">= 0.10" } }, + "node_modules/is": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", + "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==", + "engines": { + "node": "*" + } + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", diff --git a/package.json b/package.json index 9cc3bf042..34c373a32 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@fastify/http-proxy": "^10.0.1", "@fastify/rate-limit": "^10.1.1", "@fastify/websocket": "^11.0.1", + "@google-cloud/bigquery": "^7.9.1", "@google-cloud/opentelemetry-resource-util": "^2.4.0", "@google-cloud/pubsub": "^4.8.0", "@google-cloud/storage": "^7.13.0", diff --git a/src/cio.ts b/src/cio.ts index 9b657b484..3fa706cfd 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -1,6 +1,7 @@ import { CustomerIORequestError, TrackClient } from 'customerio-node'; import { ChangeObject } from './types'; import { + ConnectionManager, User, UserPersonalizedDigest, UserPersonalizedDigestType, @@ -89,15 +90,26 @@ export async function identifyUserStreak({ } } -export async function identifyUser({ - con, - cio, - user, -}: { - con: DataSource; - cio: TrackClient; - user: ChangeObject; -}): Promise { +export const generateIdentifyObject = ( + con: ConnectionManager, + user: ChangeObject, +) => { + const { id } = user; + const changed = JSON.parse(JSON.stringify(user)); + const identify = generateIdentifyAttributes(con, changed); + + return { + action: 'identify', + type: 'person', + identifiers: { id }, + attributes: identify, + }; +}; + +export const generateIdentifyAttributes = async ( + con: ConnectionManager, + user: ChangeObject, +) => { const dup = { ...user }; const id = dup.id; for (const field of OMIT_FIELDS) { @@ -115,24 +127,38 @@ export async function identifyUser({ }), ]); + return { + ...camelCaseToSnakeCase(dup), + first_name: getFirstName(dup.name), + created_at: dateToCioTimestamp(debeziumTimeToDate(dup.createdAt)), + updated_at: dup.updatedAt + ? dateToCioTimestamp(debeziumTimeToDate(dup.updatedAt)) + : undefined, + referral_link: genericInviteURL, + [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Marketing}`]: + user.acceptedMarketing, + [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Notifications}`]: + user.notificationEmail, + [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Digest}`]: + !!personalizedDigest, + [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Follow}`]: + user.followingEmail, + }; +}; + +export async function identifyUser({ + con, + cio, + user, +}: { + con: DataSource; + cio: TrackClient; + user: ChangeObject; +}): Promise { + const data = generateIdentifyAttributes(con, user); + try { - await cio.identify(id, { - ...camelCaseToSnakeCase(dup), - first_name: getFirstName(dup.name), - created_at: dateToCioTimestamp(debeziumTimeToDate(dup.createdAt)), - updated_at: dup.updatedAt - ? dateToCioTimestamp(debeziumTimeToDate(dup.updatedAt)) - : undefined, - referral_link: genericInviteURL, - [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Marketing}`]: - user.acceptedMarketing, - [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Notifications}`]: - user.notificationEmail, - [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Digest}`]: - !!personalizedDigest, - [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Follow}`]: - user.followingEmail, - }); + await cio.identify(user.id, data); } catch (err) { if (err instanceof CustomerIORequestError && err.statusCode === 400) { logger.warn({ err, user }, 'failed to update user in cio'); diff --git a/src/common/async.ts b/src/common/async.ts new file mode 100644 index 000000000..055600a80 --- /dev/null +++ b/src/common/async.ts @@ -0,0 +1,23 @@ +const DEFAULT_BATCH_LIMIT = 40_000; // postgresql limit is around 65k, to be safe, let's run at 40k. + +type ForceStop = boolean; + +interface BlockingBatchRunnerOptions { + data: T[]; + batchLimit?: number; + runner: (current: T[]) => Promise; +} + +export const blockingBatchRunner = async ({ + batchLimit = DEFAULT_BATCH_LIMIT, + data, + runner, +}: BlockingBatchRunnerOptions) => { + for (let i = 0; i < data.length; i += batchLimit) { + const current = data.slice(i, i + batchLimit); + const shouldStop = await runner(current); + if (shouldStop) { + break; + } + } +}; diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts new file mode 100644 index 000000000..d08144836 --- /dev/null +++ b/src/cron/validateActiveUsers.ts @@ -0,0 +1,159 @@ +import { Cron } from './cron'; +import { + User, + UserPersonalizedDigest, + UserPersonalizedDigestSendType, +} from '../entity'; +import { In } from 'typeorm'; +import { BigQuery } from '@google-cloud/bigquery'; +import { blockingBatchRunner } from '../common/async'; +import { setTimeout } from 'node:timers/promises'; +import { cio, generateIdentifyObject } from '../cio'; +import { updateFlagsStatement } from '../common'; + +enum ActiveState { + SixWeeksAgo = 'six_weeks_ago', + TwelveWeeksAgo = 'twelve_weeks_ago', + Active = 'active', +} + +interface UserBq { + state: string; + id: string; +} + +const ITEMS_PER_DESTROY = 4000; +const ITEMS_PER_IDENTIFY = 250; + +const cron: Cron = { + name: 'validate-active-users', + handler: async (con) => { + const bigquery = new BigQuery(); + const ds = bigquery.dataset('id'); + const [usersFromBq] = await ds.table('table id').get(); // replace with actual fetch + const inactiveUsers: string[] = []; + const downgradeUsers: string[] = []; + const reactivateUsers: string[] = []; + + // sort users from bq into active, inactive, downgrade, and reactivate + for (const user of usersFromBq) { + if (user.state === ActiveState.SixWeeksAgo) { + inactiveUsers.push(user.id); + } else if (user.state === ActiveState.TwelveWeeksAgo) { + downgradeUsers.push(user.id); + } else if (user.state === ActiveState.Active) { + reactivateUsers.push(user.id); + } + } + + // update users in db: reactivated + await blockingBatchRunner({ + data: reactivateUsers, + runner: async (current) => { + const validReactivateUsers = await con.getRepository(User).find({ + select: ['id'], + where: { id: In(current), cioRegistered: false }, + }); + + if (validReactivateUsers.length === 0) { + return true; + } + + await blockingBatchRunner({ + batchLimit: ITEMS_PER_IDENTIFY, + data: validReactivateUsers.map((u) => ({ id: u.id })), + runner: async (ids) => { + const users = await con + .getRepository(User) + .find({ where: { id: In(ids.map(({ id }) => id)) } }); + + const data = users.map((user) => + generateIdentifyObject(con, JSON.parse(JSON.stringify(user))), + ); + + await cio.request.post('/users', { batch: data }); + + await con + .getRepository(User) + .update({ id: In(ids) }, { cioRegistered: true }); + + await setTimeout(20); // wait for a bit to avoid rate limiting + }, + }); + + await con + .getRepository(User) + .update( + { id: In(validReactivateUsers.map((u) => u.id)) }, + { cioRegistered: true }, + ); + + await setTimeout(200); + }, + }); + + // inactive for 12 weeks: remove from CIO + await blockingBatchRunner({ + data: inactiveUsers, + runner: async (current) => { + const validInactiveUsers = await con.getRepository(User).find({ + select: ['id'], + where: { id: In(current), cioRegistered: true }, + }); + + if (validInactiveUsers.length === 0) { + return true; + } + + await blockingBatchRunner({ + batchLimit: ITEMS_PER_DESTROY, + data: validInactiveUsers.map((u) => ({ id: u.id })), + runner: async (ids) => { + const data = ids.map((id) => ({ + action: 'destroy', + type: 'person', + identifiers: { id }, + })); + + await cio.request.post('/users', data); + + await con + .getRepository(User) + .update({ id: In(ids) }, { cioRegistered: false }); + + await setTimeout(20); + }, + }); + }, + }); + + // inactive for 6 weeks: downgrade from daily to weekly digest + await blockingBatchRunner({ + data: downgradeUsers, + runner: async (current) => { + const validDowngradeUsers = await con + .getRepository(User) + .createQueryBuilder('u') + .select('id') + .innerJoin(UserPersonalizedDigest, 'upd', 'u.id = upd."userId"') + .where('u.id IN (:...ids)', { ids: current }) + .andWhere(`upd.flags->>'sendType' = 'daily'`) + .getRawMany>(); + + // set digest to weekly on Wednesday 9am + await con.getRepository(UserPersonalizedDigest).update( + { userId: In(validDowngradeUsers.map(({ id }) => id)) }, + { + preferredDay: 3, + preferredHour: 9, + flags: updateFlagsStatement({ + sendType: UserPersonalizedDigestSendType.weekly, + }), + }, + ); + }, + }); + }, +}; + +export default cron; diff --git a/src/entity/user/User.ts b/src/entity/user/User.ts index a88844582..482434714 100644 --- a/src/entity/user/User.ts +++ b/src/entity/user/User.ts @@ -131,6 +131,9 @@ export class User { @Column({ default: false }) devcardEligible: boolean; + @Column({ default: true }) + cioRegistered: boolean; + @Column({ type: 'text', nullable: true, default: DEFAULT_TIMEZONE }) timezone?: string; diff --git a/src/migration/1734294820179-UserCioRegistered.ts b/src/migration/1734294820179-UserCioRegistered.ts new file mode 100644 index 000000000..0bfdd0b4f --- /dev/null +++ b/src/migration/1734294820179-UserCioRegistered.ts @@ -0,0 +1,19 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class UserCioRegistered1734294820179 implements MigrationInterface { + name = 'UserCioRegistered1734294820179'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "user" ADD "cioRegistered" boolean NOT NULL DEFAULT true`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_user_cioRegistered" ON "user" ("cioRegistered") `, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_user_cioRegistered"`); + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "cioRegistered"`); + } +} From 92971dacbf551a29067ee890dc9acb067107b9ab Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 15:18:58 +0800 Subject: [PATCH 02/45] feat: retry policy --- src/common/async.ts | 33 ++++++++++++++++++++++++++++++++- src/cron/validateActiveUsers.ts | 10 +++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/common/async.ts b/src/common/async.ts index 055600a80..31be78aa4 100644 --- a/src/common/async.ts +++ b/src/common/async.ts @@ -1,4 +1,13 @@ -const DEFAULT_BATCH_LIMIT = 40_000; // postgresql limit is around 65k, to be safe, let's run at 40k. +import { + circuitBreaker, + ConsecutiveBreaker, + ExponentialBackoff, + handleAll, + retry, + wrap, +} from 'cockatiel'; + +const DEFAULT_BATCH_LIMIT = 40_000; // postgresql params limit is around 65k, to be safe, let's run at 40k. type ForceStop = boolean; @@ -21,3 +30,25 @@ export const blockingBatchRunner = async ({ } } }; + +export const callWithRetryDefault = (callback: () => Promise) => { + // Create a retry policy that'll try whatever function we execute 3 + // times with a randomized exponential backoff. + const retryPolicy = retry(handleAll, { + maxAttempts: 3, + backoff: new ExponentialBackoff(), + }); + + // Create a circuit breaker that'll stop calling the executed function for 10 + // seconds if it fails 5 times in a row. This can give time for e.g. a database + // to recover without getting tons of traffic. + const circuitBreakerPolicy = circuitBreaker(handleAll, { + halfOpenAfter: 10 * 1000, + breaker: new ConsecutiveBreaker(5), + }); + + // Combine these! Create a policy that retries 3 times, calling through the circuit breaker + const retryWithBreaker = wrap(retryPolicy, circuitBreakerPolicy); + + return retryWithBreaker.execute(callback); +}; diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index d08144836..bab970964 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -6,7 +6,7 @@ import { } from '../entity'; import { In } from 'typeorm'; import { BigQuery } from '@google-cloud/bigquery'; -import { blockingBatchRunner } from '../common/async'; +import { blockingBatchRunner, callWithRetryDefault } from '../common/async'; import { setTimeout } from 'node:timers/promises'; import { cio, generateIdentifyObject } from '../cio'; import { updateFlagsStatement } from '../common'; @@ -71,7 +71,9 @@ const cron: Cron = { generateIdentifyObject(con, JSON.parse(JSON.stringify(user))), ); - await cio.request.post('/users', { batch: data }); + await callWithRetryDefault(() => + cio.request.post('/users', { batch: data }), + ); await con .getRepository(User) @@ -115,7 +117,9 @@ const cron: Cron = { identifiers: { id }, })); - await cio.request.post('/users', data); + await callWithRetryDefault(() => + cio.request.post('/users', { batch: data }), + ); await con .getRepository(User) From bd96bc1e43bb79f873b7b23a20fb451d24b1e476 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 15:31:36 +0800 Subject: [PATCH 03/45] refactor: function name --- src/cio.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cio.ts b/src/cio.ts index 3fa706cfd..6c7038c56 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -96,7 +96,7 @@ export const generateIdentifyObject = ( ) => { const { id } = user; const changed = JSON.parse(JSON.stringify(user)); - const identify = generateIdentifyAttributes(con, changed); + const identify = getIdentifyAttributes(con, changed); return { action: 'identify', @@ -106,7 +106,7 @@ export const generateIdentifyObject = ( }; }; -export const generateIdentifyAttributes = async ( +export const getIdentifyAttributes = async ( con: ConnectionManager, user: ChangeObject, ) => { @@ -155,7 +155,7 @@ export async function identifyUser({ cio: TrackClient; user: ChangeObject; }): Promise { - const data = generateIdentifyAttributes(con, user); + const data = getIdentifyAttributes(con, user); try { await cio.identify(user.id, data); From 3b66822955056648104b76f17220c78acc8e3621 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 16:07:13 +0800 Subject: [PATCH 04/45] feat: bq query --- src/common/googleCloud.ts | 119 ++++++++++++++++++++++++++++++++ src/cron/validateActiveUsers.ts | 32 +-------- 2 files changed, 122 insertions(+), 29 deletions(-) diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index 3ee525459..1581090bc 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -1,6 +1,7 @@ import { Storage, DownloadOptions } from '@google-cloud/storage'; import { PropsParameters } from '../types'; import path from 'path'; +import { BigQuery } from '@google-cloud/bigquery'; export const downloadFile = async ({ url, @@ -29,3 +30,121 @@ export const downloadJsonFile = async ({ return JSON.parse(result); }; + +enum ActiveState { + SixWeeksAgo = 'six_weeks_ago', + TwelveWeeksAgo = 'twelve_weeks_ago', + Active = 'active', +} + +interface UserActiveState { + current_state: ActiveState; + previous_state: ActiveState; + primary_user_id: string; +} + +const bigquery = new BigQuery(); + +export const queryFromBq = async ( + query: string, +): Promise => { + // Queries the U.S. given names dataset for the state of Texas. + + // For all options, see https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query + // Location must match that of the dataset(s) referenced in the query. + const options = { query, location: 'US' }; + + // Run the query as a job + const [job] = await bigquery.createQueryJob(options); + console.log(`Job ${job.id} started.`); + + // Wait for the query to finish + const [rows] = await job.getQueryResults(); + + // Print the results + console.log('Rows:'); + rows.forEach((row) => console.log(row)); + + return rows; +}; + +export const userActiveStateQuery = ` + with d as ( + select uss.primary_user_id, + min(last_app_timestamp) as last_app_timestamp, + min(registration_timestamp) as registration_timestamp, + + min( + case when period_end between date('2024-12-07' - interval 6*7 day) and '2024-12-07' then '1. active_last_6w' + when period_end between date('2024-12-07' - interval 12*7 day) and date('2024-12-07' - interval 6*7 + 1 day) then '2. active_7w_12w' + when date(u.last_app_timestamp) < date('2024-12-07' - interval 12*7 day) then '3. active_12w+' + when date(u.registration_timestamp) < date('2024-12-07' - interval 12*7 day) then '3. active_12w+' + else '4. never_active' end + ) as previous_state, + + min( + case when period_end between date('2024-12-08' - interval 6*7 day) and '2024-12-08' then '1. active_last_6w' + when period_end between date('2024-12-08' - interval 12*7 day) and date('2024-12-08' - interval 6*7 + 1 day) then '2. active_7w_12w' + when date(u.last_app_timestamp) < date('2024-12-08' - interval 12*7 day) then '3. active_12w+' + when date(u.registration_timestamp) < date('2024-12-08' - interval 12*7 day) then '3. active_12w+' + else '4. never_active' end + ) as current_state, + + + from analytics.user as u + left join analytics.user_state_sparse as uss on uss.primary_user_id = u.primary_user_id + and uss.period_end between date('2024-12-07' - interval 12* 7 day) and '2024-12-08' + and uss.period = 'daily' + and uss.app_active_state = 'active' + and uss.registration_state = 'registered' + where u.registration_timestamp is not null + and u.last_app_timestamp is not null + and u.is_spam = false + group by 1 + ) + -- select previous_state, current_state, count(*) as ctr + -- from d + -- where current_state != previous_state + -- group by 1,2 + -- order by 1,2 + + select * + from d + where previous_state != current_state +`; + +interface GetUsersActiveState { + inactiveUsers: string[]; + downgradeUsers: string[]; + reactivateUsers: string[]; +} + +export const getUsersActiveState = async (): Promise => { + const usersFromBq = await queryFromBq(userActiveStateQuery); + const inactiveUsers: string[] = []; + const downgradeUsers: string[] = []; + const reactivateUsers: string[] = []; + + // sort users from bq into active, inactive, downgrade, and reactivate + for (const user of usersFromBq) { + if ( + user.previous_state === ActiveState.Active && + user.current_state === ActiveState.SixWeeksAgo + ) { + downgradeUsers.push(user.primary_user_id); + } else if ( + user.previous_state === ActiveState.SixWeeksAgo && + user.current_state === ActiveState.TwelveWeeksAgo + ) { + inactiveUsers.push(user.primary_user_id); + } else if ( + user.current_state === ActiveState.Active || + (user.previous_state === ActiveState.TwelveWeeksAgo && + user.current_state === ActiveState.SixWeeksAgo) + ) { + reactivateUsers.push(user.primary_user_id); + } + } + + return { inactiveUsers, downgradeUsers, reactivateUsers }; +}; diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index bab970964..38dfb2c49 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -5,22 +5,11 @@ import { UserPersonalizedDigestSendType, } from '../entity'; import { In } from 'typeorm'; -import { BigQuery } from '@google-cloud/bigquery'; import { blockingBatchRunner, callWithRetryDefault } from '../common/async'; import { setTimeout } from 'node:timers/promises'; import { cio, generateIdentifyObject } from '../cio'; import { updateFlagsStatement } from '../common'; - -enum ActiveState { - SixWeeksAgo = 'six_weeks_ago', - TwelveWeeksAgo = 'twelve_weeks_ago', - Active = 'active', -} - -interface UserBq { - state: string; - id: string; -} +import { getUsersActiveState } from '../common/googleCloud'; const ITEMS_PER_DESTROY = 4000; const ITEMS_PER_IDENTIFY = 250; @@ -28,23 +17,8 @@ const ITEMS_PER_IDENTIFY = 250; const cron: Cron = { name: 'validate-active-users', handler: async (con) => { - const bigquery = new BigQuery(); - const ds = bigquery.dataset('id'); - const [usersFromBq] = await ds.table('table id').get(); // replace with actual fetch - const inactiveUsers: string[] = []; - const downgradeUsers: string[] = []; - const reactivateUsers: string[] = []; - - // sort users from bq into active, inactive, downgrade, and reactivate - for (const user of usersFromBq) { - if (user.state === ActiveState.SixWeeksAgo) { - inactiveUsers.push(user.id); - } else if (user.state === ActiveState.TwelveWeeksAgo) { - downgradeUsers.push(user.id); - } else if (user.state === ActiveState.Active) { - reactivateUsers.push(user.id); - } - } + const { reactivateUsers, inactiveUsers, downgradeUsers } = + await getUsersActiveState(); // update users in db: reactivated await blockingBatchRunner({ From b9048d8165d9b56a888015a850cf4c7d5e67bdb7 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 17:01:44 +0800 Subject: [PATCH 05/45] feat: query from bq --- .infra/application.properties | 2 +- src/common/googleCloud.ts | 44 +++++++++++---------------------- src/cron/validateActiveUsers.ts | 14 ++++++++--- 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/.infra/application.properties b/.infra/application.properties index 67345e099..7389ac41d 100644 --- a/.infra/application.properties +++ b/.infra/application.properties @@ -9,7 +9,7 @@ debezium.source.database.password=%database_pass% debezium.source.database.dbname=%database_dbname% debezium.source.database.server.name=api debezium.source.table.include.list=public.comment,public.user_comment,public.comment_mention,public.source_request,public.post,public.user,public.post_report,public.source_feed,public.settings,public.reputation_event,public.submission,public.user_state,public.notification_v2,public.source_member,public.feature,public.source,public.post_mention,public.content_image,public.comment_report,public.user_post,public.banner,public.post_relation,public.marketing_cta,public.squad_public_request,public.user_streak,public.bookmark,public.user_company,public.source_report,public.user_top_reader,public.source_post_moderation -debezium.source.column.exclude.list=public.post.tsv,public.post.placeholder,public.source.flags,public.user_top_reader.image +debezium.source.column.exclude.list=public.post.tsv,public.post.placeholder,public.source.flags,public.user_top_reader.image,public.user.cioRegistered,public.user.acceptedMarketing,public.user.followingEmail,public.user.notificationEmail debezium.source.skip.messages.without.change=true debezium.source.plugin.name=pgoutput debezium.source.heartbeat.interval.ms=60000 diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index 1581090bc..e48b5912e 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -2,6 +2,7 @@ import { Storage, DownloadOptions } from '@google-cloud/storage'; import { PropsParameters } from '../types'; import path from 'path'; import { BigQuery } from '@google-cloud/bigquery'; +import { Query } from '@google-cloud/bigquery/build/src/bigquery'; export const downloadFile = async ({ url, @@ -31,15 +32,9 @@ export const downloadJsonFile = async ({ return JSON.parse(result); }; -enum ActiveState { - SixWeeksAgo = 'six_weeks_ago', - TwelveWeeksAgo = 'twelve_weeks_ago', - Active = 'active', -} - interface UserActiveState { - current_state: ActiveState; - previous_state: ActiveState; + current_state: string; + previous_state: string; primary_user_id: string; } @@ -48,23 +43,11 @@ const bigquery = new BigQuery(); export const queryFromBq = async ( query: string, ): Promise => { - // Queries the U.S. given names dataset for the state of Texas. - - // For all options, see https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query - // Location must match that of the dataset(s) referenced in the query. - const options = { query, location: 'US' }; + const options: Query = { query }; - // Run the query as a job const [job] = await bigquery.createQueryJob(options); - console.log(`Job ${job.id} started.`); - - // Wait for the query to finish const [rows] = await job.getQueryResults(); - // Print the results - console.log('Rows:'); - rows.forEach((row) => console.log(row)); - return rows; }; @@ -128,21 +111,22 @@ export const getUsersActiveState = async (): Promise => { // sort users from bq into active, inactive, downgrade, and reactivate for (const user of usersFromBq) { if ( - user.previous_state === ActiveState.Active && - user.current_state === ActiveState.SixWeeksAgo + user.current_state.includes('active_7w_12w') && + user.previous_state.includes('active_last_6w') ) { downgradeUsers.push(user.primary_user_id); } else if ( - user.previous_state === ActiveState.SixWeeksAgo && - user.current_state === ActiveState.TwelveWeeksAgo + user.current_state.includes('active_last_6w') && + !user.previous_state.includes('active_last_6w') ) { - inactiveUsers.push(user.primary_user_id); + reactivateUsers.push(user.primary_user_id); } else if ( - user.current_state === ActiveState.Active || - (user.previous_state === ActiveState.TwelveWeeksAgo && - user.current_state === ActiveState.SixWeeksAgo) + (user.current_state.includes('active_12w+') && + !user.previous_state.includes('active_12w+')) || + (user.current_state.includes('never_active') && + !user.previous_state.includes('never_active')) ) { - reactivateUsers.push(user.primary_user_id); + inactiveUsers.push(user.primary_user_id); } } diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index 38dfb2c49..42d5390b0 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -20,7 +20,7 @@ const cron: Cron = { const { reactivateUsers, inactiveUsers, downgradeUsers } = await getUsersActiveState(); - // update users in db: reactivated + // reactivated users: add to CIO await blockingBatchRunner({ data: reactivateUsers, runner: async (current) => { @@ -95,9 +95,15 @@ const cron: Cron = { cio.request.post('/users', { batch: data }), ); - await con - .getRepository(User) - .update({ id: In(ids) }, { cioRegistered: false }); + await con.getRepository(User).update( + { id: In(ids) }, + { + cioRegistered: false, + acceptedMarketing: false, + followingEmail: false, + notificationEmail: false, + }, + ); await setTimeout(20); }, From a9adb0244cb5ea95ada955240349f7583d35cd24 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 22:12:50 +0800 Subject: [PATCH 06/45] test: cron job --- __tests__/cron/validateActiveUsers.ts | 271 ++++++++++++++++++++++++++ src/cio.ts | 4 +- src/cron/validateActiveUsers.ts | 14 +- 3 files changed, 281 insertions(+), 8 deletions(-) create mode 100644 __tests__/cron/validateActiveUsers.ts diff --git a/__tests__/cron/validateActiveUsers.ts b/__tests__/cron/validateActiveUsers.ts new file mode 100644 index 000000000..2c6e55440 --- /dev/null +++ b/__tests__/cron/validateActiveUsers.ts @@ -0,0 +1,271 @@ +import { crons } from '../../src/cron/index'; +import cron from '../../src/cron/validateActiveUsers'; +import { expectSuccessfulCron, saveFixtures } from '../helpers'; +import * as gcp from '../../src/common/googleCloud'; +import * as cioModule from '../../src/cio'; +import { DataSource, In, JsonContains } from 'typeorm'; +import createOrGetConnection from '../../src/db'; +import { + User, + UserPersonalizedDigest, + UserPersonalizedDigestSendType, +} from '../../src/entity'; +import { badUsersFixture, plusUsersFixture, usersFixture } from '../fixture'; +import { updateFlagsStatement } from '../../src/common'; + +let con: DataSource; + +beforeEach(async () => { + con = await createOrGetConnection(); + jest.clearAllMocks(); + jest.resetModules(); + await saveFixtures(con, User, [ + ...usersFixture, + ...plusUsersFixture, + ...badUsersFixture, + ]); +}); + +describe('validateActiveUsers', () => { + beforeEach(async () => { + await saveFixtures(con, User, usersFixture); + }); + + it('should be registered', () => { + const registeredWorker = crons.find((item) => item.name === cron.name); + + expect(registeredWorker).toBeDefined(); + }); +}); + +describe('users for downgrade', () => { + it('should not do anything if users do not have digest subscription', async () => { + jest.spyOn(gcp, 'getUsersActiveState').mockResolvedValue({ + reactivateUsers: [], + inactiveUsers: [], + downgradeUsers: ['4', '1'], + }); + + await con.getRepository(UserPersonalizedDigest).delete({}); + await expectSuccessfulCron(cron); + + const digests = await con.getRepository(UserPersonalizedDigest).find(); + expect(digests.length).toEqual(0); + }); + + it('should downgrade daily digest to weekly digest', async () => { + const downgradeUsers = ['4', '1']; + + jest.spyOn(gcp, 'getUsersActiveState').mockResolvedValue({ + reactivateUsers: [], + inactiveUsers: [], + downgradeUsers, + }); + + await con.getRepository(UserPersonalizedDigest).update( + {}, + { + preferredDay: 1, + preferredHour: 4, + flags: updateFlagsStatement({ + digestSendType: UserPersonalizedDigestSendType.workdays, + }), + }, + ); + await expectSuccessfulCron(cron); + + const digests = await con.getRepository(UserPersonalizedDigest).find({ + where: { + flags: JsonContains({ + digestSendType: UserPersonalizedDigestSendType.weekly, + }), + }, + }); + const downgradedOnly = digests.every( + ({ userId, preferredDay, preferredHour }) => + downgradeUsers.includes(userId) && + preferredDay === 3 && + preferredHour === 9, + ); + expect(downgradedOnly).toEqual(true); + }); +}); + +describe('users for removal', () => { + it('should not do anything if users are removed to CIO already', async () => { + jest.spyOn(gcp, 'getUsersActiveState').mockResolvedValue({ + reactivateUsers: ['1', '2'], + inactiveUsers: ['3', '5'], + downgradeUsers: ['4'], + }); + + const postSpy = jest + .spyOn(cioModule.cioV2.request, 'post') + .mockResolvedValue({}); + + await con + .getRepository(User) + .update({ id: In(['3', '5']) }, { cioRegistered: false }); + + await expectSuccessfulCron(cron); + + expect(postSpy).not.toHaveBeenCalled(); + }); + + it('should send removal to cio', async () => { + jest.spyOn(gcp, 'getUsersActiveState').mockResolvedValue({ + reactivateUsers: ['1', '2'], + inactiveUsers: ['3', '5', 'vordr'], + downgradeUsers: ['4'], + }); + + const postSpy = jest + .spyOn(cioModule.cioV2.request, 'post') + .mockResolvedValue({}); + + await con + .getRepository(User) + .update({ id: In(['vordr']) }, { cioRegistered: false }); + + await expectSuccessfulCron(cron); + + const batch = [ + { + action: 'destroy', + type: 'person', + identifiers: { id: '3' }, + }, + { + action: 'destroy', + type: 'person', + identifiers: { id: '5' }, + }, + ]; + + expect(postSpy).toHaveBeenCalledWith('/users', { batch }); + + const fromRemovalOnly = postSpy.mock.calls[0][1].batch.every( + ({ identifiers }) => ['3', '5'].includes(identifiers.id), + ); + expect(fromRemovalOnly).toBeTruthy(); + + const unRegisteredOnly = postSpy.mock.calls[0][1].batch.every( + ({ identifiers }) => !['vordr'].includes(identifiers.id), + ); + expect(unRegisteredOnly).toBeTruthy(); + + const unregistered = await con + .getRepository(User) + .findOne({ select: ['cioRegistered'], where: { id: '3' } }); + expect(unregistered.cioRegistered).toEqual(false); + }); +}); + +describe('users for reactivation', () => { + it('should not do anything if users are registered to CIO already', async () => { + jest.spyOn(gcp, 'getUsersActiveState').mockResolvedValue({ + reactivateUsers: ['1', '2'], + inactiveUsers: ['3'], + downgradeUsers: ['4'], + }); + const postSpy = jest + .spyOn(cioModule.cioV2.request, 'post') + .mockResolvedValue({}); + + // to stop running removal of `inactiveUsers` + await con.getRepository(User).update({ id: '3' }, { cioRegistered: false }); + + // default value for `cioRegistered` is true + await expectSuccessfulCron(cron); + + expect(postSpy).not.toHaveBeenCalled(); + }); + + it('should send reactivation to cio', async () => { + jest.spyOn(gcp, 'getUsersActiveState').mockResolvedValue({ + reactivateUsers: ['1', '2'], + inactiveUsers: ['3'], + downgradeUsers: ['4'], + }); + + const postSpy = jest + .spyOn(cioModule.cioV2.request, 'post') + .mockResolvedValue({}); + + // to stop running removal of `inactiveUsers` + await con + .getRepository(User) + .update({ id: In(['3', '1']) }, { cioRegistered: false }); + + // default value for `cioRegistered` is true + await expectSuccessfulCron(cron); + + const batch = [ + { + action: 'identify', + type: 'person', + identifiers: { id: '1' }, + attributes: { + name: 'Ido', + email: 'ido@daily.dev', + image: 'https://daily.dev/ido.jpg', + cover: null, + company: null, + title: null, + accepted_marketing: false, + reputation: 10, + username: 'idoshamun', + twitter: null, + github: 'idogithub', + roadmap: null, + threads: null, + codepen: null, + reddit: null, + stackoverflow: null, + youtube: null, + linkedin: null, + mastodon: null, + portfolio: null, + hashnode: null, + cio_registered: false, + timezone: 'Etc/UTC', + week_start: 1, + created_at: NaN, + updated_at: undefined, + referral_id: null, + referral_origin: null, + acquisition_channel: null, + experience_level: null, + flags: {}, + language: null, + default_feed_id: null, + subscription_flags: {}, + permalink: 'http://localhost:5002/idoshamun', + first_name: 'Ido', + referral_link: 'http://localhost:5002/join?cid=generic&userid=1', + 'cio_subscription_preferences.topics.topic_4': false, + 'cio_subscription_preferences.topics.topic_7': true, + 'cio_subscription_preferences.topics.topic_8': true, + 'cio_subscription_preferences.topics.topic_9': true, + }, + }, + ]; + + expect(postSpy).toHaveBeenCalledWith('/users', { batch }); + + const fromReactivateUserOnly = postSpy.mock.calls[0][1].batch.every( + ({ identifiers }) => ['1'].includes(identifiers.id), + ); + expect(fromReactivateUserOnly).toBeTruthy(); + + const registeredOnly = postSpy.mock.calls[0][1].batch.every( + ({ identifiers }) => identifiers.id !== '3', + ); + expect(registeredOnly).toBeTruthy(); + + const reactivated = await con + .getRepository(User) + .findOne({ select: ['cioRegistered'], where: { id: '1' } }); + expect(reactivated.cioRegistered).toEqual(true); + }); +}); diff --git a/src/cio.ts b/src/cio.ts index 6c7038c56..721981d31 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -90,13 +90,13 @@ export async function identifyUserStreak({ } } -export const generateIdentifyObject = ( +export const generateIdentifyObject = async ( con: ConnectionManager, user: ChangeObject, ) => { const { id } = user; const changed = JSON.parse(JSON.stringify(user)); - const identify = getIdentifyAttributes(con, changed); + const identify = await getIdentifyAttributes(con, changed); return { action: 'identify', diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index 42d5390b0..7a8b6dcce 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -7,7 +7,7 @@ import { import { In } from 'typeorm'; import { blockingBatchRunner, callWithRetryDefault } from '../common/async'; import { setTimeout } from 'node:timers/promises'; -import { cio, generateIdentifyObject } from '../cio'; +import { cioV2, generateIdentifyObject } from '../cio'; import { updateFlagsStatement } from '../common'; import { getUsersActiveState } from '../common/googleCloud'; @@ -41,12 +41,14 @@ const cron: Cron = { .getRepository(User) .find({ where: { id: In(ids.map(({ id }) => id)) } }); - const data = users.map((user) => - generateIdentifyObject(con, JSON.parse(JSON.stringify(user))), + const data = await Promise.all( + users.map((user) => + generateIdentifyObject(con, JSON.parse(JSON.stringify(user))), + ), ); await callWithRetryDefault(() => - cio.request.post('/users', { batch: data }), + cioV2.request.post('/users', { batch: data }), ); await con @@ -83,7 +85,7 @@ const cron: Cron = { await blockingBatchRunner({ batchLimit: ITEMS_PER_DESTROY, - data: validInactiveUsers.map((u) => ({ id: u.id })), + data: validInactiveUsers.map(({ id }) => id), runner: async (ids) => { const data = ids.map((id) => ({ action: 'destroy', @@ -92,7 +94,7 @@ const cron: Cron = { })); await callWithRetryDefault(() => - cio.request.post('/users', { batch: data }), + cioV2.request.post('/users', { batch: data }), ); await con.getRepository(User).update( From f2d68b00575b0ffd9ffbf022b21c7ac7e16d0a0f Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 22:16:14 +0800 Subject: [PATCH 07/45] chore: prepare cron job --- .infra/crons.ts | 6 +++++- __tests__/cron/validateActiveUsers.ts | 22 +++++++++++----------- src/cron/index.ts | 2 ++ 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.infra/crons.ts b/.infra/crons.ts index 304299efa..65139b47f 100644 --- a/.infra/crons.ts +++ b/.infra/crons.ts @@ -59,6 +59,10 @@ export const crons: Cron[] = [ name: 'generic-referral-reminder', schedule: '12 3 * * *', }, + // { + // name: 'validate-active-users', + // schedule: '15 4 * * *', + // }, { name: 'update-source-tag-view', schedule: '20 3 * * 0', @@ -109,5 +113,5 @@ export const crons: Cron[] = [ { name: 'calculate-top-readers', schedule: '0 2 1 * *', - } + }, ]; diff --git a/__tests__/cron/validateActiveUsers.ts b/__tests__/cron/validateActiveUsers.ts index 2c6e55440..78cfb7d6e 100644 --- a/__tests__/cron/validateActiveUsers.ts +++ b/__tests__/cron/validateActiveUsers.ts @@ -26,17 +26,17 @@ beforeEach(async () => { ]); }); -describe('validateActiveUsers', () => { - beforeEach(async () => { - await saveFixtures(con, User, usersFixture); - }); - - it('should be registered', () => { - const registeredWorker = crons.find((item) => item.name === cron.name); - - expect(registeredWorker).toBeDefined(); - }); -}); +// describe('validateActiveUsers', () => { +// beforeEach(async () => { +// await saveFixtures(con, User, usersFixture); +// }); +// +// it('should be registered', () => { +// const registeredWorker = crons.find((item) => item.name === cron.name); +// +// expect(registeredWorker).toBeDefined(); +// }); +// }); describe('users for downgrade', () => { it('should not do anything if users do not have digest subscription', async () => { diff --git a/src/cron/index.ts b/src/cron/index.ts index ee48180d3..f7fb8ac69 100644 --- a/src/cron/index.ts +++ b/src/cron/index.ts @@ -13,6 +13,7 @@ import updateHighlightedViews from './updateHighlightedViews'; import hourlyNotifications from './hourlyNotifications'; import updateCurrentStreak from './updateCurrentStreak'; import syncSubscriptionWithCIO from './syncSubscriptionWithCIO'; +// import validateActiveUsers from './validateActiveUsers'; import { updateSourcePublicThreshold } from './updateSourcePublicThreshold'; import { cleanZombieUserCompany } from './cleanZombieUserCompany'; import { calculateTopReaders } from './calculateTopReaders'; @@ -35,4 +36,5 @@ export const crons: Cron[] = [ cleanZombieUserCompany, updateSourcePublicThreshold, calculateTopReaders, + // validateActiveUsers, ]; From bb510ff7bc864814ec39eab4208a6d80e5f9c218 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 22:16:59 +0800 Subject: [PATCH 08/45] chore: no registered test --- __tests__/cron/validateActiveUsers.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/__tests__/cron/validateActiveUsers.ts b/__tests__/cron/validateActiveUsers.ts index 78cfb7d6e..05e34b254 100644 --- a/__tests__/cron/validateActiveUsers.ts +++ b/__tests__/cron/validateActiveUsers.ts @@ -26,17 +26,17 @@ beforeEach(async () => { ]); }); -// describe('validateActiveUsers', () => { -// beforeEach(async () => { -// await saveFixtures(con, User, usersFixture); -// }); -// -// it('should be registered', () => { -// const registeredWorker = crons.find((item) => item.name === cron.name); -// -// expect(registeredWorker).toBeDefined(); -// }); -// }); +describe('validateActiveUsers', () => { + beforeEach(async () => { + await saveFixtures(con, User, usersFixture); + }); + + it('should NOT be registered yet', () => { + const registeredWorker = crons.find((item) => item.name === cron.name); + + expect(registeredWorker).not.toBeDefined(); + }); +}); describe('users for downgrade', () => { it('should not do anything if users do not have digest subscription', async () => { From 6af995bd2e7a0f482fd4e42ed474fc1ca67c4e3a Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 22:26:42 +0800 Subject: [PATCH 09/45] fix: missing await --- src/cio.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cio.ts b/src/cio.ts index 721981d31..ec67d2c9a 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -155,7 +155,7 @@ export async function identifyUser({ cio: TrackClient; user: ChangeObject; }): Promise { - const data = getIdentifyAttributes(con, user); + const data = await getIdentifyAttributes(con, user); try { await cio.identify(user.id, data); From 0c0e4a305e0304dafa4a7c35105cf885b0ab673c Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 22:31:03 +0800 Subject: [PATCH 10/45] refactor: ordering of variables --- src/common/googleCloud.ts | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index e48b5912e..f94b503cd 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -32,25 +32,6 @@ export const downloadJsonFile = async ({ return JSON.parse(result); }; -interface UserActiveState { - current_state: string; - previous_state: string; - primary_user_id: string; -} - -const bigquery = new BigQuery(); - -export const queryFromBq = async ( - query: string, -): Promise => { - const options: Query = { query }; - - const [job] = await bigquery.createQueryJob(options); - const [rows] = await job.getQueryResults(); - - return rows; -}; - export const userActiveStateQuery = ` with d as ( select uss.primary_user_id, @@ -102,6 +83,25 @@ interface GetUsersActiveState { reactivateUsers: string[]; } +interface UserActiveState { + current_state: string; + previous_state: string; + primary_user_id: string; +} + +const bigquery = new BigQuery(); + +export const queryFromBq = async ( + query: string, +): Promise => { + const options: Query = { query }; + + const [job] = await bigquery.createQueryJob(options); + const [rows] = await job.getQueryResults(); + + return rows; +}; + export const getUsersActiveState = async (): Promise => { const usersFromBq = await queryFromBq(userActiveStateQuery); const inactiveUsers: string[] = []; From 4df7881f17811ae44820352846a91cc589822ab0 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 22:44:07 +0800 Subject: [PATCH 11/45] refactor: code cleanup --- src/cron/validateActiveUsers.ts | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index 7a8b6dcce..0768c2af0 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -35,11 +35,11 @@ const cron: Cron = { await blockingBatchRunner({ batchLimit: ITEMS_PER_IDENTIFY, - data: validReactivateUsers.map((u) => ({ id: u.id })), + data: validReactivateUsers.map(({ id }) => id), runner: async (ids) => { const users = await con .getRepository(User) - .find({ where: { id: In(ids.map(({ id }) => id)) } }); + .find({ where: { id: In(ids) } }); const data = await Promise.all( users.map((user) => @@ -58,15 +58,6 @@ const cron: Cron = { await setTimeout(20); // wait for a bit to avoid rate limiting }, }); - - await con - .getRepository(User) - .update( - { id: In(validReactivateUsers.map((u) => u.id)) }, - { cioRegistered: true }, - ); - - await setTimeout(200); }, }); @@ -107,7 +98,7 @@ const cron: Cron = { }, ); - await setTimeout(20); + await setTimeout(20); // wait for a bit to avoid rate limiting }, }); }, From 80731186f990a4aa6e1abdd7274460cf84e99b62 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Wed, 18 Dec 2024 22:46:09 +0800 Subject: [PATCH 12/45] refactor: better naming --- src/cron/validateActiveUsers.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index 0768c2af0..a4467ccac 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -23,10 +23,10 @@ const cron: Cron = { // reactivated users: add to CIO await blockingBatchRunner({ data: reactivateUsers, - runner: async (current) => { + runner: async (batch) => { const validReactivateUsers = await con.getRepository(User).find({ select: ['id'], - where: { id: In(current), cioRegistered: false }, + where: { id: In(batch), cioRegistered: false }, }); if (validReactivateUsers.length === 0) { @@ -64,10 +64,10 @@ const cron: Cron = { // inactive for 12 weeks: remove from CIO await blockingBatchRunner({ data: inactiveUsers, - runner: async (current) => { + runner: async (batch) => { const validInactiveUsers = await con.getRepository(User).find({ select: ['id'], - where: { id: In(current), cioRegistered: true }, + where: { id: In(batch), cioRegistered: true }, }); if (validInactiveUsers.length === 0) { From 17dba79f28b3f55f740bf55eb2a6df732d50ab4e Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 19 Dec 2024 17:22:02 +0800 Subject: [PATCH 13/45] fix: going through missed days --- src/common/constants.ts | 2 + src/common/googleCloud.ts | 46 ++++++---- src/common/mailing.ts | 131 ++++++++++++++++++++++++++- src/common/users.ts | 5 +- src/cron/validateActiveUsers.ts | 151 +++++++------------------------- 5 files changed, 193 insertions(+), 142 deletions(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index ad2484d7e..fbd4f15b8 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -4,3 +4,5 @@ export const ONE_DAY_IN_SECONDS = ONE_HOUR_IN_SECONDS * 24; export const ONE_WEEK_IN_SECONDS = ONE_DAY_IN_SECONDS * 7; export const MAX_FOLLOWERS_LIMIT = 5_000; + +export const SUCCESSFUL_CIO_SYNC_DATE = 'successful_cio_sync_date'; diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index f94b503cd..d752edce9 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -3,6 +3,7 @@ import { PropsParameters } from '../types'; import path from 'path'; import { BigQuery } from '@google-cloud/bigquery'; import { Query } from '@google-cloud/bigquery/build/src/bigquery'; +import { subDays } from 'date-fns'; export const downloadFile = async ({ url, @@ -32,6 +33,7 @@ export const downloadJsonFile = async ({ return JSON.parse(result); }; +// TODO: turn this into a get query function so we can pass the dates to supply the query export const userActiveStateQuery = ` with d as ( select uss.primary_user_id, @@ -39,25 +41,25 @@ export const userActiveStateQuery = ` min(registration_timestamp) as registration_timestamp, min( - case when period_end between date('2024-12-07' - interval 6*7 day) and '2024-12-07' then '1. active_last_6w' - when period_end between date('2024-12-07' - interval 12*7 day) and date('2024-12-07' - interval 6*7 + 1 day) then '2. active_7w_12w' - when date(u.last_app_timestamp) < date('2024-12-07' - interval 12*7 day) then '3. active_12w+' - when date(u.registration_timestamp) < date('2024-12-07' - interval 12*7 day) then '3. active_12w+' + case when period_end between date(@run_date - interval 6*7 day) and @run_date then '1. active_last_6w' + when period_end between date(@run_date - interval 12*7 day) and date(@run_date - interval 6*7 + 1 day) then '2. active_7w_12w' + when date(u.last_app_timestamp) < date(@run_date - interval 12*7 day) then '3. active_12w+' + when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3. active_12w+' else '4. never_active' end ) as previous_state, min( - case when period_end between date('2024-12-08' - interval 6*7 day) and '2024-12-08' then '1. active_last_6w' - when period_end between date('2024-12-08' - interval 12*7 day) and date('2024-12-08' - interval 6*7 + 1 day) then '2. active_7w_12w' - when date(u.last_app_timestamp) < date('2024-12-08' - interval 12*7 day) then '3. active_12w+' - when date(u.registration_timestamp) < date('2024-12-08' - interval 12*7 day) then '3. active_12w+' + case when period_end between date(@previous_date - interval 6*7 day) and @previous_date then '1. active_last_6w' + when period_end between date(@previous_date - interval 12*7 day) and date(@previous_date - interval 6*7 + 1 day) then '2. active_7w_12w' + when date(u.last_app_timestamp) < date(@previous_date - interval 12*7 day) then '3. active_12w+' + when date(u.registration_timestamp) < date(@previous_date - interval 12*7 day) then '3. active_12w+' else '4. never_active' end ) as current_state, from analytics.user as u left join analytics.user_state_sparse as uss on uss.primary_user_id = u.primary_user_id - and uss.period_end between date('2024-12-07' - interval 12* 7 day) and '2024-12-08' + and uss.period_end between date(@run_date - interval 12* 7 day) and @previous_date and uss.period = 'daily' and uss.app_active_state = 'active' and uss.registration_state = 'registered' @@ -77,13 +79,20 @@ export const userActiveStateQuery = ` where previous_state != current_state `; -interface GetUsersActiveState { +export const getUserActiveStateQuery = (runDate: Date): Query => { + const run_date = runDate.toISOString().split('T')[0]; + const previous_date = subDays(runDate, 1).toISOString().split('T')[0]; + + return { query: userActiveStateQuery, params: { previous_date, run_date } }; +}; + +export interface GetUsersActiveState { inactiveUsers: string[]; downgradeUsers: string[]; reactivateUsers: string[]; } -interface UserActiveState { +export interface UserActiveState { current_state: string; previous_state: string; primary_user_id: string; @@ -91,19 +100,18 @@ interface UserActiveState { const bigquery = new BigQuery(); -export const queryFromBq = async ( - query: string, -): Promise => { - const options: Query = { query }; - - const [job] = await bigquery.createQueryJob(options); +export const queryFromBq = async (query: Query): Promise => { + const [job] = await bigquery.createQueryJob(query); const [rows] = await job.getQueryResults(); return rows; }; -export const getUsersActiveState = async (): Promise => { - const usersFromBq = await queryFromBq(userActiveStateQuery); +export const getUsersActiveState = async ( + runDate: Date, +): Promise => { + const query = getUserActiveStateQuery(runDate); + const usersFromBq = await queryFromBq(query); const inactiveUsers: string[] = []; const downgradeUsers: string[] = []; const reactivateUsers: string[] = []; diff --git a/src/common/mailing.ts b/src/common/mailing.ts index d97f5257e..fc237713c 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -7,13 +7,19 @@ import { import CIORequest from 'customerio-node/dist/lib/request'; import { SendEmailRequestOptionalOptions } from 'customerio-node/lib/api/requests'; import { SendEmailRequestWithTemplate } from 'customerio-node/dist/lib/api/requests'; -import { DataSource } from 'typeorm'; +import { DataSource, In } from 'typeorm'; import { Source, User, UserPersonalizedDigest, + UserPersonalizedDigestSendType, UserPersonalizedDigestType, } from '../entity'; +import { blockingBatchRunner, callWithRetryDefault } from './async'; +import { cioV2, generateIdentifyObject } from '../cio'; +import { setTimeout } from 'node:timers/promises'; +import { updateFlagsStatement } from './utils'; +import { GetUsersActiveState } from './googleCloud'; export enum CioUnsubscribeTopic { Marketing = '4', @@ -171,3 +177,126 @@ export const addPrivateSourceJoinParams = ({ return urlObj.toString(); }; + +const ITEMS_PER_DESTROY = 4000; +const ITEMS_PER_IDENTIFY = 250; + +interface SyncSubscriptionsWithActiveStateProps { + con: DataSource; + users: GetUsersActiveState; +} + +export const syncSubscriptionsWithActiveState = async ({ + con, + users: { inactiveUsers, downgradeUsers, reactivateUsers }, +}: SyncSubscriptionsWithActiveStateProps) => { + await blockingBatchRunner({ + data: reactivateUsers, + runner: async (batch) => { + const validReactivateUsers = await con.getRepository(User).find({ + select: ['id'], + where: { id: In(batch), cioRegistered: false }, + }); + + if (validReactivateUsers.length === 0) { + return true; + } + + await blockingBatchRunner({ + batchLimit: ITEMS_PER_IDENTIFY, + data: validReactivateUsers.map(({ id }) => id), + runner: async (ids) => { + const users = await con + .getRepository(User) + .find({ where: { id: In(ids) } }); + + const data = await Promise.all( + users.map((user) => + generateIdentifyObject(con, JSON.parse(JSON.stringify(user))), + ), + ); + + await callWithRetryDefault(() => + cioV2.request.post('/users', { batch: data }), + ); + + await con + .getRepository(User) + .update({ id: In(ids) }, { cioRegistered: true }); + + await setTimeout(20); // wait for a bit to avoid rate limiting + }, + }); + }, + }); + + // inactive for 12 weeks: remove from CIO + await blockingBatchRunner({ + data: inactiveUsers, + runner: async (batch) => { + const validInactiveUsers = await con.getRepository(User).find({ + select: ['id'], + where: { id: In(batch), cioRegistered: true }, + }); + + if (validInactiveUsers.length === 0) { + return true; + } + + await blockingBatchRunner({ + batchLimit: ITEMS_PER_DESTROY, + data: validInactiveUsers.map(({ id }) => id), + runner: async (ids) => { + const data = ids.map((id) => ({ + action: 'destroy', + type: 'person', + identifiers: { id }, + })); + + await callWithRetryDefault(() => + cioV2.request.post('/users', { batch: data }), + ); + + await con.getRepository(User).update( + { id: In(ids) }, + { + cioRegistered: false, + acceptedMarketing: false, + followingEmail: false, + notificationEmail: false, + }, + ); + + await setTimeout(20); // wait for a bit to avoid rate limiting + }, + }); + }, + }); + + // inactive for 6 weeks: downgrade from daily to weekly digest + await blockingBatchRunner({ + data: downgradeUsers, + runner: async (current) => { + const validDowngradeUsers = await con + .getRepository(User) + .createQueryBuilder('u') + .select('id') + .innerJoin(UserPersonalizedDigest, 'upd', 'u.id = upd."userId"') + .where('u.id IN (:...ids)', { ids: current }) + .andWhere(`upd.flags->>'sendType' = 'daily'`) + .getRawMany>(); + + // set digest to weekly on Wednesday 9am + await con.getRepository(UserPersonalizedDigest).update( + { userId: In(validDowngradeUsers.map(({ id }) => id)) }, + { + preferredDay: 3, + preferredHour: 9, + flags: updateFlagsStatement({ + sendType: UserPersonalizedDigestSendType.weekly, + }), + }, + ); + }, + }); +}; diff --git a/src/common/users.ts b/src/common/users.ts index 0a0180e72..f5893578e 100644 --- a/src/common/users.ts +++ b/src/common/users.ts @@ -557,7 +557,10 @@ export enum LogoutReason { KratosSessionAlreadyAvailable = 'kratos session already available', } -const getAbsoluteDifferenceInDays: typeof differenceInDays = (date1, date2) => { +export const getAbsoluteDifferenceInDays: typeof differenceInDays = ( + date1, + date2, +) => { const day1 = startOfDay(date1); const day2 = startOfDay(date2); diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index a4467ccac..101272dcf 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -1,135 +1,44 @@ import { Cron } from './cron'; import { - User, - UserPersonalizedDigest, - UserPersonalizedDigestSendType, -} from '../entity'; -import { In } from 'typeorm'; -import { blockingBatchRunner, callWithRetryDefault } from '../common/async'; -import { setTimeout } from 'node:timers/promises'; -import { cioV2, generateIdentifyObject } from '../cio'; -import { updateFlagsStatement } from '../common'; + getAbsoluteDifferenceInDays, + SUCCESSFUL_CIO_SYNC_DATE, + syncSubscriptionsWithActiveState, +} from '../common'; import { getUsersActiveState } from '../common/googleCloud'; - -const ITEMS_PER_DESTROY = 4000; -const ITEMS_PER_IDENTIFY = 250; +import { getRedisObject, setRedisObject } from '../redis'; +import { DataSource } from 'typeorm'; +import { addDays } from 'date-fns'; + +const runCron = async (con: DataSource, runDate: Date) => { + const users = await getUsersActiveState(runDate); + + await syncSubscriptionsWithActiveState({ + con, + users, + }); + await setRedisObject(SUCCESSFUL_CIO_SYNC_DATE, runDate.toISOString()); +}; const cron: Cron = { name: 'validate-active-users', handler: async (con) => { - const { reactivateUsers, inactiveUsers, downgradeUsers } = - await getUsersActiveState(); - - // reactivated users: add to CIO - await blockingBatchRunner({ - data: reactivateUsers, - runner: async (batch) => { - const validReactivateUsers = await con.getRepository(User).find({ - select: ['id'], - where: { id: In(batch), cioRegistered: false }, - }); - - if (validReactivateUsers.length === 0) { - return true; - } - - await blockingBatchRunner({ - batchLimit: ITEMS_PER_IDENTIFY, - data: validReactivateUsers.map(({ id }) => id), - runner: async (ids) => { - const users = await con - .getRepository(User) - .find({ where: { id: In(ids) } }); - - const data = await Promise.all( - users.map((user) => - generateIdentifyObject(con, JSON.parse(JSON.stringify(user))), - ), - ); - - await callWithRetryDefault(() => - cioV2.request.post('/users', { batch: data }), - ); - - await con - .getRepository(User) - .update({ id: In(ids) }, { cioRegistered: true }); - - await setTimeout(20); // wait for a bit to avoid rate limiting - }, - }); - }, - }); - - // inactive for 12 weeks: remove from CIO - await blockingBatchRunner({ - data: inactiveUsers, - runner: async (batch) => { - const validInactiveUsers = await con.getRepository(User).find({ - select: ['id'], - where: { id: In(batch), cioRegistered: true }, - }); - - if (validInactiveUsers.length === 0) { - return true; - } - - await blockingBatchRunner({ - batchLimit: ITEMS_PER_DESTROY, - data: validInactiveUsers.map(({ id }) => id), - runner: async (ids) => { - const data = ids.map((id) => ({ - action: 'destroy', - type: 'person', - identifiers: { id }, - })); - - await callWithRetryDefault(() => - cioV2.request.post('/users', { batch: data }), - ); + const lastSuccessfulDate = await getRedisObject(SUCCESSFUL_CIO_SYNC_DATE); - await con.getRepository(User).update( - { id: In(ids) }, - { - cioRegistered: false, - acceptedMarketing: false, - followingEmail: false, - notificationEmail: false, - }, - ); + if (!lastSuccessfulDate) { + return runCron(con, new Date()); + } - await setTimeout(20); // wait for a bit to avoid rate limiting - }, - }); - }, - }); + const lastRunDate = new Date(lastSuccessfulDate); + const runDate = new Date(); + const difference = getAbsoluteDifferenceInDays(lastRunDate, runDate); - // inactive for 6 weeks: downgrade from daily to weekly digest - await blockingBatchRunner({ - data: downgradeUsers, - runner: async (current) => { - const validDowngradeUsers = await con - .getRepository(User) - .createQueryBuilder('u') - .select('id') - .innerJoin(UserPersonalizedDigest, 'upd', 'u.id = upd."userId"') - .where('u.id IN (:...ids)', { ids: current }) - .andWhere(`upd.flags->>'sendType' = 'daily'`) - .getRawMany>(); + if (difference === 0) { + return; + } - // set digest to weekly on Wednesday 9am - await con.getRepository(UserPersonalizedDigest).update( - { userId: In(validDowngradeUsers.map(({ id }) => id)) }, - { - preferredDay: 3, - preferredHour: 9, - flags: updateFlagsStatement({ - sendType: UserPersonalizedDigestSendType.weekly, - }), - }, - ); - }, - }); + for (let i = 1; i <= difference; i++) { + await runCron(con, addDays(lastRunDate, i)); + } }, }; From 41e7960abd4c2dd2a4e4882cc6a77c65616b46ed Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 19 Dec 2024 17:28:48 +0800 Subject: [PATCH 14/45] chore: finished todo --- src/common/googleCloud.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index d752edce9..0cca3c388 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -33,7 +33,6 @@ export const downloadJsonFile = async ({ return JSON.parse(result); }; -// TODO: turn this into a get query function so we can pass the dates to supply the query export const userActiveStateQuery = ` with d as ( select uss.primary_user_id, From a6dfe3fb5b1812a54de415bab049944670dfa52a Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 19 Dec 2024 17:42:53 +0800 Subject: [PATCH 15/45] fix: date minus 1 --- src/cron/validateActiveUsers.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index 101272dcf..e17dc4046 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -7,7 +7,7 @@ import { import { getUsersActiveState } from '../common/googleCloud'; import { getRedisObject, setRedisObject } from '../redis'; import { DataSource } from 'typeorm'; -import { addDays } from 'date-fns'; +import { addDays, subDays } from 'date-fns'; const runCron = async (con: DataSource, runDate: Date) => { const users = await getUsersActiveState(runDate); @@ -23,14 +23,14 @@ const cron: Cron = { name: 'validate-active-users', handler: async (con) => { const lastSuccessfulDate = await getRedisObject(SUCCESSFUL_CIO_SYNC_DATE); + const processingDate = subDays(new Date(), 1); if (!lastSuccessfulDate) { - return runCron(con, new Date()); + return runCron(con, processingDate); } const lastRunDate = new Date(lastSuccessfulDate); - const runDate = new Date(); - const difference = getAbsoluteDifferenceInDays(lastRunDate, runDate); + const difference = getAbsoluteDifferenceInDays(lastRunDate, processingDate); if (difference === 0) { return; From ef1308f673d33bf02a99939b7349d73150dee54c Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 19 Dec 2024 17:51:07 +0800 Subject: [PATCH 16/45] chore: updated query based on dave --- src/common/googleCloud.ts | 93 +++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index 0cca3c388..ffd00d622 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -33,49 +33,68 @@ export const downloadJsonFile = async ({ return JSON.parse(result); }; +export enum UserActiveState { + Active = '1', + InactiveSince6wAgo = '2', + InactiveSince12wAgo = '3', + NeverActive = '4', +} + export const userActiveStateQuery = ` with d as ( - select uss.primary_user_id, + select u.primary_user_id, min(last_app_timestamp) as last_app_timestamp, min(registration_timestamp) as registration_timestamp, - + min( + case + when period_end is null then '4' + when period_end between date(@previous_date - interval 6*7 day) and @previous_date then '1' + when period_end between date(@previous_date - interval 12*7 day) and date(@previous_date - interval 6*7 + 1 day) then '2' + when date(u.last_app_timestamp) < date(@previous_date - interval 12*7 day) then '3' + when date(u.registration_timestamp) < date(@previous_date - interval 12*7 day) then '3' + else '4' end + ) as previous_state, + min( + case + when period_end is null then '4' + when period_end between date(@run_date - interval 6*7 day) and @run_date then '1' + when period_end between date(@run_date - interval 12*7 day) and date(@run_date - interval 6*7 + 1 day) then '2' + when date(u.last_app_timestamp) < date(@run_date - interval 12*7 day) then '3' + when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3' + else '4' end + ) as current_state, min( - case when period_end between date(@run_date - interval 6*7 day) and @run_date then '1. active_last_6w' - when period_end between date(@run_date - interval 12*7 day) and date(@run_date - interval 6*7 + 1 day) then '2. active_7w_12w' - when date(u.last_app_timestamp) < date(@run_date - interval 12*7 day) then '3. active_12w+' - when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3. active_12w+' + case + when period_end is null then '4. never_active' + when period_end between date(@previous_date - interval 6*7 day) and @previous_date then '1. active_last_6w' + when period_end between date(@previous_date - interval 12*7 day) and date(@previous_date - interval 6*7 + 1 day) then '2. active_7w_12w' + when date(u.last_app_timestamp) < date(@previous_date - interval 12*7 day) then '3. active_12w+' + when date(u.registration_timestamp) < date(@previous_date - interval 12*7 day) then '3. active_12w+' else '4. never_active' end - ) as previous_state, - + ) as previous_state_name, min( - case when period_end between date(@previous_date - interval 6*7 day) and @previous_date then '1. active_last_6w' - when period_end between date(@previous_date - interval 12*7 day) and date(@previous_date - interval 6*7 + 1 day) then '2. active_7w_12w' - when date(u.last_app_timestamp) < date(@previous_date - interval 12*7 day) then '3. active_12w+' - when date(u.registration_timestamp) < date(@previous_date - interval 12*7 day) then '3. active_12w+' + case + when period_end is null then '4. never_active' + when period_end between date(@run_date - interval 6*7 day) and @run_date then '1. active_last_6w' + when period_end between date(@run_date - interval 12*7 day) and date(@run_date - interval 6*7 + 1 day) then '2. active_7w_12w' + when date(u.last_app_timestamp) < date(@run_date - interval 12*7 day) then '3. active_12w+' + when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3. active_12w+' else '4. never_active' end - ) as current_state, - - + ) as current_state_name, from analytics.user as u left join analytics.user_state_sparse as uss on uss.primary_user_id = u.primary_user_id - and uss.period_end between date(@run_date - interval 12* 7 day) and @previous_date + and uss.period_end between date(@previous_date - interval 12* 7 day) and @run_date and uss.period = 'daily' and uss.app_active_state = 'active' and uss.registration_state = 'registered' where u.registration_timestamp is not null - and u.last_app_timestamp is not null - and u.is_spam = false + and date(u.registration_timestamp) < @run_date group by 1 ) - -- select previous_state, current_state, count(*) as ctr - -- from d - -- where current_state != previous_state - -- group by 1,2 - -- order by 1,2 - select * from d - where previous_state != current_state + where current_state != previous_state + and previous_state != '4' `; export const getUserActiveStateQuery = (runDate: Date): Query => { @@ -91,15 +110,17 @@ export interface GetUsersActiveState { reactivateUsers: string[]; } -export interface UserActiveState { - current_state: string; - previous_state: string; +export interface UserActiveStateData { + current_state: UserActiveState; + previous_state: UserActiveState; primary_user_id: string; } const bigquery = new BigQuery(); -export const queryFromBq = async (query: Query): Promise => { +export const queryFromBq = async ( + query: Query, +): Promise => { const [job] = await bigquery.createQueryJob(query); const [rows] = await job.getQueryResults(); @@ -118,20 +139,18 @@ export const getUsersActiveState = async ( // sort users from bq into active, inactive, downgrade, and reactivate for (const user of usersFromBq) { if ( - user.current_state.includes('active_7w_12w') && - user.previous_state.includes('active_last_6w') + user.current_state === UserActiveState.InactiveSince6wAgo && + user.previous_state === UserActiveState.Active ) { downgradeUsers.push(user.primary_user_id); } else if ( - user.current_state.includes('active_last_6w') && - !user.previous_state.includes('active_last_6w') + user.current_state === UserActiveState.Active && + user.previous_state !== UserActiveState.Active ) { reactivateUsers.push(user.primary_user_id); } else if ( - (user.current_state.includes('active_12w+') && - !user.previous_state.includes('active_12w+')) || - (user.current_state.includes('never_active') && - !user.previous_state.includes('never_active')) + user.current_state === UserActiveState.InactiveSince12wAgo && + user.previous_state !== UserActiveState.InactiveSince12wAgo ) { inactiveUsers.push(user.primary_user_id); } From a7cab633d3233718838b807eaf2e775c047aad1c Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 19 Dec 2024 19:09:26 +0800 Subject: [PATCH 17/45] revert: application properties --- .infra/application.properties | 2 +- src/cron/validateActiveUsers.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.infra/application.properties b/.infra/application.properties index 1c83253ca..d34bc3058 100644 --- a/.infra/application.properties +++ b/.infra/application.properties @@ -9,7 +9,7 @@ debezium.source.database.password=%database_pass% debezium.source.database.dbname=%database_dbname% debezium.source.database.server.name=api debezium.source.table.include.list=public.comment,public.user_comment,public.comment_mention,public.source_request,public.post,public.user,public.post_report,public.source_feed,public.settings,public.reputation_event,public.submission,public.user_state,public.notification_v2,public.source_member,public.feature,public.source,public.post_mention,public.content_image,public.comment_report,public.user_post,public.banner,public.post_relation,public.marketing_cta,public.squad_public_request,public.user_streak,public.bookmark,public.user_company,public.source_report,public.user_top_reader,public.source_post_moderation -debezium.source.column.exclude.list=public.post.tsv,public.post.placeholder,public.source.flags,public.user_top_reader.image,public.user.cioRegistered,public.user.acceptedMarketing,public.user.followingEmail,public.user.notificationEmail +debezium.source.column.exclude.list=public.post.tsv,public.post.placeholder,public.source.flags,public.user_top_reader.image debezium.source.skip.messages.without.change=true debezium.source.plugin.name=pgoutput debezium.source.heartbeat.interval.ms=60000 diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index e17dc4046..1d4065c37 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -9,7 +9,7 @@ import { getRedisObject, setRedisObject } from '../redis'; import { DataSource } from 'typeorm'; import { addDays, subDays } from 'date-fns'; -const runCron = async (con: DataSource, runDate: Date) => { +const runSync = async (con: DataSource, runDate: Date) => { const users = await getUsersActiveState(runDate); await syncSubscriptionsWithActiveState({ @@ -26,7 +26,7 @@ const cron: Cron = { const processingDate = subDays(new Date(), 1); if (!lastSuccessfulDate) { - return runCron(con, processingDate); + return runSync(con, processingDate); } const lastRunDate = new Date(lastSuccessfulDate); @@ -37,7 +37,7 @@ const cron: Cron = { } for (let i = 1; i <= difference; i++) { - await runCron(con, addDays(lastRunDate, i)); + await runSync(con, addDays(lastRunDate, i)); } }, }; From b833be6691de8073211eb52a55b5ba933857fe3c Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 19 Dec 2024 19:27:51 +0800 Subject: [PATCH 18/45] fix: edge cases on success and failure --- src/cio.ts | 3 +-- src/common/async.ts | 22 +++++++++++++++++- src/common/mailing.ts | 53 +++++++++++++++++++++++++++---------------- 3 files changed, 55 insertions(+), 23 deletions(-) diff --git a/src/cio.ts b/src/cio.ts index ec67d2c9a..4fa618361 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -95,8 +95,7 @@ export const generateIdentifyObject = async ( user: ChangeObject, ) => { const { id } = user; - const changed = JSON.parse(JSON.stringify(user)); - const identify = await getIdentifyAttributes(con, changed); + const identify = await getIdentifyAttributes(con, user); return { action: 'identify', diff --git a/src/common/async.ts b/src/common/async.ts index 31be78aa4..2b732cce5 100644 --- a/src/common/async.ts +++ b/src/common/async.ts @@ -3,6 +3,8 @@ import { ConsecutiveBreaker, ExponentialBackoff, handleAll, + IFailureEvent, + ISuccessEvent, retry, wrap, } from 'cockatiel'; @@ -31,7 +33,17 @@ export const blockingBatchRunner = async ({ } }; -export const callWithRetryDefault = (callback: () => Promise) => { +interface CallWithRetryDefaultProps { + callback: () => Promise; + onSuccess?: (data: ISuccessEvent) => void; + onFailure?: (err: IFailureEvent) => void; +} + +export const callWithRetryDefault = ({ + callback, + onFailure, + onSuccess, +}: CallWithRetryDefaultProps) => { // Create a retry policy that'll try whatever function we execute 3 // times with a randomized exponential backoff. const retryPolicy = retry(handleAll, { @@ -39,6 +51,14 @@ export const callWithRetryDefault = (callback: () => Promise) => { backoff: new ExponentialBackoff(), }); + if (onFailure) { + retryPolicy.onFailure(onFailure); + } + + if (onSuccess) { + retryPolicy.onSuccess(onSuccess); + } + // Create a circuit breaker that'll stop calling the executed function for 10 // seconds if it fails 5 times in a row. This can give time for e.g. a database // to recover without getting tons of traffic. diff --git a/src/common/mailing.ts b/src/common/mailing.ts index fc237713c..726fe3a3a 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -20,6 +20,8 @@ import { cioV2, generateIdentifyObject } from '../cio'; import { setTimeout } from 'node:timers/promises'; import { updateFlagsStatement } from './utils'; import { GetUsersActiveState } from './googleCloud'; +import { ChangeObject } from '../types'; +import { logger } from '../logger'; export enum CioUnsubscribeTopic { Marketing = '4', @@ -212,17 +214,24 @@ export const syncSubscriptionsWithActiveState = async ({ const data = await Promise.all( users.map((user) => - generateIdentifyObject(con, JSON.parse(JSON.stringify(user))), + generateIdentifyObject( + con, + structuredClone(user) as unknown as ChangeObject, + ), ), ); - await callWithRetryDefault(() => - cioV2.request.post('/users', { batch: data }), - ); - - await con - .getRepository(User) - .update({ id: In(ids) }, { cioRegistered: true }); + await callWithRetryDefault({ + callback: () => cioV2.request.post('/users', { batch: data }), + onSuccess: async () => { + await con + .getRepository(User) + .update({ id: In(ids) }, { cioRegistered: true }); + }, + onFailure: (err) => { + logger.info({ err }, 'Failed to add users to CIO'); + }, + }); await setTimeout(20); // wait for a bit to avoid rate limiting }, @@ -253,19 +262,23 @@ export const syncSubscriptionsWithActiveState = async ({ identifiers: { id }, })); - await callWithRetryDefault(() => - cioV2.request.post('/users', { batch: data }), - ); - - await con.getRepository(User).update( - { id: In(ids) }, - { - cioRegistered: false, - acceptedMarketing: false, - followingEmail: false, - notificationEmail: false, + await callWithRetryDefault({ + callback: () => cioV2.request.post('/users', { batch: data }), + onSuccess: async () => { + await con.getRepository(User).update( + { id: In(ids) }, + { + cioRegistered: false, + acceptedMarketing: false, + followingEmail: false, + notificationEmail: false, + }, + ); }, - ); + onFailure: (err) => { + logger.info({ err }, 'Failed to remove users from CIO'); + }, + }); await setTimeout(20); // wait for a bit to avoid rate limiting }, From 3e718033b199746e7a98eead96c2ab640e7fc0cc Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 19 Dec 2024 19:51:21 +0800 Subject: [PATCH 19/45] refactor: to change object --- __tests__/cron/validateActiveUsers.ts | 4 +++- src/cio.ts | 12 ++++++++++-- src/common/mailing.ts | 7 ++----- src/common/typedPubsub.ts | 3 +++ 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/__tests__/cron/validateActiveUsers.ts b/__tests__/cron/validateActiveUsers.ts index 05e34b254..113bbe389 100644 --- a/__tests__/cron/validateActiveUsers.ts +++ b/__tests__/cron/validateActiveUsers.ts @@ -12,11 +12,13 @@ import { } from '../../src/entity'; import { badUsersFixture, plusUsersFixture, usersFixture } from '../fixture'; import { updateFlagsStatement } from '../../src/common'; +import { ioRedisPool } from '../../src/redis'; let con: DataSource; beforeEach(async () => { con = await createOrGetConnection(); + await ioRedisPool.execute((client) => client.flushall()); jest.clearAllMocks(); jest.resetModules(); await saveFixtures(con, User, [ @@ -230,7 +232,7 @@ describe('users for reactivation', () => { cio_registered: false, timezone: 'Etc/UTC', week_start: 1, - created_at: NaN, + created_at: 1656427727, updated_at: undefined, referral_id: null, referral_origin: null, diff --git a/src/cio.ts b/src/cio.ts index 4fa618361..a0bd3f1c3 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -126,12 +126,20 @@ export const getIdentifyAttributes = async ( }), ]); + const getDate = (value: number | string | Date) => { + if (typeof value === 'number') { + return debeziumTimeToDate(dup.createdAt); + } + + return new Date(value); + }; + return { ...camelCaseToSnakeCase(dup), first_name: getFirstName(dup.name), - created_at: dateToCioTimestamp(debeziumTimeToDate(dup.createdAt)), + created_at: dateToCioTimestamp(getDate(dup.createdAt)), updated_at: dup.updatedAt - ? dateToCioTimestamp(debeziumTimeToDate(dup.updatedAt)) + ? dateToCioTimestamp(getDate(dup.updatedAt)) : undefined, referral_link: genericInviteURL, [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Marketing}`]: diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 726fe3a3a..09a0c1f83 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -20,8 +20,8 @@ import { cioV2, generateIdentifyObject } from '../cio'; import { setTimeout } from 'node:timers/promises'; import { updateFlagsStatement } from './utils'; import { GetUsersActiveState } from './googleCloud'; -import { ChangeObject } from '../types'; import { logger } from '../logger'; +import { toChangeObject } from './typedPubsub'; export enum CioUnsubscribeTopic { Marketing = '4', @@ -214,10 +214,7 @@ export const syncSubscriptionsWithActiveState = async ({ const data = await Promise.all( users.map((user) => - generateIdentifyObject( - con, - structuredClone(user) as unknown as ChangeObject, - ), + generateIdentifyObject(con, toChangeObject(user)), ), ); diff --git a/src/common/typedPubsub.ts b/src/common/typedPubsub.ts index ffce7f188..b5d8737b3 100644 --- a/src/common/typedPubsub.ts +++ b/src/common/typedPubsub.ts @@ -119,3 +119,6 @@ export async function triggerTypedEvent( ): Promise { await publishEvent(log, pubsub.topic(topic), data); } + +export const toChangeObject = (entity: T): ChangeObject => + JSON.parse(Buffer.from(JSON.stringify(entity)).toString('utf-8').trim()); From 4ba9ff6992a97c591eca3aa6ad3b377b89e168dd Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 11:32:59 +0800 Subject: [PATCH 20/45] refactor: avoid nested looping --- src/common/async.ts | 5 ++ src/common/mailing.ts | 124 +++++++++++++++++------------------------- 2 files changed, 56 insertions(+), 73 deletions(-) diff --git a/src/common/async.ts b/src/common/async.ts index 2b732cce5..67f919bdd 100644 --- a/src/common/async.ts +++ b/src/common/async.ts @@ -26,6 +26,11 @@ export const blockingBatchRunner = async ({ }: BlockingBatchRunnerOptions) => { for (let i = 0; i < data.length; i += batchLimit) { const current = data.slice(i, i + batchLimit); + + if (current.length === 0) { + break; + } + const shouldStop = await runner(current); if (shouldStop) { break; diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 09a0c1f83..d79ff741b 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -192,94 +192,72 @@ export const syncSubscriptionsWithActiveState = async ({ con, users: { inactiveUsers, downgradeUsers, reactivateUsers }, }: SyncSubscriptionsWithActiveStateProps) => { + const validReactivateUsers = await con.getRepository(User).find({ + where: { id: In(reactivateUsers), cioRegistered: false }, + }); + + // user is active again: reactivate to CIO await blockingBatchRunner({ - data: reactivateUsers, + batchLimit: ITEMS_PER_IDENTIFY, + data: validReactivateUsers, runner: async (batch) => { - const validReactivateUsers = await con.getRepository(User).find({ - select: ['id'], - where: { id: In(batch), cioRegistered: false }, - }); - - if (validReactivateUsers.length === 0) { - return true; - } + const data = await Promise.all( + batch.map((batch) => + generateIdentifyObject(con, toChangeObject(batch)), + ), + ); - await blockingBatchRunner({ - batchLimit: ITEMS_PER_IDENTIFY, - data: validReactivateUsers.map(({ id }) => id), - runner: async (ids) => { - const users = await con + await callWithRetryDefault({ + callback: () => cioV2.request.post('/users', { batch: data }), + onSuccess: async () => { + const ids = batch.map(({ id }) => id); + await con .getRepository(User) - .find({ where: { id: In(ids) } }); - - const data = await Promise.all( - users.map((user) => - generateIdentifyObject(con, toChangeObject(user)), - ), - ); - - await callWithRetryDefault({ - callback: () => cioV2.request.post('/users', { batch: data }), - onSuccess: async () => { - await con - .getRepository(User) - .update({ id: In(ids) }, { cioRegistered: true }); - }, - onFailure: (err) => { - logger.info({ err }, 'Failed to add users to CIO'); - }, - }); - - await setTimeout(20); // wait for a bit to avoid rate limiting + .update({ id: In(ids) }, { cioRegistered: true }); + }, + onFailure: (err) => { + logger.info({ err }, 'Failed to add users to CIO'); }, }); + + await setTimeout(20); // wait for a bit to avoid rate limiting }, }); + const validInactiveUsers = await con.getRepository(User).find({ + select: ['id'], + where: { id: In(inactiveUsers), cioRegistered: true }, + }); // inactive for 12 weeks: remove from CIO await blockingBatchRunner({ - data: inactiveUsers, + batchLimit: ITEMS_PER_DESTROY, + data: validInactiveUsers.map(({ id }) => id), runner: async (batch) => { - const validInactiveUsers = await con.getRepository(User).find({ - select: ['id'], - where: { id: In(batch), cioRegistered: true }, - }); - - if (validInactiveUsers.length === 0) { - return true; - } - - await blockingBatchRunner({ - batchLimit: ITEMS_PER_DESTROY, - data: validInactiveUsers.map(({ id }) => id), - runner: async (ids) => { - const data = ids.map((id) => ({ - action: 'destroy', - type: 'person', - identifiers: { id }, - })); - - await callWithRetryDefault({ - callback: () => cioV2.request.post('/users', { batch: data }), - onSuccess: async () => { - await con.getRepository(User).update( - { id: In(ids) }, - { - cioRegistered: false, - acceptedMarketing: false, - followingEmail: false, - notificationEmail: false, - }, - ); + const data = batch.map((id) => ({ + action: 'destroy', + type: 'person', + identifiers: { id }, + })); + + await callWithRetryDefault({ + callback: () => cioV2.request.post('/users', { batch: data }), + onSuccess: async () => { + await con.getRepository(User).update( + { id: In(batch) }, + { + cioRegistered: false, + acceptedMarketing: false, + followingEmail: false, + notificationEmail: false, }, - onFailure: (err) => { - logger.info({ err }, 'Failed to remove users from CIO'); - }, - }); - - await setTimeout(20); // wait for a bit to avoid rate limiting + ); + }, + onFailure: (err) => { + logger.info({ err }, 'Failed to remove users from CIO'); }, }); + + await setTimeout(20); // wait for a bit to avoid rate limiting }, }); From 9c5c00b3095b0ba4ced86d96a766d94fea7c30fe Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 11:38:56 +0800 Subject: [PATCH 21/45] refactor: run the downgrade within a single query --- src/common/mailing.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/common/mailing.ts b/src/common/mailing.ts index d79ff741b..6237b20ee 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -7,7 +7,7 @@ import { import CIORequest from 'customerio-node/dist/lib/request'; import { SendEmailRequestOptionalOptions } from 'customerio-node/lib/api/requests'; import { SendEmailRequestWithTemplate } from 'customerio-node/dist/lib/api/requests'; -import { DataSource, In } from 'typeorm'; +import { DataSource, In, Raw } from 'typeorm'; import { Source, User, @@ -265,18 +265,12 @@ export const syncSubscriptionsWithActiveState = async ({ await blockingBatchRunner({ data: downgradeUsers, runner: async (current) => { - const validDowngradeUsers = await con - .getRepository(User) - .createQueryBuilder('u') - .select('id') - .innerJoin(UserPersonalizedDigest, 'upd', 'u.id = upd."userId"') - .where('u.id IN (:...ids)', { ids: current }) - .andWhere(`upd.flags->>'sendType' = 'daily'`) - .getRawMany>(); - // set digest to weekly on Wednesday 9am await con.getRepository(UserPersonalizedDigest).update( - { userId: In(validDowngradeUsers.map(({ id }) => id)) }, + { + userId: In(current), + flags: Raw(() => `flags->>'sendType' = 'daily'`), + }, { preferredDay: 3, preferredHour: 9, From 28c368466d4c301991aec96759ae7597c6c61974 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 12:04:12 +0800 Subject: [PATCH 22/45] fix: import issues --- src/cio.ts | 10 +++------- src/common/mailing.ts | 3 +-- src/common/typedPubsub.ts | 3 --- src/common/utils.ts | 4 ++++ 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/cio.ts b/src/cio.ts index a0bd3f1c3..2d328264b 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -7,13 +7,9 @@ import { UserPersonalizedDigestType, UserStreak, } from './entity'; -import { - camelCaseToSnakeCase, - CioUnsubscribeTopic, - debeziumTimeToDate, - getFirstName, - getShortGenericInviteLink, -} from './common'; +import { camelCaseToSnakeCase, debeziumTimeToDate } from './common/utils'; +import { CioUnsubscribeTopic, getFirstName } from './common/mailing'; +import { getShortGenericInviteLink } from './common/links'; import type { UserCompany } from './entity/UserCompany'; import type { Company } from './entity/Company'; import { DataSource } from 'typeorm'; diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 6237b20ee..9954cf6fc 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -18,10 +18,9 @@ import { import { blockingBatchRunner, callWithRetryDefault } from './async'; import { cioV2, generateIdentifyObject } from '../cio'; import { setTimeout } from 'node:timers/promises'; -import { updateFlagsStatement } from './utils'; +import { toChangeObject, updateFlagsStatement } from './utils'; import { GetUsersActiveState } from './googleCloud'; import { logger } from '../logger'; -import { toChangeObject } from './typedPubsub'; export enum CioUnsubscribeTopic { Marketing = '4', diff --git a/src/common/typedPubsub.ts b/src/common/typedPubsub.ts index b5d8737b3..ffce7f188 100644 --- a/src/common/typedPubsub.ts +++ b/src/common/typedPubsub.ts @@ -119,6 +119,3 @@ export async function triggerTypedEvent( ): Promise { await publishEvent(log, pubsub.topic(topic), data); } - -export const toChangeObject = (entity: T): ChangeObject => - JSON.parse(Buffer.from(JSON.stringify(entity)).toString('utf-8').trim()); diff --git a/src/common/utils.ts b/src/common/utils.ts index 3f6908620..8a72d13b6 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -3,6 +3,7 @@ import { zonedTimeToUtc } from 'date-fns-tz'; import { snakeCase } from 'lodash'; import { isNullOrUndefined } from './object'; import { remoteConfig } from '../remoteConfig'; +import { ChangeObject } from '../types'; const REMOVE_SPECIAL_CHARACTERS_REGEX = /[^a-zA-Z0-9-_#.]/g; @@ -138,6 +139,9 @@ export const toGQLEnum = (value: Record, name: string) => { return `enum ${name} { ${Object.values(value).join(' ')} }`; }; +export const toChangeObject = (entity: T): ChangeObject => + JSON.parse(Buffer.from(JSON.stringify(entity)).toString('utf-8').trim()); + export function camelCaseToSnakeCase( obj: Record, ): Record { From c56070e42f7c90f5ceceffdff1e109e79207ce31 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 12:09:05 +0800 Subject: [PATCH 23/45] refactor: reusable function --- src/cio.ts | 14 +++----------- src/common/utils.ts | 8 ++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/cio.ts b/src/cio.ts index 2d328264b..f78211384 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -7,7 +7,7 @@ import { UserPersonalizedDigestType, UserStreak, } from './entity'; -import { camelCaseToSnakeCase, debeziumTimeToDate } from './common/utils'; +import { camelCaseToSnakeCase, getDateBaseFromType } from './common/utils'; import { CioUnsubscribeTopic, getFirstName } from './common/mailing'; import { getShortGenericInviteLink } from './common/links'; import type { UserCompany } from './entity/UserCompany'; @@ -122,20 +122,12 @@ export const getIdentifyAttributes = async ( }), ]); - const getDate = (value: number | string | Date) => { - if (typeof value === 'number') { - return debeziumTimeToDate(dup.createdAt); - } - - return new Date(value); - }; - return { ...camelCaseToSnakeCase(dup), first_name: getFirstName(dup.name), - created_at: dateToCioTimestamp(getDate(dup.createdAt)), + created_at: dateToCioTimestamp(getDateBaseFromType(dup.createdAt)), updated_at: dup.updatedAt - ? dateToCioTimestamp(getDate(dup.updatedAt)) + ? dateToCioTimestamp(getDateBaseFromType(dup.updatedAt)) : undefined, referral_link: genericInviteURL, [`cio_subscription_preferences.topics.topic_${CioUnsubscribeTopic.Marketing}`]: diff --git a/src/common/utils.ts b/src/common/utils.ts index 8a72d13b6..405182bf8 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -156,6 +156,14 @@ export function debeziumTimeToDate(time: number): Date { return new Date(Math.floor(time / 1000)); } +export const getDateBaseFromType = (value: number | string | Date) => { + if (typeof value === 'number') { + return debeziumTimeToDate(value); + } + + return new Date(value); +}; + export const safeJSONParse = (json: string): T | undefined => { try { return JSON.parse(json); From d68a90cdf7625c9a116bd2a5c108f58357c4a222 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 12:11:15 +0800 Subject: [PATCH 24/45] refactor: unneeded columns --- src/common/googleCloud.ts | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index ffd00d622..a669a4f49 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -63,24 +63,6 @@ export const userActiveStateQuery = ` when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3' else '4' end ) as current_state, - min( - case - when period_end is null then '4. never_active' - when period_end between date(@previous_date - interval 6*7 day) and @previous_date then '1. active_last_6w' - when period_end between date(@previous_date - interval 12*7 day) and date(@previous_date - interval 6*7 + 1 day) then '2. active_7w_12w' - when date(u.last_app_timestamp) < date(@previous_date - interval 12*7 day) then '3. active_12w+' - when date(u.registration_timestamp) < date(@previous_date - interval 12*7 day) then '3. active_12w+' - else '4. never_active' end - ) as previous_state_name, - min( - case - when period_end is null then '4. never_active' - when period_end between date(@run_date - interval 6*7 day) and @run_date then '1. active_last_6w' - when period_end between date(@run_date - interval 12*7 day) and date(@run_date - interval 6*7 + 1 day) then '2. active_7w_12w' - when date(u.last_app_timestamp) < date(@run_date - interval 12*7 day) then '3. active_12w+' - when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3. active_12w+' - else '4. never_active' end - ) as current_state_name, from analytics.user as u left join analytics.user_state_sparse as uss on uss.primary_user_id = u.primary_user_id and uss.period_end between date(@previous_date - interval 12* 7 day) and @run_date @@ -123,6 +105,7 @@ export const queryFromBq = async ( ): Promise => { const [job] = await bigquery.createQueryJob(query); const [rows] = await job.getQueryResults(); + console.log('rows: ', rows); return rows; }; From 383182c0c1fb3a01f4756615f75cd7eadd28ae61 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 12:14:27 +0800 Subject: [PATCH 25/45] chore: better log message --- src/common/mailing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 9954cf6fc..1e30a5d4c 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -215,7 +215,7 @@ export const syncSubscriptionsWithActiveState = async ({ .update({ id: In(ids) }, { cioRegistered: true }); }, onFailure: (err) => { - logger.info({ err }, 'Failed to add users to CIO'); + logger.info({ err }, 'Failed to reactivate users to CIO'); }, }); From 5a6de1ff1a944fdaaa7a483579d0cc48c5e5dcfe Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 12:20:01 +0800 Subject: [PATCH 26/45] fix: removal of personalized digest --- src/common/mailing.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 1e30a5d4c..87875c6da 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -241,15 +241,20 @@ export const syncSubscriptionsWithActiveState = async ({ await callWithRetryDefault({ callback: () => cioV2.request.post('/users', { batch: data }), onSuccess: async () => { - await con.getRepository(User).update( - { id: In(batch) }, - { - cioRegistered: false, - acceptedMarketing: false, - followingEmail: false, - notificationEmail: false, - }, - ); + await Promise.all([ + con.getRepository(User).update( + { id: In(batch) }, + { + cioRegistered: false, + acceptedMarketing: false, + followingEmail: false, + notificationEmail: false, + }, + ), + con + .getRepository(UserPersonalizedDigest) + .delete({ userId: In(batch) }), + ]); }, onFailure: (err) => { logger.info({ err }, 'Failed to remove users from CIO'); From 494142e67fc6931667a629d47239a2c8a4fed668 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 12:22:09 +0800 Subject: [PATCH 27/45] fix: remove digest --- __tests__/cron/validateActiveUsers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/__tests__/cron/validateActiveUsers.ts b/__tests__/cron/validateActiveUsers.ts index 113bbe389..a92fe65c4 100644 --- a/__tests__/cron/validateActiveUsers.ts +++ b/__tests__/cron/validateActiveUsers.ts @@ -128,6 +128,8 @@ describe('users for removal', () => { await con .getRepository(User) .update({ id: In(['vordr']) }, { cioRegistered: false }); + const digests = await con.getRepository(UserPersonalizedDigest).count(); + expect(digests).toBeGreaterThan(0); await expectSuccessfulCron(cron); @@ -160,6 +162,9 @@ describe('users for removal', () => { .getRepository(User) .findOne({ select: ['cioRegistered'], where: { id: '3' } }); expect(unregistered.cioRegistered).toEqual(false); + + const removed = await con.getRepository(UserPersonalizedDigest).count(); + expect(removed).toBeLessThan(digests); }); }); From 65d54fa98264f54efa9194c03d95c6a2bb98ac53 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 12:23:33 +0800 Subject: [PATCH 28/45] fix: unnecessary comment --- __tests__/cron/validateActiveUsers.ts | 2 +- src/cron/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/__tests__/cron/validateActiveUsers.ts b/__tests__/cron/validateActiveUsers.ts index a92fe65c4..6c0aec669 100644 --- a/__tests__/cron/validateActiveUsers.ts +++ b/__tests__/cron/validateActiveUsers.ts @@ -36,7 +36,7 @@ describe('validateActiveUsers', () => { it('should NOT be registered yet', () => { const registeredWorker = crons.find((item) => item.name === cron.name); - expect(registeredWorker).not.toBeDefined(); + expect(registeredWorker).toBeDefined(); }); }); diff --git a/src/cron/index.ts b/src/cron/index.ts index f7fb8ac69..e0a270f69 100644 --- a/src/cron/index.ts +++ b/src/cron/index.ts @@ -13,7 +13,7 @@ import updateHighlightedViews from './updateHighlightedViews'; import hourlyNotifications from './hourlyNotifications'; import updateCurrentStreak from './updateCurrentStreak'; import syncSubscriptionWithCIO from './syncSubscriptionWithCIO'; -// import validateActiveUsers from './validateActiveUsers'; +import validateActiveUsers from './validateActiveUsers'; import { updateSourcePublicThreshold } from './updateSourcePublicThreshold'; import { cleanZombieUserCompany } from './cleanZombieUserCompany'; import { calculateTopReaders } from './calculateTopReaders'; @@ -36,5 +36,5 @@ export const crons: Cron[] = [ cleanZombieUserCompany, updateSourcePublicThreshold, calculateTopReaders, - // validateActiveUsers, + validateActiveUsers, ]; From b7750051e16314f8217e21000227304fceb9780e Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 13:24:34 +0800 Subject: [PATCH 29/45] fix: bq on cron --- src/common/googleCloud.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index a669a4f49..bd6539831 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -79,11 +79,14 @@ export const userActiveStateQuery = ` and previous_state != '4' `; -export const getUserActiveStateQuery = (runDate: Date): Query => { +export const getUserActiveStateQuery = ( + runDate: Date, + query = userActiveStateQuery, +): Query => { const run_date = runDate.toISOString().split('T')[0]; const previous_date = subDays(runDate, 1).toISOString().split('T')[0]; - return { query: userActiveStateQuery, params: { previous_date, run_date } }; + return { query, params: { previous_date, run_date } }; }; export interface GetUsersActiveState { @@ -110,17 +113,13 @@ export const queryFromBq = async ( return rows; }; -export const getUsersActiveState = async ( - runDate: Date, -): Promise => { - const query = getUserActiveStateQuery(runDate); - const usersFromBq = await queryFromBq(query); +export const sortUsersActiveState = (users: UserActiveStateData[]) => { const inactiveUsers: string[] = []; const downgradeUsers: string[] = []; const reactivateUsers: string[] = []; // sort users from bq into active, inactive, downgrade, and reactivate - for (const user of usersFromBq) { + for (const user of users) { if ( user.current_state === UserActiveState.InactiveSince6wAgo && user.previous_state === UserActiveState.Active @@ -141,3 +140,12 @@ export const getUsersActiveState = async ( return { inactiveUsers, downgradeUsers, reactivateUsers }; }; + +export const getUsersActiveState = async ( + runDate: Date, +): Promise => { + const query = getUserActiveStateQuery(runDate); + const usersFromBq = await queryFromBq(query); + + return sortUsersActiveState(usersFromBq); +}; From 9735a79aa093bc935d14b0af87db07fbe19df122 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 13:25:02 +0800 Subject: [PATCH 30/45] fix: limitations --- src/common/mailing.ts | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 87875c6da..6fd9bc184 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -191,25 +191,27 @@ export const syncSubscriptionsWithActiveState = async ({ con, users: { inactiveUsers, downgradeUsers, reactivateUsers }, }: SyncSubscriptionsWithActiveStateProps) => { - const validReactivateUsers = await con.getRepository(User).find({ - where: { id: In(reactivateUsers), cioRegistered: false }, - }); - // user is active again: reactivate to CIO await blockingBatchRunner({ batchLimit: ITEMS_PER_IDENTIFY, - data: validReactivateUsers, + data: reactivateUsers, runner: async (batch) => { + const users = await con.getRepository(User).find({ + where: { id: In(batch), cioRegistered: false }, + }); + + if (users.length === 0) { + return true; + } + const data = await Promise.all( - batch.map((batch) => - generateIdentifyObject(con, toChangeObject(batch)), - ), + users.map((user) => generateIdentifyObject(con, toChangeObject(user))), ); await callWithRetryDefault({ callback: () => cioV2.request.post('/users', { batch: data }), onSuccess: async () => { - const ids = batch.map(({ id }) => id); + const ids = users.map(({ id }) => id); await con .getRepository(User) .update({ id: In(ids) }, { cioRegistered: true }); @@ -223,16 +225,21 @@ export const syncSubscriptionsWithActiveState = async ({ }, }); - const validInactiveUsers = await con.getRepository(User).find({ - select: ['id'], - where: { id: In(inactiveUsers), cioRegistered: true }, - }); // inactive for 12 weeks: remove from CIO await blockingBatchRunner({ batchLimit: ITEMS_PER_DESTROY, - data: validInactiveUsers.map(({ id }) => id), + data: inactiveUsers, runner: async (batch) => { - const data = batch.map((id) => ({ + const users = await con.getRepository(User).find({ + select: ['id'], + where: { id: In(batch), cioRegistered: true }, + }); + + if (users.length === 0) { + return true; + } + + const data = users.map((id) => ({ action: 'destroy', type: 'person', identifiers: { id }, @@ -241,9 +248,10 @@ export const syncSubscriptionsWithActiveState = async ({ await callWithRetryDefault({ callback: () => cioV2.request.post('/users', { batch: data }), onSuccess: async () => { + const ids = users.map(({ id }) => id); await Promise.all([ con.getRepository(User).update( - { id: In(batch) }, + { id: In(ids) }, { cioRegistered: false, acceptedMarketing: false, @@ -253,7 +261,7 @@ export const syncSubscriptionsWithActiveState = async ({ ), con .getRepository(UserPersonalizedDigest) - .delete({ userId: In(batch) }), + .delete({ userId: In(ids) }), ]); }, onFailure: (err) => { From f19be403d6c286d623e50e26a538f644add7feb2 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 13:45:20 +0800 Subject: [PATCH 31/45] fix: only required columns --- __tests__/cron/validateActiveUsers.ts | 43 +++++---------------------- src/cio.ts | 10 +++++++ src/common/mailing.ts | 3 +- 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/__tests__/cron/validateActiveUsers.ts b/__tests__/cron/validateActiveUsers.ts index 6c0aec669..c572a1b59 100644 --- a/__tests__/cron/validateActiveUsers.ts +++ b/__tests__/cron/validateActiveUsers.ts @@ -213,47 +213,18 @@ describe('users for reactivation', () => { type: 'person', identifiers: { id: '1' }, attributes: { - name: 'Ido', - email: 'ido@daily.dev', - image: 'https://daily.dev/ido.jpg', - cover: null, - company: null, - title: null, accepted_marketing: false, - reputation: 10, - username: 'idoshamun', - twitter: null, - github: 'idogithub', - roadmap: null, - threads: null, - codepen: null, - reddit: null, - stackoverflow: null, - youtube: null, - linkedin: null, - mastodon: null, - portfolio: null, - hashnode: null, - cio_registered: false, - timezone: 'Etc/UTC', - week_start: 1, - created_at: 1656427727, - updated_at: undefined, - referral_id: null, - referral_origin: null, - acquisition_channel: null, - experience_level: null, - flags: {}, - language: null, - default_feed_id: null, - subscription_flags: {}, - permalink: 'http://localhost:5002/idoshamun', - first_name: 'Ido', - referral_link: 'http://localhost:5002/join?cid=generic&userid=1', 'cio_subscription_preferences.topics.topic_4': false, 'cio_subscription_preferences.topics.topic_7': true, 'cio_subscription_preferences.topics.topic_8': true, 'cio_subscription_preferences.topics.topic_9': true, + created_at: 1656427727, + first_name: 'Ido', + name: 'Ido', + permalink: 'http://localhost:5002/idoshamun', + referral_link: 'http://localhost:5002/join?cid=generic&userid=1', + updated_at: undefined, + username: 'idoshamun', }, }, ]; diff --git a/src/cio.ts b/src/cio.ts index f78211384..510f9b44b 100644 --- a/src/cio.ts +++ b/src/cio.ts @@ -49,6 +49,16 @@ const OMIT_FIELDS: (keyof ChangeObject)[] = [ 'followNotifications', ]; +export const CIO_REQUIRED_FIELDS: (keyof ChangeObject)[] = [ + 'username', + 'name', + 'createdAt', + 'updatedAt', + 'notificationEmail', + 'acceptedMarketing', + 'followingEmail', +]; + export async function identifyUserStreak({ cio, data, diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 6fd9bc184..448238025 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -16,7 +16,7 @@ import { UserPersonalizedDigestType, } from '../entity'; import { blockingBatchRunner, callWithRetryDefault } from './async'; -import { cioV2, generateIdentifyObject } from '../cio'; +import { CIO_REQUIRED_FIELDS, cioV2, generateIdentifyObject } from '../cio'; import { setTimeout } from 'node:timers/promises'; import { toChangeObject, updateFlagsStatement } from './utils'; import { GetUsersActiveState } from './googleCloud'; @@ -198,6 +198,7 @@ export const syncSubscriptionsWithActiveState = async ({ runner: async (batch) => { const users = await con.getRepository(User).find({ where: { id: In(batch), cioRegistered: false }, + select: CIO_REQUIRED_FIELDS.concat('id'), }); if (users.length === 0) { From 480208533e074bbfda87f23ce29cbc507add8178 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 14:08:07 +0800 Subject: [PATCH 32/45] fix: test --- __tests__/workers/userUpdatedCio.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/__tests__/workers/userUpdatedCio.ts b/__tests__/workers/userUpdatedCio.ts index 1df28fbee..c038fe6ff 100644 --- a/__tests__/workers/userUpdatedCio.ts +++ b/__tests__/workers/userUpdatedCio.ts @@ -21,6 +21,11 @@ import { usersFixture } from '../fixture/user'; jest.mock('../../src/common', () => ({ ...jest.requireActual('../../src/common'), + resubscribeUser: jest.fn(), +})); + +jest.mock('../../src/common/links', () => ({ + ...jest.requireActual('../../src/common/links'), getShortGenericInviteLink: jest.fn(), resubscribeUser: jest.fn(), })); From 2da1eaabb4cd8d8a02ed8ffc705d827bd9510275 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 14:18:04 +0800 Subject: [PATCH 33/45] chore: removal of console logs --- src/common/googleCloud.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/googleCloud.ts b/src/common/googleCloud.ts index bd6539831..b86a49e55 100644 --- a/src/common/googleCloud.ts +++ b/src/common/googleCloud.ts @@ -108,7 +108,6 @@ export const queryFromBq = async ( ): Promise => { const [job] = await bigquery.createQueryJob(query); const [rows] = await job.getQueryResults(); - console.log('rows: ', rows); return rows; }; From 280f2501fc972eb0ae41cddf83ac71660e7aee40 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 14:20:02 +0800 Subject: [PATCH 34/45] feat: bin script for initial rollout --- bin/syncCioBasedOnActivity.ts | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 bin/syncCioBasedOnActivity.ts diff --git a/bin/syncCioBasedOnActivity.ts b/bin/syncCioBasedOnActivity.ts new file mode 100644 index 000000000..2215ad7f4 --- /dev/null +++ b/bin/syncCioBasedOnActivity.ts @@ -0,0 +1,62 @@ +import { + getUserActiveStateQuery, + queryFromBq, + sortUsersActiveState, +} from '../src/common/googleCloud'; +import { subDays } from 'date-fns'; +import { + SUCCESSFUL_CIO_SYNC_DATE, + syncSubscriptionsWithActiveState, +} from '../src/common'; +import { setRedisObject } from '../src/redis'; +import createOrGetConnection from '../src/db'; + +const userActiveStateQuery = ` + with d as ( + select u.primary_user_id, + min(last_app_timestamp) as last_app_timestamp, + min(registration_timestamp) as registration_timestamp, + min( + case + when period_end is null then '4' + when period_end between date(@run_date - interval 6*7 day) and @run_date then '1' + when period_end between date(@run_date - interval 12*7 day) and date(@run_date - interval 6*7 + 1 day) then '2' + when date(u.last_app_timestamp) < date(@run_date - interval 12*7 day) then '3' + when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3' + else '4' end + ) as current_state, + from analytics.user as u + left join analytics.user_state_sparse as uss on uss.primary_user_id = u.primary_user_id + and uss.period_end between '2018-01-01' and @run_date + and uss.period = 'daily' + and uss.app_active_state = 'active' + and uss.registration_state = 'registered' + where u.registration_timestamp is not null + and date(u.registration_timestamp) < @run_date + group by 1 + ) + select COUNT(*) + from d + where current_state != '1' +`; + +const func = async () => { + const runDate = subDays(new Date(), 1); + const usersFromBqQuery = await getUserActiveStateQuery( + runDate, + userActiveStateQuery, + ); + const usersFromBq = await queryFromBq(usersFromBqQuery); + const users = sortUsersActiveState(usersFromBq); + const con = await createOrGetConnection(); + + await syncSubscriptionsWithActiveState({ + con, + users, + }); + await setRedisObject(SUCCESSFUL_CIO_SYNC_DATE, runDate.toISOString()); + + process.exit(0); +}; + +func(); From b7e6ddf5b0df3816cecf7e898db88de810b41c97 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 14:40:47 +0800 Subject: [PATCH 35/45] fix: test shape --- src/common/mailing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 448238025..0a99f50b4 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -240,7 +240,7 @@ export const syncSubscriptionsWithActiveState = async ({ return true; } - const data = users.map((id) => ({ + const data = users.map(({ id }) => ({ action: 'destroy', type: 'person', identifiers: { id }, From c6c678b9cf63d4b8bebb102dbb9a3689e47579e6 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 15:17:15 +0800 Subject: [PATCH 36/45] test: import mock --- __tests__/workers/userCreatedCio.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/__tests__/workers/userCreatedCio.ts b/__tests__/workers/userCreatedCio.ts index 9f2e0dd2c..92758bab9 100644 --- a/__tests__/workers/userCreatedCio.ts +++ b/__tests__/workers/userCreatedCio.ts @@ -12,8 +12,8 @@ import { cio } from '../../src/cio'; import { typedWorkers } from '../../src/workers'; import mocked = jest.mocked; -jest.mock('../../src/common', () => ({ - ...jest.requireActual('../../src/common'), +jest.mock('../../src/common/links', () => ({ + ...jest.requireActual('../../src/common/links'), getShortGenericInviteLink: jest.fn(), })); From 2db40204e35fef4459574698d94f09d6f9909605 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 18:30:07 +0800 Subject: [PATCH 37/45] refactor: worker to work with initial run --- bin/syncCioBasedOnActivity.ts | 54 ++------------------------------- src/common/mailing.ts | 12 +++++++- src/cron/validateActiveUsers.ts | 43 +++++++++++++++----------- 3 files changed, 38 insertions(+), 71 deletions(-) diff --git a/bin/syncCioBasedOnActivity.ts b/bin/syncCioBasedOnActivity.ts index 2215ad7f4..6a69ab274 100644 --- a/bin/syncCioBasedOnActivity.ts +++ b/bin/syncCioBasedOnActivity.ts @@ -1,60 +1,10 @@ -import { - getUserActiveStateQuery, - queryFromBq, - sortUsersActiveState, -} from '../src/common/googleCloud'; -import { subDays } from 'date-fns'; -import { - SUCCESSFUL_CIO_SYNC_DATE, - syncSubscriptionsWithActiveState, -} from '../src/common'; -import { setRedisObject } from '../src/redis'; import createOrGetConnection from '../src/db'; - -const userActiveStateQuery = ` - with d as ( - select u.primary_user_id, - min(last_app_timestamp) as last_app_timestamp, - min(registration_timestamp) as registration_timestamp, - min( - case - when period_end is null then '4' - when period_end between date(@run_date - interval 6*7 day) and @run_date then '1' - when period_end between date(@run_date - interval 12*7 day) and date(@run_date - interval 6*7 + 1 day) then '2' - when date(u.last_app_timestamp) < date(@run_date - interval 12*7 day) then '3' - when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3' - else '4' end - ) as current_state, - from analytics.user as u - left join analytics.user_state_sparse as uss on uss.primary_user_id = u.primary_user_id - and uss.period_end between '2018-01-01' and @run_date - and uss.period = 'daily' - and uss.app_active_state = 'active' - and uss.registration_state = 'registered' - where u.registration_timestamp is not null - and date(u.registration_timestamp) < @run_date - group by 1 - ) - select COUNT(*) - from d - where current_state != '1' -`; +import { syncValidateActiveUsersCron } from '../src/cron/validateActiveUsers'; const func = async () => { - const runDate = subDays(new Date(), 1); - const usersFromBqQuery = await getUserActiveStateQuery( - runDate, - userActiveStateQuery, - ); - const usersFromBq = await queryFromBq(usersFromBqQuery); - const users = sortUsersActiveState(usersFromBq); const con = await createOrGetConnection(); - await syncSubscriptionsWithActiveState({ - con, - users, - }); - await setRedisObject(SUCCESSFUL_CIO_SYNC_DATE, runDate.toISOString()); + await syncValidateActiveUsersCron(con); process.exit(0); }; diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 0a99f50b4..0750abd38 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -187,10 +187,16 @@ interface SyncSubscriptionsWithActiveStateProps { users: GetUsersActiveState; } +interface SyncSubscriptionsWithActiveState { + hasAnyFailed: boolean; +} + export const syncSubscriptionsWithActiveState = async ({ con, users: { inactiveUsers, downgradeUsers, reactivateUsers }, -}: SyncSubscriptionsWithActiveStateProps) => { +}: SyncSubscriptionsWithActiveStateProps): Promise => { + let hasAnyFailed = false; + // user is active again: reactivate to CIO await blockingBatchRunner({ batchLimit: ITEMS_PER_IDENTIFY, @@ -218,6 +224,7 @@ export const syncSubscriptionsWithActiveState = async ({ .update({ id: In(ids) }, { cioRegistered: true }); }, onFailure: (err) => { + hasAnyFailed = true; logger.info({ err }, 'Failed to reactivate users to CIO'); }, }); @@ -266,6 +273,7 @@ export const syncSubscriptionsWithActiveState = async ({ ]); }, onFailure: (err) => { + hasAnyFailed = true; logger.info({ err }, 'Failed to remove users from CIO'); }, }); @@ -294,4 +302,6 @@ export const syncSubscriptionsWithActiveState = async ({ ); }, }); + + return { hasAnyFailed }; }; diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index 1d4065c37..bd4aa9d30 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -12,33 +12,40 @@ import { addDays, subDays } from 'date-fns'; const runSync = async (con: DataSource, runDate: Date) => { const users = await getUsersActiveState(runDate); - await syncSubscriptionsWithActiveState({ + const { hasAnyFailed } = await syncSubscriptionsWithActiveState({ con, users, }); - await setRedisObject(SUCCESSFUL_CIO_SYNC_DATE, runDate.toISOString()); + + if (!hasAnyFailed) { + await setRedisObject(SUCCESSFUL_CIO_SYNC_DATE, runDate.toISOString()); + } }; -const cron: Cron = { - name: 'validate-active-users', - handler: async (con) => { - const lastSuccessfulDate = await getRedisObject(SUCCESSFUL_CIO_SYNC_DATE); - const processingDate = subDays(new Date(), 1); +export const syncValidateActiveUsersCron = async (con: DataSource) => { + const runDate = subDays(new Date(), 1); + const lastSuccessfulDate = await getRedisObject(SUCCESSFUL_CIO_SYNC_DATE); - if (!lastSuccessfulDate) { - return runSync(con, processingDate); - } + if (!lastSuccessfulDate) { + return runSync(con, runDate); + } - const lastRunDate = new Date(lastSuccessfulDate); - const difference = getAbsoluteDifferenceInDays(lastRunDate, processingDate); + const lastRunDate = new Date(lastSuccessfulDate); + const difference = getAbsoluteDifferenceInDays(lastRunDate, runDate); - if (difference === 0) { - return; - } + if (difference === 0) { + return; + } - for (let i = 1; i <= difference; i++) { - await runSync(con, addDays(lastRunDate, i)); - } + for (let i = 1; i <= difference || i > 100; i++) { + await runSync(con, addDays(lastRunDate, i)); + } +}; + +const cron: Cron = { + name: 'validate-active-users', + handler: async (con) => { + await syncValidateActiveUsersCron(con); }, }; From d0ac6847b1c385a048d5d8469c5d4d6a83d0bafe Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 20 Dec 2024 18:44:49 +0800 Subject: [PATCH 38/45] chore: ignore column --- .infra/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/.infra/application.properties b/.infra/application.properties index e10113c77..2a3302806 100644 --- a/.infra/application.properties +++ b/.infra/application.properties @@ -28,4 +28,5 @@ debezium.transforms.ReadOperationFilter.condition=!(valueSchema.field('op') && v debezium.transforms.PostsFilter.type=io.debezium.transforms.Filter debezium.transforms.PostsFilter.language=jsr223.groovy debezium.transforms.PostsFilter.condition=!(valueSchema.field('op') && value.op == 'u' && value.source.table == 'post' && value.before.views != value.after.views) +# debezium.transforms.PostsFilter.condition=!(valueSchema.field('op') && value.op == 'u' && value.source.table == 'post' && value.before.views != value.after.views) && !(valueSchema.field('op') && value.op == 'u' && value.source.table == 'user' && value.before.cioRegistered != value.after.cioRegistered && !value.after.cioRegistered) debezium.sink.type=pubsub From ec456dac75445c69ef9f7ee5fa94d643e3ddb648 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 2 Jan 2025 12:10:15 +0800 Subject: [PATCH 39/45] fix: missing transaction --- src/common/mailing.ts | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/common/mailing.ts b/src/common/mailing.ts index 0750abd38..4db52b1c3 100644 --- a/src/common/mailing.ts +++ b/src/common/mailing.ts @@ -257,20 +257,23 @@ export const syncSubscriptionsWithActiveState = async ({ callback: () => cioV2.request.post('/users', { batch: data }), onSuccess: async () => { const ids = users.map(({ id }) => id); - await Promise.all([ - con.getRepository(User).update( - { id: In(ids) }, - { - cioRegistered: false, - acceptedMarketing: false, - followingEmail: false, - notificationEmail: false, - }, - ), - con - .getRepository(UserPersonalizedDigest) - .delete({ userId: In(ids) }), - ]); + + await con.transaction(async (manager) => { + await Promise.all([ + manager.getRepository(User).update( + { id: In(ids) }, + { + cioRegistered: false, + acceptedMarketing: false, + followingEmail: false, + notificationEmail: false, + }, + ), + manager + .getRepository(UserPersonalizedDigest) + .delete({ userId: In(ids) }), + ]); + }); }, onFailure: (err) => { hasAnyFailed = true; From 772d21c434fee5ab4fe93b3b9d309e7717c01f03 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 2 Jan 2025 12:13:53 +0800 Subject: [PATCH 40/45] refactor: unnecessary limit --- src/cron/validateActiveUsers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index bd4aa9d30..b82bf754e 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -37,7 +37,7 @@ export const syncValidateActiveUsersCron = async (con: DataSource) => { return; } - for (let i = 1; i <= difference || i > 100; i++) { + for (let i = 1; i <= difference; i++) { await runSync(con, addDays(lastRunDate, i)); } }; From dc0aa64de319024d7831aa60780599a461798bd4 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 2 Jan 2025 15:59:36 +0800 Subject: [PATCH 41/45] feat: initial rollout through csv --- bin/cioSyncBasedOnActivity.ts | 66 +++ bin/cio_initial_rollout.csv | 1001 +++++++++++++++++++++++++++++++++ bin/syncCioBasedOnActivity.ts | 12 - 3 files changed, 1067 insertions(+), 12 deletions(-) create mode 100644 bin/cioSyncBasedOnActivity.ts create mode 100644 bin/cio_initial_rollout.csv delete mode 100644 bin/syncCioBasedOnActivity.ts diff --git a/bin/cioSyncBasedOnActivity.ts b/bin/cioSyncBasedOnActivity.ts new file mode 100644 index 000000000..a211092bd --- /dev/null +++ b/bin/cioSyncBasedOnActivity.ts @@ -0,0 +1,66 @@ +import createOrGetConnection from '../src/db'; +import fs from 'fs'; +import { parse } from 'csv-parse'; +import { syncSubscriptionsWithActiveState } from '../src/common'; +import { + GetUsersActiveState, + UserActiveState, +} from '../src/common/googleCloud'; + +const func = async () => { + const csvFilePath = process.argv[2]; + + if (!csvFilePath) { + throw new Error('CSV file path is required'); + } + + const users: GetUsersActiveState = { + inactiveUsers: [], + downgradeUsers: [], + reactivateUsers: [], + }; + + const stream = fs + .createReadStream(csvFilePath) + .pipe(parse({ delimiter: ',', from_line: 2 })); + + stream.on('error', (err) => { + console.error('failed to read file: ', err.message); + }); + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + stream.on('data', function ([id, rawStatus]) { + if (!id || !rawStatus) { + return; + } + + const status = rawStatus.toString() as UserActiveState; + + if (status === UserActiveState.InactiveSince6wAgo) { + users.downgradeUsers.push(id); + } + + if ( + status === UserActiveState.InactiveSince12wAgo || + status === UserActiveState.NeverActive + ) { + users.inactiveUsers.push(id); + } + }); + + await new Promise((resolve) => { + stream.on('end', resolve); + }); + + console.log('running cron sync function'); + + const con = await createOrGetConnection(); + + await syncSubscriptionsWithActiveState({ con, users }); + + console.log('finished sync'); + + process.exit(0); +}; + +func(); diff --git a/bin/cio_initial_rollout.csv b/bin/cio_initial_rollout.csv new file mode 100644 index 000000000..31195bb50 --- /dev/null +++ b/bin/cio_initial_rollout.csv @@ -0,0 +1,1001 @@ +primary_user_id,current_state +az8FNk0UMkw96oDZA82Z5,3 +AhzuQ07YKr1rLH6fgOT4x,3 +3w7y33Wwbtsij773wxjXj,3 +5aR3jOAV2JCPm5NxqSNmU,3 +T28Fu221qqs1GFYflMHjy,3 +7gdIEdaVOkIYk4LXRImnT,4 +q9DOlUL5hQ59VOhEp9Ejy,3 +jg88HSeDwcykxusfLdtcG,3 +FoknsDjsXUD7wLKGBzjiL,3 +YAak1lkTckI88zogtSMt2,3 +mAhoGtiop11h2qYZcvnL2,3 +VNqsaD3AuBR3uvKVShrhL,3 +IECUXEP4l7zZpRitgzJua,3 +YN7zFeQOs8xbaFY1BMdKs,3 +L6tSbs6e9r4fSxwkeQDiN,3 +g08dDEbZvhymfJ8NdNCGh,2 +wVfFxDZ94VPs9T81yUSdp,4 +Ce8CjWX379JwHUJGvwi8x,3 +cUCs8UQjTZZnHku0Rkaof,3 +IyXdMhONK9lkWNwRtCeNi,3 +Ol91I5ZmUgoJnsqvxMGNW,2 +QlOcIOwvetuP9SeueeoDb,3 +nles1DZndMZANupSxkMTA,3 +fit2fHeQEsjxkz03AHJL2,3 +5yAz5nuYuUL85sY52q2kH,2 +0B9mPtoydui80AUgO1UQq,3 +wytZ9STwI9dK4KcNmRGns,3 +yy9EFzmtRCbKMAtxrrf8J,2 +SmLuSlHT8OzIUsFpXsxGC,3 +EQJY6Z6Yw,3 +Z63lMAzFsa3Pj4yKfthz8,3 +lxLTh48tVmpRkL4wHSALL,3 +0tRMgmpn3Ie6SsfwMJeql,3 +tqob0TJfsWAqjIF99NESl,3 +WF0dzY5AFwduhnycqPPB6,4 +1vmUk7andjBhAEITGVzDg,3 +oonwvwfRyZWzKN6mrs5Fi,2 +IuQvyu7rJAsJ5GteZbWqb,2 +cqZ2bfxJ4jhLjIx9mKCPt,3 +AwoQE1Up4EBp6MYwPZQmN,2 +Wqf4lM8NLJYYjilXxAjyw,2 +fObVKiG1UMNuq00IU9XEs,3 +Bo359Aw8l3bZbkGj47CaV,3 +q4wYsZ4j4rClKMvlGk5cx,4 +C1A7BVCc81mRVWN2WGItX,3 +ohwNSwO1rt9h56smMJbu5,2 +tCkEYVZbdsWVSXBn8K3rz,4 +IxwckloTr71niTdF71F4J,3 +TzDgYaUkATjx2E7x172CQ,3 +GBXpJIOIa,4 +54Yg9xTzO4FtCM6go6IcE,2 +KKNJEr8Ro6hKabrpKvPSq,3 +7a2wEeycXy2AIhKn5ADON,3 +jw7cSYwqHH8frpuEoy2QN,3 +xOTUVOQwTjcsULPdwgwDx,2 +DK8f5DKoxRdpABrTa9wiX,2 +PBjyLRtCk4mjkbvxStZil,3 +1X7xfHBQv6Mhvtc4ssIcZ,3 +8J06TpR6H4PDkLNNH4Jqf,3 +5cS2fTOw2KCtqS0jdReBf,2 +xod31vTNWKbQR6NzspxhD,3 +Um49BzohhlbCycyk0bFkd,3 +N1CsFMU9H9MknD0uPvuGM,3 +HKtv6TXst0FzCDmhQrbBp,3 +kdBeJSyTdlFtNkvV5NayB,3 +8vBt8jjF7jnaShpdZSiiC,2 +ptwYHjcH39SICG8RB3KE0,3 +UjEgIUm51Xg1fk4LC6tZ3,2 +1WrvuikAKpg2ljxe1YjUl,3 +sNuM5ltqQgQpf34Gx8FZi,2 +YdK2CZDtiJTNdqwPP7x6y,3 +w5rFqa4t5CdrP9bHSWOOW,2 +-LCj1vhHo,4 +ZOyA9kkdXF14uPHZgpLYp,3 +Tyf2ufNdw2CgguXwH5N6V,3 +LJEAvyJVFo4pT0vFi2eNM,3 +HHH8uEQQnN61MTDiwlpHA,3 +wWUbvaUFE15aNum9jH98t,4 +8ekbkvCzkCLvlXDQVFLxf,3 +esoSYtyl9n3btS2bK2wQt,3 +69v6xazlloi41le4VHzOW,3 +erzWwhqFq9bA5ZuxVrB4o,2 +YSRifJeDBolJ2b4Hlz0ky,2 +ObZT5A3Fv,3 +bnCMwRbwTEXwTt1GnSYba,3 +EMgtAjtgz,4 +Zk2IuwrZIRzoM2hYuIOyr,3 +Qb5MmVYvUojibN5wALxjF,4 +ETAYhmxdCkKWoVifP0UdL,3 +iOJyTAOwgqXTM7RSpSv8L,2 +AfGBb0UHbRA4qShn2GbRf,3 +UnsPkhqtU,4 +12Px7wX9ZYDPHb4TH18f0,3 +mAhtDk0P7IXynnd0BcQ9f,3 +zpsp6V2OVb3AqCIkeiPms,3 +Zh8k8kjL4IstUfHW8LbW9,3 +yYb0rhp6jZ8JoIwkR6XnD,3 +HCcSU0NbrHDcqbOqnYjgR,3 +8NWRny70pcDtYhsPVq8gf,3 +MzWlbjHfiZdYwY764Tsek,3 +L4HbmLJ9lVWnrdorWKLzC,3 +swMKX55B0AfWN1b9cSNnz,3 +o9pATf3D1y9pUcRVEbGrX,3 +Sb8BuSCPls15lXmAzJ5ER,3 +TLEhUP4nipYGVfEN23qgX,3 +x0ogy7X61KEaRKXzSOcsi,3 +2aP29F2BnUT95JP8PRq0T,2 +YE302CdP73rHZRTx7hy5O,3 +ANyyt295wGiODlE3rKacc,2 +oDkvjModL8oE519Cy6tlh,3 +N2jn9IyNOsz7mMC2ZJgPd,3 +CmELVwG6j467mf8ZDmhBJ,2 +1UPPN9lW3htp1iLNOGr4W,3 +3UshSLRY35IEWrFfn02Jk,3 +RFzJM2tpinoPtQL8Sg3Zz,2 +DU7qk8PEM7tTITTXQqPvX,2 +tTaAUJZZ6yECnaSwRk9GL,3 +koJ9cGAdoa8RMrYy27KQg,2 +upOOgLXsmIK9ujwJMZct9,3 +dmryvmYBdCFVAhoYX37WY,3 +zkQt9mgUyFalcgwxKphCe,4 +5VdMn1LgCVLjFxtDhV5Nc,3 +ZPWr51ffCqaw2JWsjDYG4,4 +HXQrLOBJ7MePLTMtfP2e3,2 +h8uYpXtOcMOSspX76mwbt,2 +8HP9mB13xbAOLdUAWEZBV,3 +qh1HVYwQGZXiAj44y1sTX,2 +POBsA05cTrdhgdKxYNuiY,2 +dVhmo30GfAuh7w0Y4FhLv,3 +Osp6Pj4iuCjATEilNgeGX,3 +58YNG1Y0Q9YXQeXH87bLm,3 +4CEco4r8LDYNniBjjAbo3,2 +tSacZBT6yNlArrjL2TFBl,2 +K1lbv8lh9Zbk2oBbvzBKH,2 +aFv2NpLepOgVGjJAhzsbr,3 +CjvH2Pfh9Zc6I0tC52bGX,2 +nv5dKNz3qVwQ7LCf9ZbYB,2 +dYi7IB0CHzsiFfcST48sk,3 +XvGBpR9BfoBzZwiNONvw2,3 +L9BSbgLQMFSXJqNVvOImH,3 +KrNwxvrQ2GQpQ9Xx5PWdL,2 +KoBQ2g5Iyqhm70Ve6ug5I,3 +PFQ0f9qMQeyHE2uSVrQT0,3 +Dj4XbM8LzhOr8nWXw2Mvw,3 +ur0NhNC1gttTYy828ilxV,3 +b0CODskvminv17js1AzHK,3 +W5Hpw1VhYV1wfOpezbkq5,2 +dXuEm6RkpybJukWUMgyUh,2 +0wUpZKrLWFFIuAUM0PfCL,2 +FESAZLhD8lGzuF1j0ef97,2 +vJ5LzKV4kQHN70as02Bk3,2 +pk9fBEhnfgUZwNRkWchgg,3 +aTnt1zLr8zd8MORbkEbUw,3 +SzC20qb743hp0qMkMX1T3,3 +O4lLyRBTqhN4RCJrkSgGt,3 +8A3VGBJA6koTqolKwhrLq,3 +2M4SaontEgTKAHpDCQ4FN,2 +pajZXP2H8bOS4xdrKOwjd,3 +34knwtY29ZY5ZkcSym7pT,2 +AV26ot26keMYUpe2nqaJX,2 +OCk6FeC4AeBtFc09hCpqZ,3 +KSlsSIEluFWkyzW2Qo4G8,3 +VknRqUXh3KYhn4nSZemVl,3 +oT3wm5PGQromPxwDil6ak,2 +5odv5tzJEMpnsfTTJKtYL,2 +kuuDyIttBznOkn6E4eW4J,3 +Io2yYaOg6KHu2echBFVS2,3 +jQ4UC8WmRpeYISl8E3fcX,3 +wP965CQ6UZme8xONUiTPh,3 +RuXs1Q3r4epzLNlSL4kyN,3 +sGLrSI4KFtkZmVtnbH5bP,2 +JgFAdZpMq88l4TeJaeMr6,3 +FgjlPtGSyuASGjoMRx0hF,3 +WJMB5px99ai4Xynx6We3K,3 +GfOfMpEUu7bv8hT5GPq1K,3 +lu8EPcV5WrpOZk5j8ImfB,4 +T8BdqJXHY9ZEwAxammEb4,3 +SBvoCFoaGwq1w0CvoBbAX,3 +37AmRD266zVId5SQm91FL,2 +UxpR2BJ9fMmowjzIPYofl,3 +zLqneG5Si7aHH9UXQuH8P,3 +qQ7YxkPbJqk7Xgf3NhVUu,3 +cwkYLWcXFLXGOU3vbPoqd,3 +fu0DHRbfYHFh8d3cYhYlL,3 +Utw7s9r7deQoKLnDBlcCI,3 +OGAEo7ea15lFJ64BpNopG,2 +kH7MRxXq7tllx42AYdwcF,3 +TTbFxGJJE5IO425Rr54K1,3 +FNwr8mtB9dDCfYl03n5ZX,3 +9DX7pFpUvwXFuGZslK3qp,3 +dCIhdsGO5NlXpi6vl8QxN,3 +0kezV2fBIatkCPgAvQ8vb,3 +HFcMmzjXvr2P7rFoyHJ9p,4 +cnBQm7KJN9BZXJpcyALK4,3 +6rY2AG9jg472DkhmXiKwK,3 +NFbX9dRY6P2NDBgnDVANG,3 +KjTuldnc1hiXfcOlgyjd3,3 +qDlU5LuBvteEbQ0AR1LIV,2 +1Hfujff0CGYePZwEjxTGO,3 +TTeue3Y5LsEt1UKNlQ0tw,2 +igJUqGo8jRUXbbRlpgIRi,3 +3l5qd3g8tJlarHlYMt87j,3 +LBMh79soxKZsNoyHjfSRz,3 +RqoRDQUZRwolIqqCvFMXE,3 +DxwrW8wKSNR8qqIHakcan,3 +OlCQNlPnfzSEymqndpM3B,3 +gA6Hm6VIFR4RIeCgqbj7o,3 +7fok0Hr2PuDQoUUZg9Gmm,3 +f1S8ORmrVyI0cvDcERuMu,3 +AbHveWQaXFBBRPgKaoulY,3 +qb2bBZ5uNgNuUmAqYO3wa,4 +doisUbe29hjwak7adishl,3 +1QKy88vq0ZeSHDolNkG5q,3 +UXtuNxBGgZGv9ElMauiHG,2 +LWIIdjXyl,4 +SacH5QLD21iVcSgt9S2lR,4 +dAvPVGKWlGRcggLuEcjFw,2 +1zTfQM2HiZRYp6A3yEdE8,3 +W3o5rSgg5zoLbVDiwjAEM,3 +ZYdeaQB6IL92dPkLguTmw,3 +hFw0m1r9ilooiFXCyIQmJ,3 +vzsPKzxRn,4 +O2TcvxR51fJfCYYyHfUEF,2 +Cs2fhjBKBTcG3HXRjxkvL,3 +8UGNUuKpAFgNhdS3YOQPw,2 +hB8VGEF6bJPt3iOYDxaBf,2 +W7nqA9SVCmfE3JAY6gJrO,3 +WTS0Vk9VHFNjlxC9fFKgG,2 +tBFWr7wf5pULN5Epqj4UH,3 +2cTUu4LABVI0Wx6FosMvu,3 +KuFdK8QCQYjczSfuofmhW,3 +1YV1pmvujYVmsaVqAlVke,2 +9r7waxLX8AkN73OuDi7pI,3 +cdeDgYhfX,4 +fFe_5yUrA,4 +vzbtoF7hXmqgAtMdNaaI2,2 +ZHSNmXx7xuYFKjGCOmGQv,3 +TQQSCCXO37zUHlEZa3Nxj,2 +IY4aoW89pDxDxnZauh4UD,3 +usi3Id5feZOFK3PCbVQVW,3 +17YhabOfMX8wiKoTWaBsy,3 +fFZCMIQC4py4sP02YDBb1,3 +i5Z9iNelBrx41Z8LOJGEu,4 +H1gFmWT16KpmufpsFZchp,2 +zRO4XxupgzGdNHZ3HM54u,3 +D2v5bMLKkVae74gDKLELk,3 +OETj6tTDmdCriyKVYrPEU,3 +dMYaMVBEgpXSYSebFzBL3,2 +11jIVeHJujAjXVFfADq4V,3 +szbhyJjKy8WuHURTzJeVf,2 +zlH2Fim36qc7ct6mPLvrL,3 +GjSZ0AEUIGarFpXbwBw8K,2 +ena4JMVcFpURmdv3SqpUa,3 +xaufTPMdOou6iFqKHvgAl,3 +alJDvXyeFhILmbwxNXsIY,4 +KBzEDI3lGXOWh0iiJPyFZ,3 +f8592499bed84f6ba53a3e5bde39dcc8,4 +CkjwMowFQ,4 +wH6zMdRVh3NhSHVfXOrhj,3 +AKwaNG7SvRRXPRQnuRA9I,3 +SwXcBd824nWK2D8YFhylg,3 +0WA1VMvXlr4q3kKaIQuU7,3 +XZfzzlPpv0pLmY8CuDQf2,3 +mG3QOfNPHTp81N35CsQyS,3 +2U6thoRjwn,3 +BHTf13n53TEoXMfScSv7d,2 +5oTHhngpKGU67fCKaDLuc,2 +zCjdmdOXISSo474rf5PIP,4 +tfcujciKC8PFBRhEF16bR,3 +B7kILFByHEMHykR970ap6,3 +Jbnd80SnxJkcq6bvSmJd6,3 +DK9tok3jXerNpaLGJTQLT,3 +OrXNqNIkeZl5L4KMioWLh,3 +cXGZ6eFY38oZovqTC2GLz,3 +TQXco3uyKkPckDHZizNZs,3 +MBp009j1KrkoAEgE7lWTe,3 +XeKGLyP7ETrazhD4W6kAw,3 +cRNeNd8ghzbtmZHcDF9Cu,3 +ilvodfEcPIQvjh3XCZW27,2 +yYng5O93U1GEKpTwsWDSs,3 +ECAHxoBVm0uQHaiQj0rjP,3 +7UUcssIRz,4 +C8gt7De3OBW6ZaIBBwtQs,2 +Kq56LhUnFOUlIyeTmleQv,3 +G5SJUCkPxc0XbratKnRtW,3 +gk-be4G25,4 +jMRIAHVRg8zUb0UHZN42k,3 +I0xrCW2nzqRjD4T442fHZ,2 +sBYKm5UeP5RO7ALyp3Mhh,3 +MEg5NCQBoFNdLgBCa52yw,3 +pmlMVOgqEbEmhxjH5f771,2 +vbSJQQ1WqWHak6AH2qF5q,3 +y5lz7CWQIpDNpcPuMaLJN,3 +OHurAAr4GAMfUoM85jIzY,4 +1EXSFAhTDDQ9RrrVAPHTz,3 +MXMSpAXJTrtb2mRUzQ4Bq,3 +s5lwJOXvSsGCAss3OQsOG,2 +sqFNMmcXym8ToCCGFUC4K,3 +I7wbW3URQayBQ15zjIjdB,3 +5UFj4XhTkAhATsX2iBNcM,3 +kJwU8vsdRnrINAyOTTHwq,3 +OY1vuht7aidTLzCfmxjdL,2 +VjOUgA5bnF0ClXf0TMQBy,3 +YL5qNoBFKR7ltk61MhNLq,3 +iHffpmd05JA4FhU6SsGp2,3 +j8QHlGOvEsr7siPqblzF5,3 +wPTbl762ywyT6iNEagKv3,3 +t6FdciaIKV1APuTGRz6cT,3 +kfKnLr5kW8YxRFikiXoTX,3 +TCR1dKrKr2rEw9Iyvzwk0,2 +rthQKAhjfox6oCKz3uh7Q,3 +xPwdRV2yVDMZskRxsV3VQ,3 +pWvJ0YX7olZji1RCNbrPi,2 +78LJkkJ3tOLIlAWgCA33l,3 +cqc8P8Qd19r3r3HmSa68G,3 +DfJX4j02assfQPGWZWkRk,2 +mY9v4Z2AiU05axwowDoU1,3 +35XX6VRRNPzWuDS4YA6qw,3 +Z00WcIigubLXUJqcHidQc,3 +r8E0HNDqFriUKniLQClVt,3 +3h0L50rOBggKea9Az79D8,3 +6yTuunqN4Pk3M7sKo9j7A,3 +C41vLXCSEVzu8F4QIARcC,2 +PnNFNfeHuW5LRJaByouS6,2 +eG1uPshRszH8Sh1aXBQpM,3 +oap1TZwqTI14q5z0q7vEJ,3 +3ROSSJELbcSeOwBU0yhdS,3 +N87EKtHiqQAlEARABobUE,3 +6B74iXHE8rArcNgRTruER,3 +IxfjARLiFU6f9ndtorhE2,3 +NC1vaA01OAW95JiMFShAq,2 +WckYNv65RWAhcw6OmxrFD,3 +f7Lcn9sXbNYRtZ8Z8TpNh,3 +bYhPF99PqfnVAClVGYwkM,3 +5Qc6BUNgrV5atyUREZaNA,3 +FdFggYFfCxR0KXQZ0WpAl,2 +H5f4065QbkRDajwHycE19,2 +f7ZJc4nzTyFWq1nnyJC21,4 +YMoN335auEhKBSyxPL5B5,3 +hsJUO6hPFd6JGI4oOFCS1,3 +zvDHrEOApR0JZZVGYDYLO,3 +Ts8AsjOvXqpg9oBCfj8op,3 +XBTCnmTbHF9vwq4Jmm6TX,2 +Nym4iYhgOppNDAEbNI8am,3 +pa3Uq2a5j4s7rTPznT6SD,3 +M0pmOh5dMj8BawnfFcoNp,3 +wOCQP9JiAAcI8a78va874,3 +dGLvLoJ5MRFRMY7f9p31Z,2 +XvktteOgw2vlxA40l0F2g,2 +nsTJwcLEnG2qBQEb9566j,3 +Tna8xWh59I4gkjcmYLtyP,3 +K7YRkR7waHQanA2kRGymu,3 +bSQhBDTXDaz6m4ISTHQa4,3 +jQsfsBRaKJNShrczRTi8i,4 +Lo8odZEbRFWNtkXoMG4ro,3 +sC8bShZtB,4 +nak9XCusJWJiKe4E6GaNt,2 +kJOwMhz1lPfukuSvshlYh,3 +HP6il4EJG,4 +v8LP8xcSeokPEZWtMrWCG,3 +2AY0JxlJl6s7f1DVTUBgj,4 +TjQKrGx6wTliESr2sIfl7,2 +lMrStGlUvFqohboYdC5cC,3 +KnuP0V29RcaGguuodOglt,3 +V56ZyWme09IXUWEtR20AK,2 +ruOMvUWzOnYVnWuWe7RLP,3 +WsnQJoUmKfrd0xBsV0U2t,2 +4xbkb6yadf25gI20CSCKy,3 +LDDDDAQtYedsIMrhlye26,2 +kNIlycYN8kZNxQ1a72uLW,3 +eSCIYn8IsvFMmKiQA8HXX,3 +RwxGNmt5iaCGUD1HtLZ8y,3 +JbzgI1JAnXLRAPpoye2WB,3 +sYvt2BxJFoCLuFp3atO6L,2 +YFxuwgZm3nSKareqvCsKt,3 +OWeA14MvwTXozqYNyaYMv,3 +tTQUyzEW70mQBvSnUMfcI,2 +W2nWWU3LAEbv3T5Kz65Ua,3 +tMJBCQORbVu0XO0HNl5C8,3 +M8mSvWJt5,4 +p1Bq2mPm4YX3et6iiazOe,2 +rNm0LfQ1qGkUJK3Z186zh,3 +LJ2YW1WBKFdmJgnF63Ar8,2 +7pEXekejJaqOkjkYLgEMe,3 +n4oZ4znCwUqCGHSnGtEU0,2 +qSANLxaVQom9t7FyW9c8Q,2 +fYDllZFsXdMXjJhbKj1j5,2 +3Miig67m3D1mSyGm0Ksqd,2 +UV5xNFQ2c5PzA2sHYSu2J,3 +p7xtqPKXGaZTYeApjnwph,3 +oVgyCDfU9u8YJstSARDwR,3 +N1QHcGDcMDp1ZBKV9dISi,3 +GC5FZK40mNkKBbxZxUw6U,3 +15lUO7zqpbYbELpybrrBo,3 +rtiEfRgfYSQsCeJiWy0bE,3 +DiCbJxzGqVqYCbpkx7kyv,3 +m2W8sLhdouhpDbxeRTj0w,3 +LxHG8IWCZrHY7UFwbfqKg,3 +Dys6WfpMr2XgermWO67KY,3 +xTOOscfp6tJMURhM7nAtz,3 +nNhC5PywLhT1scv7kDRyd,2 +O7zXkBUrHedY0Lu2pW5Mb,3 +nX8yVvib2Q4zSDcMHBd12,2 +m7pVBRJVqx1qMf5stnZed,3 +vTLtxDUqdtCsFN3GKrefs,3 +yv84d5fjDvdPQBIHyX3bj,3 +eg12TBRL8pMVWeVUE6HpF,3 +UUyPUFnF95tken1Twfktk,3 +w0ZJBsYEMYa2LF7TweTOV,4 +rboQ1M9vPQHqSnlEx3V1o,4 +9SsrJptyWXzVKnezTl9g1,3 +qjUwUdYxWW7S0VqERc0Pi,3 +8oyHTsDVnLi0ROyWmgxB6,2 +8wBL8705egTSgt6JWwr14,3 +IYtxYnnQPZij3OSqfu1g9,3 +Q7dYpJFuGLP2emZR0hgdu,3 +VqQECaxbnBSAuYeRwp6xX,3 +goqEngNoqwayiPiJcLV6Z,2 +baIjyPaEXc1P3MKXARPbD,3 +rEw3CrkXjQRJnTUaT6cEN,3 +KxftJIhhQ,4 +iuJA6reYilZzG5TZvXDS5,2 +Dxi6GemLWay8HmGZ7RZSf,2 +mT50p48YIJ333AgM75ojA,3 +O2e4odCdx6O10O7ydOoCF,3 +xc32SnGBpPxs2A264Kw7a,2 +b65vZ8v0yXvr73sG9Exek,2 +WGGPKg2U2,4 +rBN68MDnmoO3DtX2PhEfv,2 +ZTKWc7tlxrZ8k4O065utb,3 +hTDhe9NwabSlFQtT4KNyR,3 +0mLMhTIp45u3R3A5kzyue,3 +ie0dviAA8ZBSQEj7o56pI,3 +lL2k1XuIimOJVSxlMAKrK,3 +tT0nNyMqetA7opFWtg5Fi,3 +KuyIP2eKcmb6nlIkpGZf1,3 +NnTqSh7d9mfLrIz9kIFxX,2 +SmMbLoQiejN9AY9HHUsCB,3 +f7kq3s0HbjVjYmkcxTKx0,3 +N5wjaBK7JCGXb5m1Cr2Ad,2 +YpaOIHgs7ZwUOPVxMBq8v,3 +1auNJSoGjg7UogvaWrZDd,3 +ShbrH007qsPXK3o4spySX,3 +CNbr5SQcjM99ErDuhnnmm,3 +6sEHo0idQCjtzIBzvolgQ,2 +yo9OtaBMLn99Y9xkxWATd,2 +PLQk1qnH3CCoj3wD5ouyd,2 +fYnLz2GnVubCa8YIakFmh,3 +N1W8OBxPmTPquCOXKDVRi,3 +0pppsXoBP,4 +pWDz9xrwluSwB4MLtHgSB,3 +O31B9smuSClQXtb6lPWml,3 +PpEwd9ZDs4v833NARaYXP,3 +WsXrltT2lF7h0XSTZyjdC,3 +bbFTqVUhgEu9LjQM8Sk9q,3 +A0rDAr3OKscytPNWZboV5,2 +psraK5lEb8xgOgG1V7QNI,3 +EJaD6LtENLj8NAP1fBgLg,3 +wuaMiI6pZZGWF42PNfR9d,3 +pAu4cWNeBbOzFtEZ6yjUN,3 +9cLX9lZoJ2uThM2D2ksM0,2 +4i6PpDMXB9o0zcmZjXaGG,3 +s2u5sJyv7pGqdX9ZHaFS2,3 +HXB3zWn2SwtxUO6t9WK09,3 +d0uj4P8mYhXaqpeLL7YuK,3 +j7HpmOf40Pce5kbQqsqNG,3 +jHdHnFWKTzI4ZDYdy7AwJ,3 +fVWklyolnYpMBQwpf8QIj,3 +tOxhrxUDcw7SCVrjAEe2y,2 +I1nU0I8prMjSK9iZvK8E1,4 +ptNA9uR0OhL8kmWEUPk13,3 +cPxkS46xbT9ySxF2nuWbf,2 +ymweN87j79k2nrTrTrLpy,2 +PZh3y00u2WgTWsdUKVOTe,2 +eQhOaP26ehlc8NRmcKo0G,3 +C2ZnjevleQUbioED7DW0C,3 +6u4FXNOAaGnYmJckkz17D,2 +laSBsKTAj,4 +sSCcd9XnhRYOvmBN03Z8t,3 +nYgqraEJZTm9HrfWmLbEp,3 +178ed546fc7448c788eb61127e40bc08,4 +M96mQopyrtlZvw3nIpIfS,3 +h1CwuhgMvHIxNPm7hzDaw,3 +cCzinCwXP,3 +Z0wMM9i6ghRCosnGoehYa,3 +mV3iKMTkIJn6pMfoRZg5K,3 +X6mswYSXvvaMMjC5M99CH,3 +pNdK6ehRFIlHXlUA6Ju3p,3 +JalV2ZiR7P85HeyabSDdC,3 +K3VM0w0TlSamrY1efyRNp,3 +fL4KAqPQsgxoZzGnNJwwZ,3 +iiAKrYxnbX0OdlLjSi5hw,3 +sxfBbXDDmwbbtwCACcTk8,3 +MULkse9tgPhRvUSPZWMAR,4 +ffn6LheFUXOWppcGzIlre,3 +GrsZpCBy4bLZe8bcAhJWQ,3 +X03H8BNM93XN67pmqQs6l,2 +h7R2WIjCUXJecumik1bcQ,3 +pbBrjymZ1FwHPx5dwkXWj,3 +JWQliUIgwBd9bclRDWHgV,3 +jIaKrp4jBCA8b6J7uZg0X,3 +TQQOXzzRxI0a2oBwjJeCl,3 +4MU1GWiyJ7yus6V1hvGD7,3 +ZFXBZzvbyhLISBk69p5OJ,3 +gyaHiqcoxaWTMiSRHemdB,3 +oRDzk0newKgoMf4vABF38,3 +7YMj4glxN,4 +nWON9ymt1Ha7gCuAlIfli,3 +PxW1dJKLmAoDEFUnpGgyh,3 +rYS3jaF3hwULCzuVDe73N,3 +0J9RfWvHfZutiWWO9KlaA,3 +SF2YmCndtuMcPdNfhgDaP,3 +YgTPX7bI7cTii8vqsQp1h,3 +OT4CCzRQTs1w8wHvBjbcX,2 +SgERnHOHnPalyajm0jASc,3 +GUYSuDkesdASWVBPIyib3,3 +hCF0pHXCIGZiU5PhTTlTJ,3 +FkFy1dA8XYUIWC6cboRrw,3 +S2hKVUzwIbmu4GWpFtc0P,3 +t11tQe18HbCCL58m9y5zx,2 +9uS4M6E0HbrP5PRny7hmy,2 +1xVyptYOk39Pl7N2DxmUy,3 +CGMm2nd3Tic6Z0FuMetIF,3 +qspgKFvhXXAvbLmAH8vZN,2 +7ryk4iCXv5B5oK7FGxyYs,2 +eca0861593e1431f811302dc12e90342,3 +MICz63MyjmWGQc6eZF7I7,2 +DHtMen8ggwH5qRPXOujSK,3 +gVbLeQIIOfU0y1l2o39sB,2 +te9tGEhMdBlcyN9mCVIEl,3 +02AaOAt5wMK7HUHaWbPD9,3 +tKXSNtd0mCJ1YZQbTBfhg,3 +44V9oUNdyQohUPFsJUJ3s,3 +BKmlJEO7e0SpggASOEi5U,2 +FGZb3skcUVjgbZY9uSv8p,2 +OEQNKkxrT38f52xdGRmka,4 +eS8idDPGe5jjLLFzdBnB6,3 +P3DWVvsWS1Y3QESrORTjh,3 +k3Uw8tgIFtYI0mxL3gRyb,2 +NLXsp7HCQoYQHTmVn49Zy,3 +08ST2mz8TPDvGPQtBCOVx,3 +HCXhrjsFGqfAWxljHpm6A,4 +ofXW20bGdMZ6CmTwbpSip,2 +i49kdq78b2GIyZCtA5wzu,3 +mj0NNbPH1wFIxi2eudEZW,3 +HhrjUxhvvcsSjVccb92EL,3 +08GHmINtr,4 +YJCc8QNrH,4 +ShozDOPd40AgCqDjnphtc,2 +NaR8nnpzWa39zoDvaUMT2,3 +vzwVadXv9cZ3ChNVplbGQ,3 +aabmwY9Qb1gw6oP7cKyBv,2 +Z2ZMgy20uzMCTkqX2vwJP,3 +26PkgYJDEJNvgnB8if2u1,3 +LaBfL5hcxFdSJIbtOvUPd,2 +OrZoJyshfBXeqlZpForZi,3 +PndAxHJfsivvN7AiwWsU2,2 +W0czytj7tbQWDN0u0gvTR,2 +5N9zLq9RWOWryx2vrOSag,3 +a1ZRZkNTVYNwWe1d2h6Zj,3 +uyjxq7ZZ2yzs8EXPNeT0G,3 +LPDuKCYatMfinqpnGUCMO,3 +z7ucEhtGxBkB2vtvq4WqR,3 +eKveWTHfKA2dlWsW1RqH7,3 +BUiSoJdPpYAfKuuOJf5Os,3 +hGwkKo42goYWhVJCd5qxo,4 +8GTSva4DqR1aHPzkj6rLo,2 +7yS89zlQaRCTinL9mlyod,3 +cKejCt6ga8OnnNHsHuMFk,3 +iivT3MG6fmEilu8OtRuuP,3 +kpEDaLVn6HULGDYr3jA0l,3 +OmLheuybivu40u2Ycn6ij,3 +TMfoNvyUGn4K0EerLJWr8,3 +CruZLvzfB6YrGw6IsgyLQ,3 +0hivm9FMxerAyhX6gRpfp,3 +Y54EkArDfb3f6fALz3bC8,3 +FbEg8WpznCXW5T5m5WG0b,3 +cNyAfNmJkwH7nMW2ZKp2F,3 +yYSYdQIqJIdhsHkxI6gbg,3 +d0N6g6KnHFMJ23xLcKEmP,3 +20Y6gdaq2BavNNaqi7Jyy,3 +m6oASLNCQEQ3dlowp2xWM,3 +PNR93CS4g6AmHNu736khR,3 +foEthZtVBHjW9bBZ7coAJ,3 +MsxctB4fz6gkTRJvZGs7a,3 +QL5HvbU6Cy8HHuxPiYv9l,2 +mn8J5El9F2YiZ0SbxSKqx,3 +tZb1Ldr41Oy37XWrF0zFS,3 +Cu6cPrgHWv8xTKRk6qGUc,3 +X8XJoK5wF6jacvn0MZNbc,3 +BZsdIp1w3dd52vQq0pIpL,3 +64ZdSoPr3STm3G9ln56YZ,3 +y84JN6sqIDXKCIyJDfkUQ,2 +CdnUbOqLy,3 +4OpcjVIFyjGypbEMKGiXP,3 +JRvr6zcxbC7B1cTJzCFbd,3 +Vfz8eIt1VrIg4mDFFNVWa,3 +SwS2nVx8CsbTUEhZO1WWN,2 +qj9VfyNNjmLh9kUGJJghV,3 +jAoE9G0LyhcHycjPfWPX8,3 +W3LEJVLjXHLMd9VjN3KND,4 +gBBVuSM8aVkwwyKinKTFL,4 +MCHMnXAsPErBex5Oij42x,3 +mi816kklnXr1yt03Atm60,2 +ioweiTFHc,3 +fkdedSCVV1cBlOXlKlGUy,4 +FekyZe8CeiongEvPcE9hb,3 +AoXaEHoXr6COXNet3SX34,3 +Bac1ff0Qn8Nud9x9uue15,2 +i6BZFYe1XlQtNCLtSKc9E,3 +h8OqUMq8SNhGCHM3VhRKF,3 +RJ2jtRbrylLM516LTtRoo,2 +YUWQJegS1oQUzh9ocqQ9o,3 +NRpGj0KIZsSb85WNDdJex,3 +l4hRyJzTAqc3rgtXfIsYJ,2 +72LUQPmjPsvJ4uzFmfwyQ,3 +sbqesanUpMJ0lXEPNQOxk,2 +wbXEGC1HRVIX09cK0vo5o,3 +H2BcQK37wAu436SMwRXs9,3 +cyb1dU6kpmqoPpNtXDo1M,3 +ytdYEMLGvcjATsYow5ijP,3 +he38RINHglEIT0m0g6MmY,3 +xVitHZUacfCZ85HDGNn0J,3 +5aWPGGkCBRd8ja5H9antE,2 +0LhVbr2AwNGFP3IVXEpoL,3 +2TmfsvhtQZTlBZsayXCUP,2 +C7pkJHazRK4OTGPQj8qXD,3 +0tTL7KqCOoedmwlQBFoXu,4 +hTjrYDQbjluld3YC4WUq6,2 +UFOlWOrHi8xxYTl6EKVWZ,3 +xgh7zPA27kztrhj7R3cdh,3 +6m3e5xLrFY2354fTxwO5M,3 +4bdrJLRLOSDjgCHXvSi3j,3 +FRHJeYDa5jMytXOkFFWXn,3 +PiqViUukjDfHzTuoqNZKg,3 +DMPyzK5V151sgMdhGCscB,3 +4B4RIyyp0lDfAkWWIEsrB,3 +5zumKmzqaXHeYi12rWB4m,2 +GWF3ko1gcksnPKA8yWCYp,3 +ZumEC5UcRxAwmNu1usu9w,2 +czr2C1aa4VLV8UXhnp9VA,2 +474HgEkmkstZRkFZHsoz1,4 +4ke2BjMRCUWxjGpVAgO06,3 +Zu1T6gUMeRqKYyt7DgvhM,4 +gMx9G7CHFxxz4gL3AyU8l,3 +HYCQiw24ydvu0gbGPPbso,2 +X9n1H6SjSbPttItR9Lqa4,3 +2l15qqIuyxzUdMPi4MINS,3 +FH4FSCnn8Jm8AnpAgGEhH,3 +vP0ww5Uy3xSukOKfEajFW,3 +73HlZi3YZQh2fAlrv2Rbv,3 +s4Gvtyif3oGb1DXet7Lw9,4 +AiRFGnAWXV256Gbre35d3,3 +E0X3QMwQtLg5ZGMdIOX0b,2 +72ixbr9CCOM4Sro8PVMd3,3 +ojubvnSWoZawauJ2G4BvU,3 +n4Pmazq9GqqDL7o4tDTdi,3 +j4HFz6FAWVSjmU5tfYmsp,3 +HIHI0lGiLSDAtxt9WBmru,2 +RiAWNmso3,4 +G7Ol5LXIC4NG4eaR0p3Mj,3 +oaYG9bmhb0JN9FLlPHI3O,3 +hfbJe0zFYPW9VVzmuAGtQ,2 +nfg52YXaNVXcCvw5IQ6EF,3 +pEg02tQnYmJQkaa6R1an1,3 +LJLFrP08gvhf5kVqBAg9k,3 +fN4qEOviztWDEzUJV9OJM,2 +9FCIXax8KiVu2JYLfZXfu,3 +0gA9kbdFNbmPaGiKbjO1E,3 +016L12RCpcczzDENbYuwO,3 +D72xkNLpDoNhqdZxj4Vts,2 +iLXtKTF6FjN4hxNdIjxh1,3 +BbHdGc3YFguGQaEw1Aruj,3 +qORBFeNzv,4 +wzxEW9abIlu8Z6BDoftKE,3 +KI7kVhOgo3nBqa8f3JXI0,3 +47NTxiLOZqpqFd4RC2Bt0,3 +zA1SVIYNEgvZFvdjfpMUE,3 +EjXNSDcgyFA4E2WZMfYOW,2 +f3kAL69FvYqqMgiiNJbDB,2 +9QPkzETC77RanE2goI3wH,3 +Aur2av37w5pUEjocySCcz,3 +zUTzLN6uDz5Ac2hXxI3Oi,3 +tStk4P1pvQvQoqFRU2NiM,3 +U8wHaHeEVP3ZfAHCqbuRj,2 +zfPGHk1ta5NVxs2w5BSIx,3 +GZCCP5e1vZar4jKtuUxkB,3 +vpykGlIpyfK50lM1GfKWh,4 +joDdVM30zGiDn6TkWh0pD,2 +EZLo65YYb9Rpcu527BAxF,4 +XZEgomY1aYlHOH9bxDpKj,2 +EYF6Gbxa86OJuUZplinPG,3 +wDpstqw7sf4qBiV3EwKyK,2 +1jziqul2MJFXF3hnuZXBr,3 +zB6K7yocKt1HrDXHJLaZi,3 +uwo6fZ58HrmGbN2UrOhDV,2 +eusTTeAC4FaSfjP16jrt7,3 +seIioRTI3XbtHTHkMPzYV,3 +g02thPC74GU4JUXnZi7SI,3 +3lMbs8xosQsuDpceev8jZ,2 +u2P8hqwRFA3jggHYz1fe7,2 +Nn1bpWbLcrzljEFvyRH8p,4 +C44vQAeDuwXRIk5C1L8k1,3 +mrGWd57MhBbDU8WsOEJW7,3 +RbwAWJwX0sYIJWItpFgQz,3 +24yo3sscqUYfXDvsegxQ1,3 +IQbLalmyu6ZbSugItn94X,3 +z4SUyHffQKjwrHrvO1m0K,2 +YYI6lJrYa5AeZoZuu4m98,2 +WdT4Qqn76KkVLxGJkYO38,3 +tDyt9viBBhcRd1FNXzLFv,3 +vmG2VXU8Gye1xIenPhJ8E,2 +6askwYFFTzpYbR6hOEw2h,4 +xqMb0FqJvGM885zuIpn3i,2 +ns5zcUU5XsrluuCrMRJW2,3 +ANPI3TtrQyKGlCgMBV5YD,3 +Jwzrf2fxPRydKqRPresNw,2 +q5KP6AY8gI1guMIkfjjrs,2 +syu3AAB9EiUDODJSDSoKf,3 +tLmk47uAxP3xklZZnYPo3,3 +uHK5QsHPAXA0EIxqswCEj,2 +Xql3xZcApIRTADBL1ACDT,3 +FvLSruFBPlnnfRpbN4XOH,3 +bpBYwW9L9ZcdBxnNs8AEQ,2 +FhXsG65VcAy6ZD0vIF9df,3 +6PDRfLcS6q,4 +uuCtzoTzLR9v2BOdANveB,3 +XA52iS6Ph,4 +RdsEiIZZ4qeRYLaCLQy4U,2 +17NaNY5IQWGu3yrElZsT0,3 +iWsXCiRVqMtHkwG3DlO4e,3 +2b21Pzbby5cixFXnEqMbn,3 +dGQU45VW5yXjwoJWHBGH3,3 +qp679OYgm2UeyFPjpXRNv,3 +FdoPZFwS5w0W3s54UA6pH,3 +e7tCNsnwir9Awpgu6It2Y,3 +WQfrrQVSh5EUmAZuKK7Y8,2 +rZIueumSJQzK67prBmQB3,2 +QGekzwMA9gPHhlkVO0dy0,3 +gQRGEeFTHtzFUbP9o5LiQ,2 +jPWZjTBdjQLXYfjcgWjBz,3 +YVpRsfANvnbDGIzqaIw6v,3 +cuxF1hDApVuXoIAdrV3Z0,3 +dvLrGSo6Ghn9qwoT1nm7S,3 +CVxwtfuVl0OAyZ5Sy4b4o,3 +HAGqVcDATKef4R7dZILCe,4 +WEUO2UxYeyhyJadDaNYV5,3 +RmRH8CJRRlK9PdRYv2cpx,4 +jmtoV4pbQAkps87q944Cd,3 +LRq1cGaWiIVCMChAR7N7c,3 +1WTuFPclgyEMk09ZmAnA4,3 +fc6sBKH4AqOMzbtgts9TH,3 +Z44cGfAZI8rhgWB3mxprN,4 +N1haHq7VadcDdLu2iasQj,3 +PICL56yy7OoUw3jHAUz0A,3 +3U07EskkHcn3Lk3UgM0B9,3 +2XEkyBiAliDcJQTnNTzkn,3 +aUHwWsFSNqD7dTkk73SyT,3 +eQoqism0x8ezL4MS83hc1,3 +1sXXkqorMgFeIzxZa6zA2,3 +DPQadRG8WYrsEDB9j5bu2,2 +49Gn4c8DrbXrNoTDd4Jfc,3 +Lpq7ptJbAKNEOLS8F81fw,3 +AfsKmAd2JhodzlWzmyEI6,3 +5gAgFyFQfPBkniStruEru,3 +dLw2SjDErpS6zBZENeShn,3 +xRogTPsxXxyAMGO9sYfAW,3 +guVNmxuBePbr5LqGzuhFC,3 +L7oTtlXjRtrZzSmnbNPri,3 +MR88Z5LorCJXjg7TE3z6m,2 +wK0AFhPO9FPJfPtVmxVmb,3 +FojIr27rl7KthkSqfOWd4,3 +ZYnahK3Ybx88UMxpHAtgo,3 +T1P92S8WgCRjOlJIcc3tg,3 +Bj6RFtsqddtBLg6byqvd7,2 +BzADpy2Twlh2H02Orwe9Y,3 +oHlRbBkvQTp4Y7CR0fmZe,3 +l825B5oo3uupfC9ZB1nQe,2 +ttP6KvvA_,3 +zYKcQ6tbwGGquuvI73tyh,2 +Rjx7ZMAvssu12Q9H9pyye,3 +SMDEScYtBVQnm00ilLXNn,3 +trj2kJkwx2C30D3KLwvnk,3 +jVj9fXkqifwuUCRM3bcLy,3 +QGaYtykHOK40oKdRGUKix,3 +H0b7RpBhqQa5CYq6DB6uo,3 +OQU4ckVUM0pqy0RpiQ4RU,4 +wUr8heicwmJxSr3lDtWXR,4 +RlTKIT4b7tt6q9UTJwSrO,3 +WoiWpnz2xJfKtSiIJA6Jc,2 +RSO961MpXmkjVZyaGYQLM,3 +GCB37CHnfLBV5Y8sfalxN,3 +H3qhXLwT736O9mcKdLMTq,4 +4EKkNrfAL,4 +xfbDa7xIISxKVxsDmYzSK,3 +fRiCo09s4W8rrZfNpiDIq,3 +okbyz4mH1bsnFbkRJw6GO,3 +PkNB0NcufkI7fohRlN6yq,2 +M79iiQSBgGWs2tX2Bwv62,2 +wvSne48gMNV4k00sQfzml,3 +6KkGfCyxdjxkI9MNlGp24,3 +lgxIHoPqbqAVasWDVkbO2,3 +Sn52Ic6Y6LB4B2ZteZ8ko,3 +7t6wRExkOXPQVP9zHjkxi,3 +OiWCmXKtExcVTM85VFmCk,3 +Ym5tYD2R7Z6t1pxFemR57,3 +BSnmejgjFDBIsmZS3Ahuc,3 +0mxFgLOfttMxW0S7Tw2iX,2 +6NUkhnGhcr2Rtk3vZt3RZ,2 +xpH4DptcrHBIYRS6UFTzd,2 +iDIAerElrPv5ZdupK3SYm,3 +FTN6iQ9KhU2wQjKYeKAW2,2 +CS8K2Qh7ROwJsd5wyaCJD,3 +5jMjVfvhTQelnqVgfprwy,3 +-EvLWbMLg,4 +ZOAfsauQKvz5xmdqJeYvt,3 +NTj9Fdft1XI0Jp1Wx0bFk,3 +oKHpuLIxJIVsKYABVIjOe,3 +NX85xMA9svyfZ0vZ1EkRf,3 +MAMz6dABwlpnZanZIKa9i,3 +qyqAXdVyWcGCPFGXLJB0J,3 +ZHXOQGAtVeFd9wEE7LOUL,3 +bDUrDmaD5zYL59VPTmTaH,3 +Ek0F3ZHkXnW1Zm7tzZPHs,3 +iu9orEUJfv6W4MMKCx7Z2,3 +OnNEKNTzru6W39IwTi5I3,3 +cXhqmbEkDh7t9ZgXypbrr,3 +AK1i4lgP5al23uSGyEG1m,2 +tAC4bwGtOpgNqwT1O4wPo,3 +V0tBrYbvJ5m51xcZEOhua,3 +7qRrnUXsTbjZGRun1uzwA,3 +f34Oo0tFjxOp4utFXdCoy,3 +FTo3P4HdYTIi0t19WKuay,2 +JcvMKp2xVz4ujErc5x5Di,3 +OoZbIyztcMcfVxOY4wGRg,3 +Y1bs2L2oZVNerETHofdNs,3 +zPUkhd50FmmXSBp7j725s,3 +hddci0UD8kKuutIkDKZR8,2 +oth7uj5owyvFQGGQUSocI,3 +9ujMRK8aCsYl0TU8xlHS3,3 +niQfwfXb3MYsxND0rpprp,3 +79uXPxgaDiun8e5mpmVFs,3 +sLSaxpZB7K0K9BBPSICJo,3 +S1mrm3w7f99fWoGC5qLub,2 +Gji1NWTUQopyGlmy6WaZg,3 +jgSD7FVyyLnOX4jG1r8qL,2 +RkgIrZ911sddwq6DLBHfi,3 +5dgC5YKeqvInyTNHJk212,3 +J9TMhvBbF3fLWRoUVtdgY,2 +ZoVSdwfD58Are8qmUKNIC,3 +D9gMDdxmDWD9cOFREmTnQ,4 +e5B33iYkt3qkKbE0APFtZ,3 +XalS5NYkpwhAmdBpBul2q,3 +q3qMdVkdyQo4GqiXRS76D,2 +trJ8ziE2BahKPGMnyPojQ,2 +0cUeSSnmuYyroPp0eTF9T,3 +OpZKIxK6GLlFC6TwQ8Ham,4 +ZjtnWPee3dO3arRZQjGX6,3 +WNadxEfsh32vZVfiF83XK,3 +uVwEnzegptLC1A87UVj9G,3 +JiCUAPal3uJrSAG4Jvna5,2 +vIpNyHLGxHjYmxKo6sD3I,3 +FClGUjaHik4Y0fthvDicj,3 +WHuqRZUrVvRbt44eF3QTQ,3 +HUnJkTrUv3bLKwEOdXZ6k,3 +u2nPJdVtNHL5WZ83CCWkf,3 +My6xUjuQ2jCHGmxqgo01p,3 +d26JfdQKYE3bSY3vWxUQv,3 +IUplv8oK8f8jhVvPNSwED,2 +F8MkgCjxYLbm7p51YTzLD,4 +t3QmCPrVt2E7sZFD2qmq8,3 +PR9Z5rtCgwlKEZy9hwHJv,3 +K2ovHfnwJtxAAIppcyjCg,2 +WlBQ4b7myv8LV2Vh75RzV,3 +AnNOSp60Yj1YfspLwNV6k,2 +htNUJWE6j8ha53l0htKfp,3 +wbOY26NissdJ72zpzAhWI,3 +2bd96b984c864d8d975bbb6ec59596a9,3 +hyYaxVJTZBilXRVOOTJZN,2 +p25gsPwzaQSENIt0UlAa6,3 +wZ4Cc0h4U2cDmqnumKJUl,3 +UnMEvnqutgsuNA5bXxM0y,3 +3VP2YKJpiaIDr7vMynGpl,3 +hKlRJsvIC4E8bghZS0xVD,3 +0GJDeoIIMfR5SSqlpCXOH,3 +CSfTpBnBKoJeY5JEDTutW,3 +rEybjwtt9tqUfN5Pze8ja,3 +MSfUs8tysZGeU0ARUCTrs,2 +7oAJPiYlWw3lcDZw1jh33,4 +3cVCZ8jPrDYewbGqjATCa,2 +8JGmugHXfmf9VCXpV1d00,3 +RamO1KGRfhgbePOy2Rdy1,3 +OMbALrBvLoovGpzgVnEQo,3 +7fJeSDNrHny3yNRPhNl4Q,3 +YOpDvukKwwp3rmhMppJh6,3 +l42ybZ6Fv2tWRa87rnbaB,2 +RH1zbAhprGZNzNMzqgLTd,2 +kRzy9MtvY5vlxZnI70yUi,3 +d3i3zUHx7BCD04dNgHUBw,3 +SypHAnvzjE7yQb6AZ5A8Y,2 +lTXcr55EZtAWQ8ecUivbs,2 +rpYXJspcOQNR290ccSQ30,4 +NjEjnbpyGW3n6GJdAmkii,4 +BPJMZyGUfwOamrDXl9hHa,4 +w31xUfRZeG,4 +chlNnXG90aRiUt3sQIPYs,4 +A49q3vnTkPH6bxp2uaxJF,2 +ayK6nurOtgv7ftkoHd29z,3 +dbxhjDtHeSDunEz1gam5v,3 +fzOcGQxvZYk8m9uEfY3Ts,3 +kl10MeTl8MhBTj1iQ39Wr,3 +HioVyC3BKhtQDi6x0xRRx,3 +cSLAVE5Ae1Ue66aHNrqB2,3 +dJVdacd6wdrqF5Dp7gNxI,2 +QygTP1l7uQv4IkSaPqkSQ,2 +qKyQO7p1QI5OApmSQp1LR,4 +z5J2AIQfonLhT77lwfyEy,3 +xmWUKSRsSuZsmUaAkh4uc,2 +ScdluRxxxIlE20eiWSQdn,3 +4ERNfwlZMK8mXgwiplBoz,3 +sIXAJRIhr6GxS481AaXRQ,2 +pTYis0sspesfx3qbWZjnZ,3 +zn3UoJVmKRm9944n6Bx4F,2 +87IFCvPjwzOpPiFYoMmKd,2 +FgbBSirR4eKIG0LcgQ9tP,3 +huqVccQV1vddDwOxNzxlp,3 +4TDVT7KQcz2wfxW607Alp,3 +swt7wDGGiXkbClePglhqw,3 +qd68uSTagzkYLn3kmi8PA,3 +NdiKo9QZP83BdQcgH81VB,3 +T5c9gV7TNFIwUw03rVOyK,3 +yb8B6INRVHtrY7QlKH84o,2 +7bdWO6xjwdyyFue2gHBqX,2 +isPW1AaaFSB5rGDuMoypP,2 +ESSTc2gJdDzYepxDEC468,3 +rdKXNGRlXAFuiISdVh4Zf,3 +JJGJNmAY4pzvwUkfoqocy,3 +GFFnc3qpVvu28GAK4Q8TW,2 +CqiWRJCG3TpYAUBOi7ruK,3 +SqZ9fA03TNb2M5XbkXT2V,3 +b7gDHJENq43EFschU27s7,3 +vPToRc766pjYBVuXS8YtW,3 +STJVhCVrVTHuOTG1UlWPP,3 +Tr0Vg8C63SCLGpRYU2yAQ,4 +ngdwDce8FZa3NjmGuwQvQ,3 +GnOGRNKNZWmsWypDcvnZS,3 +hmkll2czTY9YzP1LXO1C3,3 +0x1wezBiNa1e5HE7PO9GM,3 +mSG4ErjZGppifqKFv4t6X,3 +JciIhowtFUp89yIIUq9fk,3 +dHQKoljory2XHAy5ntbVM,3 +s29UWyMoS3vy4udKaQS6K,3 +OY0lBj9gttfzgXJHfJKW4,3 +AYIIqKFCWiOIv4ZLMxLhl,2 +qM22A3XQ1TRgZnCMFc5C0,3 +bhU7FSTdH3RWlgwpYeBMn,3 +6SEOvx52eCJmhtGOXUtUr,3 +t1Q7Lk2lzrtsVVCpAbdWz,3 +KcGypN0BdsnfaCVcdfyKi,3 +G76xRoMpuH,2 +fxhASpWwZm8GadkJVSOYQ,3 +dhBtusxpVQ3uO7zalinNO,3 +5dgB0Ac9iQGIvWxKTKBHO,2 +QleFEGgBtUiEmqcE6c7mP,2 +kX2dRqKQO9dUKMI4SIcR7,3 +V4N8CRKLj0E4873Ti4pD9,3 +c2GQDSEImPO64xhOUOnUq,2 +7dxIiShD7P4jQazWiHFR4,3 +rXvpj8aSC28fME33uLx1w,3 +YRezkYhLFmQL4P8MpEUEt,4 +wBG1csdkVOf0pDHoeQYA2,3 +xNKmglHuGWzXTZX7BI1Dz,3 +gkwkqbSJW3TR906PRa9uZ,2 +8s1LdMslxzibA1DDSKoEy,3 +b45eaiKLA4PjRHuu8Vk3i,2 +lxpxEhRhEeOzGTvEkR22O,2 +AWEeZ2tghR09atNSQyCG4,3 +bxqm9DhvOy2IpG79PJPg8,3 +Gy5tFqTpNcQTTmeN8kVJb,2 +E4mMENv5W9fNQH0vd4q2S,2 +ePMqaUu7L8daO8k9Fgk4o,3 +SmXcLV3edQjNl3yZhLxmI,3 +WZB-N-025,4 +rxEOvaApsE1jypxLnsQls,3 +4qszAhGXIEn1rxVtVP28I,3 +wLaHW8XfKnzKDq1QsjfuP,3 +cjxoyGLQ2FADLrqQPTbw4,2 +eFlNZFK75XGAQrpNP4LmQ,3 +fEzcU3HBieSKOUmkRCrcc,3 +4Uy9opdBiqIvIksfS48dN,3 +tqXJRTDXnDxPrv3ldH0mZ,3 +6OkWS5QJwVaJ5b3mnEnQO,3 +RkMwRYe3zFmEHKEUkc04H,3 +o288GrgoQUml19pinhleK,3 +yNcc3Wdf7G7zy8zMxEPGD,3 +AAwMxqyqzNMG9WfuthKVB,3 +mXLEtqJh3BohsSHWGuH1s,3 +VJB91dhtT8T1sX5vtU8zC,3 +wFBSkSWmfHaeKIDqhAQ7b,3 +r3igKhuZ8VlP1X8uGJI5B,4 \ No newline at end of file diff --git a/bin/syncCioBasedOnActivity.ts b/bin/syncCioBasedOnActivity.ts deleted file mode 100644 index 6a69ab274..000000000 --- a/bin/syncCioBasedOnActivity.ts +++ /dev/null @@ -1,12 +0,0 @@ -import createOrGetConnection from '../src/db'; -import { syncValidateActiveUsersCron } from '../src/cron/validateActiveUsers'; - -const func = async () => { - const con = await createOrGetConnection(); - - await syncValidateActiveUsersCron(con); - - process.exit(0); -}; - -func(); From ac4b96e4545937b9425da83085f06d1e5434294d Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 2 Jan 2025 16:34:44 +0800 Subject: [PATCH 42/45] chore: minor code cleanups --- bin/cioSyncBasedOnActivity.ts | 1 - src/cron/validateActiveUsers.ts | 36 +++++++++++++++------------------ 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/bin/cioSyncBasedOnActivity.ts b/bin/cioSyncBasedOnActivity.ts index a211092bd..e71464100 100644 --- a/bin/cioSyncBasedOnActivity.ts +++ b/bin/cioSyncBasedOnActivity.ts @@ -28,7 +28,6 @@ const func = async () => { console.error('failed to read file: ', err.message); }); - // eslint-disable-next-line @typescript-eslint/no-unused-vars stream.on('data', function ([id, rawStatus]) { if (!id || !rawStatus) { return; diff --git a/src/cron/validateActiveUsers.ts b/src/cron/validateActiveUsers.ts index b82bf754e..b5add63ed 100644 --- a/src/cron/validateActiveUsers.ts +++ b/src/cron/validateActiveUsers.ts @@ -22,30 +22,26 @@ const runSync = async (con: DataSource, runDate: Date) => { } }; -export const syncValidateActiveUsersCron = async (con: DataSource) => { - const runDate = subDays(new Date(), 1); - const lastSuccessfulDate = await getRedisObject(SUCCESSFUL_CIO_SYNC_DATE); +const cron: Cron = { + name: 'validate-active-users', + handler: async (con) => { + const runDate = subDays(new Date(), 1); + const lastSuccessfulDate = await getRedisObject(SUCCESSFUL_CIO_SYNC_DATE); - if (!lastSuccessfulDate) { - return runSync(con, runDate); - } + if (!lastSuccessfulDate) { + return runSync(con, runDate); + } - const lastRunDate = new Date(lastSuccessfulDate); - const difference = getAbsoluteDifferenceInDays(lastRunDate, runDate); + const lastRunDate = new Date(lastSuccessfulDate); + const difference = getAbsoluteDifferenceInDays(lastRunDate, runDate); - if (difference === 0) { - return; - } + if (difference === 0) { + return; + } - for (let i = 1; i <= difference; i++) { - await runSync(con, addDays(lastRunDate, i)); - } -}; - -const cron: Cron = { - name: 'validate-active-users', - handler: async (con) => { - await syncValidateActiveUsersCron(con); + for (let i = 1; i <= difference; i++) { + await runSync(con, addDays(lastRunDate, i)); + } }, }; From f4f036ca77d8aaa2548dc76b7c3f13641eb369e8 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 2 Jan 2025 17:13:35 +0800 Subject: [PATCH 43/45] chore: uncomment ignoring of cio registered --- .infra/application.properties | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.infra/application.properties b/.infra/application.properties index 2a3302806..4f332f5df 100644 --- a/.infra/application.properties +++ b/.infra/application.properties @@ -27,6 +27,5 @@ debezium.transforms.ReadOperationFilter.language=jsr223.groovy debezium.transforms.ReadOperationFilter.condition=!(valueSchema.field('op') && value.op == 'r') debezium.transforms.PostsFilter.type=io.debezium.transforms.Filter debezium.transforms.PostsFilter.language=jsr223.groovy -debezium.transforms.PostsFilter.condition=!(valueSchema.field('op') && value.op == 'u' && value.source.table == 'post' && value.before.views != value.after.views) -# debezium.transforms.PostsFilter.condition=!(valueSchema.field('op') && value.op == 'u' && value.source.table == 'post' && value.before.views != value.after.views) && !(valueSchema.field('op') && value.op == 'u' && value.source.table == 'user' && value.before.cioRegistered != value.after.cioRegistered && !value.after.cioRegistered) +debezium.transforms.PostsFilter.condition=!(valueSchema.field('op') && value.op == 'u' && value.source.table == 'post' && value.before.views != value.after.views) && !(valueSchema.field('op') && value.op == 'u' && value.source.table == 'user' && value.before.cioRegistered != value.after.cioRegistered && !value.after.cioRegistered) debezium.sink.type=pubsub From 6cdf31c113c93b209838052c8dba4e47b1dccd38 Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Thu, 2 Jan 2025 17:17:56 +0800 Subject: [PATCH 44/45] chore: unnecessary csv file --- bin/cio_initial_rollout.csv | 1001 ----------------------------------- 1 file changed, 1001 deletions(-) delete mode 100644 bin/cio_initial_rollout.csv diff --git a/bin/cio_initial_rollout.csv b/bin/cio_initial_rollout.csv deleted file mode 100644 index 31195bb50..000000000 --- a/bin/cio_initial_rollout.csv +++ /dev/null @@ -1,1001 +0,0 @@ -primary_user_id,current_state -az8FNk0UMkw96oDZA82Z5,3 -AhzuQ07YKr1rLH6fgOT4x,3 -3w7y33Wwbtsij773wxjXj,3 -5aR3jOAV2JCPm5NxqSNmU,3 -T28Fu221qqs1GFYflMHjy,3 -7gdIEdaVOkIYk4LXRImnT,4 -q9DOlUL5hQ59VOhEp9Ejy,3 -jg88HSeDwcykxusfLdtcG,3 -FoknsDjsXUD7wLKGBzjiL,3 -YAak1lkTckI88zogtSMt2,3 -mAhoGtiop11h2qYZcvnL2,3 -VNqsaD3AuBR3uvKVShrhL,3 -IECUXEP4l7zZpRitgzJua,3 -YN7zFeQOs8xbaFY1BMdKs,3 -L6tSbs6e9r4fSxwkeQDiN,3 -g08dDEbZvhymfJ8NdNCGh,2 -wVfFxDZ94VPs9T81yUSdp,4 -Ce8CjWX379JwHUJGvwi8x,3 -cUCs8UQjTZZnHku0Rkaof,3 -IyXdMhONK9lkWNwRtCeNi,3 -Ol91I5ZmUgoJnsqvxMGNW,2 -QlOcIOwvetuP9SeueeoDb,3 -nles1DZndMZANupSxkMTA,3 -fit2fHeQEsjxkz03AHJL2,3 -5yAz5nuYuUL85sY52q2kH,2 -0B9mPtoydui80AUgO1UQq,3 -wytZ9STwI9dK4KcNmRGns,3 -yy9EFzmtRCbKMAtxrrf8J,2 -SmLuSlHT8OzIUsFpXsxGC,3 -EQJY6Z6Yw,3 -Z63lMAzFsa3Pj4yKfthz8,3 -lxLTh48tVmpRkL4wHSALL,3 -0tRMgmpn3Ie6SsfwMJeql,3 -tqob0TJfsWAqjIF99NESl,3 -WF0dzY5AFwduhnycqPPB6,4 -1vmUk7andjBhAEITGVzDg,3 -oonwvwfRyZWzKN6mrs5Fi,2 -IuQvyu7rJAsJ5GteZbWqb,2 -cqZ2bfxJ4jhLjIx9mKCPt,3 -AwoQE1Up4EBp6MYwPZQmN,2 -Wqf4lM8NLJYYjilXxAjyw,2 -fObVKiG1UMNuq00IU9XEs,3 -Bo359Aw8l3bZbkGj47CaV,3 -q4wYsZ4j4rClKMvlGk5cx,4 -C1A7BVCc81mRVWN2WGItX,3 -ohwNSwO1rt9h56smMJbu5,2 -tCkEYVZbdsWVSXBn8K3rz,4 -IxwckloTr71niTdF71F4J,3 -TzDgYaUkATjx2E7x172CQ,3 -GBXpJIOIa,4 -54Yg9xTzO4FtCM6go6IcE,2 -KKNJEr8Ro6hKabrpKvPSq,3 -7a2wEeycXy2AIhKn5ADON,3 -jw7cSYwqHH8frpuEoy2QN,3 -xOTUVOQwTjcsULPdwgwDx,2 -DK8f5DKoxRdpABrTa9wiX,2 -PBjyLRtCk4mjkbvxStZil,3 -1X7xfHBQv6Mhvtc4ssIcZ,3 -8J06TpR6H4PDkLNNH4Jqf,3 -5cS2fTOw2KCtqS0jdReBf,2 -xod31vTNWKbQR6NzspxhD,3 -Um49BzohhlbCycyk0bFkd,3 -N1CsFMU9H9MknD0uPvuGM,3 -HKtv6TXst0FzCDmhQrbBp,3 -kdBeJSyTdlFtNkvV5NayB,3 -8vBt8jjF7jnaShpdZSiiC,2 -ptwYHjcH39SICG8RB3KE0,3 -UjEgIUm51Xg1fk4LC6tZ3,2 -1WrvuikAKpg2ljxe1YjUl,3 -sNuM5ltqQgQpf34Gx8FZi,2 -YdK2CZDtiJTNdqwPP7x6y,3 -w5rFqa4t5CdrP9bHSWOOW,2 --LCj1vhHo,4 -ZOyA9kkdXF14uPHZgpLYp,3 -Tyf2ufNdw2CgguXwH5N6V,3 -LJEAvyJVFo4pT0vFi2eNM,3 -HHH8uEQQnN61MTDiwlpHA,3 -wWUbvaUFE15aNum9jH98t,4 -8ekbkvCzkCLvlXDQVFLxf,3 -esoSYtyl9n3btS2bK2wQt,3 -69v6xazlloi41le4VHzOW,3 -erzWwhqFq9bA5ZuxVrB4o,2 -YSRifJeDBolJ2b4Hlz0ky,2 -ObZT5A3Fv,3 -bnCMwRbwTEXwTt1GnSYba,3 -EMgtAjtgz,4 -Zk2IuwrZIRzoM2hYuIOyr,3 -Qb5MmVYvUojibN5wALxjF,4 -ETAYhmxdCkKWoVifP0UdL,3 -iOJyTAOwgqXTM7RSpSv8L,2 -AfGBb0UHbRA4qShn2GbRf,3 -UnsPkhqtU,4 -12Px7wX9ZYDPHb4TH18f0,3 -mAhtDk0P7IXynnd0BcQ9f,3 -zpsp6V2OVb3AqCIkeiPms,3 -Zh8k8kjL4IstUfHW8LbW9,3 -yYb0rhp6jZ8JoIwkR6XnD,3 -HCcSU0NbrHDcqbOqnYjgR,3 -8NWRny70pcDtYhsPVq8gf,3 -MzWlbjHfiZdYwY764Tsek,3 -L4HbmLJ9lVWnrdorWKLzC,3 -swMKX55B0AfWN1b9cSNnz,3 -o9pATf3D1y9pUcRVEbGrX,3 -Sb8BuSCPls15lXmAzJ5ER,3 -TLEhUP4nipYGVfEN23qgX,3 -x0ogy7X61KEaRKXzSOcsi,3 -2aP29F2BnUT95JP8PRq0T,2 -YE302CdP73rHZRTx7hy5O,3 -ANyyt295wGiODlE3rKacc,2 -oDkvjModL8oE519Cy6tlh,3 -N2jn9IyNOsz7mMC2ZJgPd,3 -CmELVwG6j467mf8ZDmhBJ,2 -1UPPN9lW3htp1iLNOGr4W,3 -3UshSLRY35IEWrFfn02Jk,3 -RFzJM2tpinoPtQL8Sg3Zz,2 -DU7qk8PEM7tTITTXQqPvX,2 -tTaAUJZZ6yECnaSwRk9GL,3 -koJ9cGAdoa8RMrYy27KQg,2 -upOOgLXsmIK9ujwJMZct9,3 -dmryvmYBdCFVAhoYX37WY,3 -zkQt9mgUyFalcgwxKphCe,4 -5VdMn1LgCVLjFxtDhV5Nc,3 -ZPWr51ffCqaw2JWsjDYG4,4 -HXQrLOBJ7MePLTMtfP2e3,2 -h8uYpXtOcMOSspX76mwbt,2 -8HP9mB13xbAOLdUAWEZBV,3 -qh1HVYwQGZXiAj44y1sTX,2 -POBsA05cTrdhgdKxYNuiY,2 -dVhmo30GfAuh7w0Y4FhLv,3 -Osp6Pj4iuCjATEilNgeGX,3 -58YNG1Y0Q9YXQeXH87bLm,3 -4CEco4r8LDYNniBjjAbo3,2 -tSacZBT6yNlArrjL2TFBl,2 -K1lbv8lh9Zbk2oBbvzBKH,2 -aFv2NpLepOgVGjJAhzsbr,3 -CjvH2Pfh9Zc6I0tC52bGX,2 -nv5dKNz3qVwQ7LCf9ZbYB,2 -dYi7IB0CHzsiFfcST48sk,3 -XvGBpR9BfoBzZwiNONvw2,3 -L9BSbgLQMFSXJqNVvOImH,3 -KrNwxvrQ2GQpQ9Xx5PWdL,2 -KoBQ2g5Iyqhm70Ve6ug5I,3 -PFQ0f9qMQeyHE2uSVrQT0,3 -Dj4XbM8LzhOr8nWXw2Mvw,3 -ur0NhNC1gttTYy828ilxV,3 -b0CODskvminv17js1AzHK,3 -W5Hpw1VhYV1wfOpezbkq5,2 -dXuEm6RkpybJukWUMgyUh,2 -0wUpZKrLWFFIuAUM0PfCL,2 -FESAZLhD8lGzuF1j0ef97,2 -vJ5LzKV4kQHN70as02Bk3,2 -pk9fBEhnfgUZwNRkWchgg,3 -aTnt1zLr8zd8MORbkEbUw,3 -SzC20qb743hp0qMkMX1T3,3 -O4lLyRBTqhN4RCJrkSgGt,3 -8A3VGBJA6koTqolKwhrLq,3 -2M4SaontEgTKAHpDCQ4FN,2 -pajZXP2H8bOS4xdrKOwjd,3 -34knwtY29ZY5ZkcSym7pT,2 -AV26ot26keMYUpe2nqaJX,2 -OCk6FeC4AeBtFc09hCpqZ,3 -KSlsSIEluFWkyzW2Qo4G8,3 -VknRqUXh3KYhn4nSZemVl,3 -oT3wm5PGQromPxwDil6ak,2 -5odv5tzJEMpnsfTTJKtYL,2 -kuuDyIttBznOkn6E4eW4J,3 -Io2yYaOg6KHu2echBFVS2,3 -jQ4UC8WmRpeYISl8E3fcX,3 -wP965CQ6UZme8xONUiTPh,3 -RuXs1Q3r4epzLNlSL4kyN,3 -sGLrSI4KFtkZmVtnbH5bP,2 -JgFAdZpMq88l4TeJaeMr6,3 -FgjlPtGSyuASGjoMRx0hF,3 -WJMB5px99ai4Xynx6We3K,3 -GfOfMpEUu7bv8hT5GPq1K,3 -lu8EPcV5WrpOZk5j8ImfB,4 -T8BdqJXHY9ZEwAxammEb4,3 -SBvoCFoaGwq1w0CvoBbAX,3 -37AmRD266zVId5SQm91FL,2 -UxpR2BJ9fMmowjzIPYofl,3 -zLqneG5Si7aHH9UXQuH8P,3 -qQ7YxkPbJqk7Xgf3NhVUu,3 -cwkYLWcXFLXGOU3vbPoqd,3 -fu0DHRbfYHFh8d3cYhYlL,3 -Utw7s9r7deQoKLnDBlcCI,3 -OGAEo7ea15lFJ64BpNopG,2 -kH7MRxXq7tllx42AYdwcF,3 -TTbFxGJJE5IO425Rr54K1,3 -FNwr8mtB9dDCfYl03n5ZX,3 -9DX7pFpUvwXFuGZslK3qp,3 -dCIhdsGO5NlXpi6vl8QxN,3 -0kezV2fBIatkCPgAvQ8vb,3 -HFcMmzjXvr2P7rFoyHJ9p,4 -cnBQm7KJN9BZXJpcyALK4,3 -6rY2AG9jg472DkhmXiKwK,3 -NFbX9dRY6P2NDBgnDVANG,3 -KjTuldnc1hiXfcOlgyjd3,3 -qDlU5LuBvteEbQ0AR1LIV,2 -1Hfujff0CGYePZwEjxTGO,3 -TTeue3Y5LsEt1UKNlQ0tw,2 -igJUqGo8jRUXbbRlpgIRi,3 -3l5qd3g8tJlarHlYMt87j,3 -LBMh79soxKZsNoyHjfSRz,3 -RqoRDQUZRwolIqqCvFMXE,3 -DxwrW8wKSNR8qqIHakcan,3 -OlCQNlPnfzSEymqndpM3B,3 -gA6Hm6VIFR4RIeCgqbj7o,3 -7fok0Hr2PuDQoUUZg9Gmm,3 -f1S8ORmrVyI0cvDcERuMu,3 -AbHveWQaXFBBRPgKaoulY,3 -qb2bBZ5uNgNuUmAqYO3wa,4 -doisUbe29hjwak7adishl,3 -1QKy88vq0ZeSHDolNkG5q,3 -UXtuNxBGgZGv9ElMauiHG,2 -LWIIdjXyl,4 -SacH5QLD21iVcSgt9S2lR,4 -dAvPVGKWlGRcggLuEcjFw,2 -1zTfQM2HiZRYp6A3yEdE8,3 -W3o5rSgg5zoLbVDiwjAEM,3 -ZYdeaQB6IL92dPkLguTmw,3 -hFw0m1r9ilooiFXCyIQmJ,3 -vzsPKzxRn,4 -O2TcvxR51fJfCYYyHfUEF,2 -Cs2fhjBKBTcG3HXRjxkvL,3 -8UGNUuKpAFgNhdS3YOQPw,2 -hB8VGEF6bJPt3iOYDxaBf,2 -W7nqA9SVCmfE3JAY6gJrO,3 -WTS0Vk9VHFNjlxC9fFKgG,2 -tBFWr7wf5pULN5Epqj4UH,3 -2cTUu4LABVI0Wx6FosMvu,3 -KuFdK8QCQYjczSfuofmhW,3 -1YV1pmvujYVmsaVqAlVke,2 -9r7waxLX8AkN73OuDi7pI,3 -cdeDgYhfX,4 -fFe_5yUrA,4 -vzbtoF7hXmqgAtMdNaaI2,2 -ZHSNmXx7xuYFKjGCOmGQv,3 -TQQSCCXO37zUHlEZa3Nxj,2 -IY4aoW89pDxDxnZauh4UD,3 -usi3Id5feZOFK3PCbVQVW,3 -17YhabOfMX8wiKoTWaBsy,3 -fFZCMIQC4py4sP02YDBb1,3 -i5Z9iNelBrx41Z8LOJGEu,4 -H1gFmWT16KpmufpsFZchp,2 -zRO4XxupgzGdNHZ3HM54u,3 -D2v5bMLKkVae74gDKLELk,3 -OETj6tTDmdCriyKVYrPEU,3 -dMYaMVBEgpXSYSebFzBL3,2 -11jIVeHJujAjXVFfADq4V,3 -szbhyJjKy8WuHURTzJeVf,2 -zlH2Fim36qc7ct6mPLvrL,3 -GjSZ0AEUIGarFpXbwBw8K,2 -ena4JMVcFpURmdv3SqpUa,3 -xaufTPMdOou6iFqKHvgAl,3 -alJDvXyeFhILmbwxNXsIY,4 -KBzEDI3lGXOWh0iiJPyFZ,3 -f8592499bed84f6ba53a3e5bde39dcc8,4 -CkjwMowFQ,4 -wH6zMdRVh3NhSHVfXOrhj,3 -AKwaNG7SvRRXPRQnuRA9I,3 -SwXcBd824nWK2D8YFhylg,3 -0WA1VMvXlr4q3kKaIQuU7,3 -XZfzzlPpv0pLmY8CuDQf2,3 -mG3QOfNPHTp81N35CsQyS,3 -2U6thoRjwn,3 -BHTf13n53TEoXMfScSv7d,2 -5oTHhngpKGU67fCKaDLuc,2 -zCjdmdOXISSo474rf5PIP,4 -tfcujciKC8PFBRhEF16bR,3 -B7kILFByHEMHykR970ap6,3 -Jbnd80SnxJkcq6bvSmJd6,3 -DK9tok3jXerNpaLGJTQLT,3 -OrXNqNIkeZl5L4KMioWLh,3 -cXGZ6eFY38oZovqTC2GLz,3 -TQXco3uyKkPckDHZizNZs,3 -MBp009j1KrkoAEgE7lWTe,3 -XeKGLyP7ETrazhD4W6kAw,3 -cRNeNd8ghzbtmZHcDF9Cu,3 -ilvodfEcPIQvjh3XCZW27,2 -yYng5O93U1GEKpTwsWDSs,3 -ECAHxoBVm0uQHaiQj0rjP,3 -7UUcssIRz,4 -C8gt7De3OBW6ZaIBBwtQs,2 -Kq56LhUnFOUlIyeTmleQv,3 -G5SJUCkPxc0XbratKnRtW,3 -gk-be4G25,4 -jMRIAHVRg8zUb0UHZN42k,3 -I0xrCW2nzqRjD4T442fHZ,2 -sBYKm5UeP5RO7ALyp3Mhh,3 -MEg5NCQBoFNdLgBCa52yw,3 -pmlMVOgqEbEmhxjH5f771,2 -vbSJQQ1WqWHak6AH2qF5q,3 -y5lz7CWQIpDNpcPuMaLJN,3 -OHurAAr4GAMfUoM85jIzY,4 -1EXSFAhTDDQ9RrrVAPHTz,3 -MXMSpAXJTrtb2mRUzQ4Bq,3 -s5lwJOXvSsGCAss3OQsOG,2 -sqFNMmcXym8ToCCGFUC4K,3 -I7wbW3URQayBQ15zjIjdB,3 -5UFj4XhTkAhATsX2iBNcM,3 -kJwU8vsdRnrINAyOTTHwq,3 -OY1vuht7aidTLzCfmxjdL,2 -VjOUgA5bnF0ClXf0TMQBy,3 -YL5qNoBFKR7ltk61MhNLq,3 -iHffpmd05JA4FhU6SsGp2,3 -j8QHlGOvEsr7siPqblzF5,3 -wPTbl762ywyT6iNEagKv3,3 -t6FdciaIKV1APuTGRz6cT,3 -kfKnLr5kW8YxRFikiXoTX,3 -TCR1dKrKr2rEw9Iyvzwk0,2 -rthQKAhjfox6oCKz3uh7Q,3 -xPwdRV2yVDMZskRxsV3VQ,3 -pWvJ0YX7olZji1RCNbrPi,2 -78LJkkJ3tOLIlAWgCA33l,3 -cqc8P8Qd19r3r3HmSa68G,3 -DfJX4j02assfQPGWZWkRk,2 -mY9v4Z2AiU05axwowDoU1,3 -35XX6VRRNPzWuDS4YA6qw,3 -Z00WcIigubLXUJqcHidQc,3 -r8E0HNDqFriUKniLQClVt,3 -3h0L50rOBggKea9Az79D8,3 -6yTuunqN4Pk3M7sKo9j7A,3 -C41vLXCSEVzu8F4QIARcC,2 -PnNFNfeHuW5LRJaByouS6,2 -eG1uPshRszH8Sh1aXBQpM,3 -oap1TZwqTI14q5z0q7vEJ,3 -3ROSSJELbcSeOwBU0yhdS,3 -N87EKtHiqQAlEARABobUE,3 -6B74iXHE8rArcNgRTruER,3 -IxfjARLiFU6f9ndtorhE2,3 -NC1vaA01OAW95JiMFShAq,2 -WckYNv65RWAhcw6OmxrFD,3 -f7Lcn9sXbNYRtZ8Z8TpNh,3 -bYhPF99PqfnVAClVGYwkM,3 -5Qc6BUNgrV5atyUREZaNA,3 -FdFggYFfCxR0KXQZ0WpAl,2 -H5f4065QbkRDajwHycE19,2 -f7ZJc4nzTyFWq1nnyJC21,4 -YMoN335auEhKBSyxPL5B5,3 -hsJUO6hPFd6JGI4oOFCS1,3 -zvDHrEOApR0JZZVGYDYLO,3 -Ts8AsjOvXqpg9oBCfj8op,3 -XBTCnmTbHF9vwq4Jmm6TX,2 -Nym4iYhgOppNDAEbNI8am,3 -pa3Uq2a5j4s7rTPznT6SD,3 -M0pmOh5dMj8BawnfFcoNp,3 -wOCQP9JiAAcI8a78va874,3 -dGLvLoJ5MRFRMY7f9p31Z,2 -XvktteOgw2vlxA40l0F2g,2 -nsTJwcLEnG2qBQEb9566j,3 -Tna8xWh59I4gkjcmYLtyP,3 -K7YRkR7waHQanA2kRGymu,3 -bSQhBDTXDaz6m4ISTHQa4,3 -jQsfsBRaKJNShrczRTi8i,4 -Lo8odZEbRFWNtkXoMG4ro,3 -sC8bShZtB,4 -nak9XCusJWJiKe4E6GaNt,2 -kJOwMhz1lPfukuSvshlYh,3 -HP6il4EJG,4 -v8LP8xcSeokPEZWtMrWCG,3 -2AY0JxlJl6s7f1DVTUBgj,4 -TjQKrGx6wTliESr2sIfl7,2 -lMrStGlUvFqohboYdC5cC,3 -KnuP0V29RcaGguuodOglt,3 -V56ZyWme09IXUWEtR20AK,2 -ruOMvUWzOnYVnWuWe7RLP,3 -WsnQJoUmKfrd0xBsV0U2t,2 -4xbkb6yadf25gI20CSCKy,3 -LDDDDAQtYedsIMrhlye26,2 -kNIlycYN8kZNxQ1a72uLW,3 -eSCIYn8IsvFMmKiQA8HXX,3 -RwxGNmt5iaCGUD1HtLZ8y,3 -JbzgI1JAnXLRAPpoye2WB,3 -sYvt2BxJFoCLuFp3atO6L,2 -YFxuwgZm3nSKareqvCsKt,3 -OWeA14MvwTXozqYNyaYMv,3 -tTQUyzEW70mQBvSnUMfcI,2 -W2nWWU3LAEbv3T5Kz65Ua,3 -tMJBCQORbVu0XO0HNl5C8,3 -M8mSvWJt5,4 -p1Bq2mPm4YX3et6iiazOe,2 -rNm0LfQ1qGkUJK3Z186zh,3 -LJ2YW1WBKFdmJgnF63Ar8,2 -7pEXekejJaqOkjkYLgEMe,3 -n4oZ4znCwUqCGHSnGtEU0,2 -qSANLxaVQom9t7FyW9c8Q,2 -fYDllZFsXdMXjJhbKj1j5,2 -3Miig67m3D1mSyGm0Ksqd,2 -UV5xNFQ2c5PzA2sHYSu2J,3 -p7xtqPKXGaZTYeApjnwph,3 -oVgyCDfU9u8YJstSARDwR,3 -N1QHcGDcMDp1ZBKV9dISi,3 -GC5FZK40mNkKBbxZxUw6U,3 -15lUO7zqpbYbELpybrrBo,3 -rtiEfRgfYSQsCeJiWy0bE,3 -DiCbJxzGqVqYCbpkx7kyv,3 -m2W8sLhdouhpDbxeRTj0w,3 -LxHG8IWCZrHY7UFwbfqKg,3 -Dys6WfpMr2XgermWO67KY,3 -xTOOscfp6tJMURhM7nAtz,3 -nNhC5PywLhT1scv7kDRyd,2 -O7zXkBUrHedY0Lu2pW5Mb,3 -nX8yVvib2Q4zSDcMHBd12,2 -m7pVBRJVqx1qMf5stnZed,3 -vTLtxDUqdtCsFN3GKrefs,3 -yv84d5fjDvdPQBIHyX3bj,3 -eg12TBRL8pMVWeVUE6HpF,3 -UUyPUFnF95tken1Twfktk,3 -w0ZJBsYEMYa2LF7TweTOV,4 -rboQ1M9vPQHqSnlEx3V1o,4 -9SsrJptyWXzVKnezTl9g1,3 -qjUwUdYxWW7S0VqERc0Pi,3 -8oyHTsDVnLi0ROyWmgxB6,2 -8wBL8705egTSgt6JWwr14,3 -IYtxYnnQPZij3OSqfu1g9,3 -Q7dYpJFuGLP2emZR0hgdu,3 -VqQECaxbnBSAuYeRwp6xX,3 -goqEngNoqwayiPiJcLV6Z,2 -baIjyPaEXc1P3MKXARPbD,3 -rEw3CrkXjQRJnTUaT6cEN,3 -KxftJIhhQ,4 -iuJA6reYilZzG5TZvXDS5,2 -Dxi6GemLWay8HmGZ7RZSf,2 -mT50p48YIJ333AgM75ojA,3 -O2e4odCdx6O10O7ydOoCF,3 -xc32SnGBpPxs2A264Kw7a,2 -b65vZ8v0yXvr73sG9Exek,2 -WGGPKg2U2,4 -rBN68MDnmoO3DtX2PhEfv,2 -ZTKWc7tlxrZ8k4O065utb,3 -hTDhe9NwabSlFQtT4KNyR,3 -0mLMhTIp45u3R3A5kzyue,3 -ie0dviAA8ZBSQEj7o56pI,3 -lL2k1XuIimOJVSxlMAKrK,3 -tT0nNyMqetA7opFWtg5Fi,3 -KuyIP2eKcmb6nlIkpGZf1,3 -NnTqSh7d9mfLrIz9kIFxX,2 -SmMbLoQiejN9AY9HHUsCB,3 -f7kq3s0HbjVjYmkcxTKx0,3 -N5wjaBK7JCGXb5m1Cr2Ad,2 -YpaOIHgs7ZwUOPVxMBq8v,3 -1auNJSoGjg7UogvaWrZDd,3 -ShbrH007qsPXK3o4spySX,3 -CNbr5SQcjM99ErDuhnnmm,3 -6sEHo0idQCjtzIBzvolgQ,2 -yo9OtaBMLn99Y9xkxWATd,2 -PLQk1qnH3CCoj3wD5ouyd,2 -fYnLz2GnVubCa8YIakFmh,3 -N1W8OBxPmTPquCOXKDVRi,3 -0pppsXoBP,4 -pWDz9xrwluSwB4MLtHgSB,3 -O31B9smuSClQXtb6lPWml,3 -PpEwd9ZDs4v833NARaYXP,3 -WsXrltT2lF7h0XSTZyjdC,3 -bbFTqVUhgEu9LjQM8Sk9q,3 -A0rDAr3OKscytPNWZboV5,2 -psraK5lEb8xgOgG1V7QNI,3 -EJaD6LtENLj8NAP1fBgLg,3 -wuaMiI6pZZGWF42PNfR9d,3 -pAu4cWNeBbOzFtEZ6yjUN,3 -9cLX9lZoJ2uThM2D2ksM0,2 -4i6PpDMXB9o0zcmZjXaGG,3 -s2u5sJyv7pGqdX9ZHaFS2,3 -HXB3zWn2SwtxUO6t9WK09,3 -d0uj4P8mYhXaqpeLL7YuK,3 -j7HpmOf40Pce5kbQqsqNG,3 -jHdHnFWKTzI4ZDYdy7AwJ,3 -fVWklyolnYpMBQwpf8QIj,3 -tOxhrxUDcw7SCVrjAEe2y,2 -I1nU0I8prMjSK9iZvK8E1,4 -ptNA9uR0OhL8kmWEUPk13,3 -cPxkS46xbT9ySxF2nuWbf,2 -ymweN87j79k2nrTrTrLpy,2 -PZh3y00u2WgTWsdUKVOTe,2 -eQhOaP26ehlc8NRmcKo0G,3 -C2ZnjevleQUbioED7DW0C,3 -6u4FXNOAaGnYmJckkz17D,2 -laSBsKTAj,4 -sSCcd9XnhRYOvmBN03Z8t,3 -nYgqraEJZTm9HrfWmLbEp,3 -178ed546fc7448c788eb61127e40bc08,4 -M96mQopyrtlZvw3nIpIfS,3 -h1CwuhgMvHIxNPm7hzDaw,3 -cCzinCwXP,3 -Z0wMM9i6ghRCosnGoehYa,3 -mV3iKMTkIJn6pMfoRZg5K,3 -X6mswYSXvvaMMjC5M99CH,3 -pNdK6ehRFIlHXlUA6Ju3p,3 -JalV2ZiR7P85HeyabSDdC,3 -K3VM0w0TlSamrY1efyRNp,3 -fL4KAqPQsgxoZzGnNJwwZ,3 -iiAKrYxnbX0OdlLjSi5hw,3 -sxfBbXDDmwbbtwCACcTk8,3 -MULkse9tgPhRvUSPZWMAR,4 -ffn6LheFUXOWppcGzIlre,3 -GrsZpCBy4bLZe8bcAhJWQ,3 -X03H8BNM93XN67pmqQs6l,2 -h7R2WIjCUXJecumik1bcQ,3 -pbBrjymZ1FwHPx5dwkXWj,3 -JWQliUIgwBd9bclRDWHgV,3 -jIaKrp4jBCA8b6J7uZg0X,3 -TQQOXzzRxI0a2oBwjJeCl,3 -4MU1GWiyJ7yus6V1hvGD7,3 -ZFXBZzvbyhLISBk69p5OJ,3 -gyaHiqcoxaWTMiSRHemdB,3 -oRDzk0newKgoMf4vABF38,3 -7YMj4glxN,4 -nWON9ymt1Ha7gCuAlIfli,3 -PxW1dJKLmAoDEFUnpGgyh,3 -rYS3jaF3hwULCzuVDe73N,3 -0J9RfWvHfZutiWWO9KlaA,3 -SF2YmCndtuMcPdNfhgDaP,3 -YgTPX7bI7cTii8vqsQp1h,3 -OT4CCzRQTs1w8wHvBjbcX,2 -SgERnHOHnPalyajm0jASc,3 -GUYSuDkesdASWVBPIyib3,3 -hCF0pHXCIGZiU5PhTTlTJ,3 -FkFy1dA8XYUIWC6cboRrw,3 -S2hKVUzwIbmu4GWpFtc0P,3 -t11tQe18HbCCL58m9y5zx,2 -9uS4M6E0HbrP5PRny7hmy,2 -1xVyptYOk39Pl7N2DxmUy,3 -CGMm2nd3Tic6Z0FuMetIF,3 -qspgKFvhXXAvbLmAH8vZN,2 -7ryk4iCXv5B5oK7FGxyYs,2 -eca0861593e1431f811302dc12e90342,3 -MICz63MyjmWGQc6eZF7I7,2 -DHtMen8ggwH5qRPXOujSK,3 -gVbLeQIIOfU0y1l2o39sB,2 -te9tGEhMdBlcyN9mCVIEl,3 -02AaOAt5wMK7HUHaWbPD9,3 -tKXSNtd0mCJ1YZQbTBfhg,3 -44V9oUNdyQohUPFsJUJ3s,3 -BKmlJEO7e0SpggASOEi5U,2 -FGZb3skcUVjgbZY9uSv8p,2 -OEQNKkxrT38f52xdGRmka,4 -eS8idDPGe5jjLLFzdBnB6,3 -P3DWVvsWS1Y3QESrORTjh,3 -k3Uw8tgIFtYI0mxL3gRyb,2 -NLXsp7HCQoYQHTmVn49Zy,3 -08ST2mz8TPDvGPQtBCOVx,3 -HCXhrjsFGqfAWxljHpm6A,4 -ofXW20bGdMZ6CmTwbpSip,2 -i49kdq78b2GIyZCtA5wzu,3 -mj0NNbPH1wFIxi2eudEZW,3 -HhrjUxhvvcsSjVccb92EL,3 -08GHmINtr,4 -YJCc8QNrH,4 -ShozDOPd40AgCqDjnphtc,2 -NaR8nnpzWa39zoDvaUMT2,3 -vzwVadXv9cZ3ChNVplbGQ,3 -aabmwY9Qb1gw6oP7cKyBv,2 -Z2ZMgy20uzMCTkqX2vwJP,3 -26PkgYJDEJNvgnB8if2u1,3 -LaBfL5hcxFdSJIbtOvUPd,2 -OrZoJyshfBXeqlZpForZi,3 -PndAxHJfsivvN7AiwWsU2,2 -W0czytj7tbQWDN0u0gvTR,2 -5N9zLq9RWOWryx2vrOSag,3 -a1ZRZkNTVYNwWe1d2h6Zj,3 -uyjxq7ZZ2yzs8EXPNeT0G,3 -LPDuKCYatMfinqpnGUCMO,3 -z7ucEhtGxBkB2vtvq4WqR,3 -eKveWTHfKA2dlWsW1RqH7,3 -BUiSoJdPpYAfKuuOJf5Os,3 -hGwkKo42goYWhVJCd5qxo,4 -8GTSva4DqR1aHPzkj6rLo,2 -7yS89zlQaRCTinL9mlyod,3 -cKejCt6ga8OnnNHsHuMFk,3 -iivT3MG6fmEilu8OtRuuP,3 -kpEDaLVn6HULGDYr3jA0l,3 -OmLheuybivu40u2Ycn6ij,3 -TMfoNvyUGn4K0EerLJWr8,3 -CruZLvzfB6YrGw6IsgyLQ,3 -0hivm9FMxerAyhX6gRpfp,3 -Y54EkArDfb3f6fALz3bC8,3 -FbEg8WpznCXW5T5m5WG0b,3 -cNyAfNmJkwH7nMW2ZKp2F,3 -yYSYdQIqJIdhsHkxI6gbg,3 -d0N6g6KnHFMJ23xLcKEmP,3 -20Y6gdaq2BavNNaqi7Jyy,3 -m6oASLNCQEQ3dlowp2xWM,3 -PNR93CS4g6AmHNu736khR,3 -foEthZtVBHjW9bBZ7coAJ,3 -MsxctB4fz6gkTRJvZGs7a,3 -QL5HvbU6Cy8HHuxPiYv9l,2 -mn8J5El9F2YiZ0SbxSKqx,3 -tZb1Ldr41Oy37XWrF0zFS,3 -Cu6cPrgHWv8xTKRk6qGUc,3 -X8XJoK5wF6jacvn0MZNbc,3 -BZsdIp1w3dd52vQq0pIpL,3 -64ZdSoPr3STm3G9ln56YZ,3 -y84JN6sqIDXKCIyJDfkUQ,2 -CdnUbOqLy,3 -4OpcjVIFyjGypbEMKGiXP,3 -JRvr6zcxbC7B1cTJzCFbd,3 -Vfz8eIt1VrIg4mDFFNVWa,3 -SwS2nVx8CsbTUEhZO1WWN,2 -qj9VfyNNjmLh9kUGJJghV,3 -jAoE9G0LyhcHycjPfWPX8,3 -W3LEJVLjXHLMd9VjN3KND,4 -gBBVuSM8aVkwwyKinKTFL,4 -MCHMnXAsPErBex5Oij42x,3 -mi816kklnXr1yt03Atm60,2 -ioweiTFHc,3 -fkdedSCVV1cBlOXlKlGUy,4 -FekyZe8CeiongEvPcE9hb,3 -AoXaEHoXr6COXNet3SX34,3 -Bac1ff0Qn8Nud9x9uue15,2 -i6BZFYe1XlQtNCLtSKc9E,3 -h8OqUMq8SNhGCHM3VhRKF,3 -RJ2jtRbrylLM516LTtRoo,2 -YUWQJegS1oQUzh9ocqQ9o,3 -NRpGj0KIZsSb85WNDdJex,3 -l4hRyJzTAqc3rgtXfIsYJ,2 -72LUQPmjPsvJ4uzFmfwyQ,3 -sbqesanUpMJ0lXEPNQOxk,2 -wbXEGC1HRVIX09cK0vo5o,3 -H2BcQK37wAu436SMwRXs9,3 -cyb1dU6kpmqoPpNtXDo1M,3 -ytdYEMLGvcjATsYow5ijP,3 -he38RINHglEIT0m0g6MmY,3 -xVitHZUacfCZ85HDGNn0J,3 -5aWPGGkCBRd8ja5H9antE,2 -0LhVbr2AwNGFP3IVXEpoL,3 -2TmfsvhtQZTlBZsayXCUP,2 -C7pkJHazRK4OTGPQj8qXD,3 -0tTL7KqCOoedmwlQBFoXu,4 -hTjrYDQbjluld3YC4WUq6,2 -UFOlWOrHi8xxYTl6EKVWZ,3 -xgh7zPA27kztrhj7R3cdh,3 -6m3e5xLrFY2354fTxwO5M,3 -4bdrJLRLOSDjgCHXvSi3j,3 -FRHJeYDa5jMytXOkFFWXn,3 -PiqViUukjDfHzTuoqNZKg,3 -DMPyzK5V151sgMdhGCscB,3 -4B4RIyyp0lDfAkWWIEsrB,3 -5zumKmzqaXHeYi12rWB4m,2 -GWF3ko1gcksnPKA8yWCYp,3 -ZumEC5UcRxAwmNu1usu9w,2 -czr2C1aa4VLV8UXhnp9VA,2 -474HgEkmkstZRkFZHsoz1,4 -4ke2BjMRCUWxjGpVAgO06,3 -Zu1T6gUMeRqKYyt7DgvhM,4 -gMx9G7CHFxxz4gL3AyU8l,3 -HYCQiw24ydvu0gbGPPbso,2 -X9n1H6SjSbPttItR9Lqa4,3 -2l15qqIuyxzUdMPi4MINS,3 -FH4FSCnn8Jm8AnpAgGEhH,3 -vP0ww5Uy3xSukOKfEajFW,3 -73HlZi3YZQh2fAlrv2Rbv,3 -s4Gvtyif3oGb1DXet7Lw9,4 -AiRFGnAWXV256Gbre35d3,3 -E0X3QMwQtLg5ZGMdIOX0b,2 -72ixbr9CCOM4Sro8PVMd3,3 -ojubvnSWoZawauJ2G4BvU,3 -n4Pmazq9GqqDL7o4tDTdi,3 -j4HFz6FAWVSjmU5tfYmsp,3 -HIHI0lGiLSDAtxt9WBmru,2 -RiAWNmso3,4 -G7Ol5LXIC4NG4eaR0p3Mj,3 -oaYG9bmhb0JN9FLlPHI3O,3 -hfbJe0zFYPW9VVzmuAGtQ,2 -nfg52YXaNVXcCvw5IQ6EF,3 -pEg02tQnYmJQkaa6R1an1,3 -LJLFrP08gvhf5kVqBAg9k,3 -fN4qEOviztWDEzUJV9OJM,2 -9FCIXax8KiVu2JYLfZXfu,3 -0gA9kbdFNbmPaGiKbjO1E,3 -016L12RCpcczzDENbYuwO,3 -D72xkNLpDoNhqdZxj4Vts,2 -iLXtKTF6FjN4hxNdIjxh1,3 -BbHdGc3YFguGQaEw1Aruj,3 -qORBFeNzv,4 -wzxEW9abIlu8Z6BDoftKE,3 -KI7kVhOgo3nBqa8f3JXI0,3 -47NTxiLOZqpqFd4RC2Bt0,3 -zA1SVIYNEgvZFvdjfpMUE,3 -EjXNSDcgyFA4E2WZMfYOW,2 -f3kAL69FvYqqMgiiNJbDB,2 -9QPkzETC77RanE2goI3wH,3 -Aur2av37w5pUEjocySCcz,3 -zUTzLN6uDz5Ac2hXxI3Oi,3 -tStk4P1pvQvQoqFRU2NiM,3 -U8wHaHeEVP3ZfAHCqbuRj,2 -zfPGHk1ta5NVxs2w5BSIx,3 -GZCCP5e1vZar4jKtuUxkB,3 -vpykGlIpyfK50lM1GfKWh,4 -joDdVM30zGiDn6TkWh0pD,2 -EZLo65YYb9Rpcu527BAxF,4 -XZEgomY1aYlHOH9bxDpKj,2 -EYF6Gbxa86OJuUZplinPG,3 -wDpstqw7sf4qBiV3EwKyK,2 -1jziqul2MJFXF3hnuZXBr,3 -zB6K7yocKt1HrDXHJLaZi,3 -uwo6fZ58HrmGbN2UrOhDV,2 -eusTTeAC4FaSfjP16jrt7,3 -seIioRTI3XbtHTHkMPzYV,3 -g02thPC74GU4JUXnZi7SI,3 -3lMbs8xosQsuDpceev8jZ,2 -u2P8hqwRFA3jggHYz1fe7,2 -Nn1bpWbLcrzljEFvyRH8p,4 -C44vQAeDuwXRIk5C1L8k1,3 -mrGWd57MhBbDU8WsOEJW7,3 -RbwAWJwX0sYIJWItpFgQz,3 -24yo3sscqUYfXDvsegxQ1,3 -IQbLalmyu6ZbSugItn94X,3 -z4SUyHffQKjwrHrvO1m0K,2 -YYI6lJrYa5AeZoZuu4m98,2 -WdT4Qqn76KkVLxGJkYO38,3 -tDyt9viBBhcRd1FNXzLFv,3 -vmG2VXU8Gye1xIenPhJ8E,2 -6askwYFFTzpYbR6hOEw2h,4 -xqMb0FqJvGM885zuIpn3i,2 -ns5zcUU5XsrluuCrMRJW2,3 -ANPI3TtrQyKGlCgMBV5YD,3 -Jwzrf2fxPRydKqRPresNw,2 -q5KP6AY8gI1guMIkfjjrs,2 -syu3AAB9EiUDODJSDSoKf,3 -tLmk47uAxP3xklZZnYPo3,3 -uHK5QsHPAXA0EIxqswCEj,2 -Xql3xZcApIRTADBL1ACDT,3 -FvLSruFBPlnnfRpbN4XOH,3 -bpBYwW9L9ZcdBxnNs8AEQ,2 -FhXsG65VcAy6ZD0vIF9df,3 -6PDRfLcS6q,4 -uuCtzoTzLR9v2BOdANveB,3 -XA52iS6Ph,4 -RdsEiIZZ4qeRYLaCLQy4U,2 -17NaNY5IQWGu3yrElZsT0,3 -iWsXCiRVqMtHkwG3DlO4e,3 -2b21Pzbby5cixFXnEqMbn,3 -dGQU45VW5yXjwoJWHBGH3,3 -qp679OYgm2UeyFPjpXRNv,3 -FdoPZFwS5w0W3s54UA6pH,3 -e7tCNsnwir9Awpgu6It2Y,3 -WQfrrQVSh5EUmAZuKK7Y8,2 -rZIueumSJQzK67prBmQB3,2 -QGekzwMA9gPHhlkVO0dy0,3 -gQRGEeFTHtzFUbP9o5LiQ,2 -jPWZjTBdjQLXYfjcgWjBz,3 -YVpRsfANvnbDGIzqaIw6v,3 -cuxF1hDApVuXoIAdrV3Z0,3 -dvLrGSo6Ghn9qwoT1nm7S,3 -CVxwtfuVl0OAyZ5Sy4b4o,3 -HAGqVcDATKef4R7dZILCe,4 -WEUO2UxYeyhyJadDaNYV5,3 -RmRH8CJRRlK9PdRYv2cpx,4 -jmtoV4pbQAkps87q944Cd,3 -LRq1cGaWiIVCMChAR7N7c,3 -1WTuFPclgyEMk09ZmAnA4,3 -fc6sBKH4AqOMzbtgts9TH,3 -Z44cGfAZI8rhgWB3mxprN,4 -N1haHq7VadcDdLu2iasQj,3 -PICL56yy7OoUw3jHAUz0A,3 -3U07EskkHcn3Lk3UgM0B9,3 -2XEkyBiAliDcJQTnNTzkn,3 -aUHwWsFSNqD7dTkk73SyT,3 -eQoqism0x8ezL4MS83hc1,3 -1sXXkqorMgFeIzxZa6zA2,3 -DPQadRG8WYrsEDB9j5bu2,2 -49Gn4c8DrbXrNoTDd4Jfc,3 -Lpq7ptJbAKNEOLS8F81fw,3 -AfsKmAd2JhodzlWzmyEI6,3 -5gAgFyFQfPBkniStruEru,3 -dLw2SjDErpS6zBZENeShn,3 -xRogTPsxXxyAMGO9sYfAW,3 -guVNmxuBePbr5LqGzuhFC,3 -L7oTtlXjRtrZzSmnbNPri,3 -MR88Z5LorCJXjg7TE3z6m,2 -wK0AFhPO9FPJfPtVmxVmb,3 -FojIr27rl7KthkSqfOWd4,3 -ZYnahK3Ybx88UMxpHAtgo,3 -T1P92S8WgCRjOlJIcc3tg,3 -Bj6RFtsqddtBLg6byqvd7,2 -BzADpy2Twlh2H02Orwe9Y,3 -oHlRbBkvQTp4Y7CR0fmZe,3 -l825B5oo3uupfC9ZB1nQe,2 -ttP6KvvA_,3 -zYKcQ6tbwGGquuvI73tyh,2 -Rjx7ZMAvssu12Q9H9pyye,3 -SMDEScYtBVQnm00ilLXNn,3 -trj2kJkwx2C30D3KLwvnk,3 -jVj9fXkqifwuUCRM3bcLy,3 -QGaYtykHOK40oKdRGUKix,3 -H0b7RpBhqQa5CYq6DB6uo,3 -OQU4ckVUM0pqy0RpiQ4RU,4 -wUr8heicwmJxSr3lDtWXR,4 -RlTKIT4b7tt6q9UTJwSrO,3 -WoiWpnz2xJfKtSiIJA6Jc,2 -RSO961MpXmkjVZyaGYQLM,3 -GCB37CHnfLBV5Y8sfalxN,3 -H3qhXLwT736O9mcKdLMTq,4 -4EKkNrfAL,4 -xfbDa7xIISxKVxsDmYzSK,3 -fRiCo09s4W8rrZfNpiDIq,3 -okbyz4mH1bsnFbkRJw6GO,3 -PkNB0NcufkI7fohRlN6yq,2 -M79iiQSBgGWs2tX2Bwv62,2 -wvSne48gMNV4k00sQfzml,3 -6KkGfCyxdjxkI9MNlGp24,3 -lgxIHoPqbqAVasWDVkbO2,3 -Sn52Ic6Y6LB4B2ZteZ8ko,3 -7t6wRExkOXPQVP9zHjkxi,3 -OiWCmXKtExcVTM85VFmCk,3 -Ym5tYD2R7Z6t1pxFemR57,3 -BSnmejgjFDBIsmZS3Ahuc,3 -0mxFgLOfttMxW0S7Tw2iX,2 -6NUkhnGhcr2Rtk3vZt3RZ,2 -xpH4DptcrHBIYRS6UFTzd,2 -iDIAerElrPv5ZdupK3SYm,3 -FTN6iQ9KhU2wQjKYeKAW2,2 -CS8K2Qh7ROwJsd5wyaCJD,3 -5jMjVfvhTQelnqVgfprwy,3 --EvLWbMLg,4 -ZOAfsauQKvz5xmdqJeYvt,3 -NTj9Fdft1XI0Jp1Wx0bFk,3 -oKHpuLIxJIVsKYABVIjOe,3 -NX85xMA9svyfZ0vZ1EkRf,3 -MAMz6dABwlpnZanZIKa9i,3 -qyqAXdVyWcGCPFGXLJB0J,3 -ZHXOQGAtVeFd9wEE7LOUL,3 -bDUrDmaD5zYL59VPTmTaH,3 -Ek0F3ZHkXnW1Zm7tzZPHs,3 -iu9orEUJfv6W4MMKCx7Z2,3 -OnNEKNTzru6W39IwTi5I3,3 -cXhqmbEkDh7t9ZgXypbrr,3 -AK1i4lgP5al23uSGyEG1m,2 -tAC4bwGtOpgNqwT1O4wPo,3 -V0tBrYbvJ5m51xcZEOhua,3 -7qRrnUXsTbjZGRun1uzwA,3 -f34Oo0tFjxOp4utFXdCoy,3 -FTo3P4HdYTIi0t19WKuay,2 -JcvMKp2xVz4ujErc5x5Di,3 -OoZbIyztcMcfVxOY4wGRg,3 -Y1bs2L2oZVNerETHofdNs,3 -zPUkhd50FmmXSBp7j725s,3 -hddci0UD8kKuutIkDKZR8,2 -oth7uj5owyvFQGGQUSocI,3 -9ujMRK8aCsYl0TU8xlHS3,3 -niQfwfXb3MYsxND0rpprp,3 -79uXPxgaDiun8e5mpmVFs,3 -sLSaxpZB7K0K9BBPSICJo,3 -S1mrm3w7f99fWoGC5qLub,2 -Gji1NWTUQopyGlmy6WaZg,3 -jgSD7FVyyLnOX4jG1r8qL,2 -RkgIrZ911sddwq6DLBHfi,3 -5dgC5YKeqvInyTNHJk212,3 -J9TMhvBbF3fLWRoUVtdgY,2 -ZoVSdwfD58Are8qmUKNIC,3 -D9gMDdxmDWD9cOFREmTnQ,4 -e5B33iYkt3qkKbE0APFtZ,3 -XalS5NYkpwhAmdBpBul2q,3 -q3qMdVkdyQo4GqiXRS76D,2 -trJ8ziE2BahKPGMnyPojQ,2 -0cUeSSnmuYyroPp0eTF9T,3 -OpZKIxK6GLlFC6TwQ8Ham,4 -ZjtnWPee3dO3arRZQjGX6,3 -WNadxEfsh32vZVfiF83XK,3 -uVwEnzegptLC1A87UVj9G,3 -JiCUAPal3uJrSAG4Jvna5,2 -vIpNyHLGxHjYmxKo6sD3I,3 -FClGUjaHik4Y0fthvDicj,3 -WHuqRZUrVvRbt44eF3QTQ,3 -HUnJkTrUv3bLKwEOdXZ6k,3 -u2nPJdVtNHL5WZ83CCWkf,3 -My6xUjuQ2jCHGmxqgo01p,3 -d26JfdQKYE3bSY3vWxUQv,3 -IUplv8oK8f8jhVvPNSwED,2 -F8MkgCjxYLbm7p51YTzLD,4 -t3QmCPrVt2E7sZFD2qmq8,3 -PR9Z5rtCgwlKEZy9hwHJv,3 -K2ovHfnwJtxAAIppcyjCg,2 -WlBQ4b7myv8LV2Vh75RzV,3 -AnNOSp60Yj1YfspLwNV6k,2 -htNUJWE6j8ha53l0htKfp,3 -wbOY26NissdJ72zpzAhWI,3 -2bd96b984c864d8d975bbb6ec59596a9,3 -hyYaxVJTZBilXRVOOTJZN,2 -p25gsPwzaQSENIt0UlAa6,3 -wZ4Cc0h4U2cDmqnumKJUl,3 -UnMEvnqutgsuNA5bXxM0y,3 -3VP2YKJpiaIDr7vMynGpl,3 -hKlRJsvIC4E8bghZS0xVD,3 -0GJDeoIIMfR5SSqlpCXOH,3 -CSfTpBnBKoJeY5JEDTutW,3 -rEybjwtt9tqUfN5Pze8ja,3 -MSfUs8tysZGeU0ARUCTrs,2 -7oAJPiYlWw3lcDZw1jh33,4 -3cVCZ8jPrDYewbGqjATCa,2 -8JGmugHXfmf9VCXpV1d00,3 -RamO1KGRfhgbePOy2Rdy1,3 -OMbALrBvLoovGpzgVnEQo,3 -7fJeSDNrHny3yNRPhNl4Q,3 -YOpDvukKwwp3rmhMppJh6,3 -l42ybZ6Fv2tWRa87rnbaB,2 -RH1zbAhprGZNzNMzqgLTd,2 -kRzy9MtvY5vlxZnI70yUi,3 -d3i3zUHx7BCD04dNgHUBw,3 -SypHAnvzjE7yQb6AZ5A8Y,2 -lTXcr55EZtAWQ8ecUivbs,2 -rpYXJspcOQNR290ccSQ30,4 -NjEjnbpyGW3n6GJdAmkii,4 -BPJMZyGUfwOamrDXl9hHa,4 -w31xUfRZeG,4 -chlNnXG90aRiUt3sQIPYs,4 -A49q3vnTkPH6bxp2uaxJF,2 -ayK6nurOtgv7ftkoHd29z,3 -dbxhjDtHeSDunEz1gam5v,3 -fzOcGQxvZYk8m9uEfY3Ts,3 -kl10MeTl8MhBTj1iQ39Wr,3 -HioVyC3BKhtQDi6x0xRRx,3 -cSLAVE5Ae1Ue66aHNrqB2,3 -dJVdacd6wdrqF5Dp7gNxI,2 -QygTP1l7uQv4IkSaPqkSQ,2 -qKyQO7p1QI5OApmSQp1LR,4 -z5J2AIQfonLhT77lwfyEy,3 -xmWUKSRsSuZsmUaAkh4uc,2 -ScdluRxxxIlE20eiWSQdn,3 -4ERNfwlZMK8mXgwiplBoz,3 -sIXAJRIhr6GxS481AaXRQ,2 -pTYis0sspesfx3qbWZjnZ,3 -zn3UoJVmKRm9944n6Bx4F,2 -87IFCvPjwzOpPiFYoMmKd,2 -FgbBSirR4eKIG0LcgQ9tP,3 -huqVccQV1vddDwOxNzxlp,3 -4TDVT7KQcz2wfxW607Alp,3 -swt7wDGGiXkbClePglhqw,3 -qd68uSTagzkYLn3kmi8PA,3 -NdiKo9QZP83BdQcgH81VB,3 -T5c9gV7TNFIwUw03rVOyK,3 -yb8B6INRVHtrY7QlKH84o,2 -7bdWO6xjwdyyFue2gHBqX,2 -isPW1AaaFSB5rGDuMoypP,2 -ESSTc2gJdDzYepxDEC468,3 -rdKXNGRlXAFuiISdVh4Zf,3 -JJGJNmAY4pzvwUkfoqocy,3 -GFFnc3qpVvu28GAK4Q8TW,2 -CqiWRJCG3TpYAUBOi7ruK,3 -SqZ9fA03TNb2M5XbkXT2V,3 -b7gDHJENq43EFschU27s7,3 -vPToRc766pjYBVuXS8YtW,3 -STJVhCVrVTHuOTG1UlWPP,3 -Tr0Vg8C63SCLGpRYU2yAQ,4 -ngdwDce8FZa3NjmGuwQvQ,3 -GnOGRNKNZWmsWypDcvnZS,3 -hmkll2czTY9YzP1LXO1C3,3 -0x1wezBiNa1e5HE7PO9GM,3 -mSG4ErjZGppifqKFv4t6X,3 -JciIhowtFUp89yIIUq9fk,3 -dHQKoljory2XHAy5ntbVM,3 -s29UWyMoS3vy4udKaQS6K,3 -OY0lBj9gttfzgXJHfJKW4,3 -AYIIqKFCWiOIv4ZLMxLhl,2 -qM22A3XQ1TRgZnCMFc5C0,3 -bhU7FSTdH3RWlgwpYeBMn,3 -6SEOvx52eCJmhtGOXUtUr,3 -t1Q7Lk2lzrtsVVCpAbdWz,3 -KcGypN0BdsnfaCVcdfyKi,3 -G76xRoMpuH,2 -fxhASpWwZm8GadkJVSOYQ,3 -dhBtusxpVQ3uO7zalinNO,3 -5dgB0Ac9iQGIvWxKTKBHO,2 -QleFEGgBtUiEmqcE6c7mP,2 -kX2dRqKQO9dUKMI4SIcR7,3 -V4N8CRKLj0E4873Ti4pD9,3 -c2GQDSEImPO64xhOUOnUq,2 -7dxIiShD7P4jQazWiHFR4,3 -rXvpj8aSC28fME33uLx1w,3 -YRezkYhLFmQL4P8MpEUEt,4 -wBG1csdkVOf0pDHoeQYA2,3 -xNKmglHuGWzXTZX7BI1Dz,3 -gkwkqbSJW3TR906PRa9uZ,2 -8s1LdMslxzibA1DDSKoEy,3 -b45eaiKLA4PjRHuu8Vk3i,2 -lxpxEhRhEeOzGTvEkR22O,2 -AWEeZ2tghR09atNSQyCG4,3 -bxqm9DhvOy2IpG79PJPg8,3 -Gy5tFqTpNcQTTmeN8kVJb,2 -E4mMENv5W9fNQH0vd4q2S,2 -ePMqaUu7L8daO8k9Fgk4o,3 -SmXcLV3edQjNl3yZhLxmI,3 -WZB-N-025,4 -rxEOvaApsE1jypxLnsQls,3 -4qszAhGXIEn1rxVtVP28I,3 -wLaHW8XfKnzKDq1QsjfuP,3 -cjxoyGLQ2FADLrqQPTbw4,2 -eFlNZFK75XGAQrpNP4LmQ,3 -fEzcU3HBieSKOUmkRCrcc,3 -4Uy9opdBiqIvIksfS48dN,3 -tqXJRTDXnDxPrv3ldH0mZ,3 -6OkWS5QJwVaJ5b3mnEnQO,3 -RkMwRYe3zFmEHKEUkc04H,3 -o288GrgoQUml19pinhleK,3 -yNcc3Wdf7G7zy8zMxEPGD,3 -AAwMxqyqzNMG9WfuthKVB,3 -mXLEtqJh3BohsSHWGuH1s,3 -VJB91dhtT8T1sX5vtU8zC,3 -wFBSkSWmfHaeKIDqhAQ7b,3 -r3igKhuZ8VlP1X8uGJI5B,4 \ No newline at end of file From 47a6b42304b99a67e8799aabd66011226ed4fb9d Mon Sep 17 00:00:00 2001 From: Lee Hansel Solevilla Date: Fri, 3 Jan 2025 18:10:56 +0800 Subject: [PATCH 45/45] fix: condition to not process anything related to cio registered --- .infra/application.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.infra/application.properties b/.infra/application.properties index 4f332f5df..67b5c853d 100644 --- a/.infra/application.properties +++ b/.infra/application.properties @@ -27,5 +27,5 @@ debezium.transforms.ReadOperationFilter.language=jsr223.groovy debezium.transforms.ReadOperationFilter.condition=!(valueSchema.field('op') && value.op == 'r') debezium.transforms.PostsFilter.type=io.debezium.transforms.Filter debezium.transforms.PostsFilter.language=jsr223.groovy -debezium.transforms.PostsFilter.condition=!(valueSchema.field('op') && value.op == 'u' && value.source.table == 'post' && value.before.views != value.after.views) && !(valueSchema.field('op') && value.op == 'u' && value.source.table == 'user' && value.before.cioRegistered != value.after.cioRegistered && !value.after.cioRegistered) +debezium.transforms.PostsFilter.condition=!(valueSchema.field('op') && value.op == 'u' && value.source.table == 'post' && value.before.views != value.after.views) && !(valueSchema.field('op') && value.op == 'u' && value.source.table == 'user' && value.before.cioRegistered != value.after.cioRegistered) debezium.sink.type=pubsub