Skip to content

Remove unused collections #6820

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: newarchitecture
Choose a base branch
from
Open

Remove unused collections #6820

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
11e772c
Remove unused collections
ar2rsawseen Oct 17, 2025
41b166c
Merge branch 'newarchitecture' into ar2rsawseen/newarchitecture
ar2rsawseen Oct 29, 2025
13e46ab
[crashes] fix installation
ar2rsawseen Oct 30, 2025
50e8486
Add log for tests
ar2rsawseen Oct 30, 2025
5e06e8b
[crashes] more logs
ar2rsawseen Oct 31, 2025
9536dd0
[crashes] fix
ar2rsawseen Oct 31, 2025
813d25c
[logger] fix tests
ar2rsawseen Oct 31, 2025
a75d8a9
[compliance-hub] fix tests
ar2rsawseen Oct 31, 2025
1459c44
[compliance-hub] remove consent_history usage
ar2rsawseen Oct 31, 2025
2b0d81c
Merge branch 'newarchitecture' into ar2rsawseen/newarchitecture
ar2rsawseen Oct 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions api/parts/data/usage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -868,19 +868,6 @@ function processMetrics(user, uniqueLevelsZero, uniqueLevelsMonth, params, done)
}
}

if (!isNewUser) {
/*
If metricChanges object contains a uid this means we have at least one metric that has changed
in this begin_session so we'll insert it into metric_changesAPPID collection.
Inserted document has below format;

{ "uid" : "1", "ts" : 1463778143, "d" : { "o" : "iPhone1", "n" : "iPhone2" }, "av" : { "o" : "1:0", "n" : "1:1" } }
*/
if (plugins.getConfig("api", params.app && params.app.plugins, true).metric_changes && metricChanges.uid && params.qstring.begin_session) {
common.db.collection('metric_changes' + params.app_id).insert(metricChanges, function() {});
}
}

if (done) {
done();
}
Expand Down
146 changes: 65 additions & 81 deletions api/parts/mgmt/app_users.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,80 +267,78 @@ usersApi.delete = function(app_id, query, params, callback) {
console.log("Error generating list of uids", err0, res);
}
if (res && res[0] && res[0].uid && res[0].uid.length) {
common.db.collection("metric_changes" + app_id).remove({uid: {$in: res[0].uid}}, function() {
plugins.dispatch("/i/app_users/delete", {
app_id: app_id,
query: query,
uids: res[0].uid,
params: params
}, function(_, otherPluginResults) {
const rejectReasons = otherPluginResults.reduce((acc, result) => {
if (result.status === "rejected") {
acc.push((result.reason && result.reason.message) || '');
}
plugins.dispatch("/i/app_users/delete", {
app_id: app_id,
query: query,
uids: res[0].uid,
params: params
}, function(_, otherPluginResults) {
const rejectReasons = otherPluginResults.reduce((acc, result) => {
if (result.status === "rejected") {
acc.push((result.reason && result.reason.message) || '');
}

return acc;
}, []);
return acc;
}, []);

if (rejectReasons.length > 0) {
log.e("User deletion failed\n%j", rejectReasons.join("\n"));
common.returnMessage(params, 500, { errorMessage: "User deletion failed. Failed to delete some data related to this user." });
return;
}
if (rejectReasons.length > 0) {
log.e("User deletion failed\n%j", rejectReasons.join("\n"));
common.returnMessage(params, 500, { errorMessage: "User deletion failed. Failed to delete some data related to this user." });
return;
}

//remove from drill_events
plugins.dispatch("/core/delete_granular_data", {
db: "countly_drill",
collection: "drill_events",
query: { a: app_id + "", uid: { $in: res[0].uid } }
});
//remove from drill_events
plugins.dispatch("/core/delete_granular_data", {
db: "countly_drill",
collection: "drill_events",
query: { a: app_id + "", uid: { $in: res[0].uid } }
});

common.db.collection("app_users" + app_id).remove({uid: {$in: res[0].uid}}, function(err) {
if (res[0].exported) {
//delete exports if exist
for (let i = 0;i < res[0].exported.length; i++) {
let id = res[0].exported[i].split("/");
id = id[id.length - 1]; //last one is filename
id = id.substr(id.length - 7);

deleteMyExport(id).then(
function() {},
function(err5) {
console.log(err5);
}
);
}
}
//deleting userimages(if they exist);
if (res[0].picture) {
for (let i = 0;i < res[0].picture.length; i++) {
//remove /userimages/
let id = res[0].picture[i].substr(12, res[0].picture[i].length - 12);
var pp = path.resolve(__dirname, './../../../frontend/express/public/userimages/' + id);
countlyFs.deleteFile("userimages", pp, {id: id}, function(err6) {
if (err6) {
console.log(err6);
}
});
}
common.db.collection("app_users" + app_id).remove({uid: {$in: res[0].uid}}, function(err) {
if (res[0].exported) {
//delete exports if exist
for (let i = 0;i < res[0].exported.length; i++) {
let id = res[0].exported[i].split("/");
id = id[id.length - 1]; //last one is filename
id = id.substr(id.length - 7);

deleteMyExport(id).then(
function() {},
function(err5) {
console.log(err5);
}
);
}
try {
fs.appendFileSync(path.resolve(__dirname, './../../../log/deletedUsers' + app_id + '.txt'), res[0].uid.join("\n") + "\n", "utf-8");
}
//deleting userimages(if they exist);
if (res[0].picture) {
for (let i = 0;i < res[0].picture.length; i++) {
//remove /userimages/
let id = res[0].picture[i].substr(12, res[0].picture[i].length - 12);
var pp = path.resolve(__dirname, './../../../frontend/express/public/userimages/' + id);
countlyFs.deleteFile("userimages", pp, {id: id}, function(err6) {
if (err6) {
console.log(err6);
}
});
}
catch (err2) {
console.log(err2);
}
try {
fs.appendFileSync(path.resolve(__dirname, './../../../log/deletedUsers' + app_id + '.txt'), res[0].uid.join("\n") + "\n", "utf-8");
}
catch (err2) {
console.log(err2);
}
plugins.dispatch("/systemlogs", {
params: params,
action: "app_user_deleted",
data: {
app_id: app_id,
query: JSON.stringify(query),
uids: res[0].uid,
}
plugins.dispatch("/systemlogs", {
params: params,
action: "app_user_deleted",
data: {
app_id: app_id,
query: JSON.stringify(query),
uids: res[0].uid,
}
});
callback(err, res[0].uid);
});
callback(err, res[0].uid);
});
});
}
Expand Down Expand Up @@ -758,7 +756,6 @@ usersApi.merge = async function(app_id, newAppUser, new_id, old_id, new_device_i
var iid = app_id + "_" + newAppUser.uid + "_" + oldAppUser.uid;
try {
await common.db.collection('app_user_merges').updateOne({"_id": iid, "cc": {"$ne": true}}, {'$set': {"u": true}}, {upsert: false});
await common.db.collection("metric_changes" + app_id).updateMany({uid: oldAppUser.uid}, {'$set': {uid: newAppUserP.uid}});
}
catch (e) {
log.e("Failed metric changes update in app_users merge", e);
Expand Down Expand Up @@ -1144,21 +1141,8 @@ usersApi.export = function(app_id, query, params, callback) {
// }

//try deleting old export
deleteMyExport(export_id).then(function(err) {
if (err) {
log.e(err);
}
deleteMyExport(export_id).then(function() {
log.d("old export deleted");
return new Promise(function(resolve) {
log.d("collection marked");
//export data from metric_changes

export_safely({projection: {"appUserExport": 0}, export_id: export_id, app_id: app_id, args: [...dbargs, "--collection", "metric_changes" + app_id, "-q", '{"uid":{"$in": ["' + res[0].uid.join('","') + '"]}}', "--out", export_folder + "/metric_changes" + app_id + ".json"]}).finally(function() {
resolve();
});
});
}).then(function() {
log.d("metric_changes exported");
//export data from app_users
return export_safely({projection: {"appUserExport": 0}, export_id: export_id, app_id: app_id, args: [...dbargs, "--collection", "app_users" + app_id, "-q", '{"uid":{"$in": ["' + res[0].uid.join('","') + '"]}}', "--out", export_folder + "/app_users" + app_id + ".json"]});

Expand Down
13 changes: 0 additions & 13 deletions api/parts/mgmt/apps.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,8 +289,6 @@ appsApi.createApp = async function(params) {
common.db.collection('app_users' + app.ops[0]._id).ensureIndex({"lac": -1}, { background: true }, function() {});
common.db.collection('app_users' + app.ops[0]._id).ensureIndex({"tsd": 1}, { background: true }, function() {});
common.db.collection('app_users' + app.ops[0]._id).ensureIndex({"did": 1}, { background: true }, function() {});
common.db.collection('metric_changes' + app.ops[0]._id).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, function() {});
common.db.collection('metric_changes' + app.ops[0]._id).ensureIndex({uid: 1}, { background: true }, function() {});
plugins.dispatch("/i/apps/create", {
params: params,
appId: app.ops[0]._id,
Expand Down Expand Up @@ -857,10 +855,6 @@ function deleteAllAppData(appId, fromAppDelete, params, app) {
}
common.db.collection('app_users' + appId).drop(function() {
if (!fromAppDelete) {
common.db.collection('metric_changes' + appId).drop(function() {
common.db.collection('metric_changes' + appId).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, function() {});
common.db.collection('metric_changes' + appId).ensureIndex({uid: 1}, { background: true }, function() {});
});
//Removes old app_user_merges collection
common.db.collection('app_user_merges' + appId).drop(function() {});
if (params.qstring.args.period === "reset") {
Expand All @@ -879,7 +873,6 @@ function deleteAllAppData(appId, fromAppDelete, params, app) {
}
}
else {
common.db.collection('metric_changes' + appId).drop(function() {});
common.db.collection('app_user_merges' + appId).drop(function() {});
plugins.dispatch("/i/apps/delete", {
params: params,
Expand Down Expand Up @@ -986,12 +979,6 @@ function deletePeriodAppData(appId, fromAppDelete, params, app) {
*/
common.db.collection('app_users' + appId).update({ls: {$lte: oldestTimestampWanted}}, {$set: {ls: 1}}, function() {});

/*
Remove all metric changes that happened before oldestTimestampWanted since we no longer need
old metric changes
*/
common.db.collection('metric_changes' + appId).remove({ts: {$lte: oldestTimestampWanted}}, function() {});

plugins.dispatch("/i/apps/clear", {
params: params,
appId: appId,
Expand Down
4 changes: 1 addition & 3 deletions bin/scripts/add_indexes.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,7 @@ pluginManager.dbConnection().then((countlyDb) => {
() => countlyDb.collection('app_users' + app._id).ensureIndex({"lac": -1}, { background: true }, cb),
() => countlyDb.collection('app_users' + app._id).ensureIndex({"tsd": 1}, { background: true }, cb),
() => countlyDb.collection('app_users' + app._id).ensureIndex({"did": 1}, { background: true }, cb),
() => countlyDb.collection('app_users' + app._id).dropIndex("lac_1_ls_1", cb),
() => countlyDb.collection('metric_changes' + app._id).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, cb),
() => countlyDb.collection('metric_changes' + app._id).ensureIndex({uid: 1}, { background: true }, cb)
() => countlyDb.collection('app_users' + app._id).dropIndex("lac_1_ls_1", cb)
];

totalParallelJobs = parallelJobs.length;
Expand Down
6 changes: 0 additions & 6 deletions bin/scripts/data-cleanup/remove_old_data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,6 @@ pluginManager.dbConnection().then(async(db) => {
var count = await db.collection(`consent_history${app._id}`).countDocuments({ ts: { $lt: lastUnixTimestamp } });
console.log("", count, "consents to be deleted");
checkThreshold(count);
count = await db.collection(`metric_changes${app._id}`).countDocuments({ ts: { $lt: lastUnixTimestamp } });
console.log("", count, "metric changes to be deleted");
checkThreshold(count);
count = await db.collection(`eventTimes${app._id}`).countDocuments({ ts: { $lt: lastUnixTimestamp * 1000 } });
console.log("", count, "event times to be deleted");
checkThreshold(count);
Expand All @@ -64,9 +61,6 @@ pluginManager.dbConnection().then(async(db) => {
var res = await db.collection(`consent_history${app._id}`).deleteMany({ ts: { $lt: lastUnixTimestamp } });
console.log("", res, "consents deleted");
await sleep(SLEEP);
res = await db.collection(`metric_changes${app._id}`).deleteMany({ ts: { $lt: lastUnixTimestamp } });
console.log("", res, "metric changes deleted");
await sleep(SLEEP);
res = await db.collection(`eventTimes${app._id}`).deleteMany({ ts: { $lt: lastUnixTimestamp * 1000 } });
console.log("", res, "event times deleted");
await sleep(SLEEP);
Expand Down
1 change: 0 additions & 1 deletion bin/scripts/expire-data/countly_single_app_expireDataBatches.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ var errorCn = 0;
var process = {
drill_events: true,
/*app_crashes:true,
metric_changes:true,
consent_history:true,
feedback:true,
symbolication_jobs:true,
Expand Down
1 change: 0 additions & 1 deletion bin/scripts/expire-data/mongo_expireData.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ var COUNTLY_DRILL = 'countly_drill',
var PROCESS = [
/^drill_events\.*/,
/^app_crashes\.*/,
/^metric_changes\.*/,
/^consent_history\.*/,
/^feedback[^_]*/,
/^symbolication_jobs/,
Expand Down
1 change: 0 additions & 1 deletion bin/scripts/expire-data/old_scripts/countly_single_app_expireDataBatches.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ var errorCn = 0;
var process = {
drill_events: true,
/*app_crashes:true,
metric_changes:true,
consent_history:true,
feedback:true,
symbolication_jobs:true,
Expand Down
15 changes: 1 addition & 14 deletions bin/scripts/fix-data/process_merges.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,6 @@ function mergeUserData(db, user, resolve) {
db.collection('app_users' + app_id).update({_id: newAppUser._id}, {'$set': newAppUser}, function(err6) {
//Dispatch to other plugins only after callback.
if (!err6) {
//update metric changes document
db.collection("metric_changes" + app_id).update({uid: oldAppUser.uid}, {'$set': {uid: newAppUser.uid}}, {multi: true}, function(err7) {
if (err7) {
console.log("Failed metric changes update in app_users merge", err7);
}
});
//delete old app users document
db.collection('app_users' + app_id).remove({_id: oldAppUser._id}, function(errRemoving) {
if (errRemoving) {
Expand All @@ -145,14 +139,7 @@ function mergeUserData(db, user, resolve) {
if (err0) {
console.log(err0);
}
db.collection("metric_changes" + app_id).update({uid: olduid}, {'$set': {uid: usersApi.merged_to}}, {multi: true}, function(err7) {
if (err7) {
console.log("Failed metric changes update in app_users merge", err7);
}
else {
usersApi.mergeOtherPlugins({db: db, app_id: app_id, newAppUser: {uid: user.merged_to}, oldAppUser: {uid: olduid}, updateFields: {"cc": true, "mc": true}, mergeDoc: user}, resolve);
}
});
usersApi.mergeOtherPlugins({db: db, app_id: app_id, newAppUser: {uid: user.merged_to}, oldAppUser: {uid: olduid}, updateFields: {"cc": true, "mc": true}, mergeDoc: user}, resolve);
});
}
else {
Expand Down
1 change: 0 additions & 1 deletion jobServer/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,6 @@ if (require.main === module) {
// total_users: true,
// export_limit: 10000,
prevent_duplicate_requests: true,
// metric_changes: true,
offline_mode: false,
// reports_regenerate_interval: 3600,
// send_test_email: "",
Expand Down
Loading
Loading