Skip to content

Commit 244afea

Browse files
msukkarimsukkari
committed
add back names in logs for repo and connection sync
1 parent 1ab22e3 commit 244afea

File tree

3 files changed

+70
-47
lines changed

3 files changed

+70
-47
lines changed

‎packages/backend/src/connectionManager.ts

Lines changed: 36 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const QUEUE_NAME = 'connectionSyncQueue';
2020

2121
type JobPayload = {
2222
connectionId: number,
23+
connectionName: string,
2324
orgId: number,
2425
config: ConnectionConfig,
2526
};
@@ -60,12 +61,13 @@ export class ConnectionManager implements IConnectionManager {
6061

6162
await this.queue.add('connectionSyncJob', {
6263
connectionId: connection.id,
64+
connectionName: connection.name,
6365
orgId: connection.orgId,
6466
config: connectionConfig,
6567
});
66-
this.logger.info(`Added job to queue for connection ${connection.id}`);
68+
this.logger.info(`Added job to queue for connection ${connection.name} (id: ${connection.id})`);
6769
}).catch((err: unknown) => {
68-
this.logger.error(`Failed to add job to queue for connection ${connection.id}: ${err}`);
70+
this.logger.error(`Failed to add job to queue for connection ${connection.name} (id: ${connection.id}): ${err}`);
6971
});
7072
}
7173

@@ -83,14 +85,18 @@ export class ConnectionManager implements IConnectionManager {
8385
// (or if the date isn't set for some reason).
8486
{
8587
AND: [
86-
{ OR: [
87-
{ syncStatus: ConnectionSyncStatus.SYNCED },
88-
{ syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS },
89-
]},
90-
{ OR: [
91-
{ syncedAt: null },
92-
{ syncedAt: { lt: thresholdDate } },
93-
]}
88+
{
89+
OR: [
90+
{ syncStatus: ConnectionSyncStatus.SYNCED },
91+
{ syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS },
92+
]
93+
},
94+
{
95+
OR: [
96+
{ syncedAt: null },
97+
{ syncedAt: { lt: thresholdDate } },
98+
]
99+
}
94100
]
95101
}
96102
]
@@ -103,7 +109,7 @@ export class ConnectionManager implements IConnectionManager {
103109
}
104110

105111
private async runSyncJob(job: Job<JobPayload>): Promise<JobResult> {
106-
const { config, orgId } = job.data;
112+
const { config, orgId, connectionName } = job.data;
107113
// @note: We aren't actually doing anything with this atm.
108114
const abortController = new AbortController();
109115

@@ -120,7 +126,7 @@ export class ConnectionManager implements IConnectionManager {
120126
Sentry.captureException(e);
121127
throw e;
122128
}
123-
129+
124130
// Reset the syncStatusMetadata to an empty object at the start of the sync job
125131
await this.db.connection.update({
126132
where: {
@@ -131,7 +137,7 @@ export class ConnectionManager implements IConnectionManager {
131137
syncStatusMetadata: {}
132138
}
133139
})
134-
140+
135141

136142
let result: {
137143
repoData: RepoData[],
@@ -167,7 +173,7 @@ export class ConnectionManager implements IConnectionManager {
167173
}
168174
})();
169175
} catch (err) {
170-
this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId}: ${err}`);
176+
this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId} (${connectionName}): ${err}`);
171177
Sentry.captureException(err);
172178

173179
if (err instanceof BackendException) {
@@ -191,7 +197,7 @@ export class ConnectionManager implements IConnectionManager {
191197
syncStatusMetadata: { notFound }
192198
}
193199
});
194-
200+
195201
// Filter out any duplicates by external_id and external_codeHostUrl.
196202
repoData = repoData.filter((repo, index, self) => {
197203
return index === self.findIndex(r =>
@@ -218,7 +224,7 @@ export class ConnectionManager implements IConnectionManager {
218224
}
219225
});
220226
const deleteDuration = performance.now() - deleteStart;
221-
this.logger.info(`Deleted all RepoToConnection records for connection ${job.data.connectionId} in ${deleteDuration}ms`);
227+
this.logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`);
222228

223229
const totalUpsertStart = performance.now();
224230
for (const repo of repoData) {
@@ -235,10 +241,10 @@ export class ConnectionManager implements IConnectionManager {
235241
create: repo,
236242
})
237243
const upsertDuration = performance.now() - upsertStart;
238-
this.logger.info(`Upserted repo ${repo.external_id} in ${upsertDuration}ms`);
244+
this.logger.info(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`);
239245
}
240246
const totalUpsertDuration = performance.now() - totalUpsertStart;
241-
this.logger.info(`Upserted ${repoData.length} repos in ${totalUpsertDuration}ms`);
247+
this.logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`);
242248
}, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS });
243249

244250
return {
@@ -248,18 +254,20 @@ export class ConnectionManager implements IConnectionManager {
248254

249255

250256
private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) {
251-
this.logger.info(`Connection sync job ${job.id} completed`);
257+
this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`);
252258
const { connectionId } = job.data;
253259

254260
let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
255261
where: { id: connectionId },
256262
select: { syncStatusMetadata: true }
257263
}))?.syncStatusMetadata as Record<string, unknown> ?? {};
258-
const { notFound } = syncStatusMetadata as { notFound: {
259-
users: string[],
260-
orgs: string[],
261-
repos: string[],
262-
}};
264+
const { notFound } = syncStatusMetadata as {
265+
notFound: {
266+
users: string[],
267+
orgs: string[],
268+
repos: string[],
269+
}
270+
};
263271

264272
await this.db.connection.update({
265273
where: {
@@ -268,8 +276,8 @@ export class ConnectionManager implements IConnectionManager {
268276
data: {
269277
syncStatus:
270278
notFound.users.length > 0 ||
271-
notFound.orgs.length > 0 ||
272-
notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
279+
notFound.orgs.length > 0 ||
280+
notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
273281
syncedAt: new Date()
274282
}
275283
})
@@ -281,7 +289,7 @@ export class ConnectionManager implements IConnectionManager {
281289
}
282290

283291
private async onSyncJobFailed(job: Job<JobPayload> | undefined, err: unknown) {
284-
this.logger.info(`Connection sync job failed with error: ${err}`);
292+
this.logger.info(`Connection sync job for connection ${job?.data.connectionName} (id: ${job?.data.connectionId}, jobId: ${job?.id}) failed with error: ${err}`);
285293
Sentry.captureException(err, {
286294
tags: {
287295
connectionid: job?.data.connectionId,
@@ -312,7 +320,7 @@ export class ConnectionManager implements IConnectionManager {
312320
}
313321
} else {
314322
syncStatusMetadata = {
315-
...syncStatusMetadata,
323+
...syncStatusMetadata,
316324
error: 'UNKNOWN',
317325
}
318326
}

‎packages/backend/src/repoCompileUtils.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@ import { getGerritReposFromConfig } from "./gerrit.js";
66
import { Prisma, PrismaClient } from '@sourcebot/db';
77
import { WithRequired } from "./types.js"
88
import { marshalBool } from "./utils.js";
9+
import { createLogger } from './logger.js';
910
import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
1011
import { RepoMetadata } from './types.js';
1112
import path from 'path';
1213

1314
export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
1415

16+
const logger = createLogger('RepoCompileUtils');
17+
1518
export const compileGithubConfig = async (
1619
config: GithubConnectionConfig,
1720
connectionId: number,
@@ -39,6 +42,8 @@ export const compileGithubConfig = async (
3942
const repoName = path.join(repoNameRoot, repoDisplayName);
4043
const cloneUrl = new URL(repo.clone_url!);
4144

45+
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
46+
4247
const record: RepoData = {
4348
external_id: repo.id.toString(),
4449
external_codeHostType: 'github',
@@ -110,6 +115,8 @@ export const compileGitlabConfig = async (
110115
const repoDisplayName = project.path_with_namespace;
111116
const repoName = path.join(repoNameRoot, repoDisplayName);
112117

118+
logger.debug(`Found gitlab repo ${repoDisplayName} with webUrl: ${projectUrl}`);
119+
113120
const record: RepoData = {
114121
external_id: project.id.toString(),
115122
external_codeHostType: 'gitlab',
@@ -177,6 +184,8 @@ export const compileGiteaConfig = async (
177184
const repoDisplayName = repo.full_name!;
178185
const repoName = path.join(repoNameRoot, repoDisplayName);
179186

187+
logger.debug(`Found gitea repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
188+
180189
const record: RepoData = {
181190
external_id: repo.id!.toString(),
182191
external_codeHostType: 'gitea',
@@ -246,11 +255,15 @@ export const compileGerritConfig = async (
246255
const webLink = project.web_links[0];
247256
const webUrl = webLink.url;
248257

258+
logger.debug(`Found gerrit repo ${project.name} with webUrl: ${webUrl}`);
259+
249260
// Handle case where webUrl is just a gitiles path
250261
// https://github.com/GerritCodeReview/plugins_gitiles/blob/5ee7f57/src/main/java/com/googlesource/gerrit/plugins/gitiles/GitilesWeblinks.java#L50
251262
if (webUrl.startsWith('/plugins/gitiles/')) {
263+
logger.debug(`WebUrl is a gitiles path, joining with hostUrl: ${webUrl}`);
252264
return path.join(hostUrl, webUrl);
253265
} else {
266+
logger.debug(`WebUrl is not a gitiles path, returning as is: ${webUrl}`);
254267
return webUrl;
255268
}
256269
})();

‎packages/backend/src/repoManager.ts

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -140,10 +140,12 @@ export class RepoManager implements IRepoManager {
140140
{
141141
AND: [
142142
{ repoIndexingStatus: RepoIndexingStatus.INDEXED },
143-
{ OR: [
144-
{ indexedAt: null },
145-
{ indexedAt: { lt: thresholdDate } },
146-
]}
143+
{
144+
OR: [
145+
{ indexedAt: null },
146+
{ indexedAt: { lt: thresholdDate } },
147+
]
148+
}
147149
]
148150
}
149151
]
@@ -201,27 +203,27 @@ export class RepoManager implements IRepoManager {
201203

202204
const repoPath = getRepoPath(repo, this.ctx);
203205
const metadata = repoMetadataSchema.parse(repo.metadata);
204-
206+
205207
// If the repo was already in the indexing state, this job was likely killed and picked up again. As a result,
206208
// to ensure the repo state is valid, we delete the repo if it exists so we get a fresh clone
207209
if (repoAlreadyInIndexingState && existsSync(repoPath)) {
208210
this.logger.info(`Deleting repo directory ${repoPath} during sync because it was already in the indexing state`);
209-
await promises.rm(repoPath, { recursive: true, force: true });
211+
await promises.rm(repoPath, { recursive: true, force: true });
210212
}
211213

212214
if (existsSync(repoPath)) {
213-
this.logger.info(`Fetching ${repo.id}...`);
215+
this.logger.info(`Fetching ${repo.displayName}...`);
214216

215217
const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
216-
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
218+
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`)
217219
}));
218220
fetchDuration_s = durationMs / 1000;
219221

220222
process.stdout.write('\n');
221-
this.logger.info(`Fetched ${repo.name} in ${fetchDuration_s}s`);
223+
this.logger.info(`Fetched ${repo.displayName} in ${fetchDuration_s}s`);
222224

223225
} else {
224-
this.logger.info(`Cloning ${repo.id}...`);
226+
this.logger.info(`Cloning ${repo.displayName}...`);
225227

226228
const token = await this.getTokenForRepo(repo, this.db);
227229
const cloneUrl = new URL(repo.cloneUrl);
@@ -240,12 +242,12 @@ export class RepoManager implements IRepoManager {
240242
}
241243

242244
const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, ({ method, stage, progress }) => {
243-
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
245+
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`)
244246
}));
245247
cloneDuration_s = durationMs / 1000;
246248

247249
process.stdout.write('\n');
248-
this.logger.info(`Cloned ${repo.id} in ${cloneDuration_s}s`);
250+
this.logger.info(`Cloned ${repo.displayName} in ${cloneDuration_s}s`);
249251
}
250252

251253
// Regardless of clone or fetch, always upsert the git config for the repo.
@@ -255,10 +257,10 @@ export class RepoManager implements IRepoManager {
255257
await upsertGitConfig(repoPath, metadata.gitConfig);
256258
}
257259

258-
this.logger.info(`Indexing ${repo.id}...`);
260+
this.logger.info(`Indexing ${repo.displayName}...`);
259261
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
260262
const indexDuration_s = durationMs / 1000;
261-
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);
263+
this.logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`);
262264

263265
return {
264266
fetchDuration_s,
@@ -268,7 +270,7 @@ export class RepoManager implements IRepoManager {
268270
}
269271

270272
private async runIndexJob(job: Job<RepoIndexingPayload>) {
271-
this.logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.id}`);
273+
this.logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.displayName}`);
272274
const repo = job.data.repo as RepoWithConnections;
273275

274276
// We have to use the existing repo object to get the repoIndexingStatus because the repo object
@@ -332,7 +334,7 @@ export class RepoManager implements IRepoManager {
332334
}
333335

334336
private async onIndexJobCompleted(job: Job<RepoIndexingPayload>) {
335-
this.logger.info(`Repo index job ${job.id} completed`);
337+
this.logger.info(`Repo index job for repo ${job.data.repo.displayName} (id: ${job.data.repo.id}, jobId: ${job.id}) completed`);
336338
this.promClient.activeRepoIndexingJobs.dec();
337339
this.promClient.repoIndexingSuccessTotal.inc();
338340

@@ -348,7 +350,7 @@ export class RepoManager implements IRepoManager {
348350
}
349351

350352
private async onIndexJobFailed(job: Job<RepoIndexingPayload> | undefined, err: unknown) {
351-
this.logger.info(`Repo index job failed (id: ${job?.id ?? 'unknown'}) with error: ${err}`);
353+
this.logger.info(`Repo index job for repo ${job?.data.repo.displayName} (id: ${job?.data.repo.id}, jobId: ${job?.id}) failed with error: ${err}`);
352354
Sentry.captureException(err, {
353355
tags: {
354356
repoId: job?.data.repo.id,
@@ -468,7 +470,7 @@ export class RepoManager implements IRepoManager {
468470
const repoPath = getRepoPath(repo, this.ctx);
469471
if (existsSync(repoPath)) {
470472
this.logger.info(`Deleting repo directory ${repoPath}`);
471-
await promises.rm(repoPath, { recursive: true, force: true });
473+
await promises.rm(repoPath, { recursive: true, force: true });
472474
}
473475

474476
// delete shards
@@ -542,7 +544,7 @@ export class RepoManager implements IRepoManager {
542544
});
543545
});
544546
}
545-
547+
546548
public async dispose() {
547549
this.indexWorker.close();
548550
this.indexQueue.close();

0 commit comments

Comments
 (0)