Skip to content
This repository was archived by the owner on Sep 30, 2024. It is now read-only.

Commit 014cad1

Browse files
eseligereseliger
authored
Move phabricator syncer into repo-updater (#59789)
1 parent ab8746e commit 014cad1

10 files changed

Lines changed: 155 additions & 111 deletions

File tree

‎cmd/repo-updater/internal/phabricator/BUILD.bazel‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library")
2+
3+
go_library(
4+
name = "phabricator",
5+
srcs = ["phabricator.go"],
6+
importpath = "github.com/sourcegraph/sourcegraph/cmd/repo-updater/internal/phabricator",
7+
visibility = ["//cmd/repo-updater:__subpackages__"],
8+
deps = [
9+
"//internal/actor",
10+
"//internal/conf",
11+
"//internal/database",
12+
"//internal/extsvc",
13+
"//internal/extsvc/phabricator",
14+
"//internal/goroutine",
15+
"//internal/httpcli",
16+
"//internal/repos",
17+
"//internal/types",
18+
"//lib/errors",
19+
"//schema",
20+
"@com_github_prometheus_client_golang//prometheus",
21+
"@com_github_prometheus_client_golang//prometheus/promauto",
22+
"@com_github_sourcegraph_log//:log",
23+
],
24+
)

‎cmd/repo-updater/internal/phabricator/phabricator.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package phabricator
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"github.com/prometheus/client_golang/prometheus"
8+
"github.com/prometheus/client_golang/prometheus/promauto"
9+
"github.com/sourcegraph/log"
10+
11+
"github.com/sourcegraph/sourcegraph/internal/actor"
12+
"github.com/sourcegraph/sourcegraph/internal/conf"
13+
"github.com/sourcegraph/sourcegraph/internal/database"
14+
"github.com/sourcegraph/sourcegraph/internal/extsvc"
15+
"github.com/sourcegraph/sourcegraph/internal/extsvc/phabricator"
16+
"github.com/sourcegraph/sourcegraph/internal/goroutine"
17+
"github.com/sourcegraph/sourcegraph/internal/httpcli"
18+
"github.com/sourcegraph/sourcegraph/internal/repos"
19+
"github.com/sourcegraph/sourcegraph/internal/types"
20+
"github.com/sourcegraph/sourcegraph/lib/errors"
21+
"github.com/sourcegraph/sourcegraph/schema"
22+
)
23+
24+
const (
25+
tagID = "id"
26+
)
27+
28+
var (
29+
phabricatorUpdateTime = promauto.NewGaugeVec(prometheus.GaugeOpts{
30+
Name: "src_repoupdater_time_last_phabricator_sync",
31+
Help: "The last time a comprehensive Phabricator sync finished",
32+
}, []string{tagID})
33+
)
34+
35+
// NewRepositorySyncWorker runs the worker that syncs repositories from Phabricator to Sourcegraph.
36+
func NewRepositorySyncWorker(ctx context.Context, db database.DB, logger log.Logger, s repos.Store) goroutine.BackgroundRoutine {
37+
cf := httpcli.NewExternalClientFactory(
38+
httpcli.NewLoggingMiddleware(logger),
39+
)
40+
41+
return goroutine.NewPeriodicGoroutine(
42+
actor.WithInternalActor(ctx),
43+
goroutine.HandlerFunc(func(ctx context.Context) error {
44+
phabs, err := s.ExternalServiceStore().List(ctx, database.ExternalServicesListOptions{
45+
Kinds: []string{extsvc.KindPhabricator},
46+
})
47+
if err != nil {
48+
return errors.Wrap(err, "unable to fetch Phabricator connections")
49+
}
50+
51+
var errs error
52+
53+
for _, phab := range phabs {
54+
src, err := repos.NewPhabricatorSource(ctx, logger, phab, cf)
55+
if err != nil {
56+
errs = errors.Append(errs, errors.Wrap(err, "failed to instantiate PhabricatorSource"))
57+
continue
58+
}
59+
60+
repos, err := repos.ListAll(ctx, src)
61+
if err != nil {
62+
errs = errors.Append(errs, errors.Wrap(err, "error fetching Phabricator repos"))
63+
continue
64+
}
65+
66+
err = updatePhabRepos(ctx, db, repos)
67+
if err != nil {
68+
errs = errors.Append(errs, errors.Wrap(err, "error updating Phabricator repos"))
69+
continue
70+
}
71+
72+
cfg, err := phab.Configuration(ctx)
73+
if err != nil {
74+
errs = errors.Append(errs, errors.Wrap(err, "failed to parse Phabricator config"))
75+
continue
76+
}
77+
78+
phabricatorUpdateTime.WithLabelValues(
79+
cfg.(*schema.PhabricatorConnection).Url,
80+
).Set(float64(time.Now().Unix()))
81+
}
82+
83+
return errs
84+
}),
85+
goroutine.WithName("repo-updater.phabricator-repository-syncer"),
86+
goroutine.WithDescription("periodically syncs repositories from Phabricator to Sourcegraph"),
87+
goroutine.WithIntervalFunc(func() time.Duration {
88+
return conf.RepoListUpdateInterval()
89+
}),
90+
)
91+
}
92+
93+
// updatePhabRepos ensures that all provided repositories exist in the phabricator_repos table.
94+
func updatePhabRepos(ctx context.Context, db database.DB, repos []*types.Repo) error {
95+
for _, r := range repos {
96+
repo := r.Metadata.(*phabricator.Repo)
97+
_, err := db.Phabricator().CreateOrUpdate(ctx, repo.Callsign, r.Name, r.ExternalRepo.ServiceID)
98+
if err != nil {
99+
return err
100+
}
101+
}
102+
return nil
103+
}

‎cmd/repo-updater/shared/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ go_library(
1414
deps = [
1515
"//cmd/frontend/envvar",
1616
"//cmd/frontend/globals",
17+
"//cmd/repo-updater/internal/phabricator",
1718
"//cmd/repo-updater/internal/repoupdater",
1819
"//cmd/repo-updater/internal/scheduler",
1920
"//internal/actor",

‎cmd/repo-updater/shared/main.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323

2424
"github.com/sourcegraph/sourcegraph/cmd/frontend/envvar"
2525
"github.com/sourcegraph/sourcegraph/cmd/frontend/globals"
26+
"github.com/sourcegraph/sourcegraph/cmd/repo-updater/internal/phabricator"
2627
"github.com/sourcegraph/sourcegraph/cmd/repo-updater/internal/repoupdater"
2728
"github.com/sourcegraph/sourcegraph/cmd/repo-updater/internal/scheduler"
2829
"github.com/sourcegraph/sourcegraph/internal/actor"
@@ -133,16 +134,16 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
133134
routines := []goroutine.BackgroundRoutine{
134135
makeHTTPServer(logger, server),
135136
newUnclonedReposManager(ctx, logger, envvar.SourcegraphDotComMode(), updateScheduler, store),
136-
repos.NewPhabricatorRepositorySyncWorker(ctx, db, log.Scoped("PhabricatorRepositorySyncWorker"), store),
137+
phabricator.NewRepositorySyncWorker(ctx, db, log.Scoped("PhabricatorRepositorySyncWorker"), store),
137138
// Run git fetches scheduler
138139
updateScheduler,
139140
}
140141

141142
routines = append(routines,
142143
syncer.Routines(ctx, store, repos.RunOptions{
143-
EnqueueInterval: repos.ConfRepoListUpdateInterval,
144+
EnqueueInterval: conf.RepoListUpdateInterval,
144145
IsDotCom: envvar.SourcegraphDotComMode(),
145-
MinSyncInterval: repos.ConfRepoListUpdateInterval,
146+
MinSyncInterval: conf.RepoListUpdateInterval,
146147
})...,
147148
)
148149

‎internal/conf/computed.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1233,3 +1233,25 @@ func fireworksDefaultMaxPromptTokens(model string) int {
12331233

12341234
return 4_000
12351235
}
1236+
1237+
// RepoListUpdateInterval returns the repository list update interval.
1238+
//
1239+
// If the RepoListUpdateInterval site configuration setting is 0, it defaults to 1 minute.
1240+
func RepoListUpdateInterval() time.Duration {
1241+
v := Get().RepoListUpdateInterval
1242+
if v == 0 { // default to 1 minute
1243+
v = 1
1244+
}
1245+
return time.Duration(v) * time.Minute
1246+
}
1247+
1248+
// RepoConcurrentExternalServiceSyncers returns the number of concurrent external service syncers.
1249+
//
1250+
// If the RepoConcurrentExternalServiceSyncers site configuration setting is 0, it defaults to 3.
1251+
func RepoConcurrentExternalServiceSyncers() int {
1252+
v := Get().RepoConcurrentExternalServiceSyncers
1253+
if v <= 0 {
1254+
return 3
1255+
}
1256+
return v
1257+
}

‎internal/repos/BUILD.bazel‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ go_library(
88
"azuredevops.go",
99
"bitbucketcloud.go",
1010
"bitbucketserver.go",
11-
"conf.go",
1211
"discoverable_sources.go",
1312
"doc.go",
1413
"exclude.go",

‎internal/repos/conf.go‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎internal/repos/metrics.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,12 @@ import (
88
const (
99
tagFamily = "family"
1010
tagOwner = "owner"
11-
tagID = "id"
1211
tagSuccess = "success"
1312
tagState = "state"
1413
tagReason = "reason"
1514
)
1615

1716
var (
18-
phabricatorUpdateTime = promauto.NewGaugeVec(prometheus.GaugeOpts{
19-
Name: "src_repoupdater_time_last_phabricator_sync",
20-
Help: "The last time a comprehensive Phabricator sync finished",
21-
}, []string{tagID})
22-
2317
lastSync = promauto.NewGaugeVec(prometheus.GaugeOpts{
2418
Name: "src_repoupdater_syncer_sync_last_time",
2519
Help: "The last time a sync finished",

‎internal/repos/phabricator.go‎

Lines changed: 0 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,14 @@ package repos
33
import (
44
"context"
55
"sync"
6-
"time"
76

87
"github.com/goware/urlx"
98

109
"github.com/sourcegraph/log"
1110

12-
"github.com/sourcegraph/sourcegraph/internal/actor"
1311
"github.com/sourcegraph/sourcegraph/internal/api"
14-
"github.com/sourcegraph/sourcegraph/internal/database"
1512
"github.com/sourcegraph/sourcegraph/internal/extsvc"
1613
"github.com/sourcegraph/sourcegraph/internal/extsvc/phabricator"
17-
"github.com/sourcegraph/sourcegraph/internal/goroutine"
1814
"github.com/sourcegraph/sourcegraph/internal/httpcli"
1915
"github.com/sourcegraph/sourcegraph/internal/jsonc"
2016
"github.com/sourcegraph/sourcegraph/internal/types"
@@ -193,73 +189,3 @@ func (s *PhabricatorSource) client(ctx context.Context) (*phabricator.Client, er
193189
s.cli, err = phabricator.NewClient(ctx, s.conn.Url, s.conn.Token, hc)
194190
return s.cli, err
195191
}
196-
197-
// NewPhabricatorRepositorySyncWorker runs the worker that syncs repositories from Phabricator to Sourcegraph.
198-
func NewPhabricatorRepositorySyncWorker(ctx context.Context, db database.DB, logger log.Logger, s Store) goroutine.BackgroundRoutine {
199-
cf := httpcli.NewExternalClientFactory(
200-
httpcli.NewLoggingMiddleware(logger),
201-
)
202-
203-
return goroutine.NewPeriodicGoroutine(
204-
actor.WithInternalActor(ctx),
205-
goroutine.HandlerFunc(func(ctx context.Context) error {
206-
phabs, err := s.ExternalServiceStore().List(ctx, database.ExternalServicesListOptions{
207-
Kinds: []string{extsvc.KindPhabricator},
208-
})
209-
if err != nil {
210-
return errors.Wrap(err, "unable to fetch Phabricator connections")
211-
}
212-
213-
var errs error
214-
215-
for _, phab := range phabs {
216-
src, err := NewPhabricatorSource(ctx, logger, phab, cf)
217-
if err != nil {
218-
errs = errors.Append(errs, errors.Wrap(err, "failed to instantiate PhabricatorSource"))
219-
continue
220-
}
221-
222-
repos, err := ListAll(ctx, src)
223-
if err != nil {
224-
errs = errors.Append(errs, errors.Wrap(err, "error fetching Phabricator repos"))
225-
continue
226-
}
227-
228-
err = updatePhabRepos(ctx, db, repos)
229-
if err != nil {
230-
errs = errors.Append(errs, errors.Wrap(err, "error updating Phabricator repos"))
231-
continue
232-
}
233-
234-
cfg, err := phab.Configuration(ctx)
235-
if err != nil {
236-
errs = errors.Append(errs, errors.Wrap(err, "failed to parse Phabricator config"))
237-
continue
238-
}
239-
240-
phabricatorUpdateTime.WithLabelValues(
241-
cfg.(*schema.PhabricatorConnection).Url,
242-
).Set(float64(time.Now().Unix()))
243-
}
244-
245-
return errs
246-
}),
247-
goroutine.WithName("repo-updater.phabricator-repository-syncer"),
248-
goroutine.WithDescription("periodically syncs repositories from Phabricator to Sourcegraph"),
249-
goroutine.WithIntervalFunc(func() time.Duration {
250-
return ConfRepoListUpdateInterval()
251-
}),
252-
)
253-
}
254-
255-
// updatePhabRepos ensures that all provided repositories exist in the phabricator_repos table.
256-
func updatePhabRepos(ctx context.Context, db database.DB, repos []*types.Repo) error {
257-
for _, r := range repos {
258-
repo := r.Metadata.(*phabricator.Repo)
259-
_, err := db.Phabricator().CreateOrUpdate(ctx, repo.Callsign, r.Name, r.ExternalRepo.ServiceID)
260-
if err != nil {
261-
return err
262-
}
263-
}
264-
return nil
265-
}

‎internal/repos/syncer.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ func (s *Syncer) Routines(ctx context.Context, store Store, opts RunOptions) []g
9090
minSyncInterval: opts.MinSyncInterval,
9191
}, SyncWorkerOptions{
9292
WorkerInterval: opts.DequeueInterval,
93-
NumHandlers: ConfRepoConcurrentExternalServiceSyncers(),
93+
NumHandlers: conf.RepoConcurrentExternalServiceSyncers(),
9494
CleanupOldJobs: true,
9595
},
9696
)

0 commit comments

Comments
 (0)