18
18
*/
19
19
20
20
/*
21
- * Copyright (c) 2018, 2022 , Oracle and/or its affiliates. All rights reserved.
21
+ * Copyright (c) 2018, 2023 , Oracle and/or its affiliates. All rights reserved.
22
22
*/
23
23
package org .opengrok .suggest ;
24
24
34
34
import org .apache .lucene .util .BytesRef ;
35
35
import org .opengrok .suggest .query .SuggesterPrefixQuery ;
36
36
import org .opengrok .suggest .query .SuggesterQuery ;
37
+ import org .opengrok .suggest .util .Progress ;
37
38
38
39
import java .io .Closeable ;
39
40
import java .io .File ;
@@ -93,6 +94,7 @@ public final class Suggester implements Closeable {
93
94
private final int timeThreshold ;
94
95
95
96
private final int rebuildParallelismLevel ;
97
+ private final boolean isPrintProgress ;
96
98
97
99
private volatile boolean rebuilding ;
98
100
private volatile boolean terminating ;
@@ -123,7 +125,9 @@ public final class Suggester implements Closeable {
123
125
* @param allowedFields fields for which should the suggester be enabled,
124
126
* if {@code null} then enabled for all fields
125
127
* @param timeThreshold time in milliseconds after which the suggestions requests should time out
128
+ * @param rebuildParallelismLevel parallelism level for rebuild
126
129
* @param registry meter registry
130
+ * @param isPrintProgress whether to report progress for initialization and rebuild
127
131
*/
128
132
public Suggester (
129
133
final File suggesterDir ,
@@ -134,7 +138,8 @@ public Suggester(
134
138
final Set <String > allowedFields ,
135
139
final int timeThreshold ,
136
140
final int rebuildParallelismLevel ,
137
- MeterRegistry registry ) {
141
+ MeterRegistry registry ,
142
+ boolean isPrintProgress ) {
138
143
if (suggesterDir == null ) {
139
144
throw new IllegalArgumentException ("Suggester needs to have directory specified" );
140
145
}
@@ -152,6 +157,7 @@ public Suggester(
152
157
this .allowedFields = new HashSet <>(allowedFields );
153
158
this .timeThreshold = timeThreshold ;
154
159
this .rebuildParallelismLevel = rebuildParallelismLevel ;
160
+ this .isPrintProgress = isPrintProgress ;
155
161
156
162
suggesterRebuildTimer = Timer .builder ("suggester.rebuild.latency" ).
157
163
description ("suggester rebuild latency" ).
@@ -180,17 +186,20 @@ public void init(final Collection<NamedIndexDir> luceneIndexes) {
180
186
181
187
ExecutorService executor = Executors .newWorkStealingPool (rebuildParallelismLevel );
182
188
183
- for (NamedIndexDir indexDir : luceneIndexes ) {
184
- if (terminating ) {
185
- LOGGER .log (Level .INFO , "Terminating suggester initialization" );
186
- return ;
189
+ try (Progress progress = new Progress (LOGGER , "suggester initialization" , luceneIndexes .size (),
190
+ Level .INFO , isPrintProgress )) {
191
+ for (NamedIndexDir indexDir : luceneIndexes ) {
192
+ if (terminating ) {
193
+ LOGGER .log (Level .INFO , "Terminating suggester initialization" );
194
+ return ;
195
+ }
196
+ submitInitIfIndexExists (executor , indexDir , progress );
187
197
}
188
- submitInitIfIndexExists (executor , indexDir );
189
- }
190
198
191
- shutdownAndAwaitTermination (executor , start , suggesterInitTimer ,
192
- "Suggester successfully initialized" );
193
- initDone .countDown ();
199
+ shutdownAndAwaitTermination (executor , start , suggesterInitTimer ,
200
+ "Suggester successfully initialized" );
201
+ initDone .countDown ();
202
+ }
194
203
}
195
204
}
196
205
@@ -206,10 +215,11 @@ public void waitForInit(long timeout, TimeUnit unit) throws InterruptedException
206
215
}
207
216
}
208
217
209
- private void submitInitIfIndexExists (final ExecutorService executorService , final NamedIndexDir indexDir ) {
218
+ private void submitInitIfIndexExists (final ExecutorService executorService , final NamedIndexDir indexDir ,
219
+ Progress progress ) {
210
220
try {
211
221
if (indexExists (indexDir .path )) {
212
- executorService .submit (getInitRunnable (indexDir ));
222
+ executorService .submit (getInitRunnable (indexDir , progress ));
213
223
} else {
214
224
LOGGER .log (Level .FINE , "Index in {0} directory does not exist, skipping..." , indexDir );
215
225
}
@@ -218,7 +228,7 @@ private void submitInitIfIndexExists(final ExecutorService executorService, fina
218
228
}
219
229
}
220
230
221
- private Runnable getInitRunnable (final NamedIndexDir indexDir ) {
231
+ private Runnable getInitRunnable (final NamedIndexDir indexDir , Progress progress ) {
222
232
return () -> {
223
233
try {
224
234
if (terminating ) {
@@ -239,6 +249,7 @@ private Runnable getInitRunnable(final NamedIndexDir indexDir) {
239
249
240
250
Duration d = Duration .between (start , Instant .now ());
241
251
LOGGER .log (Level .FINE , "Finished initialization of {0}, took {1}" , new Object [] {indexDir , d });
252
+ progress .increment ();
242
253
} catch (Exception e ) {
243
254
LOGGER .log (Level .SEVERE , String .format ("Could not initialize suggester data for %s" , indexDir ), e );
244
255
}
@@ -300,17 +311,20 @@ public void rebuild(final Collection<NamedIndexDir> indexDirs) {
300
311
301
312
ExecutorService executor = Executors .newWorkStealingPool (rebuildParallelismLevel );
302
313
303
- for (NamedIndexDir indexDir : indexDirs ) {
304
- SuggesterProjectData data = this .projectData .get (indexDir .name );
305
- if (data != null ) {
306
- executor .submit (getRebuildRunnable (data ));
307
- } else {
308
- submitInitIfIndexExists (executor , indexDir );
314
+ try (Progress progress = new Progress (LOGGER , "suggester rebuild" , indexDirs .size (),
315
+ Level .INFO , isPrintProgress )) {
316
+ for (NamedIndexDir indexDir : indexDirs ) {
317
+ SuggesterProjectData data = this .projectData .get (indexDir .name );
318
+ if (data != null ) {
319
+ executor .submit (getRebuildRunnable (data , progress ));
320
+ } else {
321
+ submitInitIfIndexExists (executor , indexDir , progress );
322
+ }
309
323
}
310
- }
311
324
312
- shutdownAndAwaitTermination (executor , start , suggesterRebuildTimer ,
313
- "Suggesters for " + indexDirs + " were successfully rebuilt" );
325
+ shutdownAndAwaitTermination (executor , start , suggesterRebuildTimer ,
326
+ "Suggesters for " + indexDirs + " were successfully rebuilt" );
327
+ }
314
328
}
315
329
316
330
rebuildLock .lock ();
@@ -341,7 +355,7 @@ public void waitForRebuild(long timeout, TimeUnit unit) throws InterruptedExcept
341
355
}
342
356
}
343
357
344
- private Runnable getRebuildRunnable (final SuggesterProjectData data ) {
358
+ private Runnable getRebuildRunnable (final SuggesterProjectData data , Progress progress ) {
345
359
return () -> {
346
360
try {
347
361
if (terminating ) {
@@ -354,6 +368,7 @@ private Runnable getRebuildRunnable(final SuggesterProjectData data) {
354
368
355
369
Duration d = Duration .between (start , Instant .now ());
356
370
LOGGER .log (Level .FINE , "Rebuild of {0} finished, took {1}" , new Object [] {data , d });
371
+ progress .increment ();
357
372
} catch (Exception e ) {
358
373
LOGGER .log (Level .SEVERE , "Could not rebuild suggester" , e );
359
374
}
0 commit comments