diff --git a/backend/app/AppComponents.scala b/backend/app/AppComponents.scala index 9d4df49b..62d2af23 100644 --- a/backend/app/AppComponents.scala +++ b/backend/app/AppComponents.scala @@ -176,7 +176,7 @@ class AppComponents(context: Context, config: Config) val workspacesController = new Workspaces(authControllerComponents, annotations, esResources, manifest) val commentsController = new Comments(authControllerComponents, manifest, esResources, annotations) val usersController = new Users(authControllerComponents, userProvider) - val pagesController = new PagesController(authControllerComponents, manifest, esResources, pages2, annotations, previewStorage) + val pagesController = new PagesController(authControllerComponents, manifest, esResources, pages2, annotations, previewStorage, materializer) val workerControl = config.aws match { case Some(awsDiscoveryConfig) => diff --git a/backend/app/controllers/api/Authentication.scala b/backend/app/controllers/api/Authentication.scala index 7a4206d3..4e451c14 100644 --- a/backend/app/controllers/api/Authentication.scala +++ b/backend/app/controllers/api/Authentication.scala @@ -1,5 +1,11 @@ package controllers.api +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.{Materializer, OverflowStrategy} +import akka.stream.scaladsl.{Source, SourceQueueWithComplete} +import model.Uri +import model.index.FrontendPage import pdi.jwt.JwtSession._ import pdi.jwt.JwtTime import play.api.libs.json.Json @@ -11,6 +17,7 @@ import utils.attempt._ import utils.auth._ import utils.auth.providers.UserProvider import utils.controller.{AuthControllerComponents, OptionalAuthApiController} + import java.time.Clock import play.api.Configuration @@ -23,6 +30,25 @@ class Authentication(override val controllerComponents: AuthControllerComponents Right(Ok("Ok")) } + def liveClock() = noAuth.ApiAction { + import java.time.ZonedDateTime + import java.time.format.DateTimeFormatter + import javax.inject.Singleton + import akka.stream.scaladsl.Source + import play.api.http.ContentTypes + import play.api.libs.EventSource + import play.api.mvc._ + + import scala.concurrent.duration._ + + val df: DateTimeFormatter = DateTimeFormatter.ofPattern("HH mm ss") + val tickSource = Source.tick(0.millis, 100.millis, "TICK") + val source = tickSource.map { (tick) => + df.format(ZonedDateTime.now()) + } + Right(Ok.chunked(source via EventSource.flow).as(ContentTypes.EVENT_STREAM)) + } + def token() = noAuth.ApiAction.attempt { implicit request:Request[AnyContent] => val time = Epoch.now for { diff --git a/backend/app/controllers/api/PagesController.scala b/backend/app/controllers/api/PagesController.scala index 7e53dc4e..de13a227 100644 --- a/backend/app/controllers/api/PagesController.scala +++ b/backend/app/controllers/api/PagesController.scala @@ -1,10 +1,17 @@ package controllers.api +import akka.NotUsed +import akka.actor.{ActorSystem, ClassicActorSystemProvider} +import akka.stream.scaladsl.SourceQueueWithComplete +import akka.stream.{Materializer, OverflowStrategy} +import akka.stream.scaladsl.Source import commands.{GetPagePreview, GetResource, ResourceFetchMode} import model.frontend.{Chips, HighlightableText, TextHighlight} import model.index.{FrontendPage, HighlightForSearchNavigation, PageHighlight} import model.{Language, Languages, Uri} import org.apache.pdfbox.pdmodel.PDDocument +import play.api.http.ContentTypes +import play.api.libs.EventSource import play.api.libs.json.Json import play.api.mvc.{ResponseHeader, Result} import services.ObjectStorage @@ -16,8 +23,10 @@ import utils.PDFUtil import utils.attempt.Attempt import utils.controller.{AuthApiController, AuthControllerComponents} +import scala.concurrent.Future + class PagesController(val controllerComponents: AuthControllerComponents, manifest: Manifest, - index: Index, pagesService: Pages2, annotations: Annotations, previewStorage: ObjectStorage) extends AuthApiController { + index: Index, pagesService: Pages2, annotations: Annotations, previewStorage: ObjectStorage, materializer: Materializer) extends AuthApiController { def getPageCount(uri: Uri) = ApiAction.attempt { req => @@ -118,6 +127,11 @@ class PagesController(val controllerComponents: AuthControllerComponents, manife } } + private def createSourceOfHighlights(): (SourceQueueWithComplete[FrontendPage], Source[FrontendPage, NotUsed]) = { + val initialSourceOfStatuses = Source.queue[FrontendPage](100, OverflowStrategy.dropHead) + initialSourceOfStatuses.preMaterialize()(materializer) + } + private def getHighlights(uri: Uri, query: String, username: String, isSearch: Boolean): Attempt[Seq[HighlightForSearchNavigation]] = { val searchQuery = if (isSearch) Some(query) else None val findQuery = if (isSearch) None else Some(query) @@ -138,6 +152,22 @@ class PagesController(val controllerComponents: AuthControllerComponents, manife } } + private def getHighlightsStream(sourceQueue: SourceQueueWithComplete[FrontendPage], uri: Uri, query: String, username: String, isSearch: Boolean): Attempt[Unit] = { + val searchQuery = if (isSearch) Some(query) else None + val findQuery = if (isSearch) None else Some(query) + for { + pagesWithHits <- pagesService.findInPages(uri, query) + } yield { + pagesWithHits.foreach(page => { + frontendPageFromQuery(uri, page, username, searchQuery, findQuery).map { frontendPage => + sourceQueue.offer(frontendPage) + } + }) + + () + } + } + // This endpoint is used to get highlights for "find in document" on-demand queries. def findInDocument(uri: Uri, q: String) = ApiAction.attempt { req => getHighlights(uri, q, req.user.username, isSearch = false).map(highlights => @@ -145,6 +175,24 @@ class PagesController(val controllerComponents: AuthControllerComponents, manife ) } + def findInDocumentStream(uri: Uri, q: String) = ApiAction { req => + val (sourceQueue, sourceOfFrontendPages) = createSourceOfHighlights() + getHighlightsStream(sourceQueue, uri, q, req.user.username, isSearch = false) + + val sourceOfHighlights = sourceOfFrontendPages.map(frontendPage => { + (for { + highlight <- frontendPage.highlights + } yield { + HighlightForSearchNavigation.fromPageHighlight(frontendPage.page, highlight.index, highlight) + }).toString + }) + + Right(Ok.chunked(sourceOfHighlights via EventSource.flow) + .as(ContentTypes.EVENT_STREAM) + .withHeaders("Cache-Control" -> "no-cache") + .withHeaders("Connection" -> "keep-alive")) + } + // This endpoint is used to get highlights for the "search across documents" query which // should be fixed for the lifetime of the page viewer of a given document. // It behaves identically to the findInDocument endpoint, except that it expects its query to be in diff --git a/backend/conf/routes b/backend/conf/routes index 1862f813..acfdd6ef 100644 --- a/backend/conf/routes +++ b/backend/conf/routes @@ -36,6 +36,7 @@ GET /api/pages/preview/:language/:uri/:pageNumber cont GET /api/pages2/:uri/pageCount controllers.api.PagesController.getPageCount(uri: model.Uri) GET /api/pages2/:uri/find controllers.api.PagesController.findInDocument(uri: model.Uri, q: String) +GET /api/pages2/:uri/find-stream controllers.api.PagesController.findInDocumentStream(uri: model.Uri, q: String) GET /api/pages2/:uri/search controllers.api.PagesController.searchInDocument(uri: model.Uri, q: String) GET /api/pages2/:uri/:pageNumber/preview controllers.api.PagesController.getPagePreview(uri: model.Uri, pageNumber: Int) GET /api/pages2/:uri/:pageNumber/text controllers.api.PagesController.getPageData(uri: model.Uri, pageNumber: Int, sq: Option[String], fq: Option[String]) @@ -92,6 +93,8 @@ GET /api/keepalive cont GET /api/config controllers.frontend.App.configuration() + + + NOCSRF GET /healthcheck controllers.api.Authentication.healthcheck() @@ -102,6 +105,8 @@ GET /setup cont + NOCSRF PUT /setup controllers.genesis.Genesis.doSetup() +GET /clock controllers.api.Authentication.liveClock() + # === Web Client === GET / controllers.frontend.App.index() GET /third-party/*file controllers.Assets.at(path="/public/third-party", file)