From b045479228af625de9cbd8ccaf6ec25b847a7419 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Tue, 22 Sep 2020 17:45:12 +0100 Subject: [PATCH 1/3] SEARCH-2434 Update lucene and remove alfresco-legacy-lucene --- pom.xml | 15 +- .../opencmis/search/CMISQueryServiceImpl.java | 245 - .../alfresco/repo/blog/BlogServiceImpl.java | 2 +- .../alfresco/repo/links/LinksServiceImpl.java | 52 +- .../org/alfresco/repo/search/LuceneUtils.java | 128 + .../repo/search/QueryParserException.java | 82 + .../impl/lucene/AbstractLuceneBase.java | 355 -- ...stractLuceneIndexerAndSearcherFactory.java | 2238 -------- .../lucene/AbstractLuceneIndexerImpl.java | 814 --- .../impl/lucene/ClosingIndexSearcher.java | 69 - .../lucene/FilterIndexReaderByStringId.java | 395 -- .../lucene/LuceneCategoryServiceImpl.java | 127 +- .../search/impl/lucene/LuceneIndexer.java | 44 - .../search/impl/lucene/LuceneResultSet.java | 355 -- .../impl/lucene/LuceneResultSetRow.java | 166 - .../lucene/LuceneResultSetRowIterator.java | 59 - .../search/impl/lucene/LuceneSearcher.java | 72 - .../search/impl/lucene/SolrJSONResultSet.java | 3 +- .../search/impl/lucene/index/IndexEntry.java | 171 - .../search/impl/lucene/index/IndexEvent.java | 94 - .../search/impl/lucene/index/IndexInfo.java | 4527 ----------------- .../impl/lucene/index/IndexMonitor.java | 108 - .../search/impl/lucene/index/IndexType.java | 55 - .../impl/lucene/index/ReferenceCounting.java | 69 - ...nceCountingReadOnlyIndexReaderFactory.java | 752 --- .../impl/lucene/index/TransactionStatus.java | 533 -- .../impl/lucene/LuceneQueryEngine.java | 300 -- .../solr/AbstractSolrAdminHTTPClient.java | 4 +- .../solr/AbstractSolrQueryHTTPClient.java | 4 +- ...DynamicSolrStoreMappingWrapperFactory.java | 60 +- .../solr/ExplicitSolrStoreMappingWrapper.java | 4 +- .../search/impl/solr/SolrAdminHTTPClient.java | 62 +- .../SolrChildApplicationContextFactory.java | 54 +- .../repo/search/impl/solr/SolrClientUtil.java | 6 +- .../search/impl/solr/SolrQueryHTTPClient.java | 28 +- .../search/impl/solr/SolrSQLHttpClient.java | 6 +- .../repo/search/results/SortedResultSet.java | 54 +- .../security/authority/AuthorityDAOImpl.java | 2 - .../alfresco/repo/site/SiteServiceImpl.java | 6 +- .../alfresco/repo/solr/SOLRAdminClient.java | 6 +- .../apache/lucene/index/TermInfosReader.java | 376 -- .../org/apache/lucene/search/TermQuery.java | 205 - .../org/apache/lucene/search/TermScorer.java | 215 - .../lucene/store/AlfrescoFSDirectory.java | 31 - .../org/apache/lucene/store/FSDirectory.java | 749 --- .../alfresco/core-services-context.xml | 14 - .../index-tracking-context.xml.sample | 4 - ...ecific-index-and-search-context.xml.sample | 48 - .../java/org/alfresco/AllUnitTestsSuite.java | 2 +- .../opencmis/search/OpenCmisQueryTest.java | 32 +- .../model/filefolder/HiddenAspectTest.java | 1 - .../alfresco/repo/search/LuceneUtilsTest.java | 62 + .../impl/solr/SolrSQLHttpClientTest.java | 4 +- 53 files changed, 500 insertions(+), 13369 deletions(-) delete mode 100644 src/main/java/org/alfresco/opencmis/search/CMISQueryServiceImpl.java create mode 100644 src/main/java/org/alfresco/repo/search/LuceneUtils.java create mode 100644 src/main/java/org/alfresco/repo/search/QueryParserException.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneBase.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/ClosingIndexSearcher.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/FilterIndexReaderByStringId.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSet.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSetRow.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSetRowIterator.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/LuceneSearcher.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexEvent.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexMonitor.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java delete mode 100644 src/main/java/org/alfresco/repo/search/impl/querymodel/impl/lucene/LuceneQueryEngine.java delete mode 100644 src/main/java/org/apache/lucene/index/TermInfosReader.java delete mode 100644 src/main/java/org/apache/lucene/search/TermQuery.java delete mode 100644 src/main/java/org/apache/lucene/search/TermScorer.java delete mode 100644 src/main/java/org/apache/lucene/store/AlfrescoFSDirectory.java delete mode 100644 src/main/java/org/apache/lucene/store/FSDirectory.java delete mode 100644 src/main/resources/alfresco/extension/index-tracking-context.xml.sample delete mode 100644 src/main/resources/alfresco/extension/language-specific-index-and-search-context.xml.sample create mode 100644 src/test/java/org/alfresco/repo/search/LuceneUtilsTest.java diff --git a/pom.xml b/pom.xml index 022f5ced70..358b1cae1b 100644 --- a/pom.xml +++ b/pom.xml @@ -36,10 +36,9 @@ 11 - 8.152 + search-2434-1 8.49 - 6.2 6.2 7.1 1.1 @@ -161,11 +160,6 @@ - - org.alfresco - alfresco-legacy-lucene - ${dependency.alfresco-legacy-lucene.version} - org.alfresco alfresco-jlan-embed @@ -1129,13 +1123,6 @@ - - org.alfresco - alfresco-legacy-lucene - ${dependency.alfresco-legacy-lucene.version} - tests - test - org.postgresql postgresql diff --git a/src/main/java/org/alfresco/opencmis/search/CMISQueryServiceImpl.java b/src/main/java/org/alfresco/opencmis/search/CMISQueryServiceImpl.java deleted file mode 100644 index f3132f1f22..0000000000 --- a/src/main/java/org/alfresco/opencmis/search/CMISQueryServiceImpl.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.opencmis.search; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.alfresco.opencmis.dictionary.CMISDictionaryService; -import org.alfresco.opencmis.search.CMISQueryOptions.CMISQueryMode; -import org.alfresco.repo.search.impl.lucene.PagingLuceneResultSet; -import org.alfresco.repo.search.impl.querymodel.Query; -import org.alfresco.repo.search.impl.querymodel.QueryEngine; -import org.alfresco.repo.search.impl.querymodel.QueryEngineResults; -import org.alfresco.repo.search.impl.querymodel.QueryModelException; -import org.alfresco.repo.security.permissions.impl.acegi.FilteringResultSet; -import org.alfresco.service.cmr.dictionary.DictionaryService; -import org.alfresco.service.cmr.repository.NodeRef; -import org.alfresco.service.cmr.repository.NodeService; -import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.service.cmr.search.LimitBy; -import org.alfresco.service.cmr.search.QueryConsistency; -import org.alfresco.service.cmr.search.ResultSet; -import org.alfresco.util.Pair; -import org.apache.chemistry.opencmis.commons.enums.BaseTypeId; -import org.apache.chemistry.opencmis.commons.enums.CapabilityJoin; -import org.apache.chemistry.opencmis.commons.enums.CapabilityQuery; - -/** - * @author andyh - */ -public class CMISQueryServiceImpl implements CMISQueryService -{ - private CMISDictionaryService cmisDictionaryService; - - private QueryEngine luceneQueryEngine; - private QueryEngine dbQueryEngine; - - private NodeService nodeService; - - private DictionaryService alfrescoDictionaryService; - - public void setOpenCMISDictionaryService(CMISDictionaryService cmisDictionaryService) - { - this.cmisDictionaryService = cmisDictionaryService; - } - - /** - * @param queryEngine - * the luceneQueryEngine to set - */ - public void setLuceneQueryEngine(QueryEngine queryEngine) - { - this.luceneQueryEngine = queryEngine; - } - - /** - * @param queryEngine - * the dbQueryEngine to set - */ - public void setDbQueryEngine(QueryEngine queryEngine) - { - this.dbQueryEngine = queryEngine; - } - - /** - * @param nodeService - * the nodeService to set - */ - public void setNodeService(NodeService nodeService) - { - this.nodeService = nodeService; - } - - /** - * @param alfrescoDictionaryService - * the Alfresco Dictionary Service to set - */ - public void setAlfrescoDictionaryService(DictionaryService alfrescoDictionaryService) - { - this.alfrescoDictionaryService = alfrescoDictionaryService; - } - - public CMISResultSet query(CMISQueryOptions options) - { - Pair resultPair = executeQuerySwitchingImpl(options); - - Query query = resultPair.getFirst(); - QueryEngineResults results = resultPair.getSecond(); - - Map wrapped = new HashMap(); - Map, ResultSet> map = results.getResults(); - for (Set group : map.keySet()) - { - ResultSet current = map.get(group); - for (String selector : group) - { - wrapped.put(selector, filterNotExistingNodes(current)); - } - } - LimitBy limitBy = null; - if ((null != results.getResults()) && !results.getResults().isEmpty() - && (null != results.getResults().values()) && !results.getResults().values().isEmpty()) - { - limitBy = results.getResults().values().iterator().next().getResultSetMetaData().getLimitedBy(); - } - CMISResultSet cmis = new CMISResultSet(wrapped, options, limitBy, nodeService, query, cmisDictionaryService, - alfrescoDictionaryService); - return cmis; - } - - private Pair executeQuerySwitchingImpl(CMISQueryOptions options) - { - switch (options.getQueryConsistency()) - { - case TRANSACTIONAL_IF_POSSIBLE : - { - try - { - return executeQueryUsingEngine(dbQueryEngine, options); - } - catch(QueryModelException qme) - { - return executeQueryUsingEngine(luceneQueryEngine, options); - } - } - case TRANSACTIONAL : - { - return executeQueryUsingEngine(dbQueryEngine, options); - } - case EVENTUAL : - case DEFAULT : - default : - { - return executeQueryUsingEngine(luceneQueryEngine, options); - } - } - } - - private Pair executeQueryUsingEngine(QueryEngine queryEngine, CMISQueryOptions options) - { - CapabilityJoin joinSupport = getJoinSupport(); - if (options.getQueryMode() == CMISQueryOptions.CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS) - { - joinSupport = CapabilityJoin.INNERANDOUTER; - } - - // TODO: Refactor to avoid duplication of valid scopes here and in - // CMISQueryParser - - BaseTypeId[] validScopes = (options.getQueryMode() == CMISQueryMode.CMS_STRICT) ? CmisFunctionEvaluationContext.STRICT_SCOPES - : CmisFunctionEvaluationContext.ALFRESCO_SCOPES; - CmisFunctionEvaluationContext functionContext = new CmisFunctionEvaluationContext(); - functionContext.setCmisDictionaryService(cmisDictionaryService); - functionContext.setNodeService(nodeService); - functionContext.setValidScopes(validScopes); - - CMISQueryParser parser = new CMISQueryParser(options, cmisDictionaryService, joinSupport); - QueryConsistency queryConsistency = options.getQueryConsistency(); - if (queryConsistency == QueryConsistency.DEFAULT) - { - options.setQueryConsistency(QueryConsistency.EVENTUAL); - } - - Query query = parser.parse(queryEngine.getQueryModelFactory(), functionContext); - QueryEngineResults queryEngineResults = queryEngine.executeQuery(query, options, functionContext); - - return new Pair(query, queryEngineResults); - } - - /* MNT-8804 filter ResultSet for nodes with corrupted indexes */ - private ResultSet filterNotExistingNodes(ResultSet resultSet) - { - if (resultSet instanceof PagingLuceneResultSet) - { - ResultSet wrapped = ((PagingLuceneResultSet)resultSet).getWrapped(); - - if (wrapped instanceof FilteringResultSet) - { - FilteringResultSet filteringResultSet = (FilteringResultSet)wrapped; - - for (int i = 0; i < filteringResultSet.length(); i++) - { - NodeRef nodeRef = filteringResultSet.getNodeRef(i); - /* filter node if it does not exist */ - if (!nodeService.exists(nodeRef)) - { - filteringResultSet.setIncluded(i, false); - } - } - } - } - - return resultSet; - } - - public CMISResultSet query(String query, StoreRef storeRef) - { - CMISQueryOptions options = new CMISQueryOptions(query, storeRef); - return query(options); - } - - public boolean getPwcSearchable() - { - return true; - } - - public boolean getAllVersionsSearchable() - { - return false; - } - - public CapabilityQuery getQuerySupport() - { - return CapabilityQuery.BOTHCOMBINED; - } - - public CapabilityJoin getJoinSupport() - { - return CapabilityJoin.NONE; - } -} diff --git a/src/main/java/org/alfresco/repo/blog/BlogServiceImpl.java b/src/main/java/org/alfresco/repo/blog/BlogServiceImpl.java index da7d6d5a33..0b5939b0da 100644 --- a/src/main/java/org/alfresco/repo/blog/BlogServiceImpl.java +++ b/src/main/java/org/alfresco/repo/blog/BlogServiceImpl.java @@ -46,7 +46,7 @@ import org.alfresco.repo.blog.cannedqueries.GetBlogPostsCannedQuery; import org.alfresco.repo.blog.cannedqueries.GetBlogPostsCannedQueryFactory; import org.alfresco.repo.content.MimetypeMap; -import org.alfresco.repo.search.impl.lucene.LuceneUtils; +import org.alfresco.repo.search.LuceneUtils; import org.alfresco.repo.security.authentication.AuthenticationUtil; import org.alfresco.repo.security.permissions.AccessDeniedException; import org.alfresco.repo.site.SiteServiceImpl; diff --git a/src/main/java/org/alfresco/repo/links/LinksServiceImpl.java b/src/main/java/org/alfresco/repo/links/LinksServiceImpl.java index 93bf691db4..2cc8dedd66 100644 --- a/src/main/java/org/alfresco/repo/links/LinksServiceImpl.java +++ b/src/main/java/org/alfresco/repo/links/LinksServiceImpl.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.links; import java.io.Serializable; @@ -42,7 +42,7 @@ import org.alfresco.repo.node.getchildren.GetChildrenAuditableCannedQuery; import org.alfresco.repo.node.getchildren.GetChildrenAuditableCannedQueryFactory; import org.alfresco.repo.query.NodeBackedEntity; -import org.alfresco.repo.search.impl.lucene.LuceneUtils; +import org.alfresco.repo.search.LuceneUtils; import org.alfresco.repo.site.SiteServiceImpl; import org.alfresco.service.cmr.dictionary.DictionaryService; import org.alfresco.service.cmr.links.LinkInfo; diff --git a/src/main/java/org/alfresco/repo/search/LuceneUtils.java b/src/main/java/org/alfresco/repo/search/LuceneUtils.java new file mode 100644 index 0000000000..d3eb2665bb --- /dev/null +++ b/src/main/java/org/alfresco/repo/search/LuceneUtils.java @@ -0,0 +1,128 @@ +/* + * #%L + * Alfresco Legacy Lucene + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.repo.search; + +import java.text.SimpleDateFormat; +import java.util.Date; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.dictionary.PropertyDefinition; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.namespace.QName; + +/** + * Lucene utils + * + * @author Andy + * + */ +public class LuceneUtils +{ + /** + * This is the date string format as required by Lucene e.g. "1970\\-01\\-01T00:00:00" + * @since 4.0 + */ + private static final SimpleDateFormat LUCENE_DATETIME_FORMAT = new SimpleDateFormat("yyyy\\-MM\\-dd'T'HH:mm:ss"); + + /** + * Returns a date string in the format required by Lucene. + * + * @since 4.0 + */ + public static String getLuceneDateString(Date date) + { + return LUCENE_DATETIME_FORMAT.format(date); + } + /** + * This method creates a Lucene query fragment which constrains the specified dateProperty to a range + * given by the fromDate and toDate parameters. + * + * @param fromDate the start of the date range (defaults to 1970-01-01 00:00:00 if null). + * @param toDate the end of the date range (defaults to 3000-12-31 00:00:00 if null). + * @param dateProperty the Alfresco property value to check against the range (must be a valid Date or DateTime property). + * + * @return the Lucene query fragment. + * + * @throws NullPointerException if dateProperty is null or if the dateProperty is not recognised by the system. + * @throws IllegalArgumentException if dateProperty refers to a property that is not of type {@link DataTypeDefinition#DATE} or {@link DataTypeDefinition#DATETIME}. + */ + public static String createDateRangeQuery(Date fromDate, Date toDate, QName dateProperty, + DictionaryService dictionaryService, NamespaceService namespaceService) + { + // Some sanity checking of the date property. + if (dateProperty == null) + { + throw new NullPointerException("dateProperty cannot be null"); + } + PropertyDefinition propDef = dictionaryService.getProperty(dateProperty); + if (propDef == null) + { + throw new NullPointerException("dateProperty '" + dateProperty + "' not recognised."); + } + else + { + final QName propDefType = propDef.getDataType().getName(); + if ( !DataTypeDefinition.DATE.equals(propDefType) && + !DataTypeDefinition.DATETIME.equals(propDefType)) + { + throw new IllegalArgumentException("Illegal property type '" + dateProperty + "' [" + propDefType + "]"); + } + } + + QName propertyName = propDef.getName(); + final String shortFormQName = propertyName.toPrefixString(namespaceService); + final String prefix = shortFormQName.substring(0, shortFormQName.indexOf(QName.NAMESPACE_PREFIX)); + final String localName = propertyName.getLocalName(); + + + // I can see potential issues with using 1970 and 3000 as default dates, but this is what the previous + // JavaScript controllers/libs did and I'll reproduce it here. + final String ZERO_DATE = "1970\\-01\\-01T00:00:00"; + final String FUTURE_DATE = "3000\\-12\\-31T00:00:00"; + + StringBuilder luceneQuery = new StringBuilder(); + luceneQuery.append(" +@").append(prefix).append("\\:").append(localName).append(":["); + if (fromDate != null) + { + luceneQuery.append(LuceneUtils.getLuceneDateString(fromDate)); + } + else + { + luceneQuery.append(ZERO_DATE); + } + luceneQuery.append(" TO "); + if (toDate != null) + { + luceneQuery.append(LuceneUtils.getLuceneDateString(toDate)); + } + else + { + luceneQuery.append(FUTURE_DATE); + } + luceneQuery.append("] "); + return luceneQuery.toString(); + } +} diff --git a/src/main/java/org/alfresco/repo/search/QueryParserException.java b/src/main/java/org/alfresco/repo/search/QueryParserException.java new file mode 100644 index 0000000000..b2151927fe --- /dev/null +++ b/src/main/java/org/alfresco/repo/search/QueryParserException.java @@ -0,0 +1,82 @@ +/* + * #%L + * Alfresco Legacy Lucene + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.repo.search; + +import org.alfresco.error.AlfrescoRuntimeException; + +/** + * @author Andy + * + */ +public class QueryParserException extends AlfrescoRuntimeException +{ + + /** + * + */ + private static final long serialVersionUID = 4886993838297301968L; + + /** + * @param msgId + */ + public QueryParserException(String msgId) + { + super(msgId); + // TODO Auto-generated constructor stub + } + + /** + * @param msgId + * @param msgParams + */ + public QueryParserException(String msgId, Object[] msgParams) + { + super(msgId, msgParams); + // TODO Auto-generated constructor stub + } + + /** + * @param msgId + * @param cause + */ + public QueryParserException(String msgId, Throwable cause) + { + super(msgId, cause); + // TODO Auto-generated constructor stub + } + + /** + * @param msgId + * @param msgParams + * @param cause + */ + public QueryParserException(String msgId, Object[] msgParams, Throwable cause) + { + super(msgId, msgParams, cause); + // TODO Auto-generated constructor stub + } + +} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneBase.java b/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneBase.java deleted file mode 100644 index c5530008ee..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneBase.java +++ /dev/null @@ -1,355 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.io.File; -import java.io.IOException; -import java.util.Set; - -import org.alfresco.repo.search.IndexerException; -import org.alfresco.repo.search.impl.lucene.analysis.AlfrescoStandardAnalyser; -import org.alfresco.repo.search.impl.lucene.index.IndexInfo; -import org.alfresco.repo.search.impl.lucene.index.TransactionStatus; -import org.alfresco.repo.search.impl.lucene.index.IndexInfo.LockWork; -import org.alfresco.service.cmr.dictionary.DictionaryService; -import org.alfresco.service.cmr.repository.StoreRef; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.search.IndexSearcher; - -/** - * Common support for abstracting the lucene indexer from its configuration and management requirements. - * - *

- * This class defines where the indexes are stored. This should be via a configurable Bean property in Spring. - * - *

- * The default file structure is - *

    - *
  1. "base"/"protocol"/"name"/ for the main index - *
  2. "base"/"protocol"/"name"/deltas/"id" for transactional updates - *
  3. "base"/"protocol"/"name"/undo/"id" undo information - *
- * - *

- * The IndexWriter and IndexReader for a given index are toggled (one should be used for delete and the other for write). These are reused/closed/initialised as required. - * - *

- * The index deltas are buffered to memory and persisted in the file system as required. - * - * @author Andy Hind - * - */ - -public abstract class AbstractLuceneBase -{ - private static Log s_logger = LogFactory.getLog(AbstractLuceneBase.class); - - private IndexInfo indexInfo; - - /** - * The identifier for the store - */ - - protected StoreRef store; - - /** - * The identifier for the delta - */ - - protected String deltaId; - - private LuceneConfig config; - - private TransactionStatus status = TransactionStatus.UNKNOWN; - - // "lucene-indexes"; - - /** - * Initialise the configuration elements of the lucene store indexers and searchers. - * - * @param store StoreRef - * @param deltaId String - * @throws LuceneIndexException - */ - protected void initialise(StoreRef store, String deltaId) - throws LuceneIndexException - { - this.store = store; - this.deltaId = deltaId; - - String basePath = getBasePath(); - File baseDir = new File(basePath); - indexInfo = IndexInfo.getIndexInfo(baseDir, config); - try - { - if (this.deltaId != null) - { - if (! getStatus().equals(TransactionStatus.ACTIVE)) - { - setStatus(TransactionStatus.ACTIVE); - } - else - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Delta already set as active " + deltaId); - } - } - } - } - catch (IOException e) - { - throw new IndexerException("Failed to set delta as active"); - } - } - - /** - * Utility method to find the path to the base index - * - * @return - the base path - */ - private String getBasePath() - { - if (config.getIndexRootLocation() == null) - { - throw new IndexerException("No configuration for index location"); - } - String basePath = config.getIndexRootLocation() - + File.separator + store.getProtocol() + File.separator + store.getIdentifier() + File.separator; - return basePath; - } - - /** - * Get a searcher for the main index TODO: Split out support for the main index. We really only need this if we want to search over the changing index before it is committed - * - * @return - the searcher - * @throws LuceneIndexException - */ - - protected IndexSearcher getSearcher() throws LuceneIndexException - { - try - { - return new ClosingIndexSearcher(indexInfo.getMainIndexReferenceCountingReadOnlyIndexReader()); - } - catch (IOException e) - { - s_logger.error("Error", e); - throw new LuceneIndexException("Failed to open IndexSarcher for " + getBasePath(), e); - } - } - - protected ClosingIndexSearcher getSearcher(LuceneIndexer luceneIndexer) throws LuceneIndexException - { - // If we know the delta id we should do better - - try - { - if (luceneIndexer == null) - { - return new ClosingIndexSearcher(indexInfo.getMainIndexReferenceCountingReadOnlyIndexReader()); - } - else - { - // TODO: Create appropriate reader that lies about deletions - // from the first - // - luceneIndexer.flushPending(); - return new ClosingIndexSearcher(indexInfo.getMainIndexReferenceCountingReadOnlyIndexReader(deltaId, - luceneIndexer.getDeletions(), luceneIndexer.getContainerDeletions(), luceneIndexer - .getDeleteOnlyNodes())); - } - - } - catch (IOException e) - { - s_logger.error("Error", e); - throw new LuceneIndexException("Failed to open IndexSarcher for " + getBasePath(), e); - } - } - - /** - * Get a reader for the on file portion of the delta - * - * @return - the index reader - * @throws IOException - * @throws IOException - */ - - protected IndexReader getDeltaReader() throws LuceneIndexException, IOException - { - return indexInfo.getDeltaIndexReader(deltaId); - } - - /** - * Close the on file reader for the delta if it is open - * - * @throws IOException - * - * @throws IOException - */ - - protected void closeDeltaReader() throws LuceneIndexException, IOException - { - indexInfo.closeDeltaIndexReader(deltaId); - } - - /** - * Get the on file writer for the delta - * - * @return - the writer for the delta - * @throws IOException - * @throws IOException - */ - protected IndexWriter getDeltaWriter() throws LuceneIndexException, IOException - { - return indexInfo.getDeltaIndexWriter(deltaId, new LuceneAnalyser(dictionaryService, config.getDefaultMLIndexAnalysisMode())); - } - - /** - * Close the on disk delta writer - * - * @throws IOException - * - * @throws IOException - */ - - protected void closeDeltaWriter() throws LuceneIndexException, IOException - { - indexInfo.closeDeltaIndexWriter(deltaId); - } - - /** - * Save the in memory delta to the disk, make sure there is nothing held in memory - * - * @throws IOException - * - * @throws IOException - */ - protected void saveDelta() throws LuceneIndexException, IOException - { - // Only one should exist so we do not need error trapping to execute the - // other - closeDeltaReader(); - closeDeltaWriter(); - } - - protected void setInfo(long docs, Set deletions, Set containerDeletions, boolean deleteNodesOnly) throws IOException - { - indexInfo.setPreparedState(deltaId, deletions, containerDeletions, docs, deleteNodesOnly); - } - - protected void setStatus(TransactionStatus status) throws IOException - { - indexInfo.setStatus(deltaId, status, null, null); - this.status = status; - } - - protected TransactionStatus getStatus() - { - return status; - } - - - - private DictionaryService dictionaryService; - - protected IndexReader getReader() throws LuceneIndexException, IOException - { - return indexInfo.getMainIndexReferenceCountingReadOnlyIndexReader(); - } - - /** - * Set the dictionary service - * @param dictionaryService DictionaryService - */ - public void setDictionaryService(DictionaryService dictionaryService) - { - this.dictionaryService = dictionaryService; - } - - /** - * Get the dictionary service. - * - * @return - the service - */ - public DictionaryService getDictionaryService() - { - return dictionaryService; - } - - /** - * Set the lucene configuration options - * - * @param config LuceneConfig - */ - public void setLuceneConfig(LuceneConfig config) - { - this.config = config; - } - - /** - * Get the lucene configuration options. - * - * @return - the config options object. - */ - public LuceneConfig getLuceneConfig() - { - return config; - } - - /** - * Get the ID for the delat we are working with. - * - * @return - the id - */ - public String getDeltaId() - { - return deltaId; - } - - - /** - * Execute actions against a read only index (all write ops will block) - * - * @return - the result returned by the action. - */ - public R doReadOnly(LockWork lockWork) - { - return indexInfo.doReadOnly(lockWork); - } - - - public void deleteIndex() - { - indexInfo.delete(deltaId); - } - - -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java b/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java deleted file mode 100644 index 855c3c2c9d..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerAndSearcherFactory.java +++ /dev/null @@ -1,2238 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import javax.transaction.RollbackException; -import javax.transaction.SystemException; -import javax.transaction.Transaction; -import javax.transaction.xa.XAException; -import javax.transaction.xa.XAResource; -import javax.transaction.xa.Xid; - -import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.repo.node.NodeBulkLoader; -import org.alfresco.repo.search.IndexerException; -import org.alfresco.repo.search.MLAnalysisMode; -import org.alfresco.repo.search.QueryRegisterComponent; -import org.alfresco.repo.search.SearcherException; -import org.alfresco.repo.search.impl.lucene.index.IndexInfo; -import org.alfresco.repo.search.transaction.SimpleTransaction; -import org.alfresco.repo.search.transaction.SimpleTransactionManager; -import org.alfresco.repo.tenant.TenantService; -import org.alfresco.repo.transaction.AlfrescoTransactionSupport; -import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; -import org.alfresco.service.cmr.repository.NodeService; -import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.service.cmr.search.SearchService; -import org.alfresco.service.transaction.TransactionService; -import org.alfresco.util.GUID; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.store.Lock; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.transaction.support.TransactionSynchronizationManager; - -/** - * This class is resource manager LuceneIndexers and LuceneSearchers. It supports two phase commit inside XA - * transactions and outside transactions it provides thread local transaction support. TODO: Provide pluggable support - * for a transaction manager TODO: Integrate with Spring transactions - * - * @author andyh - */ - -public abstract class AbstractLuceneIndexerAndSearcherFactory extends AbstractIndexerAndSearcher implements LuceneIndexerAndSearcher, XAResource, ApplicationContextAware, DisposableBean -{ - private static Log logger = LogFactory.getLog(AbstractLuceneIndexerAndSearcherFactory.class); - - private int queryMaxClauses; - - private int indexerBatchSize; - - /** - * A map of active global transactions . It contains all the indexers a transaction has used, with at most one - * indexer for each store within a transaction - */ - - private Map> activeIndexersInGlobalTx = new HashMap>(); - - /** - * Suspended global transactions. - */ - private Map> suspendedIndexersInGlobalTx = new HashMap>(); - - /** - * The key under which this instance's map of indexers is stored in a (non-global) transaction - */ - private final String indexersKey = "AbstractLuceneIndexerAndSearcherFactory." + GUID.generate(); - - /** - * The default timeout for transactions TODO: Respect this - */ - - private int timeout = DEFAULT_TIMEOUT; - - /** - * Default time out value set to 10 minutes. - */ - private static final int DEFAULT_TIMEOUT = 600000; - - protected TenantService tenantService; - - private String indexRootLocation; - - private QueryRegisterComponent queryRegister; - - /** the maximum transformation time to allow atomically, defaulting to 20ms */ - private long maxAtomicTransformationTime = 20; - - private int indexerMaxFieldLength = IndexWriter.DEFAULT_MAX_FIELD_LENGTH; - - private long writeLockTimeout; - - private long commitLockTimeout; - - private String lockDirectory; - - private MLAnalysisMode defaultMLIndexAnalysisMode = MLAnalysisMode.EXACT_LANGUAGE_AND_ALL; - - private MLAnalysisMode defaultMLSearchAnalysisMode = MLAnalysisMode.EXACT_LANGUAGE_AND_ALL; - - private ThreadPoolExecutor threadPoolExecutor; - - private NodeBulkLoader bulkLoader; - - private int maxDocIdCacheSize = 10000; - - private int maxDocsForInMemoryMerge = 10000; - - private int maxDocsForInMemoryIndex = 10000; - - private double maxRamInMbForInMemoryMerge = 16.0; - - private double maxRamInMbForInMemoryIndex = 16.0; - - private int maxDocumentCacheSize = 100; - - private int maxIsCategoryCacheSize = -1; - - private int maxLinkAspectCacheSize = 10000; - - private int maxParentCacheSize = 10000; - - private int maxPathCacheSize = 10000; - - private int maxTypeCacheSize = 10000; - - private int mergerMaxMergeDocs = 1000000; - - private int mergerMergeFactor = 5; - - - private int mergerMaxBufferedDocs = IndexWriter.DISABLE_AUTO_FLUSH; - - private double mergerRamBufferSizeMb = 16.0; - - private int mergerTargetIndexCount = 5; - - private int mergerTargetOverlayCount = 5; - - private int mergerTargetOverlaysBlockingFactor = 1; - - private boolean fairLocking; - - private int termIndexInterval = IndexWriter.DEFAULT_TERM_INDEX_INTERVAL; - - private boolean useNioMemoryMapping = true; - - private int writerMaxMergeDocs = 1000000; - - private int writerMergeFactor = 5; - - private int writerMaxBufferedDocs = IndexWriter.DISABLE_AUTO_FLUSH; - - private double writerRamBufferSizeMb = 16.0; - - private boolean cacheEnabled = true; - - private boolean postSortDateTime; - - private ConfigurableApplicationContext applicationContext; - - private boolean contentIndexingEnabled = true; - - private boolean useInMemorySort = true; - - private int maxRawResultSetSizeForInMemorySort = 1000; - - private volatile boolean destroyed = false; - - /** - * Private constructor for the singleton TODO: FIt in with IOC - */ - - public AbstractLuceneIndexerAndSearcherFactory() - { - super(); - } - - /* - * (non-Javadoc) - * @seeorg.springframework.context.ApplicationContextAware#setApplicationContext(org.springframework.context. - * ApplicationContext) - */ - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException - { - this.applicationContext = (ConfigurableApplicationContext) applicationContext; - } - - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.LuceneConfig#getApplicationContext() - */ - public ConfigurableApplicationContext getApplicationContext() - { - return this.applicationContext; - } - - /** - * Set the directory that contains the indexes - * - * @param indexRootLocation String - */ - - public void setIndexRootLocation(String indexRootLocation) - { - this.indexRootLocation = indexRootLocation; - } - - /** - * Set the tenant service - * - * @param tenantService TenantService - */ - public void setTenantService(TenantService tenantService) - { - this.tenantService = tenantService; - } - - /** - * Set the query register - * - * @param queryRegister QueryRegisterComponent - */ - public void setQueryRegister(QueryRegisterComponent queryRegister) - { - this.queryRegister = queryRegister; - } - - /** - * Get the query register. - * - * @return - the query register. - */ - public QueryRegisterComponent getQueryRegister() - { - return queryRegister; - } - - /** - * Set the maximum average transformation time allowed to a transformer in order to have the transformation - * performed in the current transaction. The default is 20ms. - * - * @param maxAtomicTransformationTime - * the maximum average time that a text transformation may take in order to be performed atomically. - */ - @Override - public void setMaxAtomicTransformationTime(long maxAtomicTransformationTime) - { - this.maxAtomicTransformationTime = maxAtomicTransformationTime; - } - - /** - * Get the max time for an atomic transform - * - * @return - milliseconds as a long - */ - @Override - public long getMaxTransformationTime() - { - return maxAtomicTransformationTime; - } - - public NodeBulkLoader getBulkLoader() - { - return bulkLoader; - } - - public void setBulkLoader(NodeBulkLoader bulkLoader) - { - this.bulkLoader = bulkLoader; - } - - /** - * Check if we are in a global transactoin according to the transaction manager - * - * @return - true if in a global transaction - */ - - private boolean inGlobalTransaction() - { - try - { - return SimpleTransactionManager.getInstance().getTransaction() != null; - } - catch (SystemException e) - { - return false; - } - } - - /** - * Get the local transaction - may be null oif we are outside a transaction. - * - * @return - the transaction - * @throws IndexerException - */ - private SimpleTransaction getTransaction() throws IndexerException - { - try - { - return SimpleTransactionManager.getInstance().getTransaction(); - } - catch (SystemException e) - { - throw new IndexerException("Failed to get transaction", e); - } - } - - /** - * Get an indexer for the store to use in the current transaction for this thread of control. - * - * @param storeRef - - * the id of the store - */ - public LuceneIndexer getIndexer(StoreRef storeRef) throws IndexerException - { - storeRef = tenantService.getName(storeRef); - - // register to receive txn callbacks - // TODO: make this conditional on whether the XA stuff is being used - // directly on not - AlfrescoTransactionSupport.bindLucene(this); - - if (inGlobalTransaction()) - { - SimpleTransaction tx = getTransaction(); - // Only find indexers in the active list - Map indexers = activeIndexersInGlobalTx.get(tx); - if (indexers == null) - { - if (suspendedIndexersInGlobalTx.containsKey(tx)) - { - throw new IndexerException("Trying to obtain an index for a suspended transaction."); - } - indexers = new HashMap(); - activeIndexersInGlobalTx.put(tx, indexers); - try - { - tx.enlistResource(this); - } - // TODO: what to do in each case? - catch (IllegalStateException e) - { - throw new IndexerException("", e); - } - catch (RollbackException e) - { - throw new IndexerException("", e); - } - catch (SystemException e) - { - throw new IndexerException("", e); - } - } - LuceneIndexer indexer = indexers.get(storeRef); - if (indexer == null) - { - indexer = createIndexer(storeRef, getTransactionId(tx, storeRef)); - indexers.put(storeRef, indexer); - } - return indexer; - } - else - // A thread local transaction - { - return getThreadLocalIndexer(storeRef); - } - - } - - @SuppressWarnings("unchecked") - private LuceneIndexer getThreadLocalIndexer(StoreRef storeRef) - { - Map indexers = (Map) AlfrescoTransactionSupport.getResource(indexersKey); - if (indexers == null) - { - indexers = new HashMap(); - AlfrescoTransactionSupport.bindResource(indexersKey, indexers); - } - LuceneIndexer indexer = indexers.get(storeRef); - if (indexer == null) - { - indexer = createIndexer(storeRef, GUID.generate()); - indexers.put(storeRef, indexer); - } - return indexer; - } - - /** - * Get the transaction identifier used to store it in the transaction map. - * - * @param tx Transaction - * @param storeRef StoreRef - * @return - the transaction id - */ - @SuppressWarnings("unchecked") - private String getTransactionId(Transaction tx, StoreRef storeRef) - { - if (tx instanceof SimpleTransaction) - { - SimpleTransaction simpleTx = (SimpleTransaction) tx; - return simpleTx.getGUID(); - } - else if (TransactionSynchronizationManager.isSynchronizationActive()) - { - Map indexers = (Map) AlfrescoTransactionSupport.getResource(indexersKey); - if (indexers != null) - { - LuceneIndexer indexer = indexers.get(storeRef); - if (indexer != null) - { - return indexer.getDeltaId(); - } - } - } - return null; - } - - /** - * Encapsulate creating an indexer - * - * @param storeRef StoreRef - * @param deltaId String - * @return - the indexer made by the concrete implemntation - */ - protected abstract LuceneIndexer createIndexer(StoreRef storeRef, String deltaId); - - /** - * Encapsulate creating a searcher over the main index - */ - public LuceneSearcher getSearcher(StoreRef storeRef, boolean searchDelta) throws SearcherException - { - storeRef = tenantService.getName(storeRef); - - String deltaId = null; - LuceneIndexer indexer = null; - if (searchDelta) - { - deltaId = getTransactionId(getTransaction(), storeRef); - if (deltaId != null) - { - indexer = getIndexer(storeRef); - } - } - LuceneSearcher searcher = getSearcher(storeRef, indexer); - return searcher; - } - - /** - * Get node-based searcher (for "selectNodes / selectProperties") - */ - protected abstract SearchService getNodeSearcher() throws SearcherException; - - /** - * Get a searcher over the index and the current delta - * - * @param storeRef StoreRef - * @param indexer LuceneIndexer - * @return - the searcher made by the concrete implementation. - * @throws SearcherException - */ - - protected abstract LuceneSearcher getSearcher(StoreRef storeRef, LuceneIndexer indexer) throws SearcherException; - - /* - * XAResource implementation - */ - - public void commit(Xid xid, boolean onePhase) throws XAException - { - try - { - // TODO: Should be remembering overall state - // TODO: Keep track of prepare responses - Map indexers = activeIndexersInGlobalTx.get(xid); - if (indexers == null) - { - if (suspendedIndexersInGlobalTx.containsKey(xid)) - { - throw new XAException("Trying to commit indexes for a suspended transaction."); - } - else - { - // nothing to do - return; - } - } - - if (onePhase) - { - if (indexers.size() == 0) - { - return; - } - else if (indexers.size() == 1) - { - for (LuceneIndexer indexer : indexers.values()) - { - indexer.commit(); - } - return; - } - else - { - throw new XAException("Trying to do one phase commit on more than one index"); - } - } - else - // two phase - { - for (LuceneIndexer indexer : indexers.values()) - { - indexer.commit(); - } - return; - } - } - finally - { - activeIndexersInGlobalTx.remove(xid); - } - } - - public void end(Xid xid, int flag) throws XAException - { - Map indexers = activeIndexersInGlobalTx.get(xid); - if (indexers == null) - { - if (suspendedIndexersInGlobalTx.containsKey(xid)) - { - throw new XAException("Trying to commit indexes for a suspended transaction."); - } - else - { - // nothing to do - return; - } - } - if (flag == XAResource.TMSUSPEND) - { - activeIndexersInGlobalTx.remove(xid); - suspendedIndexersInGlobalTx.put(xid, indexers); - } - else if (flag == TMFAIL) - { - activeIndexersInGlobalTx.remove(xid); - suspendedIndexersInGlobalTx.remove(xid); - } - else if (flag == TMSUCCESS) - { - activeIndexersInGlobalTx.remove(xid); - } - } - - public void forget(Xid xid) throws XAException - { - activeIndexersInGlobalTx.remove(xid); - suspendedIndexersInGlobalTx.remove(xid); - } - - public int getTransactionTimeout() throws XAException - { - return timeout; - } - - public boolean isSameRM(XAResource xar) throws XAException - { - return (xar instanceof AbstractLuceneIndexerAndSearcherFactory); - } - - public int prepare(Xid xid) throws XAException - { - // TODO: Track state OK, ReadOnly, Exception (=> rolled back?) - Map indexers = activeIndexersInGlobalTx.get(xid); - if (indexers == null) - { - if (suspendedIndexersInGlobalTx.containsKey(xid)) - { - throw new XAException("Trying to commit indexes for a suspended transaction."); - } - else - { - // nothing to do - return XAResource.XA_OK; - } - } - boolean isPrepared = true; - boolean isModified = false; - for (LuceneIndexer indexer : indexers.values()) - { - try - { - isModified |= indexer.isModified(); - indexer.prepare(); - } - catch (IndexerException e) - { - isPrepared = false; - } - } - if (isPrepared) - { - if (isModified) - { - return XAResource.XA_OK; - } - else - { - return XAResource.XA_RDONLY; - } - } - else - { - throw new XAException("Failed to prepare: requires rollback"); - } - } - - public Xid[] recover(int arg0) throws XAException - { - // We can not rely on being able to recover at the moment - // Avoiding for performance benefits at the moment - // Assume roll back and no recovery - in the worst case we get an unused - // delta - // This should be there to avoid recovery of partial commits. - // It is difficult to see how we can mandate the same conditions. - return new Xid[0]; - } - - public void rollback(Xid xid) throws XAException - { - // TODO: What to do if all do not roll back? - try - { - Map indexers = activeIndexersInGlobalTx.get(xid); - if (indexers == null) - { - if (suspendedIndexersInGlobalTx.containsKey(xid)) - { - throw new XAException("Trying to commit indexes for a suspended transaction."); - } - else - { - // nothing to do - return; - } - } - for (LuceneIndexer indexer : indexers.values()) - { - indexer.rollback(); - } - } - finally - { - activeIndexersInGlobalTx.remove(xid); - } - } - - public boolean setTransactionTimeout(int timeout) throws XAException - { - this.timeout = timeout; - return true; - } - - public void start(Xid xid, int flag) throws XAException - { - Map active = activeIndexersInGlobalTx.get(xid); - Map suspended = suspendedIndexersInGlobalTx.get(xid); - if (flag == XAResource.TMJOIN) - { - // must be active - if ((active != null) && (suspended == null)) - { - return; - } - else - { - throw new XAException("Trying to rejoin transaction in an invalid state"); - } - - } - else if (flag == XAResource.TMRESUME) - { - // must be suspended - if ((active == null) && (suspended != null)) - { - suspendedIndexersInGlobalTx.remove(xid); - activeIndexersInGlobalTx.put(xid, suspended); - return; - } - else - { - throw new XAException("Trying to rejoin transaction in an invalid state"); - } - - } - else if (flag == XAResource.TMNOFLAGS) - { - if ((active == null) && (suspended == null)) - { - return; - } - else - { - throw new XAException("Trying to start an existing or suspended transaction"); - } - } - else - { - throw new XAException("Unkown flags for start " + flag); - } - - } - - /* - * Thread local support for transactions - */ - - /** - * Commit the transaction - */ - - @SuppressWarnings("unchecked") - public void commit() throws IndexerException - { - Map indexers = null; - try - { - indexers = (Map) AlfrescoTransactionSupport.getResource(indexersKey); - if (indexers != null) - { - for (LuceneIndexer indexer : indexers.values()) - { - if (destroyed && Thread.currentThread().isDaemon()) - { - rollback(); - throw new IndexerException("Destroyed .."); - } - else - { - try - { - indexer.commit(); - } - catch (IndexerException e) - { - rollback(); - throw e; - } - } - } - } - } - finally - { - if (indexers != null) - { - indexers.clear(); - AlfrescoTransactionSupport.unbindResource(indexersKey); - } - } - } - - /** - * Prepare the transaction TODO: Store prepare results - * - * @return - the tx code - */ - @SuppressWarnings("unchecked") - public int prepare() throws IndexerException - { - boolean isPrepared = true; - boolean isModified = false; - Map indexers = (Map) AlfrescoTransactionSupport.getResource(indexersKey); - if (indexers != null) - { - for (LuceneIndexer indexer : indexers.values()) - { - try - { - isModified |= indexer.isModified(); - indexer.prepare(); - } - catch (IndexerException e) - { - isPrepared = false; - throw new IndexerException("Failed to prepare: requires rollback", e); - } - } - } - if (isPrepared) - { - if (isModified) - { - return XAResource.XA_OK; - } - else - { - return XAResource.XA_RDONLY; - } - } - else - { - throw new IndexerException("Failed to prepare: requires rollback"); - } - } - - /** - * Roll back the transaction - */ - @SuppressWarnings("unchecked") - public void rollback() - { - Map indexers = (Map) AlfrescoTransactionSupport.getResource(indexersKey); - - if (indexers != null) - { - for (LuceneIndexer indexer : indexers.values()) - { - try - { - indexer.rollback(); - } - catch (IndexerException e) - { - - } - } - indexers.clear(); - AlfrescoTransactionSupport.unbindResource(indexersKey); - } - } - - @SuppressWarnings("unchecked") - public void flush() - { - // TODO: Needs fixing if we expose the indexer in JTA - Map indexers = (Map) AlfrescoTransactionSupport.getResource(indexersKey); - - if (indexers != null) - { - for (LuceneIndexer indexer : indexers.values()) - { - indexer.flushPending(); - } - } - } - - @Override - public String getIndexRootLocation() - { - return indexRootLocation; - } - - @Override - public int getIndexerBatchSize() - { - return indexerBatchSize; - } - - /** - * Set the batch six to use for background indexing - * - * @param indexerBatchSize int - */ - @Override - public void setIndexerBatchSize(int indexerBatchSize) - { - this.indexerBatchSize = indexerBatchSize; - } - - /** - * Get the directory where any lock files are written (by default there are none) - * - * @return - the path to the directory - */ - public String getLockDirectory() - { - return lockDirectory; - } - - public void setLockDirectory(String lockDirectory) - { - this.lockDirectory = lockDirectory; - // Set the lucene lock file via System property - // org.apache.lucene.lockDir - System.setProperty("org.apache.lucene.lockDir", lockDirectory); - // Make sure the lock directory exists - File lockDir = new File(lockDirectory); - if (!lockDir.exists()) - { - lockDir.mkdirs(); - } - // clean out any existing locks when we start up - - File[] children = lockDir.listFiles(); - if (children != null) - { - for (int i = 0; i < children.length; i++) - { - File child = children[i]; - if (child.isFile()) - { - if (child.exists() && !child.delete() && child.exists()) - { - throw new IllegalStateException("Failed to delete " + child); - } - } - } - } - } - - @Override - public int getQueryMaxClauses() - { - return queryMaxClauses; - } - - /** - * Set the max number of queries in a llucen boolean query - * - * @param queryMaxClauses int - */ - @Override - public void setQueryMaxClauses(int queryMaxClauses) - { - this.queryMaxClauses = queryMaxClauses; - BooleanQuery.setMaxClauseCount(this.queryMaxClauses); - } - - /** - * Set the lucene write lock timeout - * - * @param timeout long - */ - @Override - public void setWriteLockTimeout(long timeout) - { - this.writeLockTimeout = timeout; - } - - /** - * Set the lucene commit lock timeout (no longer used with lucene 2.1) - * - * @param timeout long - */ - @Override - public void setCommitLockTimeout(long timeout) - { - this.commitLockTimeout = timeout; - } - - /** - * Get the commit lock timout. - * - * @return - the timeout - */ - @Override - public long getCommitLockTimeout() - { - return commitLockTimeout; - } - - /** - * Get the write lock timeout - * - * @return - the timeout in ms - */ - @Override - public long getWriteLockTimeout() - { - return writeLockTimeout; - } - - /** - * Set the lock poll interval in ms - * - * @param time long - */ - @Override - public void setLockPollInterval(long time) - { - Lock.LOCK_POLL_INTERVAL = time; - } - - /** - * Get the max number of tokens in the field - * - * @return - the max tokens considered. - */ - @Override - public int getIndexerMaxFieldLength() - { - return indexerMaxFieldLength; - } - - /** - * Set the max field length. - * - * @param indexerMaxFieldLength int - */ - @Override - public void setIndexerMaxFieldLength(int indexerMaxFieldLength) - { - this.indexerMaxFieldLength = indexerMaxFieldLength; - } - - public ThreadPoolExecutor getThreadPoolExecutor() - { - return this.threadPoolExecutor; - } - - public void setThreadPoolExecutor(ThreadPoolExecutor threadPoolExecutor) - { - this.threadPoolExecutor = threadPoolExecutor; - } - - /** - * @return the useInMemorySort - */ - public boolean getUseInMemorySort() - { - return useInMemorySort; - } - - /** - * @param useInMemorySort the useInMemorySort to set - */ - public void setUseInMemorySort(boolean useInMemorySort) - { - this.useInMemorySort = useInMemorySort; - } - - /** - * @return the maxRawResultSetSizeForInMemorySort - */ - public int getMaxRawResultSetSizeForInMemorySort() - { - return maxRawResultSetSizeForInMemorySort; - } - - /** - * @param maxRawResultSetSizeForInMemorySort the maxRawResultSetSizeForInMemorySort to set - */ - public void setMaxRawResultSetSizeForInMemorySort(int maxRawResultSetSizeForInMemorySort) - { - this.maxRawResultSetSizeForInMemorySort = maxRawResultSetSizeForInMemorySort; - } - - /** - * This component is able to safely perform backups of the Lucene indexes while the server is running. - *

- * It can be run directly by calling the {@link #backup() } method, but the convenience {@link LuceneIndexBackupJob} - * can be used to call it as well. - * - * @author Derek Hulley - */ - public static class LuceneIndexBackupComponent /* implements InitializingBean */ - { - ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); - - boolean executing = false; - - private static String BACKUP_TEMP_NAME = ".indexbackup_temp"; - - private TransactionService transactionService; - - private Set factories; - - private NodeService nodeService; - - private String targetLocation; - - private boolean checkConfiguration = true; - - /** - * Default constructor - */ - public LuceneIndexBackupComponent() - { - } - - /** - * If false do not check the index configuration. - * - * @param checkConfiguration boolean - */ - public void setCheckConfiguration(boolean checkConfiguration) - { - this.checkConfiguration = checkConfiguration; - } - - /** - * Provides transactions in which to perform the work - * - * @param transactionService TransactionService - */ - public void setTransactionService(TransactionService transactionService) - { - this.transactionService = transactionService; - } - - /** - * Set the Lucene index factory that will be used to control the index locks - * - * @param factories - * the index factories - */ - public void setFactories(Set factories) - { - this.factories = factories; - } - - /** - * Used to retrieve the stores - * - * @param nodeService - * the node service - */ - public void setNodeService(NodeService nodeService) - { - this.nodeService = nodeService; - } - - /** - * Set the directory to which the backup will be copied - * - * @param targetLocation - * the backup directory - */ - public void setTargetLocation(String targetLocation) - { - this.targetLocation = targetLocation; - } - - /** - * Backup the Lucene indexes - */ - public void backup() - { - rwLock.readLock().lock(); - try - { - if (executing) - { - return; - } - } - finally - { - rwLock.readLock().unlock(); - } - - rwLock.writeLock().lock(); - try - { - if (executing) - { - return; - } - executing = true; - } - finally - { - rwLock.writeLock().unlock(); - } - - try - { - RetryingTransactionCallback backupWork = new RetryingTransactionCallback() - { - public Object execute() throws Exception - { - backupImpl(); - return null; - } - }; - transactionService.getRetryingTransactionHelper().doInTransaction(backupWork); - } - finally - { - rwLock.writeLock().lock(); - try - { - executing = false; - } - finally - { - rwLock.writeLock().unlock(); - } - } - } - - private void backupImpl() - { - // create the location to copy to - File targetDir = new File(targetLocation); - if (targetDir.exists() && !targetDir.isDirectory()) - { - throw new AlfrescoRuntimeException("Target location is a file and not a directory: " + targetDir); - } - File targetParentDir = targetDir.getParentFile(); - if (targetParentDir == null) - { - throw new AlfrescoRuntimeException("Target location may not be a root directory: " + targetDir); - } - File tempDir = new File(targetParentDir, BACKUP_TEMP_NAME); - - for (LuceneIndexerAndSearcher factory : factories) - { - ReadOnlyWork backupWork = new BackUpReadOnlyWork(factory, tempDir, targetDir); - - if (logger.isDebugEnabled()) - { - logger.debug("Backing up Lucene indexes: \n" + " Target directory: " + targetDir); - } - - factory.doReadOnly(backupWork); - - if (logger.isDebugEnabled()) - { - logger.debug("Backed up Lucene indexes: \n" + " Target directory: " + targetDir); - } - } - } - - static class BackUpReadOnlyWork implements ReadOnlyWork - { - LuceneIndexerAndSearcher factory; - - File tempDir; - - File targetDir; - - BackUpReadOnlyWork(LuceneIndexerAndSearcher factory, File tempDir, File targetDir) - { - this.factory = factory; - this.tempDir = tempDir; - this.targetDir = targetDir; - } - - public Object doWork() - { - try - { - File indexRootDir = new File(factory.getIndexRootLocation()); - // perform the copy - backupDirectory(indexRootDir, tempDir, targetDir); - return null; - } - catch (Throwable e) - { - throw new AlfrescoRuntimeException("Failed to copy Lucene index root: \n" - + " Index root: " + factory.getIndexRootLocation() + "\n" + " Target: " + targetDir, e); - } - } - - /** - * Makes a backup of the source directory via a temporary folder. - */ - private void backupDirectory(File sourceDir, File tempDir, File targetDir) throws Exception - { - if (!sourceDir.exists()) - { - // there is nothing to copy - return; - } - // delete the files from the temp directory - if (tempDir.exists()) - { - deleteDirectory(tempDir); - if (tempDir.exists()) - { - throw new AlfrescoRuntimeException("Temp directory exists and cannot be deleted: " + tempDir); - } - } - // copy to the temp directory - copyDirectory(sourceDir, tempDir, true); - // check that the temp directory was created - if (!tempDir.exists()) - { - throw new AlfrescoRuntimeException("Copy to temp location failed"); - } - // delete the target directory - deleteDirectory(targetDir); - if (targetDir.exists()) - { - throw new AlfrescoRuntimeException("Failed to delete older files from target location"); - } - // rename the temp to be the target - tempDir.renameTo(targetDir); - // make sure the rename worked - if (!targetDir.exists()) - { - throw new AlfrescoRuntimeException("Failed to rename temporary directory to target backup directory"); - } - } - - /** - * Note files can alter due to background processes so file not found is Ok - * - * @param srcDir File - * @param destDir File - * @param preserveFileDate boolean - * @throws IOException - */ - private void copyDirectory(File srcDir, File destDir, boolean preserveFileDate) throws IOException - { - if (destDir.exists()) - { - throw new IOException("Destination should be created from clean"); - } - else - { - if (!destDir.mkdirs()) - { - throw new IOException("Destination '" + destDir + "' directory cannot be created"); - } - if (preserveFileDate) - { - // OL if file not found so does not need to check - destDir.setLastModified(srcDir.lastModified()); - } - } - if (!destDir.canWrite()) - { - throw new IOException("No acces to destination directory" + destDir); - } - - File[] files = srcDir.listFiles(); - if (files != null) - { - for (int i = 0; i < files.length; i++) - { - File currentCopyTarget = new File(destDir, files[i].getName()); - if (files[i].isDirectory()) - { - // Skip any temp index file - if (files[i].getName().equals(tempDir.getName())) - { - // skip any temp back up directories - } - else if (files[i].getName().equals(targetDir.getName())) - { - // skip any back up directories - } - else - { - copyDirectory(files[i], currentCopyTarget, preserveFileDate); - } - } - else - { - copyFile(files[i], currentCopyTarget, preserveFileDate); - } - } - } - else - { - if (logger.isDebugEnabled()) - { - logger.debug("Skipping transient directory " + srcDir); - } - } - } - - private void copyFile(File srcFile, File destFile, boolean preserveFileDate) throws IOException - { - try - { - if (destFile.exists()) - { - throw new IOException("File shoud not exist " + destFile); - } - - FileInputStream input = new FileInputStream(srcFile); - try - { - FileOutputStream output = new FileOutputStream(destFile); - try - { - copy(input, output); - } - finally - { - try - { - output.close(); - } - catch (IOException io) - { - - } - } - } - finally - { - try - { - input.close(); - } - catch (IOException io) - { - - } - } - - // check copy - if (srcFile.length() != destFile.length()) - { - throw new IOException("Failed to copy full from '" + srcFile + "' to '" + destFile + "'"); - } - if (preserveFileDate) - { - destFile.setLastModified(srcFile.lastModified()); - } - } - catch (FileNotFoundException fnfe) - { - // ignore as files can go - if (logger.isDebugEnabled()) - { - logger.debug("Skipping transient file " + srcFile); - } - } - } - - public int copy(InputStream input, OutputStream output) throws IOException - { - byte[] buffer = new byte[2048 * 4]; - int count = 0; - int n = 0; - while ((n = input.read(buffer)) != -1) - { - output.write(buffer, 0, n); - count += n; - } - return count; - } - - public void deleteDirectory(File directory) throws IOException - { - if (!directory.exists()) - { - return; - } - if (!directory.isDirectory()) - { - throw new IllegalArgumentException("Not a directory " + directory); - } - - File[] files = directory.listFiles(); - if (files == null) - { - throw new IOException("Failed to delete director - no access" + directory); - } - - for (int i = 0; i < files.length; i++) - { - File file = files[i]; - - if (file.isDirectory()) - { - deleteDirectory(file); - } - else - { - if (!file.delete()) - { - throw new IOException("Unable to delete file: " + file); - } - } - } - - if (!directory.delete()) - { - throw new IOException("Unable to delete directory " + directory); - } - } - - } - - public void afterPropertiesSetXXX() throws Exception - { - RetryingTransactionCallback backupWork = new RetryingTransactionCallback() - { - public Object execute() throws Exception - { - File targetDir = new File(targetLocation).getCanonicalFile(); - - List stores; - try - { - stores = nodeService.getStores(); - } - catch (Exception e) - { - return null; - } - Set protocols = new HashSet(); - protocols.add(StoreRef.PROTOCOL_ARCHIVE); - protocols.add(StoreRef.PROTOCOL_WORKSPACE); - protocols.add("locks"); - for (StoreRef store : stores) - { - protocols.add(store.getProtocol()); - } - - for (LuceneIndexerAndSearcher factory : factories) - { - File indexRootDir = new File(factory.getIndexRootLocation()).getCanonicalFile(); - - if (indexRootDir.getCanonicalPath().startsWith(targetDir.getCanonicalPath())) - { - throw new IllegalArgumentException("Backup directory can not contain or be an index directory"); - } - if (targetDir.getCanonicalPath().startsWith(indexRootDir.getCanonicalPath())) - { - for (String name : protocols) - { - File test = new File(indexRootDir, name); - if (targetDir.getCanonicalPath().startsWith(test.getCanonicalPath())) - { - throw new IllegalArgumentException("Backup directory can not be in index directory and match a store protocol name " + targetDir); - } - } - } - // if the back up directory exists make sure it only contains directories that are store - // protocols - - if (targetDir.exists()) - { - for (File file : targetDir.listFiles()) - { - if (file.isFile()) - { - throw new IllegalArgumentException("Existing index backup does not look like the expected structure. It constains a file " - + file.getCanonicalPath()); - } - if (!protocols.contains(file.getName())) - { - throw new IllegalArgumentException( - "Existing index backup does not look like the expected structure. It constains a directory with a name that does not match a store protocol " - + file.getCanonicalPath()); - - } - } - } - - } - return null; - } - }; - - if (checkConfiguration) - { - transactionService.getRetryingTransactionHelper().doInTransaction(backupWork, true); - } - - } - } - - /** - * Job that lock uses the {@link LuceneIndexBackupComponent} to perform safe backups of the Lucene indexes. - * - * @author Derek Hulley - */ - public static class LuceneIndexBackupJob implements Job - { - - /** KEY_LUCENE_INDEX_BACKUP_COMPONENT = 'luceneIndexBackupComponent' */ - public static final String KEY_LUCENE_INDEX_BACKUP_COMPONENT = "luceneIndexBackupComponent"; - - /** - * Locks the Lucene indexes and copies them to a backup location - */ - public void execute(JobExecutionContext context) throws JobExecutionException - { - JobDataMap jobData = context.getJobDetail().getJobDataMap(); - LuceneIndexBackupComponent backupComponent = (LuceneIndexBackupComponent) jobData.get(KEY_LUCENE_INDEX_BACKUP_COMPONENT); - if (backupComponent == null) - { - throw new JobExecutionException("Missing job data: " + KEY_LUCENE_INDEX_BACKUP_COMPONENT); - } - // perform the backup - backupComponent.backup(); - } - } - - @Override - public MLAnalysisMode getDefaultMLIndexAnalysisMode() - { - return defaultMLIndexAnalysisMode; - } - - /** - * Set the ML analysis mode at index time. - * - * @param mode MLAnalysisMode - */ - @Override - public void setDefaultMLIndexAnalysisMode(MLAnalysisMode mode) - { - // defaultMLIndexAnalysisMode = MLAnalysisMode.getMLAnalysisMode(mode); - defaultMLIndexAnalysisMode = mode; - } - - @Override - public MLAnalysisMode getDefaultMLSearchAnalysisMode() - { - return defaultMLSearchAnalysisMode; - } - - /** - * Set the ML analysis mode at search time - * - * @param mode MLAnalysisMode - */ - @Override - public void setDefaultMLSearchAnalysisMode(MLAnalysisMode mode) - { - // defaultMLSearchAnalysisMode = MLAnalysisMode.getMLAnalysisMode(mode); - defaultMLSearchAnalysisMode = mode; - } - - @Override - public int getMaxDocIdCacheSize() - { - return maxDocIdCacheSize; - } - - @Override - public void setMaxDocIdCacheSize(int maxDocIdCacheSize) - { - this.maxDocIdCacheSize = maxDocIdCacheSize; - } - - @Override - public int getMaxDocsForInMemoryMerge() - { - return maxDocsForInMemoryMerge; - } - - @Override - public void setMaxDocsForInMemoryMerge(int maxDocsForInMemoryMerge) - { - this.maxDocsForInMemoryMerge = maxDocsForInMemoryMerge; - } - - @Override - public int getMaxDocumentCacheSize() - { - return maxDocumentCacheSize; - } - - @Override - public void setMaxDocumentCacheSize(int maxDocumentCacheSize) - { - this.maxDocumentCacheSize = maxDocumentCacheSize; - } - - @Override - public int getMaxIsCategoryCacheSize() - { - return maxIsCategoryCacheSize; - } - - @Override - public void setMaxIsCategoryCacheSize(int maxIsCategoryCacheSize) - { - this.maxIsCategoryCacheSize = maxIsCategoryCacheSize; - } - - @Override - public int getMaxLinkAspectCacheSize() - { - return maxLinkAspectCacheSize; - } - - @Override - public void setMaxLinkAspectCacheSize(int maxLinkAspectCacheSize) - { - this.maxLinkAspectCacheSize = maxLinkAspectCacheSize; - } - - @Override - public int getMaxParentCacheSize() - { - return maxParentCacheSize; - } - - @Override - public void setMaxParentCacheSize(int maxParentCacheSize) - { - this.maxParentCacheSize = maxParentCacheSize; - } - - @Override - public int getMaxPathCacheSize() - { - return maxPathCacheSize; - } - - @Override - public void setMaxPathCacheSize(int maxPathCacheSize) - { - this.maxPathCacheSize = maxPathCacheSize; - } - - @Override - public int getMaxTypeCacheSize() - { - return maxTypeCacheSize; - } - - @Override - public void setMaxTypeCacheSize(int maxTypeCacheSize) - { - this.maxTypeCacheSize = maxTypeCacheSize; - } - - @Override - public int getMergerMaxMergeDocs() - { - return mergerMaxMergeDocs; - } - - @Override - public void setMergerMaxMergeDocs(int mergerMaxMergeDocs) - { - this.mergerMaxMergeDocs = mergerMaxMergeDocs; - } - - @Override - public int getMergerMergeFactor() - { - return mergerMergeFactor; - } - - @Override - public void setMergerMergeFactor(int mergerMergeFactor) - { - this.mergerMergeFactor = mergerMergeFactor; - } - - @Override - public int getMergerMaxBufferedDocs() - { - return mergerMaxBufferedDocs; - } - - @Override - public void setMergerMaxBufferedDocs(int mergerMaxBufferedDocs) - { - this.mergerMaxBufferedDocs = mergerMaxBufferedDocs; - } - - @Override - public int getMergerTargetIndexCount() - { - return mergerTargetIndexCount; - } - - @Override - public void setMergerTargetIndexCount(int mergerTargetIndexCount) - { - this.mergerTargetIndexCount = mergerTargetIndexCount; - } - - @Override - public int getMergerTargetOverlayCount() - { - return mergerTargetOverlayCount; - } - - @Override - public void setMergerTargetOverlayCount(int mergerTargetOverlayCount) - { - this.mergerTargetOverlayCount = mergerTargetOverlayCount; - } - - @Override - public int getMergerTargetOverlaysBlockingFactor() - { - return mergerTargetOverlaysBlockingFactor; - } - - @Override - public void setMergerTargetOverlaysBlockingFactor(int mergerTargetOverlaysBlockingFactor) - { - this.mergerTargetOverlaysBlockingFactor = mergerTargetOverlaysBlockingFactor; - } - - @Override - public boolean getFairLocking() - { - return this.fairLocking; - } - - @Override - public void setFairLocking(boolean fairLocking) - { - this.fairLocking = fairLocking; - } - - @Override - public int getTermIndexInterval() - { - return termIndexInterval; - } - - @Override - public void setTermIndexInterval(int termIndexInterval) - { - this.termIndexInterval = termIndexInterval; - } - - @Override - public boolean getUseNioMemoryMapping() - { - return useNioMemoryMapping; - } - - @Override - public void setUseNioMemoryMapping(boolean useNioMemoryMapping) - { - this.useNioMemoryMapping = useNioMemoryMapping; - } - - @Override - public int getWriterMaxMergeDocs() - { - return writerMaxMergeDocs; - } - - @Override - public void setWriterMaxMergeDocs(int writerMaxMergeDocs) - { - this.writerMaxMergeDocs = writerMaxMergeDocs; - } - - @Override - public int getWriterMergeFactor() - { - return writerMergeFactor; - } - - @Override - public void setWriterMergeFactor(int writerMergeFactor) - { - this.writerMergeFactor = writerMergeFactor; - } - - @Override - public int getWriterMaxBufferedDocs() - { - return writerMaxBufferedDocs; - } - - @Override - public void setWriterMaxBufferedDocs(int writerMaxBufferedDocs) - { - this.writerMaxBufferedDocs = writerMaxBufferedDocs; - } - - @Override - public boolean isCacheEnabled() - { - return cacheEnabled; - } - - @Override - public void setCacheEnabled(boolean cacheEnabled) - { - this.cacheEnabled = cacheEnabled; - } - - @Override - public boolean getPostSortDateTime() - { - return postSortDateTime; - } - - @Override - public void setPostSortDateTime(boolean postSortDateTime) - { - this.postSortDateTime = postSortDateTime; - } - - /** - * @return the maxDocsForInMemoryIndex - */ - @Override - public int getMaxDocsForInMemoryIndex() - { - return maxDocsForInMemoryIndex; - } - - /** - * @param maxDocsForInMemoryIndex - * the maxDocsForInMemoryIndex to set - */ - @Override - public void setMaxDocsForInMemoryIndex(int maxDocsForInMemoryIndex) - { - this.maxDocsForInMemoryIndex = maxDocsForInMemoryIndex; - } - - /** - * @return the maxRamInMbForInMemoryMerge - */ - @Override - public double getMaxRamInMbForInMemoryMerge() - { - return maxRamInMbForInMemoryMerge; - } - - /** - * @param maxRamInMbForInMemoryMerge - * the maxRamInMbForInMemoryMerge to set - */ - @Override - public void setMaxRamInMbForInMemoryMerge(double maxRamInMbForInMemoryMerge) - { - this.maxRamInMbForInMemoryMerge = maxRamInMbForInMemoryMerge; - } - - /** - * @return the maxRamInMbForInMemoryIndex - */ - @Override - public double getMaxRamInMbForInMemoryIndex() - { - return maxRamInMbForInMemoryIndex; - } - - /** - * @param maxRamInMbForInMemoryIndex - * the maxRamInMbForInMemoryIndex to set - */ - @Override - public void setMaxRamInMbForInMemoryIndex(double maxRamInMbForInMemoryIndex) - { - this.maxRamInMbForInMemoryIndex = maxRamInMbForInMemoryIndex; - } - - /** - * @return the mergerRamBufferSizeMb - */ - @Override - public double getMergerRamBufferSizeMb() - { - return mergerRamBufferSizeMb; - } - - /** - * @param mergerRamBufferSizeMb - * the mergerRamBufferSizeMb to set - */ - @Override - public void setMergerRamBufferSizeMb(double mergerRamBufferSizeMb) - { - this.mergerRamBufferSizeMb = mergerRamBufferSizeMb; - } - - /** - * @return the writerRamBufferSizeMb - */ - @Override - public double getWriterRamBufferSizeMb() - { - return writerRamBufferSizeMb; - } - - /** - * @param writerRamBufferSizeMb - * the writerRamBufferSizeMb to set - */ - @Override - public void setWriterRamBufferSizeMb(double writerRamBufferSizeMb) - { - this.writerRamBufferSizeMb = writerRamBufferSizeMb; - } - - - - - @Override - public boolean isContentIndexingEnabled() - { - return contentIndexingEnabled; - } - - @Override - public void setContentIndexingEnabled(boolean contentIndexingEnabled) - { - this.contentIndexingEnabled = contentIndexingEnabled; - - } - - protected LuceneQueryLanguageSPI getQueryLanguage(String name) - { - return getQueryLanguages().get(name); - } - - protected abstract List getAllStores(); - - public R doReadOnly(ReadOnlyWork lockWork) - { - // get all the available stores - List storeRefs = getAllStores(); - - IndexInfo.LockWork currentLockWork = null; - - for (int i = storeRefs.size() - 1; i >= 0; i--) - { - StoreRef currentStore = storeRefs.get(i); - - if (currentLockWork == null) - { - currentLockWork = new CoreReadOnlyWork(getIndexer(currentStore), lockWork); - } - else - { - currentLockWork = new NestingReadOnlyWork(getIndexer(currentStore), currentLockWork); - } - } - - if (currentLockWork != null) - { - try - { - return currentLockWork.doWork(); - } - catch (Throwable exception) - { - - // Re-throw the exception - if (exception instanceof RuntimeException) - { - throw (RuntimeException) exception; - } - else - { - throw new RuntimeException("Error during run with lock.", exception); - } - } - - } - else - { - return null; - } - } - - private static class NestingReadOnlyWork implements IndexInfo.LockWork - { - IndexInfo.LockWork lockWork; - - LuceneIndexer indexer; - - NestingReadOnlyWork(LuceneIndexer indexer, IndexInfo.LockWork lockWork) - { - this.indexer = indexer; - this.lockWork = lockWork; - } - - public R doWork() throws Exception - { - return indexer.doReadOnly(lockWork); - } - - public boolean canRetry() - { - return false; - } - } - - private static class CoreReadOnlyWork implements IndexInfo.LockWork - { - ReadOnlyWork lockWork; - - LuceneIndexer indexer; - - CoreReadOnlyWork(LuceneIndexer indexer, ReadOnlyWork lockWork) - { - this.indexer = indexer; - this.lockWork = lockWork; - } - - public R doWork() throws Exception - { - return indexer.doReadOnly(new IndexInfo.LockWork() - { - public R doWork() - { - try - { - return lockWork.doWork(); - } - catch (Throwable exception) - { - - // Re-throw the exception - if (exception instanceof RuntimeException) - { - throw (RuntimeException) exception; - } - else - { - throw new RuntimeException("Error during run with lock.", exception); - } - } - } - - public boolean canRetry() - { - return false; - } - }); - } - - public boolean canRetry() - { - return false; - } - } - - public static void main(String[] args) throws IOException - { - // delete a directory .... - if (args.length != 1) - { - return; - } - File file = new File(args[0]); - deleteDirectory(file); - } - - public static void deleteDirectory(File directory) throws IOException - { - if (!directory.exists()) - { - return; - } - if (!directory.isDirectory()) - { - throw new IllegalArgumentException("Not a directory " + directory); - } - - File[] files = directory.listFiles(); - if (files == null) - { - throw new IOException("Failed to delete director - no access" + directory); - } - - for (int i = 0; i < files.length; i++) - { - File file = files[i]; - - System.out.println("."); - // System.out.println("Deleting "+file.getCanonicalPath()); - if (file.isDirectory()) - { - deleteDirectory(file); - } - else - { - if (!file.delete()) - { - throw new IOException("Unable to delete file: " + file); - } - } - } - - if (!directory.delete()) - { - throw new IOException("Unable to delete directory " + directory); - } - } - - @Override - public void destroy() throws Exception - { - IndexInfo.destroy(); - destroyed = true; - } - - -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java b/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java deleted file mode 100644 index b39897ecfb..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/AbstractLuceneIndexerImpl.java +++ /dev/null @@ -1,814 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.ListIterator; -import java.util.Set; - -import javax.transaction.Status; -import javax.transaction.xa.XAResource; - -import org.alfresco.repo.node.NodeBulkLoader; -import org.alfresco.repo.search.Indexer; -import org.alfresco.repo.search.IndexerException; -import org.alfresco.repo.search.impl.lucene.index.TransactionStatus; -import org.alfresco.repo.transaction.RetryingTransactionHelper.RetryingTransactionCallback; -import org.alfresco.service.cmr.repository.InvalidNodeRefException; -import org.alfresco.service.transaction.TransactionService; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.springframework.dao.ConcurrencyFailureException; - -/** - * Common support for indexing across implementations - * - * @author andyh - * @param - - * the type used to generate the key in the index file - */ -public abstract class AbstractLuceneIndexerImpl extends AbstractLuceneBase implements Indexer -{ - /** - * Enum for indexing actions against a node - */ - protected enum Action - { - /** - * An index - */ - INDEX, - /** - * A reindex - */ - REINDEX, - /** - * A delete - */ - DELETE, - /** - * A cascaded reindex (ensures directory structre is ok) - */ - CASCADEREINDEX - } - - protected enum IndexUpdateStatus - { - /** - * Inde is unchanged - */ - UNMODIFIED, - /** - * Index is being changein in TX - */ - SYNCRONOUS, - /** - * Index is eiong changed by a background upate - */ - ASYNCHRONOUS; - } - - protected enum FTSStatus {New, Dirty, Clean}; - - protected long docs; - - // An indexer with read through activated can only see already-committed documents in the database. Useful when - // reindexing lots of old documents and not wanting to pollute the caches with stale versions of nodes. - private boolean isReadThrough; - - protected TransactionService transactionService; - protected NodeBulkLoader bulkLoader; - - public void setReadThrough(boolean isReadThrough) - { - this.isReadThrough = isReadThrough; - } - - public void setTransactionService(TransactionService transactionService) - { - this.transactionService = transactionService; - } - - /** - * @param bulkLoader object to provide node loading options - */ - public void setBulkLoader(NodeBulkLoader bulkLoader) - { - this.bulkLoader = bulkLoader; - } - - protected static class Command - { - S ref; - - Action action; - - Command(S ref, Action action) - { - this.ref = ref; - this.action = action; - } - - public String toString() - { - StringBuffer buffer = new StringBuffer(); - if (action == Action.INDEX) - { - buffer.append("Index "); - } - else if (action == Action.DELETE) - { - buffer.append("Delete "); - } - else if (action == Action.REINDEX) - { - buffer.append("Reindex "); - } - else - { - buffer.append("Unknown ... "); - } - buffer.append(ref); - return buffer.toString(); - } - - } - - /** - * No transform available - */ - public static final String NOT_INDEXED_NO_TRANSFORMATION = "nint"; - - /** - * Tranfrom failed - */ - public static final String NOT_INDEXED_TRANSFORMATION_FAILED = "nitf"; - - /** - * No content - */ - public static final String NOT_INDEXED_CONTENT_MISSING = "nicm"; - - /** - * No type conversion - */ - public static final String NOT_INDEXED_NO_TYPE_CONVERSION = "nintc"; - - /** - * Logger - */ - private static Log s_logger = LogFactory.getLog(AbstractLuceneIndexerImpl.class); - - protected static Set deletePrimary(Collection nodeRefs, IndexReader reader, boolean delete) - throws LuceneIndexException - { - - Set refs = new LinkedHashSet(); - - for (String nodeRef : nodeRefs) - { - - try - { - TermDocs td = reader.termDocs(new Term("PRIMARYPARENT", nodeRef)); - while (td.next()) - { - int doc = td.doc(); - Document document = reader.document(doc); - String[] ids = document.getValues("ID"); - refs.add(ids[ids.length - 1]); - if (delete) - { - reader.deleteDocument(doc); - } - } - td.close(); - } - catch (IOException e) - { - throw new LuceneIndexException("Failed to delete node by primary parent for " + nodeRef, e); - } - } - - return refs; - - } - - protected static Set deleteReference(Collection nodeRefs, IndexReader reader, boolean delete) - throws LuceneIndexException - { - - Set refs = new LinkedHashSet(); - - for (String nodeRef : nodeRefs) - { - - try - { - TermDocs td = reader.termDocs(new Term("PARENT", nodeRef)); - while (td.next()) - { - int doc = td.doc(); - Document document = reader.document(doc); - String[] ids = document.getValues("ID"); - refs.add(ids[ids.length - 1]); - if (delete) - { - reader.deleteDocument(doc); - } - } - td.close(); - } - catch (IOException e) - { - throw new LuceneIndexException("Failed to delete node by parent for " + nodeRef, e); - } - } - - return refs; - - } - - protected static Set deleteContainerAndBelow(String nodeRef, IndexReader reader, boolean delete, - boolean cascade) throws LuceneIndexException - { - Set refs = new LinkedHashSet(); - - try - { - if (delete) - { - reader.deleteDocuments(new Term("ID", nodeRef)); - } - refs.add(nodeRef); - if (cascade) - { - TermDocs td = reader.termDocs(new Term("ANCESTOR", nodeRef)); - while (td.next()) - { - int doc = td.doc(); - Document document = reader.document(doc); - String[] ids = document.getValues("ID"); - refs.add(ids[ids.length - 1]); - if (delete) - { - reader.deleteDocument(doc); - } - } - td.close(); - } - } - catch (IOException e) - { - throw new LuceneIndexException("Failed to delete container and below for " + nodeRef, e); - } - return refs; - } - - protected boolean locateContainer(String nodeRef, IndexReader reader) - { - boolean found = false; - try - { - TermDocs td = reader.termDocs(new Term("ID", nodeRef)); - while (td.next()) - { - int doc = td.doc(); - Document document = reader.document(doc); - if (document.getField("ISCONTAINER") != null) - { - found = true; - break; - } - } - td.close(); - } - catch (IOException e) - { - throw new LuceneIndexException("Failed to delete container and below for " + nodeRef, e); - } - return found; - } - - /** the maximum transformation time to allow atomically, defaulting to 20ms */ - protected long maxAtomicTransformationTime = 20; - - /** - * A list of all deletions we have made - at merge these deletions need to be made against the main index. TODO: - * Consider if this information needs to be persisted for recovery - */ - protected Set deletions = new LinkedHashSet(); - - /** - * A list of cascading container deletions we have made - at merge these deletions need to be made against the main index. - */ - protected Set containerDeletions = new LinkedHashSet(); - - /** - * List of pending indexing commands. - */ - protected List> commandList = new ArrayList>(10000); - - /** - * Flag to indicte if we are doing an in transactional delta or a batch update to the index. If true, we are just - * fixing up non atomically indexed things from one or more other updates. - */ - protected IndexUpdateStatus indexUpdateStatus = IndexUpdateStatus.UNMODIFIED; - - /** - * Set the max time allowed to transform content atomically - * - * @param maxAtomicTransformationTime long - */ - public void setMaxAtomicTransformationTime(long maxAtomicTransformationTime) - { - this.maxAtomicTransformationTime = maxAtomicTransformationTime; - } - - /** - * Utility method to check we are in the correct state to do work Also keeps track of the dirty flag. - * - * @throws IndexerException - * @throws LuceneIndexException - */ - - protected void checkAbleToDoWork(IndexUpdateStatus indexUpdateStatus) - { - if (this.indexUpdateStatus == IndexUpdateStatus.UNMODIFIED) - { - this.indexUpdateStatus = indexUpdateStatus; - } - else if (this.indexUpdateStatus == indexUpdateStatus) - { - return; - } - else - { - throw new IndexerException("Can not mix FTS and transactional updates"); - } - - switch (getStatus()) - { - case UNKNOWN: - try - { - setStatus(TransactionStatus.ACTIVE); - } - catch (IOException e) - { - throw new LuceneIndexException("Failed to set TX active", e); - } - break; - case ACTIVE: - // OK - break; - default: - // All other states are a problem - throw new IndexerException(buildErrorString()); - } - } - - /** - * Utility method to report errors about invalid state. - * - * @return - an error based on status - */ - private String buildErrorString() - { - StringBuilder buffer = new StringBuilder(128); - buffer.append("The indexer is unable to accept more work: "); - switch (getStatus().getStatus()) - { - case Status.STATUS_COMMITTED: - buffer.append("The indexer has been committed"); - break; - case Status.STATUS_COMMITTING: - buffer.append("The indexer is committing"); - break; - case Status.STATUS_MARKED_ROLLBACK: - buffer.append("The indexer is marked for rollback"); - break; - case Status.STATUS_PREPARED: - buffer.append("The indexer is prepared to commit"); - break; - case Status.STATUS_PREPARING: - buffer.append("The indexer is preparing to commit"); - break; - case Status.STATUS_ROLLEDBACK: - buffer.append("The indexer has been rolled back"); - break; - case Status.STATUS_ROLLING_BACK: - buffer.append("The indexer is rolling back"); - break; - case Status.STATUS_UNKNOWN: - buffer.append("The indexer is in an unknown state"); - break; - default: - break; - } - return buffer.toString(); - } - - /** - * Commit this index - * - * @throws LuceneIndexException - */ - public void commit() throws LuceneIndexException - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Starting Commit"); - } - switch (getStatus().getStatus()) - { - case Status.STATUS_COMMITTING: - throw new LuceneIndexException("Unable to commit: Transaction is committing"); - case Status.STATUS_COMMITTED: - throw new LuceneIndexException("Unable to commit: Transaction is commited "); - case Status.STATUS_ROLLING_BACK: - throw new LuceneIndexException("Unable to commit: Transaction is rolling back"); - case Status.STATUS_ROLLEDBACK: - throw new LuceneIndexException("Unable to commit: Transaction is aleady rolled back"); - case Status.STATUS_MARKED_ROLLBACK: - throw new LuceneIndexException("Unable to commit: Transaction is marked for roll back"); - case Status.STATUS_PREPARING: - throw new LuceneIndexException("Unable to commit: Transaction is preparing"); - case Status.STATUS_ACTIVE: - // special case - commit from active - prepare(); - // drop through to do the commit; - default: - if (getStatus().getStatus() != Status.STATUS_PREPARED) - { - throw new LuceneIndexException("Index must be prepared to commit"); - } - try - { - setStatus(TransactionStatus.COMMITTING); - if (isModified()) - { - doCommit(); - } - setStatus(TransactionStatus.COMMITTED); - } - catch (LuceneIndexException e) - { - // If anything goes wrong we try and do a roll back - rollback(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Commit Failed", e); - } - throw new LuceneIndexException("Commit failed", e); - } - catch (Throwable t) - { - // If anything goes wrong we try and do a roll back - rollback(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Commit Failed", t); - } - throw new LuceneIndexException("Commit failed", t); - } - finally - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Ending Commit"); - } - - // Make sure we tidy up - // deleteDelta(); - } - break; - } - } - - /** - * Prepare to commit At the moment this makes sure we have all the locks TODO: This is not doing proper - * serialisation against the index as would a data base transaction. - * - * @return the tx state - * @throws LuceneIndexException - */ - public int prepare() throws LuceneIndexException - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Starting Prepare"); - } - switch (getStatus().getStatus()) - { - case Status.STATUS_COMMITTING: - throw new IndexerException("Unable to prepare: Transaction is committing"); - case Status.STATUS_COMMITTED: - throw new IndexerException("Unable to prepare: Transaction is commited "); - case Status.STATUS_ROLLING_BACK: - throw new IndexerException("Unable to prepare: Transaction is rolling back"); - case Status.STATUS_ROLLEDBACK: - throw new IndexerException("Unable to prepare: Transaction is aleady rolled back"); - case Status.STATUS_MARKED_ROLLBACK: - throw new IndexerException("Unable to prepare: Transaction is marked for roll back"); - case Status.STATUS_PREPARING: - throw new IndexerException("Unable to prepare: Transaction is already preparing"); - case Status.STATUS_PREPARED: - throw new IndexerException("Unable to prepare: Transaction is already prepared"); - default: - try - { - setStatus(TransactionStatus.PREPARING); - if (isModified()) - { - doPrepare(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Waiting to Finish Preparing"); - } - } - setStatus(TransactionStatus.PREPARED); - return isModified() ? XAResource.XA_OK : XAResource.XA_RDONLY; - } - catch (LuceneIndexException e) - { - setRollbackOnly(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Prepare Failed", e); - } - throw new LuceneIndexException("Index failed to prepare", e); - } - catch (Throwable t) - { - // If anything goes wrong we try and do a roll back - rollback(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Prepare Failed", t); - } - throw new LuceneIndexException("Prepared failed", t); - } - finally - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + " Ending Prepare"); - } - } - } - } - - /** - * Has this index been modified? - * - * @return true if modified - */ - public boolean isModified() - { - return indexUpdateStatus != IndexUpdateStatus.UNMODIFIED; - } - - /** - * Roll back the index changes (this just means they are never added) - * - * @throws LuceneIndexException - */ - public void rollback() throws LuceneIndexException - { - switch (getStatus().getStatus()) - { - - case Status.STATUS_COMMITTED: - throw new IndexerException("Unable to roll back: Transaction is committed "); - case Status.STATUS_ROLLING_BACK: - throw new IndexerException("Unable to roll back: Transaction is rolling back"); - case Status.STATUS_ROLLEDBACK: - throw new IndexerException("Unable to roll back: Transaction is already rolled back"); - case Status.STATUS_COMMITTING: - // Can roll back during commit - default: - try - { - setStatus(TransactionStatus.ROLLINGBACK); - doRollBack(); - setStatus(TransactionStatus.ROLLEDBACK); - } - catch (IOException e) - { - throw new LuceneIndexException("rollback failed ", e); - } - break; - } - } - - /** - * Mark this index for roll back only. This action can not be reversed. It will reject all other work and only allow - * roll back. - */ - public void setRollbackOnly() - { - switch (getStatus().getStatus()) - { - case Status.STATUS_COMMITTING: - throw new IndexerException("Unable to mark for rollback: Transaction is committing"); - case Status.STATUS_COMMITTED: - throw new IndexerException("Unable to mark for rollback: Transaction is committed"); - default: - try - { - doSetRollbackOnly(); - setStatus(TransactionStatus.MARKED_ROLLBACK); - } - catch (IOException e) - { - throw new LuceneIndexException("Set rollback only failed ", e); - } - break; - } - } - - protected abstract void doPrepare() throws IOException; - - protected abstract void doCommit() throws IOException; - - protected abstract void doRollBack() throws IOException; - - protected abstract void doSetRollbackOnly() throws IOException; - - protected T2 doInReadthroughTransaction(final RetryingTransactionCallback callback) - { - if (isReadThrough) - { - return transactionService.getRetryingTransactionHelper().doInTransaction( - new RetryingTransactionCallback() - { - @Override - public T2 execute() throws Throwable - { - // ALF-18383: Regression in Lucene indexing performance in 4.x - // We accept the loss of some performance in order to ensure accuracy - // Request clean node data - if (bulkLoader != null) - { - bulkLoader.setCheckNodeConsistency(); - } - try - { - return callback.execute(); - } - catch (InvalidNodeRefException e) - { - // Turn InvalidNodeRefExceptions into retryable exceptions. - throw new ConcurrencyFailureException( - "Possible cache integrity issue during reindexing", e); - } - - } - }, true, true); - } - else - { - try - { - return callback.execute(); - } - catch (RuntimeException e) - { - throw e; - } - catch (Error e) - { - throw e; - } - catch (Throwable e) - { - throw new RuntimeException(e); - } - } - } - - protected void index(T ref) throws LuceneIndexException - { - addCommand(new Command(ref, Action.INDEX)); - } - - protected void reindex(T ref, boolean cascadeReindexDirectories) throws LuceneIndexException - { - addCommand(new Command(ref, cascadeReindexDirectories ? Action.CASCADEREINDEX : Action.REINDEX)); - } - - protected void delete(T ref) throws LuceneIndexException - { - addCommand(new Command(ref, Action.DELETE)); - } - - private void addCommand(Command command) - { - if (commandList.size() > 0) - { - Command last = commandList.get(commandList.size() - 1); - if ((last.action == command.action) && (last.ref.equals(command.ref))) - { - return; - } - } - purgeCommandList(command); - commandList.add(command); - - if (commandList.size() > getLuceneConfig().getIndexerBatchSize()) - { - flushPending(); - } - } - - private void purgeCommandList(Command command) - { - removeFromCommandList(command, command.action != Action.DELETE); - } - - private void removeFromCommandList(Command command, boolean matchExact) - { - for (ListIterator> it = commandList.listIterator(commandList.size()); it.hasPrevious(); /**/) - { - Command current = it.previous(); - if (matchExact) - { - if (current.ref.equals(command.ref)) - { - if ((current.action == command.action)) - { - it.remove(); - return; - } - // If there is an INDEX in this same transaction and the current command is a reindex, remove it and - // replace the current command with it - else if (command.action != Action.DELETE && current.action == Action.INDEX) - { - it.remove(); - command.action = Action.INDEX; - } - } - } - else - { - if (current.ref.equals(command.ref)) - { - it.remove(); - } - } - } - } - - /** - * Get the deletions - * - * @return - the ids to delete - */ - public Set getDeletions() - { - return Collections.unmodifiableSet(deletions); - } - - /** - * Get the container deletions - * - * @return - the ids to delete - */ - public Set getContainerDeletions() - { - return Collections.unmodifiableSet(containerDeletions); - } -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/ClosingIndexSearcher.java b/src/main/java/org/alfresco/repo/search/impl/lucene/ClosingIndexSearcher.java deleted file mode 100644 index 4f8976290c..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/ClosingIndexSearcher.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.io.IOException; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.store.Directory; - -public class ClosingIndexSearcher extends IndexSearcher -{ - IndexReader reader; - - public ClosingIndexSearcher(String path) throws IOException - { - super(path); - } - - public ClosingIndexSearcher(Directory directory) throws IOException - { - super(directory); - } - - public ClosingIndexSearcher(IndexReader r) - { - super(r); - this.reader = r; - } - - /*package*/ IndexReader getReader() - { - return reader; - } - - @Override - public void close() throws IOException - { - super.close(); - if(reader != null) - { - reader.close(); - } - } - -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/FilterIndexReaderByStringId.java b/src/main/java/org/alfresco/repo/search/impl/lucene/FilterIndexReaderByStringId.java deleted file mode 100644 index fdb098ab59..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/FilterIndexReaderByStringId.java +++ /dev/null @@ -1,395 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.io.IOException; -import java.util.Set; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.alfresco.error.AlfrescoRuntimeException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.FilterIndexReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.TermEnum; -import org.apache.lucene.index.TermPositions; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Searcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.OpenBitSet; - - -/** - * An index reader that filters documents from another. - * - * @author andyh - * - */ -public class FilterIndexReaderByStringId extends FilterIndexReader -{ - private static Log s_logger = LogFactory.getLog(FilterIndexReaderByStringId.class); - - private OpenBitSet deletedDocuments; - private final Set deletions; - private final Set containerDeletions; - private final boolean deleteNodesOnly; - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - private final String id; - - /** - * Apply the filter - * - * @param id String - * @param reader IndexReader - * @param deleteNodesOnly boolean - */ - public FilterIndexReaderByStringId(String id, IndexReader reader, Set deletions, Set containerDeletions, boolean deleteNodesOnly) - { - super(reader); - reader.incRef(); - this.id = id; - this.deletions = deletions; - this.containerDeletions = containerDeletions; - this.deleteNodesOnly = deleteNodesOnly; - - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Applying deletions FOR "+id +" (the index ito which these are applied is the previous one ...)"); - } - - } - - public OpenBitSet getDeletedDocuments() - { - lock.readLock().lock(); - try - { - if (deletedDocuments != null) - { - return deletedDocuments; - } - } - finally - { - lock.readLock().unlock(); - } - lock.writeLock().lock(); - try - { - if (deletedDocuments != null) - { - return deletedDocuments; - } - deletedDocuments = new OpenBitSet(in.maxDoc()); - - Searcher searcher = new IndexSearcher(in); - for (String stringRef : deletions) - { - if (!deleteNodesOnly || containerDeletions.contains(stringRef)) - { - TermDocs td = in.termDocs(new Term("ID", stringRef)); - while (td.next()) - { - deletedDocuments.set(td.doc()); - } - td.close(); - } - else - { - boolean found = false; - TermDocs td = in.termDocs(new Term("LEAFID", stringRef)); - while (td.next()) - { - deletedDocuments.set(td.doc()); - found = true; - } - td.close(); - // For backward compatibility, use old method of locating non-container docs - if (!found) - { - TermQuery query = new TermQuery(new Term("ID", stringRef)); - Hits hits = searcher.search(query); - if (hits.length() > 0) - { - for (int i = 0; i < hits.length(); i++) - { - Document doc = hits.doc(i); - // Exclude all containers except the root (which is also a node!) - Field path = doc.getField("PATH"); - if (path == null || path.stringValue().length() == 0) - { - deletedDocuments.set(hits.id(i)); - // There should only be one thing to delete - // break; - } - } - } - } - } - } - // searcher does not need to be closed, the reader is live - - for (String stringRef : containerDeletions) - { - TermDocs td = in.termDocs(new Term("ANCESTOR", stringRef)); - while (td.next()) - { - deletedDocuments.set(td.doc()); - } - td.close(); - } - return deletedDocuments; - } - catch (IOException e) - { - s_logger.error("Error initialising "+id, e); - throw new AlfrescoRuntimeException("Failed to find deleted documents to filter", e); - } - finally - { - lock.writeLock().unlock(); - } - - } - - // Prevent from actually setting the closed flag - @Override - protected void doClose() throws IOException - { - this.in.decRef(); - } - - /** - * Filter implementation - * - * @author andyh - * - */ - public class FilterTermDocs implements TermDocs - { - protected TermDocs in; - - String id; - - /** - * @param id String - * @param in TermDocs - */ - public FilterTermDocs(String id, TermDocs in) - { - this.in = in; - } - - public void seek(Term term) throws IOException - { - // Seek is left to the base implementation - in.seek(term); - } - - public void seek(TermEnum termEnum) throws IOException - { - // Seek is left to the base implementation - in.seek(termEnum); - } - - public int doc() - { - // The current document info is valid in the base implementation - return in.doc(); - } - - public int freq() - { - // The frequency is valid in the base implementation - return in.freq(); - } - - public boolean next() throws IOException - { - try - { - if (!in.next()) - { - return false; - } - OpenBitSet deletedDocuments = getDeletedDocuments(); - while (deletedDocuments.get(in.doc())) - { - if (!in.next()) - { - return false; - } - } - // Not masked - return true; - } - catch(IOException ioe) - { - s_logger.error("Error reading docs for "+id); - throw ioe; - } - } - - public int read(int[] docs, int[] freqs) throws IOException - { - int[] innerDocs = new int[docs.length]; - int[] innerFreq = new int[docs.length]; - int count = in.read(innerDocs, innerFreq); - - // Is the stream exhausted - if (count == 0) - { - return 0; - } - - OpenBitSet deletedDocuments = getDeletedDocuments(); - while (allDeleted(innerDocs, count, deletedDocuments)) - { - - count = in.read(innerDocs, innerFreq); - - // Is the stream exhausted - if (count == 0) - { - return 0; - } - } - - // Add non deleted - - int insertPosition = 0; - for (int i = 0; i < count; i++) - { - if (!deletedDocuments.get(innerDocs[i])) - { - docs[insertPosition] = innerDocs[i]; - freqs[insertPosition] = innerFreq[i]; - insertPosition++; - } - } - - return insertPosition; - } - - private boolean allDeleted(int[] docs, int fillSize, OpenBitSet deletedDocuments) - { - for (int i = 0; i < fillSize; i++) - { - if (!deletedDocuments.get(docs[i])) - { - return false; - } - } - return true; - } - - public boolean skipTo(int i) throws IOException - { - if (!in.skipTo(i)) - { - return false; - } - - OpenBitSet deletedDocuments = getDeletedDocuments(); - while (deletedDocuments.get(in.doc())) - { - if (!in.next()) - { - return false; - } - } - return true; - } - - public void close() throws IOException - { - // Leave to internal implementation - in.close(); - } - } - - /** Base class for filtering {@code TermPositions} implementations. */ - public class FilterTermPositions extends FilterTermDocs implements TermPositions - { - - TermPositions tp; - - /** - * @param id String - * @param in TermPositions - */ - public FilterTermPositions(String id, TermPositions in) - { - super(id, in); - tp = in; - } - - public int nextPosition() throws IOException - { - return tp.nextPosition(); - } - - public byte[] getPayload(byte[] data, int offset) throws IOException - { - return tp.getPayload(data, offset); - } - - public int getPayloadLength() - { - return tp.getPayloadLength(); - } - - public boolean isPayloadAvailable() - { - return tp.isPayloadAvailable(); - } - } - - @Override - public int numDocs() - { - return super.numDocs() - (int)getDeletedDocuments().cardinality(); - } - - @Override - public TermDocs termDocs() throws IOException - { - return new FilterTermDocs(id, super.termDocs()); - } - - @Override - public TermPositions termPositions() throws IOException - { - return new FilterTermPositions(id, super.termPositions()); - } -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java b/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java index c73d9433fa..b1ab08a356 100644 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java +++ b/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneCategoryServiceImpl.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.search.impl.lucene; import java.util.Collection; @@ -81,8 +81,8 @@ public class LuceneCategoryServiceImpl implements CategoryService protected DictionaryService dictionaryService; - protected IndexerAndSearcher indexerAndSearcher; - + protected IndexerAndSearcher indexerAndSearcher; + protected int queryFetchSize = 5000; /** @@ -153,12 +153,12 @@ public void setDictionaryService(DictionaryService dictionaryService) public void setIndexerAndSearcher(IndexerAndSearcher indexerAndSearcher) { this.indexerAndSearcher = indexerAndSearcher; - } + } - public void setQueryFetchSize(int queryFetchSize) { - this.queryFetchSize = queryFetchSize; - } - + public void setQueryFetchSize(int queryFetchSize) { + this.queryFetchSize = queryFetchSize; + } + public Collection getChildren(NodeRef categoryRef, Mode mode, Depth depth) { return getChildren(categoryRef, mode, depth, false, null, queryFetchSize); @@ -543,68 +543,7 @@ public void deleteClassification(StoreRef storeRef, QName aspectName) public List> getTopCategories(StoreRef storeRef, QName aspectName, int count) { - if (indexerAndSearcher instanceof LuceneIndexerAndSearcher) - { - AspectDefinition definition = dictionaryService.getAspect(aspectName); - if(definition == null) - { - throw new IllegalStateException("Unknown aspect"); - } - QName catProperty = null; - Map properties = definition.getProperties(); - for(QName pName : properties.keySet()) - { - if(pName.getNamespaceURI().equals(aspectName.getNamespaceURI())) - { - if(pName.getLocalName().equalsIgnoreCase(aspectName.getLocalName())) - { - PropertyDefinition def = properties.get(pName); - if(def.getDataType().getName().equals(DataTypeDefinition.CATEGORY)) - { - catProperty = pName; - } - } - } - } - if(catProperty == null) - { - throw new IllegalStateException("Aspect does not have category property mirroring the aspect name"); - } - - - LuceneIndexerAndSearcher lias = (LuceneIndexerAndSearcher) indexerAndSearcher; - String field = "@" + catProperty; - SearchService searchService = lias.getSearcher(storeRef, false); - if (searchService instanceof LuceneSearcher) - { - LuceneSearcher luceneSearcher = (LuceneSearcher)searchService; - List> topTerms = luceneSearcher.getTopTerms(field, count); - List> answer = new LinkedList>(); - for (Pair term : topTerms) - { - Pair toAdd; - NodeRef nodeRef = new NodeRef(term.getFirst()); - if (nodeService.exists(nodeRef)) - { - toAdd = new Pair(nodeRef, term.getSecond()); - } - else - { - toAdd = new Pair(null, term.getSecond()); - } - answer.add(toAdd); - } - return answer; - } - else - { - throw new UnsupportedOperationException("getPolularCategories is only supported for lucene indexes"); - } - } - else - { - throw new UnsupportedOperationException("getPolularCategories is only supported for lucene indexes"); - } + throw new UnsupportedOperationException(); } } diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer.java b/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer.java deleted file mode 100644 index 9eeb05e9e6..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneIndexer.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.util.Set; - -import org.alfresco.repo.search.Indexer; -import org.alfresco.repo.search.TransactionSynchronisationAwareIndexer; -import org.alfresco.repo.search.impl.lucene.index.IndexInfo; - -/** - * @author Andy Hind - */ -public interface LuceneIndexer extends Indexer, TransactionSynchronisationAwareIndexer -{ - public String getDeltaId(); - public Set getDeletions(); - public Set getContainerDeletions(); - public boolean getDeleteOnlyNodes(); - public R doReadOnly(IndexInfo.LockWork lockWork); -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSet.java b/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSet.java deleted file mode 100644 index 53b706b398..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSet.java +++ /dev/null @@ -1,355 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.List; - -import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.repo.node.NodeBulkLoader; -import org.alfresco.repo.search.AbstractResultSet; -import org.alfresco.repo.search.ResultSetRowIterator; -import org.alfresco.repo.search.SearcherException; -import org.alfresco.repo.search.SimpleResultSetMetaData; -import org.alfresco.repo.search.impl.lucene.index.CachingIndexReader; -import org.alfresco.repo.tenant.TenantService; -import org.alfresco.service.cmr.repository.ChildAssociationRef; -import org.alfresco.service.cmr.repository.NodeRef; -import org.alfresco.service.cmr.repository.NodeService; -import org.alfresco.service.cmr.search.LimitBy; -import org.alfresco.service.cmr.search.PermissionEvaluationMode; -import org.alfresco.service.cmr.search.ResultSetMetaData; -import org.alfresco.service.cmr.search.ResultSetRow; -import org.alfresco.service.cmr.search.SearchParameters; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.Searcher; - -/** - * Implementation of a ResultSet on top of Lucene Hits class. - * - * @author andyh - */ -public class LuceneResultSet extends AbstractResultSet -{ - private static int DEFAULT_BULK_FETCH_SIZE = 1000; - - /** - * The underlying hits - */ - Hits hits; - - private Searcher searcher; - - private NodeService nodeService; - - private TenantService tenantService; - - private SearchParameters searchParameters; - - private LuceneConfig config; - - private BitSet prefetch; - - private boolean bulkFetch = true; - - private int bulkFetchSize = DEFAULT_BULK_FETCH_SIZE; - - /** - * Wrap a lucene seach result with node support - * - * @param hits Hits - * @param searcher Searcher - * @param nodeService nodeService - * @param tenantService tenant service - * @param searchParameters SearchParameters - * @param config - lucene config - */ - public LuceneResultSet(Hits hits, Searcher searcher, NodeService nodeService, TenantService tenantService, SearchParameters searchParameters, - LuceneConfig config) - { - super(); - this.hits = hits; - this.searcher = searcher; - this.nodeService = nodeService; - this.tenantService = tenantService; - this.searchParameters = searchParameters; - this.config = config; - prefetch = new BitSet(hits.length()); - } - - /* - * ResultSet implementation - */ - - public ResultSetRowIterator iterator() - { - return new LuceneResultSetRowIterator(this); - } - - public int length() - { - return hits.length(); - } - - public NodeRef getNodeRef(int n) - { - try - { - prefetch(n); - // We have to get the document to resolve this - // It is possible the store ref is also stored in the index - if (searcher instanceof ClosingIndexSearcher) - { - ClosingIndexSearcher cis = (ClosingIndexSearcher) searcher; - IndexReader reader = cis.getReader(); - if (reader instanceof CachingIndexReader) - { - int id = hits.id(n); - CachingIndexReader cir = (CachingIndexReader) reader; - String sid = cir.getId(id); - return tenantService.getBaseName(new NodeRef(sid)); - } - } - - Document doc = hits.doc(n); - String id = doc.get("ID"); - return tenantService.getBaseName(new NodeRef(id)); - } - catch (IOException e) - { - throw new SearcherException("IO Error reading reading node ref from the result set", e); - } - } - - public float getScore(int n) throws SearcherException - { - try - { - return hits.score(n); - } - catch (IOException e) - { - throw new SearcherException("IO Error reading score from the result set", e); - } - } - - public Document getDocument(int n) - { - try - { - prefetch(n); - Document doc = hits.doc(n); - return doc; - } - catch (IOException e) - { - throw new SearcherException("IO Error reading reading document from the result set", e); - } - } - - private void prefetch(int n) throws IOException - { - NodeBulkLoader bulkLoader = config.getBulkLoader(); - if (!getBulkFetch() || (bulkLoader == null)) - { - // No prefetching - return; - } - if (prefetch.get(n)) - { - // The document was already processed - return; - } - // Start at 'n' and process the the next bulk set - int bulkFetchSize = getBulkFetchSize(); - List fetchList = new ArrayList(bulkFetchSize); - int totalHits = hits.length(); - for (int i = 0; i < bulkFetchSize; i++) - { - int next = n + i; - if (next >= totalHits) - { - // We've hit the end - break; - } - if (prefetch.get(next)) - { - // This one is in there already - continue; - } - // We store the node and mark it as prefetched - prefetch.set(next); - Document doc = hits.doc(next); - String nodeRefStr = doc.get("ID"); - try - { - NodeRef nodeRef = tenantService.getBaseName(new NodeRef(nodeRefStr)); - fetchList.add(nodeRef); - } - catch (AlfrescoRuntimeException e) - { - // Ignore IDs that don't parse as NodeRefs, e.g. FTSREF docs - } - } - // Now bulk fetch - if (fetchList.size() > 1) - { - bulkLoader.cacheNodes(fetchList); - } - } - - public void close() - { - try - { - searcher.close(); - } - catch (IOException e) - { - throw new SearcherException(e); - } - } - - public NodeService getNodeService() - { - return nodeService; - } - - public ResultSetRow getRow(int i) - { - if (i < length()) - { - return new LuceneResultSetRow(this, i); - } - else - { - throw new SearcherException("Invalid row"); - } - } - - public ChildAssociationRef getChildAssocRef(int n) - { - return tenantService.getBaseName(getRow(n).getChildAssocRef()); - } - - public ResultSetMetaData getResultSetMetaData() - { - return new SimpleResultSetMetaData(LimitBy.UNLIMITED, PermissionEvaluationMode.EAGER, searchParameters); - } - - public int getStart() - { - throw new UnsupportedOperationException(); - } - - public boolean hasMore() - { - throw new UnsupportedOperationException(); - } - - public TenantService getTenantService() - { - return tenantService; - } - - /** - * Bulk fetch results in the cache - * - * @param bulkFetch boolean - */ - @Override - public boolean setBulkFetch(boolean bulkFetch) - { - boolean oldBulkFetch = this.bulkFetch; - this.bulkFetch = bulkFetch; - return oldBulkFetch; - } - - /** - * Do we bulk fetch - * - * @return - true if we do - */ - @Override - public boolean getBulkFetch() - { - return bulkFetch; - } - - /** - * Set the bulk fetch size - * - * @param bulkFetchSize int - */ - @Override - public int setBulkFetchSize(int bulkFetchSize) - { - int oldBulkFetchSize = this.bulkFetchSize; - this.bulkFetchSize = bulkFetchSize; - return oldBulkFetchSize; - } - - /** - * Get the bulk fetch size. - * - * @return the fetch size - */ - @Override - public int getBulkFetchSize() - { - return bulkFetchSize; - } - - /** - * @param index int - * @return int - */ - public int doc(int index) - { - try - { - return hits.id(index); - } - catch (IOException e) - { - throw new SearcherException(e); - } - } - - /* (non-Javadoc) - * @see org.alfresco.service.cmr.search.ResultSetSPI#getNumberFound() - */ - @Override - public long getNumberFound() - { - return hits.length(); - } - -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSetRow.java b/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSetRow.java deleted file mode 100644 index f68864c48f..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSetRow.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.io.Serializable; -import java.util.Map; - -import org.alfresco.model.ContentModel; -import org.alfresco.repo.search.AbstractResultSetRow; -import org.alfresco.repo.tenant.TenantService; -import org.alfresco.service.cmr.repository.ChildAssociationRef; -import org.alfresco.service.cmr.repository.NodeRef; -import org.alfresco.service.namespace.QName; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -/** - * A row in a result set. Created on the fly. - * - * @author Andy Hind - * - */ -public class LuceneResultSetRow extends AbstractResultSetRow -{ - /** - * The current document - cached so we do not get it for each value - */ - private Document document; - - private TenantService tenantService; - - /** - * Wrap a position in a lucene Hits class with node support - * - * @param resultSet LuceneResultSet - * @param index int - */ - public LuceneResultSetRow(LuceneResultSet resultSet, int index) - { - super(resultSet, index); - - tenantService = resultSet.getTenantService(); - } - - /** - * Support to cache the document for this row - * - * @return Document - */ - public Document getDocument() - { - if (document == null) - { - document = ((LuceneResultSet) getResultSet()).getDocument(getIndex()); - } - return document; - } - - /* - * ResultSetRow implementation - */ - - protected Map getDirectProperties() - { - LuceneResultSet lrs = (LuceneResultSet) getResultSet(); - return lrs.getNodeService().getProperties(lrs.getNodeRef(getIndex())); - } - - public QName getQName() - { - Field field = getDocument().getField("QNAME"); - if (field != null) - { - String qname = field.stringValue(); - if((qname == null) || (qname.length() == 0)) - { - return null; - } - else - { - return QName.createQName(qname); - } - } - else - { - return null; - } - } - - public QName getPrimaryAssocTypeQName() - { - - Field field = getDocument().getField("PRIMARYASSOCTYPEQNAME"); - if (field != null) - { - String qname = field.stringValue(); - return QName.createQName(qname); - } - else - { - return ContentModel.ASSOC_CHILDREN; - } - } - - @Override - public ChildAssociationRef getChildAssocRef() - { - Field field = getDocument().getField("PRIMARYPARENT"); - String primaryParent = null; - if (field != null) - { - primaryParent = field.stringValue(); - } - NodeRef childNodeRef = getNodeRef(); - NodeRef parentNodeRef = primaryParent == null ? null : tenantService.getBaseName(new NodeRef(primaryParent)); - return new ChildAssociationRef(getPrimaryAssocTypeQName(), parentNodeRef, getQName(), childNodeRef); - } - - public NodeRef getNodeRef(String selectorName) - { - throw new UnsupportedOperationException(); - } - - public Map getNodeRefs() - { - throw new UnsupportedOperationException(); - } - - public float getScore(String selectorName) - { - throw new UnsupportedOperationException(); - } - - public Map getScores() - { - throw new UnsupportedOperationException(); - } - - public int doc() - { - return ((LuceneResultSet)getResultSet()).doc(getIndex()); - } -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSetRowIterator.java b/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSetRowIterator.java deleted file mode 100644 index bc291d8d2c..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneResultSetRowIterator.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import org.alfresco.repo.search.AbstractResultSetRowIterator; -import org.alfresco.service.cmr.search.ResultSetRow; - -/** - * Iterate over the rows in a LuceneResultSet - * - * @author andyh - * - */ -public class LuceneResultSetRowIterator extends AbstractResultSetRowIterator -{ - /** - * Create an iterator over the result set. Follows standard ListIterator - * conventions - * - * @param resultSet LuceneResultSet - */ - public LuceneResultSetRowIterator(LuceneResultSet resultSet) - { - super(resultSet); - } - - public ResultSetRow next() - { - return new LuceneResultSetRow((LuceneResultSet)getResultSet(), moveToNextPosition()); - } - - public ResultSetRow previous() - { - return new LuceneResultSetRow((LuceneResultSet)getResultSet(), moveToPreviousPosition()); - } -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneSearcher.java b/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneSearcher.java deleted file mode 100644 index d5be956bcb..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/LuceneSearcher.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene; - -import java.util.List; - -import org.alfresco.service.cmr.repository.NodeService; -import org.alfresco.service.cmr.search.SearchService; -import org.alfresco.service.namespace.NamespacePrefixResolver; -import org.alfresco.util.Pair; - -/** - * Lucene implementation specific entension to the seracher API - * @author andyh - * - */ -public interface LuceneSearcher extends SearchService -{ - /** - * Check if the index exists - * @return - true if it exists - */ - public boolean indexExists(); - /** - * Ste the node service - * @param nodeService NodeService - */ - public void setNodeService(NodeService nodeService); - /** - * Set the name space service - * @param namespacePrefixResolver NamespacePrefixResolver - */ - public void setNamespacePrefixResolver(NamespacePrefixResolver namespacePrefixResolver); - - /** - * Get top terms - * - * @param field String - * @param count int - * @return List - */ - public List> getTopTerms(String field, int count); - - /** - * Get a lucene searcher - * @return ClosingIndexSearcher - */ - public ClosingIndexSearcher getClosingIndexSearcher(); -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/SolrJSONResultSet.java b/src/main/java/org/alfresco/repo/search/impl/lucene/SolrJSONResultSet.java index 1096f11ae9..76647db0d3 100644 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/SolrJSONResultSet.java +++ b/src/main/java/org/alfresco/repo/search/impl/lucene/SolrJSONResultSet.java @@ -37,6 +37,7 @@ import java.util.stream.Collectors; import org.alfresco.repo.domain.node.NodeDAO; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.repo.search.SimpleResultSetMetaData; import org.alfresco.repo.search.impl.solr.facet.facetsresponse.GenericBucket; import org.alfresco.repo.search.impl.solr.facet.facetsresponse.GenericFacetResponse; @@ -176,7 +177,7 @@ public SolrJSONResultSet(JSONObject json, SearchParameters searchParameters, Nod else { // No DBID found - throw new LuceneQueryParserException("No DBID found for doc ..."); + throw new QueryParserException("No DBID found for doc ..."); } } diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java b/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java deleted file mode 100644 index 355dc06ae8..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexEntry.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene.index; - -/** - * Describes an entry in an index - * - * @author Andy Hind - */ -class IndexEntry -{ - /** - * The type of the index entry - */ - private IndexType type; - - /** - * The unique name of the index entry - */ - private String name; - - /** - * The preceeding index name. - * Allows deltas etc to apply to the index or an overlay for example. - */ - private String parentName; - - /** - * The status of the index entry - */ - private TransactionStatus status; - - /** - * If merging, the id where the result is going - */ - private String mergeId; - - private long documentCount; - - private long deletions; - - private boolean deletOnlyNodes; - - IndexEntry(IndexType type, String name, String parentName, TransactionStatus status, String mergeId, long documentCount, long deletions, boolean deletOnlyNodes) - { - this.type = type; - this.name = name; - this.parentName = parentName; - this.status = status; - this.mergeId = mergeId; - this.documentCount = documentCount; - this.deletions = deletions; - this.deletOnlyNodes = deletOnlyNodes; - } - - public String getMergeId() - { - return mergeId; - } - - public void setMergeId(String mergeId) - { - this.mergeId = mergeId; - } - - public String getName() - { - return name; - } - - public void setName(String name) - { - this.name = name; - } - - public String getParentName() - { - return parentName; - } - - public void setParentName(String parentName) - { - this.parentName = parentName; - } - - public TransactionStatus getStatus() - { - return status; - } - - public void setStatus(TransactionStatus status) - { - this.status = status; - } - - public IndexType getType() - { - return type; - } - - public void setType(IndexType type) - { - this.type = type; - } - - public long getDocumentCount() - { - return documentCount; - } - - public void setDocumentCount(long documentCount) - { - this.documentCount = documentCount; - } - - public long getDeletions() - { - return deletions; - } - - public void setDeletions(long deletions) - { - this.deletions = deletions; - } - - public boolean isDeletOnlyNodes() - { - return deletOnlyNodes; - } - - public void setDeletOnlyNodes(boolean deletOnlyNodes) - { - this.deletOnlyNodes = deletOnlyNodes; - } - - public String toString() - { - StringBuilder builder = new StringBuilder(); - builder.append(" Name=").append(getName()).append(" "); - builder.append("Type=").append(getType()).append(" "); - builder.append("Status=").append(getStatus()).append(" "); - builder.append("Docs=").append(getDocumentCount()).append(" "); - builder.append("Deletions=").append(getDeletions()).append(" "); - return builder.toString(); - } - - -} \ No newline at end of file diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexEvent.java b/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexEvent.java deleted file mode 100644 index 74e8af870d..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexEvent.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene.index; - -import org.springframework.context.ApplicationEvent; - -/** - * A class of event that notifies the listener of a significant event relating to a Lucene index. Useful for Monitoring - * purposes. - * - * @author dward - */ -public class IndexEvent extends ApplicationEvent -{ - - private static final long serialVersionUID = -4616231785087405506L; - - /** The event description. */ - private final String description; - - /** Its instance count. */ - private final int count; - - /** - * The Constructor. - * - * @param source - * the source index monitor - * @param description - * the event description - * @param count - * its instance count - */ - public IndexEvent(IndexMonitor source, String description, int count) - { - super(source); - this.description = description; - this.count = count; - } - - /** - * Gets the source index monitor. - * - * @return the index monitor - */ - public IndexMonitor getIndexMonitor() - { - return (IndexMonitor) getSource(); - } - - /** - * Gets the event description. - * - * @return the description - */ - public String getDescription() - { - return this.description; - } - - /** - * Gets the event instance count. - * - * @return the count - */ - public int getCount() - { - return this.count; - } - -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java b/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java deleted file mode 100644 index 8861b39dac..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexInfo.java +++ /dev/null @@ -1,4527 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene.index; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.io.UnsupportedEncodingException; -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.channels.FileChannel.MapMode; -import java.nio.channels.FileLock; -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.zip.CRC32; - -import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.repo.search.IndexerException; -import org.alfresco.repo.search.impl.lucene.FilterIndexReaderByStringId; -import org.alfresco.repo.search.impl.lucene.LuceneConfig; -import org.alfresco.repo.search.impl.lucene.LuceneXPathHandler; -import org.alfresco.repo.search.impl.lucene.analysis.AlfrescoStandardAnalyser; -import org.alfresco.repo.search.impl.lucene.query.PathQuery; -import org.alfresco.service.cmr.dictionary.DictionaryService; -import org.alfresco.service.namespace.NamespaceService; -import org.alfresco.util.ApplicationContextHelper; -import org.alfresco.util.GUID; -import org.alfresco.util.TraceableThreadFactory; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReader.FieldOption; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriter.MaxFieldLength; -import org.apache.lucene.index.LogDocMergePolicy; -import org.apache.lucene.index.MultiReader; -import org.apache.lucene.index.SerialMergeScheduler; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermEnum; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Searcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.RAMDirectory; -import org.jaxen.saxpath.SAXPathException; -import org.jaxen.saxpath.base.XPathReader; -import org.safehaus.uuid.UUID; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.event.ContextRefreshedEvent; - - -/** - * The information that makes up an index. IndexInfoVersion Repeated information of the form - *
    - *
  1. Index Type. - *
  2. sub-directory name. - *
  3. Status - *
      - *
    1. Indexes, sub indexes, and overlays must be committed. Status is ACTIVE, MERGING, COMPLETING_INDEX - *
    2. Delta: Transaction status - *
    3. Overlay: Transaction status - *
    - *
- * Merges always take place to new indexes so we can detect merge failure or partial merges. Or we do not know what has - * merged. Incomplete delete merging does not matter - the overlay would still exist and be treated as such. So a - * document may be deleted in the index as well as in the applied overlay. It is still correctly deleted. NOTE: Public - * methods lock as required, the private methods assume that the appropriate locks have been obtained. TODO: Write - * element status into individual directories. This would be enough for recovery if both index files are lost or - * corrupted. TODO: Tidy up index status at start up or after some time. How long would you leave a merge to run? - *

- * The index structure is duplicated to two files. If one is currupted the second is used. - *

- * TODO: - *

- *

    - *
  1. make the index sharing configurable - *
  2. use a thread pool for deletions, merging and index deletions - *
  3. something to control the maximum number of overlays to limit the number of things layered together for searching - *
  4. look at lucene locking again post 2.0, to see if it is improved - *
  5. clean up old data files (that are not old index entries) - should be a config option - *
- * - * @author Andy Hind - */ -public class IndexInfo implements IndexMonitor -{ - public static synchronized void destroy() - { - timer.cancel(); - timer = new Timer(true); - for(IndexInfo indexInfo : indexInfos.values()) - { - indexInfo.destroyInstance(); - } - indexInfos.clear(); - ReferenceCountingReadOnlyIndexReaderFactory.destroy(); - } - - public void destroyInstance() - { - getWriteLock(); - try - { - if(mainIndexReader != null) - { - try - { - ((ReferenceCounting) mainIndexReader).setInvalidForReuse(); - } - catch (IOException e) - { - // OK filed to close - } - mainIndexReader = null; - - for(IndexReader reader : referenceCountingReadOnlyIndexReaders.values()) - { - ReferenceCounting referenceCounting = (ReferenceCounting) reader; - try - { - referenceCounting.setInvalidForReuse(); - } - catch (IOException e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - - for(IndexReader reader : indexReaders.values()) - { - try - { - reader.close(); - } - catch (IOException e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - indexReaders.clear(); - - for(IndexWriter writer : indexWriters.values()) - { - try - { - writer.close(); - } - catch (CorruptIndexException e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - catch (IOException e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - indexWriters.clear(); - - if(indexInfoRAF != null) - { - try - { - indexInfoRAF.close(); - } - catch (IOException e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - if(indexInfoBackupRAF != null) - { - try - { - indexInfoBackupRAF.close(); - } - catch (IOException e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - // TODO: should set some running flag .... to abort ungoing stuff - // at the moment it will die ungracefully .... - } - finally - { - releaseWriteLock(); - } - } - - public static final String MAIN_READER = "MainReader"; - - private static Timer timer = new Timer("IndexInfo Cleaner Deamon", true); - - /** - * The logger. - */ - private static Log s_logger = LogFactory.getLog(IndexInfo.class); - - /** - * Use NIO memory mapping to wite the index control file. - */ - private static boolean useNIOMemoryMapping = true; - - /** - * The default name for the file that holds the index information - */ - private static String INDEX_INFO = "IndexInfo"; - - /** - * The default name for the back up file that holds the index information - */ - private static String INDEX_INFO_BACKUP = "IndexInfoBackup"; - - /** - * The default name for the index deletions file - */ - private static String INDEX_INFO_DELETIONS = "IndexInfoDeletions"; - - /** - * The default name for the index container deletions file - */ - private static String INDEX_INFO_CONTAINER_DELETIONS = "IndexInfoContainerDeletions"; - - /** - * What to look for to detect the previous index implementation. - */ - private static String OLD_INDEX = "index"; - - /** - * Is this index shared by more than one repository? We can make many lock optimisations if the index is not shared. - */ - private boolean indexIsShared = false; - - /** - * The directory that holds the index - */ - private File indexDirectory; - - /** - * The directory relative to the root path - */ - private String relativePath; - - /** - * The file holding the index information - */ - private RandomAccessFile indexInfoRAF; - - /** - * And its file channel - */ - private FileChannel indexInfoChannel; - - /** - * The file holding the backup index information. - */ - - private RandomAccessFile indexInfoBackupRAF; - - /** - * And its file channel - */ - private FileChannel indexInfoBackupChannel; - - /** - * The file version. Negative is not yet written. - */ - private long version = -1; - - /** - * The index entries that make up this index. Map entries are looked up by name. These are maintained in order so - * document order is maintained. - */ - private LinkedHashMap indexEntries = new LinkedHashMap(); - - /** - * Lock for the index entries - */ - private final ReentrantReadWriteLock readWriteLock; - - private ReentrantReadWriteLock readOnlyLock = new ReentrantReadWriteLock(); - - /** - * Read only index readers that also do reference counting. - */ - private HashMap referenceCountingReadOnlyIndexReaders = new HashMap(); - - /** - * Main index reader - */ - private IndexReader mainIndexReader; - private Map mainIndexReaders = new HashMap(); - - /** - * Index writers for deltas - */ - private Map indexWriters = new ConcurrentHashMap(51); - - /** - * Index Readers for deltas - */ - private Map indexReaders = new ConcurrentHashMap(51); - - /** - * Map of state transitions - */ - private EnumMap transitions = new EnumMap(TransactionStatus.class); - - /** - * The queue of files and folders to delete - */ - private ConcurrentLinkedQueue deleteQueue = new ConcurrentLinkedQueue(); - - /** - * A queue of reference counting index readers. We wait for these to become unused (ref count falls to zero) then - * the data can be removed. - */ - private ConcurrentLinkedQueue deletableReaders = new ConcurrentLinkedQueue(); - - /** - * The call that is responsible for deleting old index information from disk. - */ - private Cleaner cleaner = new Cleaner(); - - /** - * The thread that deletes old index data - */ - // private Thread cleanerThread; - /** - * The class the supports index merging and applying deletions from deltas to indexes and deltas that go before it. - */ - private Merger merger = new Merger(); - - /** - * The thread that carries out index merging and applying deletions from deltas to indexes and deltas that go before - * it. - */ - // private Thread mergerThread; - /** - * A shared empty index to use if non exist. - */ - private Directory emptyIndex = new RAMDirectory(); - - /** - * The index infor files that make up the index - */ - private static HashMap indexInfos = new HashMap(); - - // Properties that control lucene indexing - // -------------------------------------- - - // Properties for indexes that are created by transactions ... - - private int maxDocsForInMemoryMerge = 10000; - - private int maxDocsForInMemoryIndex = 10000; - - private double maxRamInMbForInMemoryMerge = 16.0; - - private double maxRamInMbForInMemoryIndex = 16.0; - - private int writerMaxBufferedDocs = IndexWriter.DISABLE_AUTO_FLUSH; - - private double writerRamBufferSizeMb = 16.0; - - private int writerMergeFactor = 5; - - private int writerMaxMergeDocs = 1000000; - - private boolean writerUseCompoundFile = true; - - // Properties for indexes created by merging - - private int mergerMaxBufferedDocs = IndexWriter.DISABLE_AUTO_FLUSH; - - private double mergerRamBufferSizeMb = 16.0; - - private int mergerMergeFactor = 5; - - private int mergerMaxMergeDocs = 1000000; - - private boolean mergerUseCompoundFile = true; - - private int mergerTargetOverlays = 5; - - private int mergerTargetIndexes = 5; - - private int mergerTargetOverlaysBlockingFactor = 1; - - private Object mergerTargetLock = new Object(); - - // To avoid deadlock (a thread with multiple deltas never proceeding to commit) we track whether each thread is - // already in the prepare phase. - private static ThreadLocal thisThreadPreparing = new ThreadLocal(); - - // Common properties for indexers - - private long writeLockTimeout = IndexWriter.WRITE_LOCK_TIMEOUT; - - private int maxFieldLength = IndexWriter.DEFAULT_MAX_FIELD_LENGTH; - - private int termIndexInterval = IndexWriter.DEFAULT_TERM_INDEX_INTERVAL; - - /** - * Control if the merger thread is active - */ - - private ThreadPoolExecutor threadPoolExecutor; - - private LuceneConfig config; - - private List applicationListeners = new LinkedList(); - - static - { - // We do not require any of the lucene in-built locking. - FSDirectory.setDisableLocks(true); - } - - /** - * - */ - public void delete(final String deltaId) - { - - getWriteLock(); - try - { - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - setStatusFromFile(); - - // If the index is not shared we can do some easy clean - // up - if (!indexIsShared) - { - HashSet deletable = new HashSet(); - // clean up - for (IndexEntry entry : indexEntries.values()) - { - if(!entry.getName().equals(deltaId)) - { - entry.setStatus(TransactionStatus.DELETABLE); - deletable.add(entry.getName()); - } - } - // Delete entries that are not required - invalidateMainReadersFromFirst(deletable); - for (String id : deletable) - { - indexEntries.remove(id); - } - - clearOldReaders(); - - cleaner.schedule(); - - merger.schedule(); - - // persist the new state - writeStatus(); - - if (mainIndexReader != null) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("... invalidating main index reader"); - } - ((ReferenceCounting) mainIndexReader).setInvalidForReuse(); - mainIndexReader = null; - } - } - return null; - } - - public boolean canRetry() - { - return false; - } - - }); - } - finally - { - releaseWriteLock(); - } - if(s_logger.isDebugEnabled()) - { - s_logger.debug("Index "+ indexDirectory+" deleted"); - } - - } - - /** - * Get the IndexInfo object based in the given directory. There is only one object per directory per JVM. - * - * @param file File - * @param config LuceneConfig - * @return IndexInfo - * @throws IndexerException - */ - public static synchronized IndexInfo getIndexInfo(File file, LuceneConfig config) throws IndexerException - { - File canonicalFile; - try - { - canonicalFile = file.getCanonicalFile(); - IndexInfo indexInfo = indexInfos.get(canonicalFile); - if (indexInfo == null) - { - indexInfo = new IndexInfo(canonicalFile, config); - indexInfos.put(canonicalFile, indexInfo); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Made " + indexInfo + " for " + file.getAbsolutePath()); - } - } - - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Got " + indexInfo + " for " + file.getAbsolutePath()); - } - return indexInfo; - } - catch (IOException e) - { - throw new IndexerException("Failed to transform a file into is canonical form", e); - } - - } - - /** - * Construct an index in the given directory. - * - * @param indexDirectory File - * @param config LuceneConfig - */ - private IndexInfo(File indexDirectory, LuceneConfig config) - { - super(); - initialiseTransitions(); - this.config = config; - - if (config != null) - { - this.readWriteLock = new ReentrantReadWriteLock(config.getFairLocking()); - this.maxFieldLength = config.getIndexerMaxFieldLength(); - this.threadPoolExecutor = config.getThreadPoolExecutor(); - IndexInfo.useNIOMemoryMapping = config.getUseNioMemoryMapping(); - this.maxDocsForInMemoryMerge = config.getMaxDocsForInMemoryMerge(); - this.maxRamInMbForInMemoryMerge = config.getMaxRamInMbForInMemoryMerge(); - this.maxDocsForInMemoryIndex = config.getMaxDocsForInMemoryIndex(); - this.maxRamInMbForInMemoryIndex = config.getMaxRamInMbForInMemoryIndex(); - this.writerMaxBufferedDocs = config.getWriterMaxBufferedDocs(); - this.writerRamBufferSizeMb = config.getWriterRamBufferSizeMb(); - this.writerMergeFactor = config.getWriterMergeFactor(); - this.writerMaxMergeDocs = config.getWriterMaxMergeDocs(); - this.mergerMaxBufferedDocs = config.getMergerMaxBufferedDocs(); - this.mergerRamBufferSizeMb = config.getMergerRamBufferSizeMb(); - this.mergerMergeFactor = config.getMergerMergeFactor(); - this.mergerMaxMergeDocs = config.getMergerMaxMergeDocs(); - this.termIndexInterval = config.getTermIndexInterval(); - this.mergerTargetOverlays = config.getMergerTargetOverlayCount(); - this.mergerTargetIndexes = config.getMergerTargetIndexCount(); - this.mergerTargetOverlaysBlockingFactor = config.getMergerTargetOverlaysBlockingFactor(); - // Work out the relative path of the index - try - { - String indexRoot = new File(config.getIndexRootLocation()).getCanonicalPath(); - this.relativePath = indexDirectory.getCanonicalPath().substring(indexRoot.length() + 1); - } - catch (IOException e) - { - throw new AlfrescoRuntimeException("Failed to determine index relative path", e); - } - } - else - { - this.readWriteLock = new ReentrantReadWriteLock(false); - - // need a default thread pool .... - TraceableThreadFactory threadFactory = new TraceableThreadFactory(); - threadFactory.setThreadDaemon(true); - threadFactory.setThreadPriority(5); - - threadPoolExecutor = new ThreadPoolExecutor(10, 10, 90, TimeUnit.SECONDS, new LinkedBlockingQueue(), threadFactory, new ThreadPoolExecutor.CallerRunsPolicy()); - - // Create a 'fake' relative path - try - { - this.relativePath = indexDirectory.getCanonicalPath(); - int sepIndex = this.relativePath.indexOf(File.separator); - if (sepIndex != -1) - { - if (this.relativePath.length() > sepIndex + 1) - { - this.relativePath = this.relativePath.substring(sepIndex + 1); - } - else - { - this.relativePath = ""; - } - } - } - catch (IOException e) - { - throw new AlfrescoRuntimeException("Failed to determine index relative path", e); - } - - } - - // Create an empty in memory index - IndexWriter writer; - try - { - writer = new IndexWriter(emptyIndex, new AlfrescoStandardAnalyser(), true, MaxFieldLength.LIMITED); - writer.setUseCompoundFile(writerUseCompoundFile); - writer.setMaxBufferedDocs(writerMaxBufferedDocs); - writer.setRAMBufferSizeMB(writerRamBufferSizeMb); - writer.setMergeFactor(writerMergeFactor); - writer.setMaxMergeDocs(writerMaxMergeDocs); - writer.setWriteLockTimeout(writeLockTimeout); - writer.setMaxFieldLength(maxFieldLength); - writer.setTermIndexInterval(termIndexInterval); - writer.setMergeScheduler(new SerialMergeScheduler()); - writer.setMergePolicy(new LogDocMergePolicy()); - writer.close(); - } - catch (IOException e) - { - throw new IndexerException("Failed to create an empty in memory index!"); - } - - this.indexDirectory = indexDirectory; - - // Make sure the directory exists - if (!this.indexDirectory.exists()) - { - if (!this.indexDirectory.mkdirs()) - { - throw new AlfrescoRuntimeException("Failed to create index directory"); - } - } - if (!this.indexDirectory.isDirectory()) - { - throw new AlfrescoRuntimeException("The index must be held in a directory"); - } - - // Create the info files. - File indexInfoFile = new File(this.indexDirectory, INDEX_INFO); - File indexInfoBackupFile = new File(this.indexDirectory, INDEX_INFO_BACKUP); - if (createFile(indexInfoFile) && createFile(indexInfoBackupFile)) - { - // If both files required creation this is a new index - version = 0; - } - - // Open the files and channels for the index info file and the backup - this.indexInfoRAF = openFile(indexInfoFile); - this.indexInfoChannel = this.indexInfoRAF.getChannel(); - - this.indexInfoBackupRAF = openFile(indexInfoBackupFile); - this.indexInfoBackupChannel = this.indexInfoBackupRAF.getChannel(); - - // If the index found no info files (i.e. it is new), check if there is - // an old style index and covert it. - if (version == 0) - { - // Check if an old style index exists - - final File oldIndex = new File(this.indexDirectory, OLD_INDEX); - if (IndexReader.indexExists(oldIndex)) - { - getWriteLock(); - try - { - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - IndexWriter writer; - try - { - writer = new IndexWriter(oldIndex, new AlfrescoStandardAnalyser(), false, MaxFieldLength.LIMITED); - writer.setUseCompoundFile(writerUseCompoundFile); - writer.setMaxBufferedDocs(writerMaxBufferedDocs); - writer.setRAMBufferSizeMB(writerRamBufferSizeMb); - writer.setMergeFactor(writerMergeFactor); - writer.setMaxMergeDocs(writerMaxMergeDocs); - writer.setWriteLockTimeout(writeLockTimeout); - writer.setMaxFieldLength(maxFieldLength); - writer.setTermIndexInterval(termIndexInterval); - writer.setMergeScheduler(new SerialMergeScheduler()); - writer.setMergePolicy(new LogDocMergePolicy()); - writer.optimize(); - long docs = writer.numDocs(); - writer.close(); - - IndexEntry entry = new IndexEntry(IndexType.INDEX, OLD_INDEX, "", TransactionStatus.COMMITTED, "", docs, 0, false); - indexEntries.put(OLD_INDEX, entry); - - writeStatus(); - - // The index exists and we should initialise the single reader - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); - } - catch (IOException e) - { - throw new IndexerException("Failed to optimise old index"); - } - return null; - } - - public boolean canRetry() - { - return false; - } - }); - } - finally - { - releaseWriteLock(); - } - - } - } - - // The index exists - else if (version == -1) - { - getWriteLock(); - try - { - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - setStatusFromFile(); - - // If the index is not shared we can do some easy clean - // up - if (!indexIsShared) - { - HashSet deletable = new HashSet(); - // clean up - for (IndexEntry entry : indexEntries.values()) - { - switch (entry.getStatus()) - { - // states which can be deleted - // We could check prepared states can be - // committed. - case ACTIVE: - case MARKED_ROLLBACK: - case NO_TRANSACTION: - case PREPARING: - case ROLLEDBACK: - case ROLLINGBACK: - case MERGE_TARGET: - case UNKNOWN: - case PREPARED: - case DELETABLE: - if (s_logger.isInfoEnabled()) - { - s_logger.info("Deleting index entry " + entry); - } - entry.setStatus(TransactionStatus.DELETABLE); - deletable.add(entry.getName()); - break; - // States which are in mid-transition which we - // can roll back to the committed state - case COMMITTED_DELETING: - case MERGE: - if (s_logger.isInfoEnabled()) - { - s_logger.info("Resetting merge to committed " + entry); - } - entry.setStatus(TransactionStatus.COMMITTED); - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); - break; - // Complete committing (which is post database - // commit) - case COMMITTING: - // do the commit - if (s_logger.isInfoEnabled()) - { - s_logger.info("Committing " + entry); - } - entry.setStatus(TransactionStatus.COMMITTED); - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); - break; - // States that require no action - case COMMITTED: - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); - break; - default: - // nothing to do - break; - } - } - // Delete entries that are not required - invalidateMainReadersFromFirst(deletable); - for (String id : deletable) - { - indexEntries.remove(id); - } - clearOldReaders(); - - cleaner.schedule(); - - merger.schedule(); - - // persist the new state - writeStatus(); - } - return null; - } - - public boolean canRetry() - { - return false; - } - - }); - } - finally - { - releaseWriteLock(); - } - } - // Need to do with file lock - must share info about other readers to support this with shared indexer - // implementation - - getWriteLock(); - try - { - LockWork work = new DeleteUnknownGuidDirectories(); - doWithFileLock(work); - } - finally - { - releaseWriteLock(); - } - - // Run the cleaner around every 20 secods - this just makes the request to the thread pool - timer.schedule(new TimerTask() - { - @Override - public void run() - { - cleaner.schedule(); - } - }, 0, 20000); - - publishDiscoveryEvent(); - } - - private class DeleteUnknownGuidDirectories implements LockWork - { - public boolean canRetry() - { - return true; - } - - public Object doWork() throws Exception - { - setStatusFromFile(); - - // If the index is not shared we can do some easy clean - // up - if (!indexIsShared) - { - // Safe to tidy up all files that look like guids that we do not know about - File[] files = indexDirectory.listFiles(); - if (files != null) - { - for (File file : files) - { - if (file.isDirectory()) - { - String id = file.getName(); - if (!indexEntries.containsKey(id) && isGUID(id)) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Deleting unused index directory " + id); - } - deleteQueue.add(id); - } - } - } - } - - } - return null; - } - } - - /** - * This method should only be called from one thread as it is bound to a transaction. - * - * @param id String - * @return IndexReader - * @throws IOException - */ - public IndexReader getDeltaIndexReader(String id) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - - // No read lock required as the delta should be bound to one thread only - // Index readers are simply thread safe - IndexReader reader = indexReaders.get(id); - if (reader == null) - { - // close index writer if required - closeDeltaIndexWriter(id); - // Check the index knows about the transaction - reader = buildAndRegisterDeltaReader(id); - indexReaders.put(id, reader); - } - return reader; - } - - private IndexReader buildAndRegisterDeltaReader(String id) throws IOException - { - IndexReader reader; - // only register on write to avoid any locking for transactions that only ever read - File location = getDeltaLocation(id); - // File location = ensureDeltaIsRegistered(id); - // Create a dummy index reader to deal with empty indexes and not - // persist these. - if (IndexReader.indexExists(location)) - { - reader = IndexReader.open(location); - } - else - { - reader = IndexReader.open(emptyIndex); - } - return reader; - } - - private File getDeltaLocation(String id) throws IOException - { - File file = new File(indexDirectory, id).getCanonicalFile(); - return file; - } - - /** - * The delta information does not need to be saved to disk. - * - * @param id String - * @return File - * @throws IOException - */ - private File ensureDeltaIsRegistered(String id) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - - // A write lock is required if we have to update the local index - // entries. - // There should only be one thread trying to access this delta. - File location = getDeltaLocation(id); - getReadLock(); - try - { - if (!indexEntries.containsKey(id)) - { - releaseReadLock(); - // release to upgrade to write lock - getWriteLock(); - try - { - // Make sure the index exists - if (!indexEntries.containsKey(id)) - { - indexEntries.put(id, new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "", 0, 0, false)); - } - - } - finally - { // Downgrade lock - getReadLock(); - releaseWriteLock(); - } - } - } - finally - { - // Release the lock - releaseReadLock(); - } - return location; - } - - /** - * Make a lucene index writer - * - * @param location File - * @param analyzer Analyzer - * @return IndexWriter - * @throws IOException - */ - private IndexWriter makeDeltaIndexWriter(File location, Analyzer analyzer) throws IOException - { - IndexWriter writer; - if (!IndexReader.indexExists(location)) - { - writer = new IndexWriter(location, analyzer, true, MaxFieldLength.LIMITED); - } - else - { - writer = new IndexWriter(location, analyzer, false, MaxFieldLength.LIMITED); - } - writer.setUseCompoundFile(writerUseCompoundFile); - writer.setMaxBufferedDocs(writerMaxBufferedDocs); - writer.setRAMBufferSizeMB(writerRamBufferSizeMb); - writer.setMergeFactor(writerMergeFactor); - writer.setMaxMergeDocs(writerMaxMergeDocs); - writer.setWriteLockTimeout(writeLockTimeout); - writer.setMaxFieldLength(maxFieldLength); - writer.setTermIndexInterval(termIndexInterval); - writer.setMergeScheduler(new SerialMergeScheduler()); - writer.setMergePolicy(new LogDocMergePolicy()); - return writer; - - } - - /** - * Manage getting a lucene index writer for transactional data - looks after registration and checking there is no - * active reader. - * - * @param id String - * @param analyzer Analyzer - * @return IndexWriter - * @throws IOException - */ - public IndexWriter getDeltaIndexWriter(String id, Analyzer analyzer) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - - // No read lock required as the delta should be bound to one thread only - IndexWriter writer = indexWriters.get(id); - if (writer == null) - { - // close index writer if required - closeDeltaIndexReader(id); - File location = ensureDeltaIsRegistered(id); - writer = makeDeltaIndexWriter(location, analyzer); - indexWriters.put(id, writer); - } - return writer; - } - - /** - * Manage closing and unregistering an index reader. - * - * @param id String - * @throws IOException - */ - public void closeDeltaIndexReader(String id) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - - // No lock required as the delta applied to one thread. The delta is - // still active. - IndexReader reader = indexReaders.remove(id); - if (reader != null) - { - reader.close(); - } - } - - /** - * Manage closing and unregistering an index writer . - * - * @param id String - * @throws IOException - */ - public void closeDeltaIndexWriter(String id) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - - // No lock required as the delta applied to one thread. The delta is - // still active. - IndexWriter writer = indexWriters.remove(id); - if (writer != null) - { - writer.close(); - } - } - - /** - * Make sure the writer and reader for TX data are closed. - * - * @param id String - * @throws IOException - */ - public void closeDelta(String id) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - closeDeltaIndexReader(id); - closeDeltaIndexWriter(id); - } - - /** - * Get the deletions for a given index (there is no check if they should be applied that is up to the calling layer) - * - * @param id String - * @throws IOException - */ - public Set getDeletions(String id) throws IOException - { - return getDeletions(id, INDEX_INFO_DELETIONS); - } - - /** - * Get the deletions for a given index (there is no check if they should be applied that is up to the calling layer) - * - * @param id String - * @param fileName String - * @return Set - * @throws IOException - */ - private Set getDeletions(String id, String fileName) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - // Check state - Set deletions = new HashSet(); - File location = new File(indexDirectory, id).getCanonicalFile(); - File file = new File(location, fileName).getCanonicalFile(); - if (!file.exists()) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("No deletions for " + id); - } - return Collections. emptySet(); - } - DataInputStream is = new DataInputStream(new BufferedInputStream(new FileInputStream(file))); - int size = is.readInt(); - for (int i = 0; i < size; i++) - { - String ref = is.readUTF(); - deletions.add(ref); - } - is.close(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("There are " + deletions.size() + " deletions for " + id); - } - return deletions; - - } - - /** - * Set the aux data for the index entry for a transactional unit of work. - * - * @param id - - * the tx id - * @param toDelete - - * noderefs that should be deleted from previous indexes (not this one) - * @param documents - - * the number of docs in the index - * @param deleteNodesOnly - - * should deletions on apply to nodes (ie not to containers) - * @throws IOException - */ - public void setPreparedState(String id, Set toDelete, Set containersToDelete, long documents, boolean deleteNodesOnly) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - // Check state - int toDeleteSize = toDelete.size(); - int containersToDeleteSize = containersToDelete.size(); - if (toDeleteSize > 0) - { - persistDeletions(id, toDelete, INDEX_INFO_DELETIONS); - } - if (containersToDeleteSize > 0) - { - persistDeletions(id, containersToDelete, INDEX_INFO_CONTAINER_DELETIONS); - } - getWriteLock(); - try - { - IndexEntry entry = indexEntries.get(id); - if (entry == null) - { - throw new IndexerException("Invalid index delta id " + id); - } - if ((entry.getStatus() != TransactionStatus.PREPARING) && (entry.getStatus() != TransactionStatus.COMMITTING)) - { - throw new IndexerException("Deletes and doc count can only be set on a preparing index"); - } - entry.setDocumentCount(documents); - entry.setDeletions(toDeleteSize + containersToDeleteSize); - entry.setDeletOnlyNodes(deleteNodesOnly); - } - finally - { - releaseWriteLock(); - } - } - - /** - * @param id String - * @param toDelete Set - * @param fileName String - * @throws IOException - * @throws FileNotFoundException - */ - private void persistDeletions(String id, Set toDelete, String fileName) throws IOException, FileNotFoundException - { - File location = new File(indexDirectory, id).getCanonicalFile(); - if (!location.exists()) - { - if (!location.mkdirs()) - { - throw new IndexerException("Failed to make index directory " + location); - } - } - // Write deletions - DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(location, fileName).getCanonicalFile()))); - os.writeInt(toDelete.size()); - for (String ref : toDelete) - { - os.writeUTF(ref); - } - os.flush(); - os.close(); - } - - private void invalidateMainReadersFromFirst(Set ids) throws IOException - { - boolean found = false; - for (String id : indexEntries.keySet()) - { - if (!found && ids.contains(id)) - { - found = true; - } - if (found) - { - IndexReader main = mainIndexReaders.remove(id); - if (main != null) - { - ((ReferenceCounting) main).setInvalidForReuse(); - } - } - } - - if (found) - { - if(mainIndexReader != null) - { - ((ReferenceCounting) mainIndexReader).setInvalidForReuse(); - mainIndexReader = null; - } - } - - } - - /** - * Get the main reader for committed index data - * - * @return IndexReader - * @throws IOException - */ - public IndexReader getMainIndexReferenceCountingReadOnlyIndexReader() throws IOException - { - getReadLock(); - try - { - // Check if we need to rebuild the main indexer as it is invalid. - // (it is shared and quick version check fails) - if (indexIsShared && !checkVersion()) - { - releaseReadLock(); - getWriteLock(); - try - { - if (mainIndexReader != null) - { - ((ReferenceCounting)mainIndexReader).setInvalidForReuse(); - } - mainIndexReader = null; - } - finally - { - getReadLock(); - releaseWriteLock(); - } - } - - // Build if required - if (mainIndexReader == null) - { - releaseReadLock(); - getWriteLock(); - try - { - if (mainIndexReader == null) - { - // Sync with disk image if required - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - return null; - } - - public boolean canRetry() - { - return true; - } - - }); - mainIndexReader = createMainIndexReader(); - - } - - } - finally - { - getReadLock(); - releaseWriteLock(); - } - } - // Manage reference counting - mainIndexReader.incRef(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Main index reader references = " + ((ReferenceCounting) mainIndexReader).getReferenceCount()); - } - - // ALF-10040: Wrap with a one-off CachingIndexReader (with cache disabled) so that LeafScorer behaves and passes through SingleFieldSelectors to the main index readers - IndexReader reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(MAIN_READER + GUID.generate(), mainIndexReader, false, config); - ReferenceCounting refCounting = (ReferenceCounting) reader; - reader.incRef(); - refCounting.setInvalidForReuse(); - return reader; - } - catch (RuntimeException e) - { - e.printStackTrace(); - throw e; - } - finally - { - releaseReadLock(); - } - } - - /** - * Get the main index reader augmented with the specified TX data As above but we add the TX data - * - * @param id String - * @param deleteOnlyNodes boolean - * @return IndexReader - * @throws IOException - */ - public IndexReader getMainIndexReferenceCountingReadOnlyIndexReader(String id, Set deletions, Set containerDeletions, boolean deleteOnlyNodes) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - getReadLock(); - try - { - if (indexIsShared && !checkVersion()) - { - releaseReadLock(); - getWriteLock(); - try - { - if (mainIndexReader != null) - { - ((ReferenceCounting)mainIndexReader).setInvalidForReuse(); - } - mainIndexReader = null; - } - finally - { - getReadLock(); - releaseWriteLock(); - } - } - - if (mainIndexReader == null) - { - releaseReadLock(); - getWriteLock(); - try - { - if (mainIndexReader == null) - { - // Sync with disk image if required - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - return null; - } - - public boolean canRetry() - { - return true; - } - - }); - mainIndexReader = createMainIndexReader(); - - } - } - finally - { - getReadLock(); - releaseWriteLock(); - } - } - // Combine the index delta with the main index - // Make sure the index is written to disk - // TODO: Should use the in memory index but we often end up forcing - // to disk anyway. - // Is it worth it? - // luceneIndexer.flushPending(); - - IndexReader deltaReader = buildAndRegisterDeltaReader(id); - IndexReader reader = null; - if ((deletions == null || deletions.size() == 0) && (containerDeletions == null || containerDeletions.size() == 0)) - { - reader = new MultiReader(new IndexReader[] { mainIndexReader, deltaReader }, false); - } - else - { - IndexReader filterReader = new FilterIndexReaderByStringId("main+id", mainIndexReader, deletions, containerDeletions, deleteOnlyNodes); - reader = new MultiReader(new IndexReader[] { filterReader, deltaReader }, false); - // Cancel out extra incRef made by MultiReader - filterReader.decRef(); - } - - // The reference count would have been incremented automatically by MultiReader / - // FilterIndexReaderByStringId - deltaReader.decRef(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Main index reader references = " + ((ReferenceCounting) mainIndexReader).getReferenceCount()); - } - reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(MAIN_READER + id, reader, false, config); - ReferenceCounting refCounting = (ReferenceCounting) reader; - reader.incRef(); - refCounting.setInvalidForReuse(); - return reader; - } - finally - { - releaseReadLock(); - } - } - - private boolean shouldBlock() - { - int pendingDeltas = 0; - int maxDeltas = mergerTargetOverlaysBlockingFactor * mergerTargetOverlays; - for (IndexEntry entry : indexEntries.values()) - { - if (entry.getType() == IndexType.DELTA) - { - TransactionStatus status = entry.getStatus(); - if (status == TransactionStatus.PREPARED || status == TransactionStatus.COMMITTING - || status.isCommitted()) - { - if (++pendingDeltas > maxDeltas) - { - return true; - } - } - } - } - return false; - } - - public void setStatus(final String id, final TransactionStatus state, final Set toDelete, final Set read) throws IOException - { - if (id == null) - { - throw new IndexerException("\"null\" is not a valid identifier for a transaction"); - } - final Transition transition = getTransition(state); - - getReadLock(); - try - { - transition.beforeWithReadLock(id, toDelete, read); - releaseReadLock(); - getWriteLock(); - try - { - // we may need to block for some deltas to be merged / rolled back - IndexInfo alreadyPreparing = thisThreadPreparing.get(); - if (state == TransactionStatus.PREPARED) - { - // To avoid deadlock (a thread with multiple deltas never proceeding to commit) we don't block if - // this thread is already in the prepare phase - if (alreadyPreparing != null) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Can't throttle - " + Thread.currentThread().getName() + " already preparing"); - } - } - else - { - while (shouldBlock()) - { - synchronized (mergerTargetLock) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("THROTTLING: " + Thread.currentThread().getName() + " " + indexEntries.size()); - } - merger.schedule(); - releaseWriteLock(); - try - { - mergerTargetLock.wait(60000); - } - catch (InterruptedException e) - { - } - } - getWriteLock(); - } - thisThreadPreparing.set(this); - } - } - else - { - // Only clear the flag when the outermost thread exits prepare - if (alreadyPreparing == this) - { - thisThreadPreparing.set(null); - } - } - - if (transition.requiresFileLock()) - { - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Start Index " + id + " state = " + state); - } - dumpInfo(); - transition.transition(id, toDelete, read); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("End Index " + id + " state = " + state); - } - dumpInfo(); - return null; - } - - public boolean canRetry() - { - return true; - } - - }); - } - else - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Start Index " + id + " state = " + state); - } - dumpInfo(); - transition.transition(id, toDelete, read); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("End Index " + id + " state = " + state); - } - dumpInfo(); - } - } - finally - { - getReadLock(); - releaseWriteLock(); - } - } - finally - { - releaseReadLock(); - } - } - - // - // Internal support for status management - // - - private Transition getTransition(TransactionStatus state) - { - Transition transition = transitions.get(state); - if (transition != null) - { - return transition; - } - else - { - throw new IndexerException("Invalid state " + state); - } - - } - - /** - * Initialise the definitions for the available transitions. - */ - private void initialiseTransitions() - { - - transitions.put(TransactionStatus.PREPARING, new PreparingTransition()); - transitions.put(TransactionStatus.PREPARED, new PreparedTransition()); - transitions.put(TransactionStatus.COMMITTING, new CommittingTransition()); - transitions.put(TransactionStatus.COMMITTED, new CommittedTransition()); - transitions.put(TransactionStatus.ROLLINGBACK, new RollingBackTransition()); - transitions.put(TransactionStatus.ROLLEDBACK, new RolledBackTransition()); - transitions.put(TransactionStatus.DELETABLE, new DeletableTransition()); - transitions.put(TransactionStatus.ACTIVE, new ActiveTransition()); - } - - /** - * API for transitions - * - * @author andyh - */ - private interface Transition - { - void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException; - - void transition(String id, Set toDelete, Set read) throws IOException; - - boolean requiresFileLock(); - } - - /** - * Transition to the perparing state - * - * @author andyh - */ - private class PreparingTransition implements Transition - { - public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException - { - // Nothing to do - } - - public void transition(String id, Set toDelete, Set read) throws IOException - { - IndexEntry entry = indexEntries.get(id); - if (entry == null) - { - throw new IndexerException("Unknown transaction " + id); - } - - if (TransactionStatus.PREPARING.follows(entry.getStatus())) - { - entry.setStatus(TransactionStatus.PREPARING); - } - else - { - throw new IndexerException("Invalid transition for " + id + " from " + entry.getStatus() + " to " + TransactionStatus.PREPARING); - } - } - - public boolean requiresFileLock() - { - return !TransactionStatus.PREPARING.isTransient(); - } - } - - /** - * Transition to the prepared state. - * - * @author andyh - */ - private class PreparedTransition implements Transition - { - public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException - { - - } - - public void transition(String id, Set toDelete, Set read) throws IOException - { - IndexEntry entry = indexEntries.get(id); - if (entry == null) - { - throw new IndexerException("Unknown transaction " + id); - } - - if (TransactionStatus.PREPARED.follows(entry.getStatus())) - { - - LinkedHashMap reordered = new LinkedHashMap(); - boolean addedPreparedEntry = false; - for (String key : indexEntries.keySet()) - { - IndexEntry current = indexEntries.get(key); - - if (!current.getStatus().canBeReordered()) - { - reordered.put(current.getName(), current); - } - else if (!addedPreparedEntry) - { - reordered.put(entry.getName(), entry); - reordered.put(current.getName(), current); - addedPreparedEntry = true; - invalidateMainReadersFromFirst(Collections.singleton(current.getName())); - } - else if (current.getName().equals(entry.getName())) - { - // skip as we are moving it - } - else - { - reordered.put(current.getName(), current); - } - } - - if (indexEntries.size() != reordered.size()) - { - indexEntries = reordered; - dumpInfo(); - throw new IndexerException("Concurrent modification error"); - } - indexEntries = reordered; - - entry.setStatus(TransactionStatus.PREPARED); - writeStatus(); - - } - else - { - throw new IndexerException("Invalid transition for " + id + " from " + entry.getStatus() + " to " + TransactionStatus.PREPARED); - } - } - - public boolean requiresFileLock() - { - return !TransactionStatus.PREPARED.isTransient(); - } - } - - private class CommittingTransition implements Transition - { - public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException - { - - } - - public void transition(String id, Set toDelete, Set read) throws IOException - { - IndexEntry entry = indexEntries.get(id); - if (entry == null) - { - throw new IndexerException("Unknown transaction " + id); - } - - if (TransactionStatus.COMMITTING.follows(entry.getStatus())) - { - entry.setStatus(TransactionStatus.COMMITTING); - } - else - { - throw new IndexerException("Invalid transition for " + id + " from " + entry.getStatus() + " to " + TransactionStatus.COMMITTING); - } - } - - public boolean requiresFileLock() - { - return !TransactionStatus.COMMITTING.isTransient(); - } - } - - private class CommittedTransition implements Transition - { - - ThreadLocal tl = new ThreadLocal(); - - public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException - { - // Make sure we have set up the reader for the data - // ... and close it so we do not up the ref count - closeDelta(id); - IndexEntry entry = indexEntries.get(id); - tl.set(buildReferenceCountingIndexReader(id, entry.getDocumentCount())); - } - - /** - * This has to be protected to allow for retry - */ - public void transition(String id, Set toDelete, Set read) throws IOException - { - IndexEntry entry = indexEntries.get(id); - if (entry == null) - { - clearOldReaders(); - cleaner.schedule(); - throw new IndexerException("Unknown transaction " + id); - } - - if (TransactionStatus.COMMITTED.follows(entry.getStatus())) - { - // Do the deletions - invalidateMainReadersFromFirst(Collections.singleton(id)); - if ((entry.getDocumentCount() + entry.getDeletions()) == 0) - { - registerReferenceCountingIndexReader(id, tl.get()); - indexEntries.remove(id); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Removed commit with no new docs and no deletions"); - } - clearOldReaders(); - cleaner.schedule(); - } - else - { - registerReferenceCountingIndexReader(id, tl.get()); - entry.setStatus(TransactionStatus.COMMITTED); - // TODO: optimise to index for no deletions - // have to allow for this in the application of deletions, - writeStatus(); - if (mainIndexReader != null) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("... invalidating main index reader"); - } - ((ReferenceCounting) mainIndexReader).setInvalidForReuse(); - mainIndexReader = null; - } - - merger.schedule(); - } - - } - else - { - throw new IndexerException("Invalid transition for " + id + " from " + entry.getStatus() + " to " + TransactionStatus.COMMITTED); - } - notifyListeners("CommittedTransactions", 1); - } - - public boolean requiresFileLock() - { - return !TransactionStatus.COMMITTED.isTransient(); - } - } - - private class RollingBackTransition implements Transition - { - public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException - { - - } - - public void transition(String id, Set toDelete, Set read) throws IOException - { - IndexEntry entry = indexEntries.get(id); - if (entry == null) - { - throw new IndexerException("Unknown transaction " + id); - } - - if (TransactionStatus.ROLLINGBACK.follows(entry.getStatus())) - { - entry.setStatus(TransactionStatus.ROLLINGBACK); - writeStatus(); - } - else - { - throw new IndexerException("Invalid transition for " + id + " from " + entry.getStatus() + " to " + TransactionStatus.ROLLINGBACK); - } - } - - public boolean requiresFileLock() - { - return !TransactionStatus.ROLLINGBACK.isTransient(); - } - } - - private class RolledBackTransition implements Transition - { - ThreadLocal tl = new ThreadLocal(); - - public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException - { - closeDelta(id); - IndexEntry entry = indexEntries.get(id); - tl.set(buildReferenceCountingIndexReader(id, entry.getDocumentCount())); - } - - public void transition(String id, Set toDelete, Set read) throws IOException - { - IndexEntry entry = indexEntries.get(id); - if (entry == null) - { - clearOldReaders(); - cleaner.schedule(); - throw new IndexerException("Unknown transaction " + id); - } - - if (TransactionStatus.ROLLEDBACK.follows(entry.getStatus())) - { - entry.setStatus(TransactionStatus.ROLLEDBACK); - writeStatus(); - - registerReferenceCountingIndexReader(id, tl.get()); - indexEntries.remove(id); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Removed rollback"); - } - clearOldReaders(); - cleaner.schedule(); - } - else - { - throw new IndexerException("Invalid transition for " + id + " from " + entry.getStatus() + " to " + TransactionStatus.ROLLEDBACK); - } - } - - public boolean requiresFileLock() - { - return !TransactionStatus.ROLLEDBACK.isTransient(); - } - } - - private class DeletableTransition implements Transition - { - public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException - { - - } - - public void transition(String id, Set toDelete, Set read) throws IOException - { - IndexEntry entry = indexEntries.get(id); - if (entry == null) - { - clearOldReaders(); - cleaner.schedule(); - throw new IndexerException("Unknown transaction " + id); - } - - if (TransactionStatus.DELETABLE.follows(entry.getStatus())) - { - invalidateMainReadersFromFirst(Collections.singleton(id)); - indexEntries.remove(id); - writeStatus(); - clearOldReaders(); - cleaner.schedule(); - } - else - { - throw new IndexerException("Invalid transition for " + id + " from " + entry.getStatus() + " to " + TransactionStatus.DELETABLE); - } - } - - public boolean requiresFileLock() - { - return !TransactionStatus.DELETABLE.isTransient(); - } - } - - private class ActiveTransition implements Transition - { - public void beforeWithReadLock(String id, Set toDelete, Set read) throws IOException - { - - } - - public void transition(String id, Set toDelete, Set read) throws IOException - { - IndexEntry entry = indexEntries.get(id); - if (entry != null) - { - if (entry.getStatus() != TransactionStatus.ACTIVE) - { - throw new IndexerException("TX Already active " + id); - } - } - - if (TransactionStatus.ACTIVE.follows(null)) - { - indexEntries.put(id, new IndexEntry(IndexType.DELTA, id, "", TransactionStatus.ACTIVE, "", 0, 0, false)); - } - else - { - throw new IndexerException("Invalid transition for " + id + " from " + entry.getStatus() + " to " + TransactionStatus.ACTIVE); - } - } - - public boolean requiresFileLock() - { - return !TransactionStatus.ACTIVE.isTransient(); - } - } - - // - // - // Internal methods for implementation support - // =========================================== - // - // These methods should all be called with the appropriate locks. - // - // - - private static boolean createFile(File file) - { - - if (!file.exists()) - { - try - { - file.createNewFile(); - return true; - } - catch (IOException e) - { - throw new AlfrescoRuntimeException("Failed to create info file", e); - } - } - return false; - } - - private static RandomAccessFile openFile(File file) - { - try - { - if (useNIOMemoryMapping) - { - return new RandomAccessFile(file, "rw"); - } - else - { - return new RandomAccessFile(file, "rws"); - } - } - catch (FileNotFoundException e) - { - throw new AlfrescoRuntimeException("Failed to open index info file", e); - } - } - - /** - * Check status must be called holding the file lock. - * - * @throws IOException - */ - private void setStatusFromFile() throws IOException - { - try - { - setStatusFromFile(indexInfoChannel); - } - catch (IOException e) - { - // The first data file is corrupt so we fall back to the back up - setStatusFromFile(indexInfoBackupChannel); - } - clearOldReaders(); - } - - private void clearOldReaders() throws IOException - { - // Find current invalid - HashSet inValid = new HashSet(); - for (String id : referenceCountingReadOnlyIndexReaders.keySet()) - { - if (!indexEntries.containsKey(id)) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(id + " is now INVALID "); - } - inValid.add(id); - } - else - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(id + " is still part of the index "); - } - } - } - // Clear invalid - clearInvalid(inValid); - } - - private void clearInvalid(Set inValid) throws IOException - { - boolean hasInvalid = false; - for (String id : inValid) - { - IndexReader reader = referenceCountingReadOnlyIndexReaders.remove(id); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("... invalidating sub reader " + id); - } - if (reader != null) - { - ReferenceCounting referenceCounting = (ReferenceCounting) reader; - referenceCounting.setInvalidForReuse(); - deletableReaders.add(reader); - hasInvalid = true; - } - } - if (hasInvalid) - { - for (String id : inValid) - { - IndexReader main = mainIndexReaders.remove(id); - if (main != null) - { - ((ReferenceCounting) main).setInvalidForReuse(); - } - } - if (mainIndexReader != null) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("... invalidating main index reader"); - } - ((ReferenceCounting) mainIndexReader).setInvalidForReuse(); - } - mainIndexReader = null; - } - } - - private IndexReader createMainIndexReader() throws IOException - { - IndexReader reader = null; - IndexReader oldReader = null; - for (String id : indexEntries.keySet()) - { - IndexEntry entry = indexEntries.get(id); - if (entry.getStatus().isCommitted()) - { - IndexReader subReader = getReferenceCountingIndexReader(id); - if (reader == null) - { - reader = subReader; - } - else - { - boolean oldReaderIsSubReader = oldReader == null; - oldReader = reader; - reader = mainIndexReaders.get(id); - if (reader == null) - { - if (entry.getType() == IndexType.INDEX) - { - reader = new MultiReader(new IndexReader[] { oldReader, subReader }, false); - } - else if (entry.getType() == IndexType.DELTA) - { - try - { - IndexReader filterReader = new FilterIndexReaderByStringId(id, oldReader, getDeletions(entry.getName(), INDEX_INFO_DELETIONS), getDeletions(entry.getName(), INDEX_INFO_CONTAINER_DELETIONS), entry.isDeletOnlyNodes()); - reader = new MultiReader(new IndexReader[] { filterReader, subReader }, false); - // Cancel out the incRef on the filter reader - filterReader.decRef(); - } - catch (IOException ioe) - { - s_logger.error("Failed building filter reader beneath " + entry.getName(), ioe); - throw ioe; - } - } - reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(id+"multi", reader, true, config); - mainIndexReaders.put(id, reader); - } - } - } - } - if (reader == null) - { - reader = IndexReader.open(emptyIndex); - } - else - { - // Keep this reader open whilst it is referenced by mainIndexReaders / referenceCountingReadOnlyIndexReaders - reader.incRef(); - } - - reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(MAIN_READER, reader, false, config); - return reader; - } - - private IndexReader getReferenceCountingIndexReader(String id) throws IOException - { - IndexReader reader = referenceCountingReadOnlyIndexReaders.get(id); - if (reader == null) - { - throw new IllegalStateException("Indexer should have been pre-built for " + id); - } - return reader; - } - - private void registerReferenceCountingIndexReader(String id, IndexReader reader) throws IOException - { - ReferenceCounting referenceCounting = (ReferenceCounting) reader; - if (!referenceCounting.getId().equals(id)) - { - throw new IllegalStateException("Registering " + referenceCounting.getId() + " as " + id); - } - // ALF-13981: Be careful not to invalidate the segment reader if we are trying to re-register exactly the same - // one (e.g. in a doWithFileLock() retry loop) - if (referenceCountingReadOnlyIndexReaders.get(id) != reader) - { - clearInvalid(Collections.singleton(id)); - referenceCountingReadOnlyIndexReaders.put(id, reader); - } - } - - private double getSizeInMb(File file) - { - long size = getSize(file); - return size/1024.0d/1024.0d; - } - - private long getSize(File file) - { - long size = 0l; - if (file == null) - { - return size; - } - if (file.isFile()) - { - return file.length(); - } - else - { - File[] files = file.listFiles(); - if(files == null) - { - return size; - } - for (File current : files) - { - if (current.isDirectory()) - { - size += getSize(current); - } - else - { - size += current.length(); - } - } - } - return size; - } - - private IndexReader buildReferenceCountingIndexReader(String id, long size) throws IOException - { - IndexReader reader; - File location = new File(indexDirectory, id).getCanonicalFile(); - double folderSize = getSizeInMb(location); - if (IndexReader.indexExists(location)) - { - if ((size < maxDocsForInMemoryIndex) && (folderSize < maxRamInMbForInMemoryIndex)) - { - RAMDirectory rd = new RAMDirectory(location); - reader = IndexReader.open(rd); - } - else - { - reader = IndexReader.open(location); - } - } - else - { - reader = IndexReader.open(emptyIndex); - } - reader = ReferenceCountingReadOnlyIndexReaderFactory.createReader(id, reader, true, config); - return reader; - } - - private boolean checkVersion() throws IOException - { - try - { - return checkVersion(indexInfoChannel); - } - catch (IOException e) - { - // The first data file is corrupt so we fall back to the back up - try - { - return checkVersion(indexInfoBackupChannel); - } - catch (IOException ee) - { - return false; - } - } - } - - private boolean checkVersion(FileChannel channel) throws IOException - { - if (channel.size() > 0) - { - channel.position(0); - ByteBuffer buffer; - - if (useNIOMemoryMapping) - { - MappedByteBuffer mbb = channel.map(MapMode.READ_ONLY, 0, 8); - mbb.load(); - buffer = mbb; - } - else - { - buffer = ByteBuffer.wrap(new byte[8]); - channel.read(buffer); - ((Buffer) buffer).position(0); - } - - ((Buffer) buffer).position(0); - long onDiskVersion = buffer.getLong(); - return (version == onDiskVersion); - } - return (version == 0); - } - - private void setStatusFromFile(FileChannel channel) throws IOException - { - if (channel.size() > 0) - { - channel.position(0); - ByteBuffer buffer; - - if (useNIOMemoryMapping) - { - MappedByteBuffer mbb = channel.map(MapMode.READ_ONLY, 0, channel.size()); - mbb.load(); - buffer = mbb; - } - else - { - buffer = ByteBuffer.wrap(new byte[(int) channel.size()]); - channel.read(buffer); - ((Buffer) buffer).position(0); - } - - ((Buffer) buffer).position(0); - long onDiskVersion = buffer.getLong(); - if (version != onDiskVersion) - { - CRC32 crc32 = new CRC32(); - crc32.update((int) (onDiskVersion >>> 32) & 0xFFFFFFFF); - crc32.update((int) (onDiskVersion >>> 0) & 0xFFFFFFFF); - int size = buffer.getInt(); - crc32.update(size); - LinkedHashMap newIndexEntries = new LinkedHashMap(); - // Not all state is saved some is specific to this index so we - // need to add the transient stuff. - // Until things are committed they are not shared unless it is - // prepared - for (int i = 0; i < size; i++) - { - String indexTypeString = readString(buffer, crc32); - IndexType indexType; - try - { - indexType = IndexType.valueOf(indexTypeString); - } - catch (IllegalArgumentException e) - { - throw new IOException("Invalid type " + indexTypeString); - } - - String name = readString(buffer, crc32); - - String parentName = readString(buffer, crc32); - - String txStatus = readString(buffer, crc32); - TransactionStatus status; - try - { - status = TransactionStatus.valueOf(txStatus); - } - catch (IllegalArgumentException e) - { - throw new IOException("Invalid status " + txStatus); - } - - String mergeId = readString(buffer, crc32); - - long documentCount = buffer.getLong(); - crc32.update((int) (documentCount >>> 32) & 0xFFFFFFFF); - crc32.update((int) (documentCount >>> 0) & 0xFFFFFFFF); - - long deletions = buffer.getLong(); - crc32.update((int) (deletions >>> 32) & 0xFFFFFFFF); - crc32.update((int) (deletions >>> 0) & 0xFFFFFFFF); - - byte deleteOnlyNodesFlag = buffer.get(); - crc32.update(deleteOnlyNodesFlag); - boolean isDeletOnlyNodes = deleteOnlyNodesFlag == 1; - - if (!status.isTransient()) - { - newIndexEntries.put(name, new IndexEntry(indexType, name, parentName, status, mergeId, documentCount, deletions, isDeletOnlyNodes)); - } - } - long onDiskCRC32 = buffer.getLong(); - if (crc32.getValue() == onDiskCRC32) - { - for (IndexEntry entry : indexEntries.values()) - { - if (entry.getStatus().isTransient()) - { - newIndexEntries.put(entry.getName(), entry); - } - } - version = onDiskVersion; - indexEntries = newIndexEntries; - } - else - { - throw new IOException("Invalid file check sum"); - } - } - } - - } - - private String readString(ByteBuffer buffer, CRC32 crc32) throws UnsupportedEncodingException - { - int size = buffer.getInt(); - byte[] bytes = new byte[size]; - buffer.get(bytes); - char[] chars = new char[size]; - for (int i = 0; i < size; i++) - { - chars[i] = (char) bytes[i]; - } - crc32.update(bytes); - return new String(chars); - } - - private void writeString(ByteBuffer buffer, CRC32 crc32, String string) throws UnsupportedEncodingException - { - char[] chars = string.toCharArray(); - byte[] bytes = new byte[chars.length]; - for (int i = 0; i < chars.length; i++) - { - if (chars[i] > 0xFF) - { - throw new UnsupportedEncodingException(); - } - bytes[i] = (byte) chars[i]; - } - buffer.putInt(bytes.length); - buffer.put(bytes); - crc32.update(bytes); - } - - private void writeStatus() throws IOException - { - version++; - writeStatusToFile(indexInfoChannel); - writeStatusToFile(indexInfoBackupChannel); - // We have a state that allows more transactions. Notify waiting threads - if (!shouldBlock()) - { - synchronized (mergerTargetLock) - { - mergerTargetLock.notifyAll(); - } - } - } - - private void writeStatusToFile(FileChannel channel) throws IOException - { - long size = getBufferSize(); - - ByteBuffer buffer; - if (useNIOMemoryMapping) - { - MappedByteBuffer mbb = channel.map(MapMode.READ_WRITE, 0, size); - mbb.load(); - buffer = mbb; - } - else - { - channel.truncate(size); - buffer = ByteBuffer.wrap(new byte[(int) size]); - } - - ((Buffer) buffer).position(0); - - buffer.putLong(version); - CRC32 crc32 = new CRC32(); - crc32.update((int) (version >>> 32) & 0xFFFFFFFF); - crc32.update((int) (version >>> 0) & 0xFFFFFFFF); - - buffer.putInt(indexEntries.size()); - crc32.update(indexEntries.size()); - - for (IndexEntry entry : indexEntries.values()) - { - String entryType = entry.getType().toString(); - writeString(buffer, crc32, entryType); - - writeString(buffer, crc32, entry.getName()); - - writeString(buffer, crc32, entry.getParentName()); - - String entryStatus = entry.getStatus().toString(); - writeString(buffer, crc32, entryStatus); - - writeString(buffer, crc32, entry.getMergeId()); - - buffer.putLong(entry.getDocumentCount()); - crc32.update((int) (entry.getDocumentCount() >>> 32) & 0xFFFFFFFF); - crc32.update((int) (entry.getDocumentCount() >>> 0) & 0xFFFFFFFF); - - buffer.putLong(entry.getDeletions()); - crc32.update((int) (entry.getDeletions() >>> 32) & 0xFFFFFFFF); - crc32.update((int) (entry.getDeletions() >>> 0) & 0xFFFFFFFF); - - buffer.put(entry.isDeletOnlyNodes() ? (byte) 1 : (byte) 0); - crc32.update(entry.isDeletOnlyNodes() ? new byte[] { (byte) 1 } : new byte[] { (byte) 0 }); - } - buffer.putLong(crc32.getValue()); - - if (useNIOMemoryMapping) - { - ((MappedByteBuffer) buffer).force(); - } - else - { - ((Buffer) buffer).rewind(); - channel.position(0); - channel.write(buffer); - } - } - - private long getBufferSize() throws IOException - { - long size = 0; - size += 8; - size += 4; - for (IndexEntry entry : indexEntries.values()) - { - String entryType = entry.getType().toString(); - size += (entryType.length()) + 4; - size += (entry.getName().length()) + 4; - size += (entry.getParentName().length()) + 4; - String entryStatus = entry.getStatus().toString(); - size += (entryStatus.length()) + 4; - size += (entry.getMergeId().length()) + 4; - size += 8; - size += 8; - size += 1; - } - size += 8; - return size; - } - - public interface LockWork - { - public Result doWork() throws Exception; - - public boolean canRetry(); - } - - public R doReadOnly(LockWork lockWork) - { - - readOnlyLock.writeLock().lock(); - try - { - getReadLock(); - try - { - return doWithFileLock(lockWork); - } - finally - { - releaseReadLock(); - } - } - finally - { - readOnlyLock.writeLock().unlock(); - } - } - - private static final int CHANNEL_OPEN_RETRIES = 5; - - private R doWithFileLock(LockWork lockWork) - { - try - { - return doWithFileLock(lockWork, CHANNEL_OPEN_RETRIES); - } - catch (Throwable e) - { - // Re-throw the exception - if (e instanceof RuntimeException) - { - throw (RuntimeException) e; - } - else - { - throw new RuntimeException("Error during run with lock.", e); - } - } - } - - /** - * Specific exception to catch channel close issues. - * - * @author Derek Hulley - * @since 2.1.3 - */ - private static class IndexInfoChannelException extends IOException - { - /** - * - */ - private static final long serialVersionUID = 1588898991653057286L; - - public IndexInfoChannelException(String msg) - { - super(msg); - } - } - - /** - * An iterative method that retries the operation in the event of the channel being closed. - * - * @param retriesRemaining - * the number of retries remaining - * @return Returns the lock work result - */ - private R doWithFileLock(LockWork lockWork, int retriesRemaining) throws Throwable - { - FileLock fileLock = null; - R result = null; - long start = 0L; - try - { - // Check that the channel is open - if (!indexInfoChannel.isOpen()) - { - if (lockWork.canRetry()) - { - throw new IndexInfoChannelException("Channel is closed. Manually triggering reopen attempts"); - } - else - { - reopenChannels(); - } - } - - if (indexIsShared) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(" ... waiting for file lock"); - start = System.nanoTime(); - } - fileLock = indexInfoChannel.lock(); - if (s_logger.isDebugEnabled()) - { - long end = System.nanoTime(); - s_logger.debug(" ... got file lock in " + ((end - start) / 10e6f) + " ms"); - } - if (!checkVersion()) - { - setStatusFromFile(); - } - } - result = lockWork.doWork(); - return result; - } - catch (IOException e) - { - if (!lockWork.canRetry()) - { - // We've done our best - s_logger.warn("This operation can not retry upon an IOException - it has to roll back to its previous state"); - throw e; - } - if (retriesRemaining == 0) - { - // We've done our best - s_logger.warn("No more channel open retries remaining"); - throw e; - } - else - { - // Attempt to reopen the channel - if (s_logger.isDebugEnabled()) - { - s_logger.debug("\n" + "Channel is closed. Will attempt to open it. \n" + " Retries remaining: " + retriesRemaining); - } - try - { - reopenChannels(); - // Loop around and try again - return doWithFileLock(lockWork, --retriesRemaining); - } - catch (Throwable ee) - { - // Report this error, but throw the original - s_logger.error("Channel reopen failed on index info files in: " + this.indexDirectory, ee); - throw e; - } - } - } - finally - { - if (fileLock != null) - { - try - { - fileLock.release(); - long end = System.nanoTime(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug(" ... released file lock after " + ((end - start) / 10e6f) + " ms"); - } - } - catch (IOException e) - { - s_logger.warn("Failed to release file lock: " + e.getMessage(), e); - } - } - } - } - - /** - * Reopens all the channels. The channels are closed first. This method is synchronized. - */ - private synchronized void reopenChannels() throws Throwable - { - try - { - indexInfoRAF.close(); - } - catch (IOException e) - { - s_logger.warn("Failed to close indexInfoRAF", e); - } - try - { - indexInfoBackupRAF.close(); - } - catch (IOException e) - { - s_logger.warn("Failed to close indexInfoRAF", e); - } - File indexInfoFile = new File(this.indexDirectory, INDEX_INFO); - File indexInfoBackupFile = new File(this.indexDirectory, INDEX_INFO_BACKUP); - - // Open the files and channels for the index info file and the backup - this.indexInfoRAF = openFile(indexInfoFile); - this.indexInfoChannel = this.indexInfoRAF.getChannel(); - - this.indexInfoBackupRAF = openFile(indexInfoBackupFile); - this.indexInfoBackupChannel = this.indexInfoBackupRAF.getChannel(); - } - - /** - * Helper to print out index information - * - * @param args String[] - * @throws Throwable - */ - public static void main(String[] args) throws Throwable - { - for (int i = 0; i < args.length; i++) - { - File indexLocation = new File(args[i]); - if (!indexLocation.exists()) - { - System.err.println("Index directory doesn't exist: " + indexLocation); - continue; - } - readIndexInfo(indexLocation); - } - } - - static Query getPathQuery(String path) throws SAXPathException - { - ApplicationContext ac = ApplicationContextHelper.getApplicationContext(); - XPathReader reader = new XPathReader(); - LuceneXPathHandler handler = new LuceneXPathHandler(); - handler.setNamespacePrefixResolver((NamespaceService) ac.getBean("namespaceService")); - handler.setDictionaryService((DictionaryService) ac.getBean("dictionaryService")); - reader.setXPathHandler(handler); - reader.parse(path); - PathQuery pathQuery = handler.getQuery(); - pathQuery.setRepeats(false); - return pathQuery; - } - - private static void readIndexInfo(File indexLocation) throws Throwable - { - long start; - long end; - IndexInfo ii = new IndexInfo(indexLocation, null); - - ii.readWriteLock.writeLock().lock(); - try - { - System.out.println("Entry List for " + indexLocation); - System.out.println(" Size = " + ii.indexEntries.size()); - int i = 0; - for (IndexEntry entry : ii.indexEntries.values()) - { - System.out.println("\t" + (i++) + "\t" + entry.toString()); - } - } - finally - { - ii.releaseWriteLock(); - } - IndexReader reader = ii.getMainIndexReferenceCountingReadOnlyIndexReader(); - System.out.println(reader.getFieldNames(FieldOption.ALL)); - - TermEnum te = reader.terms(); - while (te.next()) - { - if (te.term().field().contains("FTS")) - { - System.out.println(te.term()); - } - } - // @{http://www.alfresco.org/model/content/1.0}name:product363_ocmwbeersel - - IndexSearcher searcher = new IndexSearcher(reader); - Query query = new TermQuery(new Term("@{http://www.alfresco.org/model/content/1.0}name", "product363_ocmwbeersel")); - start = System.nanoTime(); - Hits hits = searcher.search(query); - end = System.nanoTime(); - System.out.println("@{http://www.alfresco.org/model/content/1.0}name:product363_ocmwbeersel = " + hits.length() + " in " + ((end - start) / 1e9)); - searcher.close(); - - searcher = new IndexSearcher(reader); - query = new WildcardQuery(new Term("@{http://www.alfresco.org/model/content/1.0}name", "b*")); - start = System.nanoTime(); - hits = searcher.search(query); - end = System.nanoTime(); - System.out.println("@{http://www.alfresco.org/model/content/1.0}name:b* = " + hits.length() + " in " + ((end - start) / 1e9)); - searcher.close(); - - searcher = new IndexSearcher(reader); - query = new TermQuery(new Term("@{http://www.alfresco.org/model/content/1.0}name", "be")); - start = System.nanoTime(); - hits = searcher.search(query); - end = System.nanoTime(); - System.out.println("@{http://www.alfresco.org/model/content/1.0}name:be = " + hits.length() + " in " + ((end - start) / 1e9)); - searcher.close(); - } - - /** - * Clean up support. - * - * @author Andy Hind - */ - private class Cleaner extends AbstractSchedulable - { - - String getLogName() - { - return "Index cleaner"; - } - - void runImpl() - { - - Iterator i = deletableReaders.iterator(); - while (i.hasNext()) - { - IndexReader reader = i.next(); - ReferenceCounting refCounting = (ReferenceCounting) reader; - if (refCounting.getReferenceCount() == 0) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Deleting no longer referenced " + refCounting.getId()); - s_logger.debug("... queued delete for " + refCounting.getId()); - s_logger.debug("... " + ReferenceCountingReadOnlyIndexReaderFactory.getState(refCounting.getId())); - } - getReadLock(); - try - { - if (indexEntries.containsKey(refCounting.getId())) - { - s_logger.error("ERROR - deleting live reader - " + refCounting.getId()); - } - } - finally - { - releaseReadLock(); - } - deleteQueue.add(refCounting.getId()); - i.remove(); - } - else if (s_logger.isTraceEnabled() && refCounting.getCreationTime() < System.currentTimeMillis() - 120000) - { - for (Throwable t : refCounting.getReferences()) - { - s_logger.trace(t.getMessage(), t); - } - } - - } - - Iterator j = deleteQueue.iterator(); - while (j.hasNext()) - { - String id = j.next(); - try - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Expunging " + id + " remaining " + deleteQueue.size()); - s_logger.debug("... " + ReferenceCountingReadOnlyIndexReaderFactory.getState(id)); - } - // try and delete - File location = new File(indexDirectory, id).getCanonicalFile(); - if (!deleteDirectory(location)) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("DELETE FAILED"); - } - } - else - { - j.remove(); - } - } - catch (IOException ioe) - { - s_logger.warn("Failed to delete file - invalid canonical file", ioe); - } - } - } - - ExitState recoverImpl() - { - return ExitState.DONE; - } - - private boolean deleteDirectory(File file) - { - File[] children = file.listFiles(); - if (children != null) - { - for (int i = 0; i < children.length; i++) - { - File child = children[i]; - if (child.isDirectory()) - { - deleteDirectory(child); - } - else - { - if (child.exists() && !child.delete() && child.exists()) - { - return false; - } - } - } - } - if (file.exists() && !file.delete() && file.exists()) - { - return false; - } - return true; - } - - } - - /** - * Supported by one thread. 1) If the first index is a delta we can just change it to an index. There is now here to - * apply the deletions 2) Merge indexes Combine indexes together according to the target index merge strategy. This - * is a trade off to make an optimised index but not spend too much time merging and optimising small merges. 3) - * Apply next deletion set to indexes Apply the deletions for the first delta to all the other indexes. Deletes can - * be applied with relative impunity. If any are applied they take effect as required. 1) 2) and 3) are mutually - * exclusive try in order This could be supported in another thread 4) Merge deltas Merge two index deltas together. - * Starting at the end. Several merges can be going on at once. a) Find merge b) Set state c) apply deletions to the - * previous delta d) update state e) add deletions to the previous delta deletion list f) update state - */ - - private enum MergeAction - { - NONE, MERGE_INDEX, APPLY_DELTA_DELETION, MERGE_DELTA - } - - private enum ScheduledState - { - UN_SCHEDULED, SCHEDULED, FAILED, RECOVERY_SCHEDULED - } - - private enum ExitState - { - DONE, RESCHEDULE; - } - - private abstract class AbstractSchedulable implements Schedulable, Runnable - { - ScheduledState scheduledState = ScheduledState.UN_SCHEDULED; - - - - public synchronized void schedule() - { - switch (scheduledState) - { - case FAILED: - scheduledState = ScheduledState.RECOVERY_SCHEDULED; - threadPoolExecutor.execute(this); - break; - case UN_SCHEDULED: - scheduledState = ScheduledState.SCHEDULED; - threadPoolExecutor.execute(this); - break; - case RECOVERY_SCHEDULED: - case SCHEDULED: - default: - // Nothing to do - break; - } - } - - synchronized void done() - { - switch (scheduledState) - { - case RECOVERY_SCHEDULED: - case SCHEDULED: - scheduledState = ScheduledState.UN_SCHEDULED; - break; - case FAILED: - case UN_SCHEDULED: - default: - throw new IllegalStateException(); - } - } - - private synchronized void rescheduleRecovery() - { - switch (scheduledState) - { - case RECOVERY_SCHEDULED: - threadPoolExecutor.execute(this); - break; - case SCHEDULED: - case FAILED: - case UN_SCHEDULED: - default: - throw new IllegalStateException(); - } - } - - private synchronized void fail() - { - switch (scheduledState) - { - case RECOVERY_SCHEDULED: - case SCHEDULED: - scheduledState = ScheduledState.FAILED; - break; - case FAILED: - case UN_SCHEDULED: - default: - throw new IllegalStateException(); - } - } - - public void run() - { - try - { - switch (scheduledState) - { - case RECOVERY_SCHEDULED: - ExitState reschedule = recoverImpl(); - s_logger.error(getLogName() + " has recovered - resuming ... "); - if (reschedule == ExitState.RESCHEDULE) - { - rescheduleRecovery(); - break; - } - case SCHEDULED: - if (s_logger.isDebugEnabled()) - { - s_logger.debug(getLogName() + " running ... "); - } - runImpl(); - done(); - break; - case FAILED: - case UN_SCHEDULED: - default: - throw new IllegalStateException(); - } - } - catch (Throwable t) - { - try - { - if (s_logger.isWarnEnabled()) - { - s_logger.warn(getLogName() + " failed with ", t); - } - recoverImpl(); - if (s_logger.isWarnEnabled()) - { - s_logger.warn(getLogName() + " recovered from ", t); - } - done(); - } - catch (Throwable rbt) - { - fail(); - s_logger.error(getLogName() + " failed to recover - suspending ", rbt); - } - } - } - - abstract void runImpl() throws Exception; - - abstract ExitState recoverImpl() throws Exception; - - abstract String getLogName(); - } - - private class Merger extends AbstractSchedulable - { - String getLogName() - { - return "Index merger"; - } - - @Override - void done() - { - // Reschedule if we need to, based on the current index state, that may have changed since we last got the - // read lock - getReadLock(); - try - { - synchronized (this) - { - if (decideMergeAction() != MergeAction.NONE) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(getLogName() + " rescheduling ... "); - } - switch (scheduledState) - { - case RECOVERY_SCHEDULED: - scheduledState = ScheduledState.SCHEDULED; - case SCHEDULED: - threadPoolExecutor.execute(this); - break; - case FAILED: - case UN_SCHEDULED: - default: - throw new IllegalStateException(); - } - } - else - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(getLogName() + " done "); - } - super.done(); - } - } - } - finally - { - releaseReadLock(); - } - } - - void runImpl() throws IOException - { - - // Get the read lock to decide what to do - // Single JVM to start with - MergeAction action; - - getReadLock(); - try - { - if (indexIsShared && !checkVersion()) - { - releaseReadLock(); - getWriteLock(); - try - { - // Sync with disk image if required - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - return null; - } - - public boolean canRetry() - { - return true; - } - }); - } - finally - { - getReadLock(); - releaseWriteLock(); - } - } - - action = decideMergeAction(); - } - - catch (IOException e) - { - s_logger.error("Error reading index file", e); - return; - } - finally - { - releaseReadLock(); - } - - if (s_logger.isDebugEnabled()) - { - s_logger.debug(getLogName() + " Merger applying MergeAction." + action.toString()); - } - if (action == MergeAction.APPLY_DELTA_DELETION) - { - mergeDeletions(); - } - else if (action == MergeAction.MERGE_INDEX) - { - mergeIndexes(); - } - if (s_logger.isDebugEnabled()) - { - dumpInfo(); - } - } - - private MergeAction decideMergeAction() - { - MergeAction action = MergeAction.NONE; - int indexes = 0; - boolean mergingIndexes = false; - int deltas = 0; - boolean applyingDeletions = false; - - for (IndexEntry entry : indexEntries.values()) - { - if (entry.getType() == IndexType.INDEX) - { - indexes++; - if ((entry.getStatus() == TransactionStatus.MERGE) || (entry.getStatus() == TransactionStatus.MERGE_TARGET)) - { - mergingIndexes = true; - } - } - else if (entry.getType() == IndexType.DELTA) - { - if (entry.getStatus() == TransactionStatus.COMMITTED) - { - deltas++; - } - if (entry.getStatus() == TransactionStatus.COMMITTED_DELETING) - { - applyingDeletions = true; - deltas++; - } - } - } - - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Indexes = " + indexes); - s_logger.debug("Merging = " + mergingIndexes); - s_logger.debug("Deltas = " + deltas); - s_logger.debug("Deleting = " + applyingDeletions); - } - - if (!mergingIndexes && !applyingDeletions) - { - if (indexes > mergerTargetIndexes) - { - // Try merge - action = MergeAction.MERGE_INDEX; - } - else if (deltas > mergerTargetOverlays) - { - // Try delete - action = MergeAction.APPLY_DELTA_DELETION; - } - } - return action; - } - - ExitState recoverImpl() - { - getWriteLock(); - try - { - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - setStatusFromFile(); - - // If the index is not shared we can do some easy clean - // up - if (!indexIsShared) - { - HashSet deletable = new HashSet(); - // clean up - for (IndexEntry entry : indexEntries.values()) - { - switch (entry.getStatus()) - { - // states which can be deleted - // We could check prepared states can be - // committed. - case ACTIVE: - case MARKED_ROLLBACK: - case NO_TRANSACTION: - case PREPARING: - case ROLLEDBACK: - case ROLLINGBACK: - case UNKNOWN: - case PREPARED: - case DELETABLE: - case COMMITTING: - case COMMITTED: - default: - if (s_logger.isInfoEnabled()) - { - s_logger.info("Roll back merge: leaving index entry " + entry); - } - break; - // States which are in mid-transition which we - // can roll back to the committed state - case COMMITTED_DELETING: - case MERGE: - if (s_logger.isInfoEnabled()) - { - s_logger.info("Roll back merge: Resetting merge and committed_deleting to committed " + entry); - } - entry.setStatus(TransactionStatus.COMMITTED); - break; - case MERGE_TARGET: - if (s_logger.isInfoEnabled()) - { - s_logger.info("Roll back merge: Deleting merge target " + entry); - } - entry.setStatus(TransactionStatus.DELETABLE); - deletable.add(entry.getName()); - break; - } - - // Check we have a reader registered - if (referenceCountingReadOnlyIndexReaders.get(entry.getName()) == null) - { - registerReferenceCountingIndexReader(entry.getName(), buildReferenceCountingIndexReader(entry.getName(), entry.getDocumentCount())); - } - } - - if (mainIndexReader != null) - { - ReferenceCounting rcMain = (ReferenceCounting) mainIndexReader; - if (rcMain.isInvalidForReuse()) - { - mainIndexReader = null; - } - } - - // Delete entries that are not required - invalidateMainReadersFromFirst(deletable); - for (String id : deletable) - { - indexEntries.remove(id); - } - clearOldReaders(); - - cleaner.schedule(); - - // persist the new state - writeStatus(); - } - return null; - } - - public boolean canRetry() - { - return false; - } - - }); - } - finally - { - releaseWriteLock(); - } - return ExitState.DONE; - } - - void mergeDeletions() throws IOException - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Deleting ..."); - } - - // lock for deletions - final LinkedHashMap toDelete; - LinkedHashMap indexes; - - getWriteLock(); - try - { - toDelete = doWithFileLock(new LockWork>() - { - public LinkedHashMap doWork() throws Exception - { - LinkedHashMap set = new LinkedHashMap(); - - for (IndexEntry entry : indexEntries.values()) - { - if ((entry.getType() == IndexType.INDEX) && (entry.getStatus() == TransactionStatus.MERGE)) - { - return set; - } - if ((entry.getType() == IndexType.INDEX) && (entry.getStatus() == TransactionStatus.MERGE_TARGET)) - { - return set; - } - if ((entry.getType() == IndexType.DELTA) && (entry.getStatus() == TransactionStatus.COMMITTED_DELETING)) - { - return set; - } - } - // Check it is not deleting - BREAK: for (IndexEntry entry : indexEntries.values()) - { - // skip indexes at the start - if (entry.getType() == IndexType.DELTA) - { - if (entry.getStatus() == TransactionStatus.COMMITTED) - { - entry.setStatus(TransactionStatus.COMMITTED_DELETING); - set.put(entry.getName(), entry); - } - else - { - // If not committed we stop as we can not - // span non committed. - break BREAK; - } - } - } - if (set.size() > 0) - { - writeStatus(); - } - return set; - - } - - public boolean canRetry() - { - return false; - } - - }); - } - finally - { - getReadLock(); - releaseWriteLock(); - } - - try - { - indexes = new LinkedHashMap(); - BREAK: for (IndexEntry entry : indexEntries.values()) - { - if (entry.getStatus() == TransactionStatus.COMMITTED_DELETING) - { - break BREAK; - } - indexes.put(entry.getName(), entry); - } - } - finally - { - releaseReadLock(); - } - - if (toDelete.size() == 0) - { - return; - } - // Build readers - - int size = 2 * (toDelete.size() + indexes.size()); - final HashSet invalidIndexes = new HashSet(size); - - final HashMap newIndexCounts = new HashMap(size); - - LinkedHashMap readers = new LinkedHashMap(size); - for (IndexEntry currentDelete : toDelete.values()) - { - Set deletions = getDeletions(currentDelete.getName(), INDEX_INFO_DELETIONS); - Set containerDeletions = getDeletions(currentDelete.getName(), INDEX_INFO_CONTAINER_DELETIONS); - if (!deletions.isEmpty()) - { - for (String key : indexes.keySet()) - { - IndexReader reader = getReferenceCountingIndexReader(key); - Searcher searcher = new IndexSearcher(reader); - try - { - for (String stringRef : deletions) - { - TermQuery query = new TermQuery(new Term("ID", stringRef)); - Hits hits = searcher.search(query); - if (hits.length() > 0) - { - IndexReader writeableReader = readers.get(key); - if (writeableReader == null) - { - File location = new File(indexDirectory, key).getCanonicalFile(); - if (IndexReader.indexExists(location)) - { - writeableReader = IndexReader.open(location); - } - else - { - continue; - } - readers.put(key, writeableReader); - } - - if (currentDelete.isDeletOnlyNodes() && !containerDeletions.contains(stringRef)) - { - Searcher writeableSearcher = new IndexSearcher(writeableReader); - hits = writeableSearcher.search(query); - if (hits.length() > 0) - { - for (int i = 0; i < hits.length(); i++) - { - Document doc = hits.doc(i); - // Exclude all containers except the root (which is also a node!) - Field path = doc.getField("PATH"); - if (path == null || path.stringValue().length() == 0) - { - writeableReader.deleteDocument(hits.id(i)); - invalidIndexes.add(key); - // There should only be one thing to - // delete - // break; - } - } - } - writeableSearcher.close(); - } - else - { - int deletedCount = 0; - try - { - deletedCount = writeableReader.deleteDocuments(new Term("ID", stringRef)); - } - catch (IOException ioe) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("IO Error for " + key); - throw ioe; - } - } - if (deletedCount > 0) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Deleted " + deletedCount + " from " + key + " for id " + stringRef + " remaining docs " + writeableReader.numDocs()); - } - invalidIndexes.add(key); - } - } - } - } - } - finally - { - searcher.close(); - } - } - } - if (!containerDeletions.isEmpty()) - { - for (String key : indexes.keySet()) - { - IndexReader reader = getReferenceCountingIndexReader(key); - Searcher searcher = new IndexSearcher(reader); - try - { - for (String stringRef : containerDeletions) - { - TermQuery query = new TermQuery(new Term("ANCESTOR", stringRef)); - Hits hits = searcher.search(query); - if (hits.length() > 0) - { - IndexReader writeableReader = readers.get(key); - if (writeableReader == null) - { - File location = new File(indexDirectory, key).getCanonicalFile(); - if (IndexReader.indexExists(location)) - { - writeableReader = IndexReader.open(location); - } - else - { - continue; - } - readers.put(key, writeableReader); - } - - int deletedCount = 0; - try - { - deletedCount = writeableReader.deleteDocuments(new Term("ANCESTOR", stringRef)); - } - catch (IOException ioe) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("IO Error for " + key); - throw ioe; - } - } - if (deletedCount > 0) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Deleted " + deletedCount + " from " + key + " for id " + stringRef + " remaining docs " + writeableReader.numDocs()); - } - invalidIndexes.add(key); - } - } - } - } - finally - { - searcher.close(); - } - } - } - // The delta we have just processed now must be included when we process the deletions of its successor - indexes.put(currentDelete.getName(), currentDelete); - } - - // Close all readers holding the write lock - so no one tries to - // read - getWriteLock(); - try - { - for (String key : readers.keySet()) - { - IndexReader reader = readers.get(key); - // TODO:Set the new document count - newIndexCounts.put(key, new Long(reader.numDocs())); - reader.close(); - } - } - finally - { - releaseWriteLock(); - } - - // Prebuild all readers for affected indexes - // Register them in the commit. - - final HashMap newReaders = new HashMap(); - - for (String id : invalidIndexes) - { - IndexReader reader = buildReferenceCountingIndexReader(id, newIndexCounts.get(id)); - newReaders.put(id, reader); - } - - getWriteLock(); - try - { - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - for (IndexEntry entry : toDelete.values()) - { - entry.setStatus(TransactionStatus.COMMITTED); - entry.setType(IndexType.INDEX); - entry.setDeletions(0); - } - - for (String key : newIndexCounts.keySet()) - { - Long newCount = newIndexCounts.get(key); - IndexEntry entry = indexEntries.get(key); - entry.setDocumentCount(newCount); - } - - writeStatus(); - - for (String id : invalidIndexes) - { - IndexReader reader = referenceCountingReadOnlyIndexReaders.remove(id); - if (reader != null) - { - ReferenceCounting referenceCounting = (ReferenceCounting) reader; - referenceCounting.setInvalidForReuse(); - if (s_logger.isDebugEnabled()) - { - s_logger.debug("... invalidating sub reader after applying deletions" + id); - } - } - } - for (String id : invalidIndexes) - { - IndexReader newReader = newReaders.get(id); - registerReferenceCountingIndexReader(id, newReader); - } - - // Invalidate all main index readers from the first invalid index onwards - invalidateMainReadersFromFirst(invalidIndexes); - - - if (s_logger.isDebugEnabled()) - { - for (String id : toDelete.keySet()) - { - s_logger.debug("...applied deletion for " + id); - } - s_logger.debug("...deleting done"); - } - - dumpInfo(); - - notifyListeners("MergedDeletions", toDelete.size()); - - return null; - } - - public boolean canRetry() - { - return false; - } - - }); - - } - finally - { - releaseWriteLock(); - } - } - - void mergeIndexes() throws IOException - { - - if (s_logger.isDebugEnabled()) - { - s_logger.debug("Merging..."); - } - - final LinkedHashMap toMerge; - - getWriteLock(); - try - { - toMerge = doWithFileLock(new LockWork>() - { - public LinkedHashMap doWork() throws Exception - { - LinkedHashMap set = new LinkedHashMap(); - - for (IndexEntry entry : indexEntries.values()) - { - if ((entry.getType() == IndexType.INDEX) && (entry.getStatus() == TransactionStatus.MERGE)) - { - return set; - } - if ((entry.getType() == IndexType.INDEX) && (entry.getStatus() == TransactionStatus.MERGE_TARGET)) - { - return set; - } - if ((entry.getType() == IndexType.DELTA) && (entry.getStatus() == TransactionStatus.COMMITTED_DELETING)) - { - return set; - } - } - - ArrayList mergeList = new ArrayList(); - for (IndexEntry entry : indexEntries.values()) - { - if ((entry.getType() == IndexType.INDEX) && (entry.getStatus() == TransactionStatus.COMMITTED)) - { - mergeList.add(entry); - } - } - - int position = findMergeIndex(1, mergerMaxMergeDocs, mergerTargetIndexes, mergeList); - String firstMergeId = mergeList.get(position).getName(); - - long count = 0; - String guid = null; - if (position >= 0) - { - guid = GUID.generate(); - for (int i = position; i < mergeList.size(); i++) - { - IndexEntry entry = mergeList.get(i); - count += entry.getDocumentCount(); - set.put(entry.getName(), entry); - entry.setStatus(TransactionStatus.MERGE); - entry.setMergeId(guid); - } - } - - if (set.size() > 0) - { - IndexEntry target = new IndexEntry(IndexType.INDEX, guid, "", TransactionStatus.MERGE_TARGET, guid, count, 0, false); - set.put(guid, target); - // rebuild merged index elements - LinkedHashMap reordered = new LinkedHashMap(); - invalidateMainReadersFromFirst(Collections.singleton(firstMergeId)); - for (IndexEntry current : indexEntries.values()) - { - if (current.getName().equals(firstMergeId)) - { - reordered.put(target.getName(), target); - } - reordered.put(current.getName(), current); - } - indexEntries = reordered; - writeStatus(); - } - return set; - - } - - public boolean canRetry() - { - return false; - } - - }); - } - finally - { - releaseWriteLock(); - } - - if (s_logger.isDebugEnabled()) - { - s_logger.debug("....Merging..." + (toMerge.size() - 1)); - } - - if (toMerge.size() == 0) - { - return; - } - - String mergeTargetId = null; - - long docCount = 0; - - if (toMerge.size() > 0) - { - int count = 0; - IndexReader[] readers = new IndexReader[toMerge.size() - 1]; - RAMDirectory ramDirectory = null; - IndexWriter writer = null; - - File outputLocation = null; - double mergeSize = 0; - for (IndexEntry entry : toMerge.values()) - { - File location = new File(indexDirectory, entry.getName()).getCanonicalFile(); - if (entry.getStatus() == TransactionStatus.MERGE) - { - IndexReader reader; - if (IndexReader.indexExists(location)) - { - reader = IndexReader.open(location); - } - else - { - s_logger.error("Index is missing " + entry.getName()); - reader = IndexReader.open(emptyIndex); - } - readers[count++] = reader; - docCount += entry.getDocumentCount(); - mergeSize += getSizeInMb(location); - } - else if (entry.getStatus() == TransactionStatus.MERGE_TARGET) - { - mergeTargetId = entry.getName(); - outputLocation = location; - if ((docCount < maxDocsForInMemoryMerge) && (mergeSize < maxRamInMbForInMemoryMerge)) - { - ramDirectory = new RAMDirectory(); - writer = new IndexWriter(ramDirectory, new AlfrescoStandardAnalyser(), true, MaxFieldLength.UNLIMITED); - } - else - { - writer = new IndexWriter(location, new AlfrescoStandardAnalyser(), true, MaxFieldLength.UNLIMITED); - - } - writer.setUseCompoundFile(mergerUseCompoundFile); - writer.setMaxBufferedDocs(mergerMaxBufferedDocs); - writer.setRAMBufferSizeMB(mergerRamBufferSizeMb); - writer.setMergeFactor(mergerMergeFactor); - writer.setMaxMergeDocs(mergerMaxMergeDocs); - writer.setWriteLockTimeout(writeLockTimeout); - writer.setMergeScheduler(new SerialMergeScheduler()); - writer.setMergePolicy(new LogDocMergePolicy()); - } - } - writer.addIndexes(readers); - writer.close(); - - if (ramDirectory != null) - { - String[] files = ramDirectory.list(); - Directory directory = FSDirectory.getDirectory(outputLocation, true); - for (int i = 0; i < files.length; i++) - { - // make place on ram disk - IndexOutput os = directory.createOutput(files[i]); - // read current file - IndexInput is = ramDirectory.openInput(files[i]); - // and copy to ram disk - int len = (int) is.length(); - byte[] buf = new byte[len]; - is.readBytes(buf, 0, len); - os.writeBytes(buf, len); - // graceful cleanup - is.close(); - os.close(); - } - ramDirectory.close(); - directory.close(); - } - - for (IndexReader reader : readers) - { - reader.close(); - } - } - - final String finalMergeTargetId = mergeTargetId; - IndexReader newReader = null; - getReadLock(); - try - { - newReader = buildReferenceCountingIndexReader(mergeTargetId, docCount); - } - finally - { - releaseReadLock(); - } - - final IndexReader finalNewReader = newReader; - - getWriteLock(); - try - { - doWithFileLock(new LockWork() - { - public Object doWork() throws Exception - { - HashSet toDelete = new HashSet(); - for (IndexEntry entry : toMerge.values()) - { - if (entry.getStatus() == TransactionStatus.MERGE) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("... deleting as merged " + entry.getName()); - } - toDelete.add(entry.getName()); - } - else if (entry.getStatus() == TransactionStatus.MERGE_TARGET) - { - - if (s_logger.isDebugEnabled()) - { - s_logger.debug("... committing merge target " + entry.getName()); - } - entry.setStatus(TransactionStatus.COMMITTED); - - } - } - invalidateMainReadersFromFirst(toDelete); - for (String id : toDelete) - { - indexEntries.remove(id); - } - - registerReferenceCountingIndexReader(finalMergeTargetId, finalNewReader); - - notifyListeners("MergedIndexes", toMerge.size()); - - dumpInfo(); - - writeStatus(); - - clearOldReaders(); - - cleaner.schedule(); - - return null; - } - - public boolean canRetry() - { - return false; - } - }); - } - finally - { - releaseWriteLock(); - } - - if (s_logger.isDebugEnabled()) - { - s_logger.debug("..done merging"); - } - - } - - private final int findMergeIndex(long min, long max, int target, List entries) throws IOException - { - // TODO: Support max - if (entries.size() <= target) - { - return -1; - } - - int total = 0; - for (int i = target; i < entries.size(); i++) - { - total += entries.get(i).getDocumentCount(); - } - - for (int i = target - 1; i > 0; i--) - { - total += entries.get(i).getDocumentCount(); - if (total < entries.get(i - 1).getDocumentCount()) - { - return i; - } - } - return 0; - } - - } - - private void dumpInfo() - { - if (s_logger.isDebugEnabled()) - { - int count = 0; - StringBuilder builder = new StringBuilder(); - readWriteLock.writeLock().lock(); - try - { - builder.append("\n"); - builder.append(this.getRelativePath()); - builder.append("Entry List\n"); - for (IndexEntry entry : indexEntries.values()) - { - builder.append(++count + " " + entry.toString()).append("\n"); - } - s_logger.debug(builder.toString()); - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - } - - private void getWriteLock() - { - readOnlyLock.readLock().lock(); - try - { - String threadName = null; - long start = 0l; - if (s_logger.isDebugEnabled()) - { - threadName = Thread.currentThread().getName(); - s_logger.debug("Waiting for WRITE lock - " + threadName); - start = System.nanoTime(); - } - readWriteLock.writeLock().lock(); - if (s_logger.isDebugEnabled()) - { - long end = System.nanoTime(); - s_logger.debug("...GOT WRITE LOCK - " + threadName + " - in " + ((end - start) / 10e6f) + " ms"); - } - } - finally - { - readOnlyLock.readLock().unlock(); - } - } - - private void releaseWriteLock() - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("RELEASED WRITE LOCK - " + Thread.currentThread().getName()); - } - readWriteLock.writeLock().unlock(); - } - - private void getReadLock() - { - String threadName = null; - long start = 0l; - if (s_logger.isDebugEnabled()) - { - threadName = Thread.currentThread().getName(); - s_logger.debug("Waiting for READ lock - " + threadName); - start = System.nanoTime(); - } - readWriteLock.readLock().lock(); - if (s_logger.isDebugEnabled()) - { - long end = System.nanoTime(); - s_logger.debug("...GOT READ LOCK - " + threadName + " - in " + ((end - start) / 10e6f) + " ms"); - } - } - - private void releaseReadLock() - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug("RELEASED READ LOCK - " + Thread.currentThread().getName()); - } - readWriteLock.readLock().unlock(); - } - - public String toString() - { - StringBuilder builder = new StringBuilder(); - builder.append(indexDirectory.toString()); - builder.append(" "); - builder.append(super.toString()); - return builder.toString(); - } - - private boolean isGUID(String guid) - { - try - { - UUID id = new UUID(guid); - // We have a valid guid. - return true; - } - catch (NumberFormatException e) - { - // Not a valid GUID - } - return false; - } - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.index.IndexMonitor#getRelativePath() - */ - public String getRelativePath() - { - return this.relativePath; - } - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.index.IndexMonitor#getStatusSnapshot() - */ - public Map getStatusSnapshot() - { - Map snapShot = new TreeMap(); - readWriteLock.writeLock().lock(); - try - { - for (IndexEntry entry : indexEntries.values()) - { - String stateKey = entry.getType() + "-" + entry.getStatus(); - Integer count = snapShot.get(stateKey); - snapShot.put(stateKey, count == null ? 1 : count + 1); - } - return snapShot; - } - finally - { - readWriteLock.writeLock().unlock(); - } - } - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.index.IndexMonitor#getActualSize() - */ - public long getActualSize() throws IOException - { - getReadLock(); - try - { - int size = 0; - for (IndexEntry entry : this.indexEntries.values()) - { - File location = new File(this.indexDirectory, entry.getName()).getCanonicalFile(); - File[] contents = location.listFiles(); - for (File file : contents) - { - if (file.isFile()) - { - size += file.length(); - } - } - } - return size; - } - finally - { - releaseReadLock(); - } - } - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.index.IndexMonitor#getUsedSize() - */ - public long getUsedSize() throws IOException - { - getReadLock(); - try - { - return sizeRecurse(this.indexDirectory); - } - finally - { - releaseReadLock(); - } - } - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.index.IndexMonitor#getNumberOfDocuments() - */ - public int getNumberOfDocuments() throws IOException - { - IndexReader reader = getMainIndexReferenceCountingReadOnlyIndexReader(); - try - { - return reader.numDocs(); - } - finally - { - reader.close(); - } - } - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.index.IndexMonitor#getNumberOfFields() - */ - public int getNumberOfFields() throws IOException - { - - IndexReader reader = getMainIndexReferenceCountingReadOnlyIndexReader(); - try - { - return reader.getFieldNames(IndexReader.FieldOption.ALL).size(); - } - finally - { - reader.close(); - } - } - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.index.IndexMonitor#getNumberOfIndexedFields() - */ - public int getNumberOfIndexedFields() throws IOException - { - IndexReader reader = getMainIndexReferenceCountingReadOnlyIndexReader(); - try - { - return reader.getFieldNames(IndexReader.FieldOption.INDEXED).size(); - } - finally - { - reader.close(); - } - } - - /* - * (non-Javadoc) - * - * @see org.alfresco.repo.search.impl.lucene.index.IndexMonitor#addApplicationListener(org.springframework.context. - * ApplicationListener) - */ - public void addApplicationListener(ApplicationListener listener) - { - this.applicationListeners.add(listener); - } - - private long sizeRecurse(File fileOrDir) - { - long size = 0; - if (fileOrDir.isDirectory()) - { - File[] files = fileOrDir.listFiles(); - for (File file : files) - { - size += sizeRecurse(file); - } - } - else - { - size = fileOrDir.length(); - } - return size; - } - - private void publishDiscoveryEvent() - { - if (this.config == null) - { - return; - } - final IndexEvent discoveryEvent = new IndexEvent(this, "Discovery", 1); - final ConfigurableApplicationContext applicationContext = this.config.getApplicationContext(); - try - { - applicationContext.publishEvent(discoveryEvent); - } - catch (IllegalStateException e) - { - // There's a possibility that the application context hasn't fully refreshed yet, so register a listener - // that will fire when it has - applicationContext.addApplicationListener(new ApplicationListener() - { - - public void onApplicationEvent(ApplicationEvent event) - { - if (event instanceof ContextRefreshedEvent) - { - applicationContext.publishEvent(discoveryEvent); - } - } - }); - } - } - - private void notifyListeners(String description, int count) - { - if (!this.applicationListeners.isEmpty()) - { - IndexEvent event = new IndexEvent(this, description, count); - for (ApplicationListener listener : this.applicationListeners) - { - listener.onApplicationEvent(event); - } - } - } - - interface Schedulable - { - void schedule(); - } -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexMonitor.java b/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexMonitor.java deleted file mode 100644 index ad2799f186..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexMonitor.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene.index; - -import java.io.IOException; -import java.util.Map; - -import org.springframework.context.ApplicationListener; - -/** - * An interface that exposes information about a Lucene Index and that allows registration of a listener for event - * notifications. - * - * @author dward - */ -public interface IndexMonitor -{ - /** - * Gets the relative path of the index directory. - * - * @return the relative path - */ - public String getRelativePath(); - - /** - * Gets a snapshot of the statuses of the individual entries in this index. - * - * @return a map of entry status names to entry counts - */ - public Map getStatusSnapshot(); - - /** - * Gets the actual size of the index in bytes. - * - * @return the actual size in bytes - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public long getActualSize() throws IOException; - - /** - * Gets the size used on disk by the index directory. A large discrepancy from the value returned by - * {@link #getActualSize()} may indicate that there are unused data files. - * - * @return the size on disk in bytes - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public long getUsedSize() throws IOException; - - /** - * Gets the number of documents in the index. - * - * @return the number of documents - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public int getNumberOfDocuments() throws IOException; - - /** - * Gets the number of fields known to the index. - * - * @return the number of fields - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public int getNumberOfFields() throws IOException; - - /** - * Gets the number of indexed fields. - * - * @return the number of indexed fields - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public int getNumberOfIndexedFields() throws IOException; - - /** - * Registers a listener for events on this index. - * - * @param listener - * the listener - */ - public void addApplicationListener(ApplicationListener listener); -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java b/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java deleted file mode 100644 index 714a262a0b..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/index/IndexType.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene.index; - -/** - * The type of an entry in this index. - * - * @author Andy Hind - */ -public enum IndexType -{ - /** - * Identifies the main index. This is always a fully optimised index. - */ - INDEX, - - /** - * An overlay. This is an optimised index with a deletion list. To commit an overlay requires no deletions against other indexes. Deletions are done when an overlay turns - * into or is merged into a index. Overlays are periodically merged into an index. An overlay can require or have background properties indexed. - */ - DELTA, - - /** - * A long running overlay definition against the index. Not yet supported. - * This, itself, may have transactional additions. - */ - OVERLAY, - - OVERLAY_DELTA; - - -} \ No newline at end of file diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java b/src/main/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java deleted file mode 100644 index 80b17e50f2..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCounting.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene.index; - -import java.io.IOException; -import java.util.Deque; - -/** - * Reference counting and caching for read only index access. - * - * When this object is invalid for reuse and all referees have gone the implementation should release all - * resources held (release the caches, close the index readers etc) - * - * @author andyh - * - */ -public interface ReferenceCounting -{ - public long getCreationTime(); - - public Deque getReferences(); - - /** - * Get the number of references - * @return int - */ - public int getReferenceCount(); - - /** - * Mark is invalid for reuse. - * @throws IOException - */ - public void setInvalidForReuse() throws IOException; - - /** - * Determine if valid for reuse - * @return boolean - */ - public boolean isInvalidForReuse(); - - /** - * Get the id for this reader. - * @return String - */ - public String getId(); -} \ No newline at end of file diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java b/src/main/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java deleted file mode 100644 index c354dea5a4..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/index/ReferenceCountingReadOnlyIndexReaderFactory.java +++ /dev/null @@ -1,752 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene.index; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Deque; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Timer; -import java.util.WeakHashMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.repo.search.impl.lucene.LuceneConfig; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.document.FieldSelectorResult; -import org.apache.lucene.document.Field.Index; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.index.FilterIndexReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.TermEnum; -import org.apache.lucene.util.OpenBitSet; - -public class ReferenceCountingReadOnlyIndexReaderFactory -{ - private static Log s_logger = LogFactory.getLog(ReferenceCountingReadOnlyIndexReaderFactory.class); - - private static WeakHashMap log = new WeakHashMap(); - - public static IndexReader createReader(String id, IndexReader indexReader, boolean enableCaching, LuceneConfig config) - { - ReferenceCountingReadOnlyIndexReader rc = new ReferenceCountingReadOnlyIndexReader(id, indexReader, enableCaching, config); - if (s_logger.isDebugEnabled()) - { - if (log.containsKey(id)) - { - s_logger.debug("Replacing ref counting reader for " + id); - } - s_logger.debug("Created ref counting reader for " + id + " " + rc.toString()); - log.put(new String(id), rc); // Copy the key because the RCROIR references the ID - } - return rc; - } - - public static void destroy() - { - log.clear(); - } - - public static String getState(String id) - { - if (s_logger.isDebugEnabled()) - { - ReferenceCountingReadOnlyIndexReader rc = log.get(id); - if (rc != null) - { - StringBuilder builder = new StringBuilder(); - builder.append("Id = " + rc.getId() + " Invalid = " + rc.getReferenceCount() + " invalid = " + rc.getInvalidForReuse() + " closed=" + rc.getClosed()); - return builder.toString(); - } - - } - return (""); - - } - - public static class ReferenceCountingReadOnlyIndexReader extends FilterIndexReader implements ReferenceCounting, CachingIndexReader - { - private static Log s_logger = LogFactory.getLog(ReferenceCountingReadOnlyIndexReader.class); - - private static final long serialVersionUID = 7693185658022810428L; - - private static java.lang.reflect.Field s_field; - - String id; - - int refCount = 0; - - boolean invalidForReuse = false; - - boolean allowsDeletions; - - boolean wrapper_closed = false; - - ConcurrentHashMap isCategory = new ConcurrentHashMap(); - - ConcurrentHashMap>> documentCache = new ConcurrentHashMap>>(); - - ConcurrentHashMap>> idCache = new ConcurrentHashMap>>(); - - ConcurrentHashMap> pathCache = new ConcurrentHashMap>(); - - ConcurrentHashMap> typeCache = new ConcurrentHashMap>(); - - ConcurrentHashMap>> parentCache = new ConcurrentHashMap>>(); - - ConcurrentHashMap>> linkAspectCache = new ConcurrentHashMap>>(); - - private boolean enableCaching; - - private LuceneConfig config; - - private final long creationTime; - private final Deque references; - - static - { - Class c = IndexReader.class; - try - { - s_field = c.getDeclaredField("closed"); - s_field.setAccessible(true); - } - catch (SecurityException e) - { - throw new AlfrescoRuntimeException("Reference counting index reader needs access to org.apache.lucene.index.IndexReader.closed to work correctly", e); - } - catch (NoSuchFieldException e) - { - throw new AlfrescoRuntimeException( - "Reference counting index reader needs access to org.apache.lucene.index.IndexReader.closed to work correctly (incompatible version of lucene)", e); - } - } - - ReferenceCountingReadOnlyIndexReader(String id, IndexReader indexReader, boolean enableCaching, LuceneConfig config) - { - super(indexReader); - this.creationTime = System.currentTimeMillis(); - this.references = new LinkedList(); - references.add(new Exception(this.refCount + ": " + indexReader.toString())); - this.id = id; - if (enableCaching && (config != null)) - { - this.enableCaching = config.isCacheEnabled(); - } - this.config = config; - } - - @Override - public synchronized long getCreationTime() - { - return this.creationTime; - } - - @Override - public synchronized Deque getReferences() - { - return this.references; - } - - @Override - public synchronized void incRef() - { - if (wrapper_closed) - { - throw new IllegalStateException(Thread.currentThread().getName() + "Indexer is closed " + id); - } - if (refCount++ > 0) - { - super.incRef(); - } - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + ": Reader " + id + " - increment - ref count is " + refCount + " ... " + super.toString()); - } - if (!wrapper_closed) - { - try - { - s_field.set(this, false); - } - catch (IllegalArgumentException e) - { - throw new AlfrescoRuntimeException("Failed to mark index as open ..", e); - } - catch (IllegalAccessException e) - { - throw new AlfrescoRuntimeException("Failed to mark index as open ..", e); - } - } - references.add(new Exception(this.refCount + ": " + in.toString())); - } - - private synchronized void decrementReferenceCount() throws IOException - { - refCount--; - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + ": Reader " + id + " - decrement - ref count is " + refCount + " ... " + super.toString()); - } - closeIfRequired(); - if (refCount < 0) - { - s_logger.error("Invalid reference count for Reader " + id + " is " + refCount + " ... " + super.toString()); - } - } - - private void closeIfRequired() throws IOException - { - if ((refCount == 0) && invalidForReuse && !wrapper_closed) - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + ": Reader " + id + " closed." + " ... " + super.toString()); - } - if (enableCaching) - { - // No tidy up - } - // Pass on the last decRef - super.decRef(); - - wrapper_closed = true; - - if (!references.isEmpty()) - { - references.removeLast(); - } - } - else - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() - + ": Reader " + id + " still open .... ref = " + refCount + " invalidForReuse = " + invalidForReuse + " ... " + super.toString()); - } - } - } - - public synchronized int getReferenceCount() - { - return refCount; - } - - public synchronized boolean getInvalidForReuse() - { - return invalidForReuse; - } - - public synchronized boolean getClosed() - { - return wrapper_closed; - } - - public synchronized void setInvalidForReuse() throws IOException - { - if (wrapper_closed) - { - throw new IllegalStateException(Thread.currentThread().getName() + "Indexer is closed " + id); - } - invalidForReuse = true; - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + ": Reader " + id + " set invalid for reuse" + " ... " + super.toString()); - } - closeIfRequired(); - } - - - @Override - public synchronized void decRef() throws IOException - { - if (s_logger.isDebugEnabled()) - { - s_logger.debug(Thread.currentThread().getName() + ": Reader " + id + " closing" + " ... " + super.toString()); - } - if (wrapper_closed) - { - throw new IllegalStateException(Thread.currentThread().getName() + "Indexer is closed " + id); - } - decrementReferenceCount(); - if (refCount > 0) - { - super.decRef(); - if (!references.isEmpty()) - { - references.removeLast(); - } - } - } - - /** - * We want to avoid setting the closed flag on our wrapped stream, passing on all decrefs. - **/ - @Override - protected void doClose() throws IOException - { - in.decRef(); - } - - @Override - protected void doDelete(int n) throws IOException - { - throw new UnsupportedOperationException("Delete is not supported by read only index readers"); - } - - private interface Accessor - { - T get(int n, FieldSelector fieldSelector) throws IOException; - } - - private class ListFieldAccessor implements Accessor> - { - - public List get(int n, FieldSelector fieldSelector) throws IOException - { - Document document = ReferenceCountingReadOnlyIndexReader.super.document(n, fieldSelector); - List fields = (List) document.getFields(); - ArrayList cacheable = new ArrayList(fields.size()); - cacheable.addAll(fields); - return cacheable; - } - - } - - private class MultipleValueFieldAccessor implements Accessor> - { - String fieldName; - - MultipleValueFieldAccessor(String fieldName) - { - this.fieldName = fieldName; - } - - public List get(int n, FieldSelector fieldSelector) throws IOException - { - Document document = ReferenceCountingReadOnlyIndexReader.super.document(n, fieldSelector); - Field[] fields = document.getFields(fieldName); - ArrayList cacheable = new ArrayList(fields.length); - for (Field field : fields) - { - cacheable.add(field); - } - return cacheable; - } - - } - - private class SingleValueFieldAccessor implements Accessor - { - String fieldName; - - SingleValueFieldAccessor(String fieldName) - { - this.fieldName = fieldName; - } - - public Field get(int n, FieldSelector fieldSelector) throws IOException - { - return new Field(fieldName, getStringValue(n, fieldName), Store.NO, Index.UN_TOKENIZED); - } - - } - - private final ListFieldAccessor LIST_FIELD_ACCESSOR = new ListFieldAccessor(); - - private final MultipleValueFieldAccessor MV_ID_FIELD_ACCESSOR = new MultipleValueFieldAccessor("ID"); - - private final SingleValueFieldAccessor SV_TYPE_FIELD_ACCESSOR = new SingleValueFieldAccessor("TYPE"); - - private final SingleValueFieldAccessor SV_PATH_FIELD_ACCESSOR = new SingleValueFieldAccessor("PATH"); - - private final MultipleValueFieldAccessor MV_PARENT_FIELD_ACCESSOR = new MultipleValueFieldAccessor("PARENT"); - - private final MultipleValueFieldAccessor MV_LINKASPECT_FIELD_ACCESSOR = new MultipleValueFieldAccessor("LINKASPECT"); - - private OpenBitSet nodes = null; - - private T manageCache(ConcurrentHashMap> cache, Accessor accessor, int n, FieldSelector fieldSelector, int limit) throws IOException - { - Integer key = Integer.valueOf(n); - WithUseCount value = cache.get(key); - if (value == null) - { - T made = accessor.get(n, fieldSelector); - value = new WithUseCount(made, n); - cache.put(key, value); - - // resize - - if (limit >= 0) - { - if (cache.size() >= limit) - { - HashMap> keep = new HashMap>(); - WithUseCount[] existing = new WithUseCount[0]; - synchronized (cache) - { - existing = cache.values().toArray(existing); - cache.clear(); - } - Arrays.sort(existing); - - for (WithUseCount current : existing) - { - keep.put(Integer.valueOf(current.doc), current); - if ((current.count.get() == 0) || (keep.size() > (limit / 4))) - { - break; - } - } - keep.put(key, value); - cache.putAll(keep); - } - } - } - else - { - value.count.getAndIncrement(); - } - return value.object; - } - - @SuppressWarnings("unchecked") - public Document document(int n, FieldSelector fieldSelector) throws IOException - { - if ((fieldSelector == null) && enableCaching) - { - List listOfFields = manageCache(documentCache, LIST_FIELD_ACCESSOR, n, fieldSelector, config.getMaxDocumentCacheSize()); - Document document = new Document(); - document.getFields().addAll(listOfFields); - return document; - - } - else - { - if (enableCaching && (fieldSelector instanceof SingleFieldSelector)) - { - SingleFieldSelector sfs = (SingleFieldSelector) fieldSelector; - - if (sfs.field.equals("ID") && !sfs.last) - { - List idFields = manageCache(idCache, MV_ID_FIELD_ACCESSOR, n, fieldSelector, config.getMaxDocIdCacheSize()); - Document d = new Document(); - d.getFields().addAll(idFields); - return d; - - } - if (sfs.field.equals("ISCATEGORY") && sfs.last) - { - Integer key = Integer.valueOf(n); - Boolean isCat = isCategory.get(key); - if (isCat == null) - { - isCat = (getStringValue(n, "ISCATEGORY") != null); - isCategory.put(key, isCat); - } - Document d = new Document(); - if (isCat) - { - d.add(new Field("ISCATEGORY", "T", Store.NO, Index.UN_TOKENIZED)); - } - return d; - } - if (sfs.field.equals("PATH") && sfs.last) - { - Field pathField = manageCache(pathCache, SV_PATH_FIELD_ACCESSOR, n, fieldSelector, config.getMaxPathCacheSize()); - Document d = new Document(); - d.add(pathField); - return d; - - } - if (sfs.field.equals("TYPE") && sfs.last) - { - Field typeField = manageCache(typeCache, SV_TYPE_FIELD_ACCESSOR, n, fieldSelector, config.getMaxTypeCacheSize()); - Document d = new Document(); - d.add(typeField); - return d; - - } - if (sfs.field.equals("PARENT") && !sfs.last) - { - List listOfFields = manageCache(parentCache, MV_PARENT_FIELD_ACCESSOR, n, fieldSelector, config.getMaxParentCacheSize()); - Document document = new Document(); - document.getFields().addAll(listOfFields); - return document; - - } - if (sfs.field.equals("LINKASPECT") && !sfs.last) - { - List listOfFields = manageCache(linkAspectCache, MV_LINKASPECT_FIELD_ACCESSOR, n, fieldSelector, config.getMaxLinkAspectCacheSize()); - Document document = new Document(); - document.getFields().addAll(listOfFields); - return document; - - } - - } - } - - return super.document(n, fieldSelector); - } - - public String getId() - { - return id; - } - - public boolean isInvalidForReuse() - { - return invalidForReuse; - } - - public String getId(int n) throws IOException - { - Document d = document(n, new SingleFieldSelector("ID", true)); - return d.getField("ID").stringValue(); - } - - public String getPathLinkId(int n) throws IOException - { - Document document = document(n, new SingleFieldSelector("ID", true)); - Field[] fields = document.getFields("ID"); - Field field = fields[fields.length - 1]; - return (field == null) ? null : field.stringValue(); - } - - public String[] getIds(int n) throws IOException - { - return getStringValues(n, "ID"); - } - - public String[] getLinkAspects(int n) throws IOException - { - // return getStringValues(n, "LINKASPECT"); - Document d = document(n, new SingleFieldSelector("LINKASPECT", false)); - return d.getValues("LINKASPECT"); - } - - public String[] getParents(int n) throws IOException - { - // return getStringValues(n, "PARENT"); - Document d = document(n, new SingleFieldSelector("PARENT", false)); - return d.getValues("PARENT"); - } - - public String getPath(int n) throws IOException - { - // return getStringValue(n, "PATH"); - Document d = document(n, new SingleFieldSelector("PATH", true)); - Field f = d.getField("PATH"); - return f == null ? null : f.stringValue(); - } - - public String getType(int n) throws IOException - { - // return getStringValue(n, "TYPE"); - Document d = document(n, new SingleFieldSelector("TYPE", true)); - Field f = d.getField("TYPE"); - return f == null ? null : f.stringValue(); - } - - public String getIsCategory(int n) throws IOException - { - Document d = document(n, new SingleFieldSelector("ISCATEGORY", true)); - Field f = d.getField("ISCATEGORY"); - return f == null ? null : f.stringValue(); - } - - // private String getLastStringValue(int n, String fieldName) throws IOException - // { - // Document document = document(n); - // Field[] fields = document.getFields(fieldName); - // Field field = fields[fields.length - 1]; - // return (field == null) ? null : field.stringValue(); - // } - - private String getStringValue(int n, String fieldName) throws IOException - { - Document document = document(n); - Field field = document.getField(fieldName); - return (field == null) ? null : field.stringValue(); - } - - @SuppressWarnings("unchecked") - private String[] getStringValues(int n, String fieldName) throws IOException - { - Document document = document(n); - ArrayList fields = new ArrayList(); - ArrayList answer = new ArrayList(2); - fields.addAll((List) document.getFields()); - - for (Field field : fields) - { - if (field.name().equals(fieldName)) - { - answer.add(field.stringValue()); - } - } - - return answer.toArray(new String[answer.size()]); - } - - public synchronized TermDocs getNodeDocs() throws IOException - { - if (nodes == null) - { - TermDocs nodeDocs = termDocs(new Term("ISNODE", "T")); - nodes = new OpenBitSet(); - while (nodeDocs.next()) - { - nodes.set(nodeDocs.doc()); - } - nodeDocs.close(); - } - return new TermDocSet(nodes); - } - } - - static class WithUseCount implements Comparable> - { - AtomicInteger count = new AtomicInteger(0); - - T object; - - int doc; - - WithUseCount(T object, int doc) - { - this.object = object; - this.doc = doc; - } - - public int compareTo(WithUseCount other) - { - return other.count.get() - this.count.get(); - } - } - - private static class SingleFieldSelector implements FieldSelector - { - String field; - - boolean last; - - SingleFieldSelector(String field, boolean last) - { - this.field = field; - this.last = last; - } - - public FieldSelectorResult accept(String fieldName) - { - if (fieldName.equals(field)) - { - return FieldSelectorResult.LOAD; - } - else - { - return FieldSelectorResult.NO_LOAD; - } - } - - } - - static class TermDocSet implements TermDocs - { - OpenBitSet set; - - int position = -1; - - TermDocSet(OpenBitSet set) - { - this.set = set; - } - - public void close() throws IOException - { - // Noop - } - - public int doc() - { - return position; - } - - public int freq() - { - return 1; - } - - public boolean next() throws IOException - { - position++; - position = set.nextSetBit(position); - return (position != -1); - } - - public int read(int[] docs, int[] freqs) throws IOException - { - throw new UnsupportedOperationException(); - } - - public void seek(Term term) throws IOException - { - throw new UnsupportedOperationException(); - } - - public void seek(TermEnum termEnum) throws IOException - { - throw new UnsupportedOperationException(); - } - - public boolean skipTo(int target) throws IOException - { - do - { - if (!next()) - { - return false; - } - } - while (target > doc()); - return true; - - } - - } -} diff --git a/src/main/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java b/src/main/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java deleted file mode 100644 index 71450584cd..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/lucene/index/TransactionStatus.java +++ /dev/null @@ -1,533 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.lucene.index; - -import javax.transaction.Status; - - -/** - * Status of indexes that make up the whole index. This starts with the value from javax.transaction.Status. - * - * Lifecycle --------- - * - * As a transaction starts, the delta is ACTIVE It may be MARKED_ROLLBACK -> ROLLED BACK -> PREPARING -> PREPARED -> COMMITTING -> COMMITTED... with roll back at any time - * - * If the index has any delayed indexing it commits to COMMITTED_REQUIRES_REINDEX and then the overlay can go from -> COMMITTED_REINDEXING -> COMMITTED_REINDEXED - * - * If there was no reindexing required the delat commits as COMMITTED - * - * A delta changes to an index overlay as it is committed. - * - * For an overlay in COMMITTED or COMMITTED_REINDEXED it can have its delete list applied to sub indexes. At this point it becomes a sub index. - * - * @author Andy Hind - */ - -public enum TransactionStatus -{ - - /** - * Active TX - */ - ACTIVE - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return true; - } - - public boolean canBeReordered() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == null) || (previous == ACTIVE); - } - - public int getStatus() - { - return Status.STATUS_ACTIVE; - } - }, - - /** - * TX marked for rollback - */ - MARKED_ROLLBACK - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return true; - } - - public boolean canBeReordered() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return previous.allowsRollbackOrMark(previous) || (previous == MARKED_ROLLBACK); - } - - public int getStatus() - { - return Status.STATUS_MARKED_ROLLBACK; - } - }, - - /** - * TX prepared - */ - PREPARED - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return false; - } - - public boolean canBeReordered() - { - return false; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == TransactionStatus.PREPARING) || (previous == PREPARED); - } - - public int getStatus() - { - return Status.STATUS_PREPARED; - } - }, - - /** - * TX Committed - */ - COMMITTED - { - public boolean isCommitted() - { - return true; - } - - public boolean isTransient() - { - return false; - } - - public boolean canBeReordered() - { - return false; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == TransactionStatus.COMMITTING) || (previous == COMMITTED); - } - - public int getStatus() - { - return Status.STATUS_COMMITTED; - } - }, - - /** - * TX rolled back - */ - ROLLEDBACK - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return true; - } - - public boolean canBeReordered() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == TransactionStatus.ROLLINGBACK) || (previous == ROLLEDBACK); - } - - public int getStatus() - { - return Status.STATUS_ROLLEDBACK; - } - }, - - /** - * TX state is unknown - */ - UNKNOWN - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return true; - } - - public boolean canBeReordered() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == UNKNOWN); - } - - public int getStatus() - { - return Status.STATUS_UNKNOWN; - } - }, - - /** - * No transaction - */ - NO_TRANSACTION - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return true; - } - - public boolean canBeReordered() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == NO_TRANSACTION); - } - - public int getStatus() - { - return Status.STATUS_NO_TRANSACTION; - } - }, - - /** - * TX is preparing - */ - PREPARING - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return true; - } - - public boolean canBeReordered() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == TransactionStatus.ACTIVE) || (previous == PREPARING); - } - - public int getStatus() - { - return Status.STATUS_PREPARING; - } - }, - - /** - * TX is committing - */ - COMMITTING - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return true; - } - - public boolean canBeReordered() - { - return false; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == TransactionStatus.PREPARED) || (previous == COMMITTING); - } - - public int getStatus() - { - return Status.STATUS_COMMITTING; - } - }, - - /** - * TX rolling back - */ - ROLLINGBACK - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return true; - } - - public boolean canBeReordered() - { - return true; - } - - public boolean follows(TransactionStatus previous) - { - return previous.allowsRollbackOrMark(previous) || (previous == ROLLINGBACK); - } - - public int getStatus() - { - return Status.STATUS_ROLLING_BACK; - } - }, - - /** - * This entry is the source for an active merge. The result will be in a new index. - */ - MERGE - { - public boolean isCommitted() - { - return true; - } - - public boolean isTransient() - { - return false; - } - - public boolean canBeReordered() - { - return false; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == MERGE); - } - - public int getStatus() - { - return Status.STATUS_COMMITTED; - } - }, - - /** - * A new index element that is being made by a merge. - */ - MERGE_TARGET - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return false; - } - - public boolean canBeReordered() - { - return false; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == MERGE_TARGET); - } - - public int getStatus() - { - return Status.STATUS_ACTIVE; - } - }, - - - /** - * Pending deleted are being committed to for the delta. - */ - COMMITTED_DELETING - { - public boolean isCommitted() - { - return true; - } - - public boolean isTransient() - { - return false; - } - - public boolean canBeReordered() - { - return false; - } - - public boolean follows(TransactionStatus previous) - { - return (previous == COMMITTED_DELETING); - } - - public int getStatus() - { - return Status.STATUS_COMMITTED; - } - }, - - /** - * An entry that may be deleted - */ - DELETABLE - { - public boolean isCommitted() - { - return false; - } - - public boolean isTransient() - { - return false; - } - - public boolean canBeReordered() - { - return false; - } - - public boolean follows(TransactionStatus previous) - { - return true; - } - - public int getStatus() - { - return Status.STATUS_UNKNOWN; - } - }; - - /** - * Is this a commited inex entry? - * @return - true if committed - */ - public abstract boolean isCommitted(); - - /** - * Is this transient - * @return - true if no information needs to be persisted - */ - public abstract boolean isTransient(); - - /** - * Can this be reordered with respect to other TXs - * @return - true if this can be reordered (fixed after prepare) - */ - public abstract boolean canBeReordered(); - - /** - * Can this state follow the one given? - * @param previous state - * @return - true if transition to this state is allowed - */ - public abstract boolean follows(TransactionStatus previous); - - /** - * Get the javax.transaction.Status best matching this state - * - * @return - the int TX state - */ - public abstract int getStatus(); - - private boolean allowsRollbackOrMark(TransactionStatus previous) - { - switch (previous) - { - case ACTIVE: - case MARKED_ROLLBACK: - case PREPARED: - case PREPARING: - case COMMITTING: - return true; - default: - return false; - } - } -} \ No newline at end of file diff --git a/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/lucene/LuceneQueryEngine.java b/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/lucene/LuceneQueryEngine.java deleted file mode 100644 index 4f25634104..0000000000 --- a/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/lucene/LuceneQueryEngine.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.repo.search.impl.querymodel.impl.lucene; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -import org.alfresco.repo.search.SearcherException; -import org.alfresco.repo.search.impl.lucene.ClosingIndexSearcher; -import org.alfresco.repo.search.impl.lucene.LuceneIndexerAndSearcher; -import org.alfresco.repo.search.impl.lucene.LuceneResultSet; -import org.alfresco.repo.search.impl.lucene.LuceneSearcher; -import org.alfresco.repo.search.impl.lucene.PagingLuceneResultSet; -import org.alfresco.repo.search.impl.querymodel.FunctionEvaluationContext; -import org.alfresco.repo.search.impl.querymodel.Query; -import org.alfresco.repo.search.impl.querymodel.QueryEngine; -import org.alfresco.repo.search.impl.querymodel.QueryEngineResults; -import org.alfresco.repo.search.impl.querymodel.QueryModelFactory; -import org.alfresco.repo.search.impl.querymodel.QueryOptions; -import org.alfresco.repo.search.results.SortedResultSet; -import org.alfresco.repo.tenant.TenantService; -import org.alfresco.service.cmr.dictionary.DictionaryService; -import org.alfresco.service.cmr.repository.NodeService; -import org.alfresco.service.cmr.repository.StoreRef; -import org.alfresco.service.cmr.search.LimitBy; -import org.alfresco.service.cmr.search.ResultSet; -import org.alfresco.service.cmr.search.SearchParameters; -import org.alfresco.service.cmr.search.SearchService; -import org.alfresco.service.namespace.NamespaceService; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; - -/** - * @author andyh - */ -@SuppressWarnings("deprecation") -public class LuceneQueryEngine implements QueryEngine -{ - protected static final Log logger = LogFactory.getLog(LuceneQueryEngine.class); - - private DictionaryService dictionaryService; - - private LuceneIndexerAndSearcher indexAndSearcher; - - private NodeService nodeService; - - private TenantService tenantService; - - private NamespaceService namespaceService; - - private boolean useInMemorySort = true; - - private int maxRawResultSetSizeForInMemorySort = 1000; - - /** - * @param dictionaryService - * the dictionaryService to set - */ - public void setDictionaryService(DictionaryService dictionaryService) - { - this.dictionaryService = dictionaryService; - } - - /** - * @param indexAndSearcher - * the indexAndSearcher to set - */ - public void setIndexAndSearcher(LuceneIndexerAndSearcher indexAndSearcher) - { - this.indexAndSearcher = indexAndSearcher; - } - - /** - * @param nodeService - * the nodeService to set - */ - public void setNodeService(NodeService nodeService) - { - this.nodeService = nodeService; - } - - /** - * @param tenantService - * the tenantService to set - */ - public void setTenantService(TenantService tenantService) - { - this.tenantService = tenantService; - } - - /** - * @param namespaceService - * the namespaceService to set - */ - public void setNamespaceService(NamespaceService namespaceService) - { - this.namespaceService = namespaceService; - } - - public QueryModelFactory getQueryModelFactory() - { - return new LuceneQueryModelFactory(); - } - - /** - * @return the useInMemorySort - */ - public boolean isUseInMemorySort() - { - return useInMemorySort; - } - - /** - * @param useInMemorySort the useInMemorySort to set - */ - public void setUseInMemorySort(boolean useInMemorySort) - { - this.useInMemorySort = useInMemorySort; - } - - /** - * @return the maxRawResultSetSizeForInMemorySort - */ - public int getMaxRawResultSetSizeForInMemorySort() - { - return maxRawResultSetSizeForInMemorySort; - } - - /** - * @param maxRawResultSetSizeForInMemorySort the maxRawResultSetSizeForInMemorySort to set - */ - public void setMaxRawResultSetSizeForInMemorySort(int maxRawResultSetSizeForInMemorySort) - { - this.maxRawResultSetSizeForInMemorySort = maxRawResultSetSizeForInMemorySort; - } - - public QueryEngineResults executeQuery(Query query, QueryOptions options, FunctionEvaluationContext functionContext) - { - Set selectorGroup = null; - if (query.getSource() != null) - { - List> selectorGroups = query.getSource().getSelectorGroups(functionContext); - - if (selectorGroups.size() == 0) - { - throw new UnsupportedOperationException("No selectors"); - } - - if (selectorGroups.size() > 1) - { - throw new UnsupportedOperationException("Advanced join is not supported"); - } - - selectorGroup = selectorGroups.get(0); - } - - SearchParameters searchParameters = new SearchParameters(); - if(options.getLocales().size() > 0) - { - for(Locale locale: options.getLocales()) - { - searchParameters.addLocale(locale); - } - } - searchParameters.excludeDataInTheCurrentTransaction(!options.isIncludeInTransactionData()); - searchParameters.setSkipCount(options.getSkipCount()); - searchParameters.setMaxPermissionChecks(options.getMaxPermissionChecks()); - searchParameters.setMaxPermissionCheckTimeMillis(options.getMaxPermissionCheckTimeMillis()); - searchParameters.setDefaultFieldName(options.getDefaultFieldName()); - searchParameters.setMlAnalaysisMode(options.getMlAnalaysisMode()); - if (options.getMaxItems() >= 0) - { - searchParameters.setLimitBy(LimitBy.FINAL_SIZE); - searchParameters.setLimit(options.getMaxItems()); - searchParameters.setMaxItems(options.getMaxItems()); - } - else - { - searchParameters.setLimitBy(LimitBy.UNLIMITED); - } - searchParameters.setUseInMemorySort(options.getUseInMemorySort()); - searchParameters.setMaxRawResultSetSizeForInMemorySort(options.getMaxRawResultSetSizeForInMemorySort()); - searchParameters.setBulkFetchEnabled(options.isBulkFetchEnabled()); - searchParameters.setQueryConsistency(options.getQueryConsistency()); - - try - { - StoreRef storeRef = options.getStores().get(0); - searchParameters.addStore(storeRef); - if (query instanceof LuceneQueryBuilder) - { - SearchService searchService = indexAndSearcher.getSearcher(storeRef, options.isIncludeInTransactionData()); - if (searchService instanceof LuceneSearcher) - { - LuceneSearcher luceneSearcher = (LuceneSearcher) searchService; - ClosingIndexSearcher searcher = luceneSearcher.getClosingIndexSearcher(); - LuceneQueryBuilderContext luceneContext = new LuceneQueryBuilderContextImpl(dictionaryService, namespaceService, tenantService, searchParameters, indexAndSearcher.getDefaultMLSearchAnalysisMode(), - searcher.getIndexReader()); - - @SuppressWarnings("unchecked") - LuceneQueryBuilder builder = (LuceneQueryBuilder) query; - org.apache.lucene.search.Query luceneQuery = builder.buildQuery(selectorGroup, luceneContext, functionContext); - - if(logger.isDebugEnabled()) - { - logger.debug("Executing lucene query: "+luceneQuery); - } - - Sort sort = builder.buildSort(selectorGroup, luceneContext, functionContext); - - - Hits hits = searcher.search(luceneQuery); - - boolean postSort = false;; - if(sort != null) - { - postSort = searchParameters.usePostSort(hits.length(), useInMemorySort, maxRawResultSetSizeForInMemorySort); - if(postSort == false) - { - hits = searcher.search(luceneQuery, sort); - } - } - - ResultSet answer; - ResultSet result = new LuceneResultSet(hits, searcher, nodeService, tenantService, searchParameters, indexAndSearcher); - if(postSort) - { - if(sort != null) - { - for(SortField sf : sort.getSort()) - { - searchParameters.addSort(sf.getField(), !sf.getReverse()); - } - } - - ResultSet sorted = new SortedResultSet(result, nodeService, builder.buildSortDefinitions(selectorGroup, luceneContext, functionContext), namespaceService, dictionaryService, searchParameters.getSortLocale()); - answer = sorted; - } - else - { - answer = result; - } - ResultSet rs = new PagingLuceneResultSet(answer, searchParameters, nodeService); - - Map, ResultSet> map = new HashMap, ResultSet>(1); - map.put(selectorGroup, rs); - return new QueryEngineResults(map); - } - else - { - throw new UnsupportedOperationException(); - } - } - else - { - throw new UnsupportedOperationException(); - } - } - catch (ParseException e) - { - throw new SearcherException("Failed to parse query: " + e); - } - catch (IOException e) - { - throw new SearcherException("IO exception during search", e); - } - } - -} diff --git a/src/main/java/org/alfresco/repo/search/impl/solr/AbstractSolrAdminHTTPClient.java b/src/main/java/org/alfresco/repo/search/impl/solr/AbstractSolrAdminHTTPClient.java index 31458000f4..1e6b6f6355 100644 --- a/src/main/java/org/alfresco/repo/search/impl/solr/AbstractSolrAdminHTTPClient.java +++ b/src/main/java/org/alfresco/repo/search/impl/solr/AbstractSolrAdminHTTPClient.java @@ -34,7 +34,7 @@ import javax.servlet.http.HttpServletResponse; import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpStatus; @@ -83,7 +83,7 @@ protected JSONObject getOperation(HttpClient httpClient, String url) throws Unsu } if (get.getStatusCode() != HttpServletResponse.SC_OK) { - throw new LuceneQueryParserException("Request failed " + get.getStatusCode() + " " + url.toString()); + throw new QueryParserException("Request failed " + get.getStatusCode() + " " + url.toString()); } Reader reader = new BufferedReader(new InputStreamReader(get.getResponseBodyAsStream(), get.getResponseCharSet())); diff --git a/src/main/java/org/alfresco/repo/search/impl/solr/AbstractSolrQueryHTTPClient.java b/src/main/java/org/alfresco/repo/search/impl/solr/AbstractSolrQueryHTTPClient.java index 61de986a93..a6a9ddcaea 100644 --- a/src/main/java/org/alfresco/repo/search/impl/solr/AbstractSolrQueryHTTPClient.java +++ b/src/main/java/org/alfresco/repo/search/impl/solr/AbstractSolrQueryHTTPClient.java @@ -33,7 +33,7 @@ import javax.servlet.http.HttpServletResponse; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; @@ -105,7 +105,7 @@ protected JSONObject postQuery(HttpClient httpClient, String url, JSONObject bod } if (post.getStatusCode() != HttpServletResponse.SC_OK) { - throw new LuceneQueryParserException("Request failed " + post.getStatusCode() + " " + url.toString()); + throw new QueryParserException("Request failed " + post.getStatusCode() + " " + url.toString()); } Reader reader = new BufferedReader(new InputStreamReader(post.getResponseBodyAsStream(), post.getResponseCharSet())); diff --git a/src/main/java/org/alfresco/repo/search/impl/solr/DynamicSolrStoreMappingWrapperFactory.java b/src/main/java/org/alfresco/repo/search/impl/solr/DynamicSolrStoreMappingWrapperFactory.java index 2a16d08905..bcffd9bbfb 100644 --- a/src/main/java/org/alfresco/repo/search/impl/solr/DynamicSolrStoreMappingWrapperFactory.java +++ b/src/main/java/org/alfresco/repo/search/impl/solr/DynamicSolrStoreMappingWrapperFactory.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.search.impl.solr; import java.io.UnsupportedEncodingException; @@ -32,7 +32,7 @@ import org.alfresco.httpclient.HttpClientFactory; import org.alfresco.repo.index.shard.ShardInstance; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.util.Pair; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpClient; @@ -113,9 +113,9 @@ public String getShards() if (builder.length() > 0) { builder.append(','); - } - Pair key = new Pair(instance.getHostName(), instance.getPort()); - HttpClient client = clients.get(key); + } + Pair key = new Pair(instance.getHostName(), instance.getPort()); + HttpClient client = clients.get(key); builder.append(encoder.encode(client.getHostConfiguration().getProtocol().getScheme() + "://", "UTF-8")); builder.append(encoder.encode(instance.getHostName(), "UTF-8")); builder.append(':'); @@ -131,7 +131,7 @@ public String getShards() } catch (UnsupportedEncodingException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } } diff --git a/src/main/java/org/alfresco/repo/search/impl/solr/ExplicitSolrStoreMappingWrapper.java b/src/main/java/org/alfresco/repo/search/impl/solr/ExplicitSolrStoreMappingWrapper.java index 6b65f1b6d2..226aa0a19c 100644 --- a/src/main/java/org/alfresco/repo/search/impl/solr/ExplicitSolrStoreMappingWrapper.java +++ b/src/main/java/org/alfresco/repo/search/impl/solr/ExplicitSolrStoreMappingWrapper.java @@ -34,7 +34,7 @@ import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.HttpClientFactory; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.util.Pair; import org.alfresco.util.shard.ExplicitShardingPolicy; @@ -243,7 +243,7 @@ private String getShards2() } catch (UnsupportedEncodingException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } } diff --git a/src/main/java/org/alfresco/repo/search/impl/solr/SolrAdminHTTPClient.java b/src/main/java/org/alfresco/repo/search/impl/solr/SolrAdminHTTPClient.java index ee3f578e90..d624fee591 100644 --- a/src/main/java/org/alfresco/repo/search/impl/solr/SolrAdminHTTPClient.java +++ b/src/main/java/org/alfresco/repo/search/impl/solr/SolrAdminHTTPClient.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.search.impl.solr; import java.io.BufferedReader; @@ -39,7 +39,7 @@ import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.HttpClientFactory; import org.alfresco.repo.domain.node.NodeDAO; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.repo.search.impl.lucene.SolrJSONResultSet; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; import org.alfresco.service.cmr.search.ResultSet; @@ -170,7 +170,7 @@ public JSONObject execute(String relativeHandlerPath, HashMap ar if (get.getStatusCode() != HttpServletResponse.SC_OK) { - throw new LuceneQueryParserException("Request failed " + get.getStatusCode() + " " + url.toString()); + throw new QueryParserException("Request failed " + get.getStatusCode() + " " + url.toString()); } Reader reader = new BufferedReader(new InputStreamReader(get.getResponseBodyAsStream())); @@ -185,19 +185,19 @@ public JSONObject execute(String relativeHandlerPath, HashMap ar } catch (UnsupportedEncodingException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } catch (HttpException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } catch (IOException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } catch (JSONException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } } diff --git a/src/main/java/org/alfresco/repo/search/impl/solr/SolrChildApplicationContextFactory.java b/src/main/java/org/alfresco/repo/search/impl/solr/SolrChildApplicationContextFactory.java index 0afc804d26..d488e5db6d 100644 --- a/src/main/java/org/alfresco/repo/search/impl/solr/SolrChildApplicationContextFactory.java +++ b/src/main/java/org/alfresco/repo/search/impl/solr/SolrChildApplicationContextFactory.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.search.impl.solr; import java.util.HashMap; @@ -32,7 +32,7 @@ import java.util.TreeSet; import org.alfresco.repo.management.subsystems.ChildApplicationContextFactory; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.service.cmr.repository.datatype.Duration; import org.json.JSONException; import org.json.JSONObject; @@ -219,7 +219,7 @@ else if (name.equals(SolrChildApplicationContextFactory.ARCHIVE_MEMORY)) // Did not find the property in JSON or the core is turned off return "Unavailable"; } - catch (LuceneQueryParserException lqe) + catch (QueryParserException lqe) { return "Unavailable: " + lqe.getMessage(); } diff --git a/src/main/java/org/alfresco/repo/search/impl/solr/SolrClientUtil.java b/src/main/java/org/alfresco/repo/search/impl/solr/SolrClientUtil.java index 1964a1035d..3b18b87a6e 100644 --- a/src/main/java/org/alfresco/repo/search/impl/solr/SolrClientUtil.java +++ b/src/main/java/org/alfresco/repo/search/impl/solr/SolrClientUtil.java @@ -33,7 +33,7 @@ import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.repo.index.shard.ShardInstance; import org.alfresco.repo.index.shard.ShardRegistry; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.service.cmr.search.BasicSearchParameters; import org.alfresco.service.cmr.search.SearchParameters; @@ -95,7 +95,7 @@ public static SolrStoreMappingWrapper extractMapping(StoreRef store, SolrStoreMappingWrapper mappings = mappingLookup.get(store); if (mappings == null) { - throw new LuceneQueryParserException("No solr query support for store " + store); + throw new QueryParserException("No solr query support for store " + store); } return mappings; } @@ -107,7 +107,7 @@ public static SolrStoreMappingWrapper extractMapping(StoreRef store, if (mappings == null) { - throw new LuceneQueryParserException("No solr query support for store " + store); + throw new QueryParserException("No solr query support for store " + store); } return mappings; } diff --git a/src/main/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java b/src/main/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java index d57c76fdb8..e2cb081234 100644 --- a/src/main/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java +++ b/src/main/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java @@ -54,7 +54,7 @@ import org.alfresco.repo.index.shard.ShardRegistry; import org.alfresco.repo.search.impl.QueryParserUtils; import org.alfresco.repo.search.impl.lucene.JSONResult; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.repo.search.impl.lucene.SolrJSONResultSet; import org.alfresco.repo.search.impl.lucene.SolrJsonProcessor; import org.alfresco.repo.search.impl.lucene.SolrStatsResult; @@ -318,19 +318,19 @@ public SolrStatsResult executeStatsQuery(final StatsParameters searchParameters) } catch (UnsupportedEncodingException e) { - throw new LuceneQueryParserException("stats", e); + throw new QueryParserException("stats", e); } catch (HttpException e) { - throw new LuceneQueryParserException("stats", e); + throw new QueryParserException("stats", e); } catch (IOException e) { - throw new LuceneQueryParserException("stats", e); + throw new QueryParserException("stats", e); } catch (JSONException e) { - throw new LuceneQueryParserException("stats", e); + throw new QueryParserException("stats", e); } } @@ -586,19 +586,19 @@ else if(searchParameters.getLimitBy() == LimitBy.FINAL_SIZE && searchParameters. } catch (UnsupportedEncodingException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } catch (HttpException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } catch (IOException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } catch (JSONException e) { - throw new LuceneQueryParserException("", e); + throw new QueryParserException("", e); } } @@ -1273,7 +1273,7 @@ public JSONObject execute(StoreRef storeRef, String handler, HashMap. - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.search.results; import java.io.Serializable; @@ -36,7 +36,6 @@ import java.util.Map; import org.alfresco.repo.search.SearcherException; -import org.alfresco.repo.search.impl.lucene.LuceneResultSetRow; import org.alfresco.service.cmr.dictionary.DataTypeDefinition; import org.alfresco.service.cmr.dictionary.DictionaryService; import org.alfresco.service.cmr.dictionary.PropertyDefinition; @@ -105,8 +104,7 @@ public SortedResultSet(ResultSet resultSet, NodeService nodeService, List(resultSet.length()); for (ResultSetRow row : resultSet) { - LuceneResultSetRow lrow = (LuceneResultSetRow) row; - nodeRefsAndScores.add(new NodeRefAndScore(row.getNodeRef(), row.getScore(), lrow.doc())); + nodeRefsAndScores.add(new NodeRefAndScore(row.getNodeRef(), row.getScore(), row.getIndex())); } ArrayList order = new ArrayList(); for (SortDefinition sd : sortDefinitions) diff --git a/src/main/java/org/alfresco/repo/security/authority/AuthorityDAOImpl.java b/src/main/java/org/alfresco/repo/security/authority/AuthorityDAOImpl.java index 686929878e..19948eba39 100644 --- a/src/main/java/org/alfresco/repo/security/authority/AuthorityDAOImpl.java +++ b/src/main/java/org/alfresco/repo/security/authority/AuthorityDAOImpl.java @@ -48,7 +48,6 @@ import org.alfresco.query.CannedQueryResults; import org.alfresco.query.PagingRequest; import org.alfresco.query.PagingResults; -import org.alfresco.repo.cache.AsynchronouslyRefreshedCache; import org.alfresco.util.cache.RefreshableCacheEvent; import org.alfresco.util.cache.RefreshableCacheListener; import org.alfresco.repo.cache.SimpleCache; @@ -59,7 +58,6 @@ import org.alfresco.repo.node.NodeServicePolicies; import org.alfresco.repo.policy.JavaBehaviour; import org.alfresco.repo.policy.PolicyComponent; -import org.alfresco.repo.search.impl.lucene.AbstractLuceneQueryParser; import org.alfresco.repo.security.authentication.AuthenticationUtil; import org.alfresco.repo.security.authentication.AuthenticationUtil.RunAsWork; import org.alfresco.repo.security.person.PersonServiceImpl; diff --git a/src/main/java/org/alfresco/repo/site/SiteServiceImpl.java b/src/main/java/org/alfresco/repo/site/SiteServiceImpl.java index b209b0c64b..5c32d77b81 100644 --- a/src/main/java/org/alfresco/repo/site/SiteServiceImpl.java +++ b/src/main/java/org/alfresco/repo/site/SiteServiceImpl.java @@ -77,7 +77,7 @@ import org.alfresco.repo.policy.BehaviourFilter; import org.alfresco.repo.policy.JavaBehaviour; import org.alfresco.repo.policy.PolicyComponent; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.repo.security.authentication.AuthenticationContext; import org.alfresco.repo.security.authentication.AuthenticationUtil; import org.alfresco.repo.security.authentication.AuthenticationUtil.RunAsWork; @@ -962,10 +962,10 @@ public List findSites(String filter, int size) result.add(createSiteInfo(site)); } } - catch (LuceneQueryParserException lqpe) + catch (QueryParserException lqpe) { //Log the error but suppress is from the user - logger.error("LuceneQueryParserException with findSites()", lqpe); + logger.error("QueryParserException with findSites()", lqpe); result = Collections.emptyList(); } finally diff --git a/src/main/java/org/alfresco/repo/solr/SOLRAdminClient.java b/src/main/java/org/alfresco/repo/solr/SOLRAdminClient.java index 56d9b24041..66f7b1b663 100644 --- a/src/main/java/org/alfresco/repo/solr/SOLRAdminClient.java +++ b/src/main/java/org/alfresco/repo/solr/SOLRAdminClient.java @@ -38,7 +38,7 @@ import org.alfresco.repo.index.shard.ShardRegistry; import org.alfresco.repo.search.impl.lucene.JSONAPIResult; import org.alfresco.repo.search.impl.lucene.JSONAPIResultFactory; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.repo.search.impl.lucene.SolrActionStatusResult; import org.alfresco.repo.search.impl.lucene.SolrCommandBackupResult; import org.alfresco.repo.search.impl.solr.AbstractSolrAdminHTTPClient; @@ -223,7 +223,7 @@ public JSONAPIResult executeAction(String core, JSONAPIResultFactory.ACTION acti } catch (IOException e) { - throw new LuceneQueryParserException("action", e); + throw new QueryParserException("action", e); } } @@ -289,7 +289,7 @@ public JSONAPIResult executeCommand(String core, JSONAPIResultFactory.HANDLER ha } catch (IOException e) { - throw new LuceneQueryParserException("action", e); + throw new QueryParserException("action", e); } diff --git a/src/main/java/org/apache/lucene/index/TermInfosReader.java b/src/main/java/org/apache/lucene/index/TermInfosReader.java deleted file mode 100644 index 225486d7d2..0000000000 --- a/src/main/java/org/apache/lucene/index/TermInfosReader.java +++ /dev/null @@ -1,376 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.BufferedIndexInput; -import org.apache.lucene.util.cache.Cache; -import org.apache.lucene.util.cache.SimpleLRUCache; -import org.apache.lucene.util.CloseableThreadLocal; - -/** This stores a monotonically increasing set of pairs in a - * Directory. Pairs are accessed either by Term or by ordinal position the - * set. */ - -final class TermInfosReader { - private Directory directory; - private String segment; - private FieldInfos fieldInfos; - - private CloseableThreadLocal threadResources = new CloseableThreadLocal(); - private SegmentTermEnum origEnum; - private long size; - - private Term[] indexTerms = null; - private ReentrantReadWriteLock indexTermsLock = new ReentrantReadWriteLock(); - private TermInfo[] indexInfos; - private long[] indexPointers; - - private SegmentTermEnum indexEnum; - - private int indexDivisor = 1; - private int totalIndexInterval; - - private final static int DEFAULT_CACHE_SIZE = 1024; - - /** - * Per-thread resources managed by ThreadLocal - */ - private static final class ThreadResources { - SegmentTermEnum termEnum; - - // Used for caching the least recently looked-up Terms - Cache termInfoCache; - } - - TermInfosReader(Directory dir, String seg, FieldInfos fis) - throws CorruptIndexException, IOException { - this(dir, seg, fis, BufferedIndexInput.BUFFER_SIZE); - } - - TermInfosReader(Directory dir, String seg, FieldInfos fis, int readBufferSize) - throws CorruptIndexException, IOException { - boolean success = false; - - try { - directory = dir; - segment = seg; - fieldInfos = fis; - - origEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_EXTENSION, - readBufferSize), fieldInfos, false); - size = origEnum.size; - totalIndexInterval = origEnum.indexInterval; - - indexEnum = new SegmentTermEnum(directory.openInput(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION, - readBufferSize), fieldInfos, true); - - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - } - - public int getSkipInterval() { - return origEnum.skipInterval; - } - - public int getMaxSkipLevels() { - return origEnum.maxSkipLevels; - } - - /** - *

Sets the indexDivisor, which subsamples the number - * of indexed terms loaded into memory. This has a - * similar effect as {@link - * IndexWriter#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1.

- * - * NOTE: you must call this before the term - * index is loaded. If the index is already loaded, - * an IllegalStateException is thrown. - * - + @throws IllegalStateException if the term index has - * already been loaded into memory. - */ - public void setIndexDivisor(int indexDivisor) throws IllegalStateException { - if (indexDivisor < 1) - throw new IllegalArgumentException("indexDivisor must be > 0: got " + indexDivisor); - - if (indexTerms != null) - throw new IllegalStateException("index terms are already loaded"); - - this.indexDivisor = indexDivisor; - totalIndexInterval = origEnum.indexInterval * indexDivisor; - } - - /** Returns the indexDivisor. - * @see #setIndexDivisor - */ - public int getIndexDivisor() { - return indexDivisor; - } - - final void close() throws IOException { - if (origEnum != null) - origEnum.close(); - if (indexEnum != null) - indexEnum.close(); - threadResources.close(); - } - - /** Returns the number of term/value pairs in the set. */ - final long size() { - return size; - } - - private ThreadResources getThreadResources() { - ThreadResources resources = (ThreadResources)threadResources.get(); - if (resources == null) { - resources = new ThreadResources(); - resources.termEnum = terms(); - // Cache does not have to be thread-safe, it is only used by one thread at the same time - resources.termInfoCache = new SimpleLRUCache(DEFAULT_CACHE_SIZE); - threadResources.set(resources); - } - return resources; - } - - private void ensureIndexIsRead() throws IOException { - indexTermsLock.readLock().lock(); - try { - if (indexTerms != null) { // index already read - return; // do nothing - } - } - finally { - indexTermsLock.readLock().unlock(); - } - indexTermsLock.writeLock().lock(); - try { - if (indexTerms != null) { - return; - } - int indexSize = 1+((int)indexEnum.size-1)/indexDivisor; // otherwise read index - - indexTerms = new Term[indexSize]; - indexInfos = new TermInfo[indexSize]; - indexPointers = new long[indexSize]; - - for (int i = 0; indexEnum.next(); i++) { - indexTerms[i] = indexEnum.term(); - indexInfos[i] = indexEnum.termInfo(); - indexPointers[i] = indexEnum.indexPointer; - - for (int j = 1; j < indexDivisor; j++) - if (!indexEnum.next()) - break; - } - } finally { - if (indexEnum != null) { - indexEnum.close(); - indexEnum = null; - } - indexTermsLock.writeLock().unlock(); - } - } - - /** Returns the offset of the greatest index entry which is less than or equal to term.*/ - private final int getIndexOffset(Term term) { - int lo = 0; // binary search indexTerms[] - int hi = indexTerms.length - 1; - - while (hi >= lo) { - int mid = (lo + hi) >>> 1; - int delta = term.compareTo(indexTerms[mid]); - if (delta < 0) - hi = mid - 1; - else if (delta > 0) - lo = mid + 1; - else - return mid; - } - return hi; - } - - private final void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException { - enumerator.seek(indexPointers[indexOffset], - (indexOffset * totalIndexInterval) - 1, - indexTerms[indexOffset], indexInfos[indexOffset]); - } - - /** Returns the TermInfo for a Term in the set, or null. */ - TermInfo get(Term term) throws IOException { - return get(term, true); - } - - /** Returns the TermInfo for a Term in the set, or null. */ - private TermInfo get(Term term, boolean useCache) throws IOException { - if (size == 0) return null; - - ensureIndexIsRead(); - - TermInfo ti; - ThreadResources resources = getThreadResources(); - Cache cache = null; - - if (useCache) { - cache = resources.termInfoCache; - // check the cache first if the term was recently looked up - ti = (TermInfo) cache.get(term); - if (ti != null) { - return ti; - } - } - - // optimize sequential access: first try scanning cached enum w/o seeking - SegmentTermEnum enumerator = resources.termEnum; - if (enumerator.term() != null // term is at or past current - && ((enumerator.prev() != null && term.compareTo(enumerator.prev())> 0) - || term.compareTo(enumerator.term()) >= 0)) { - int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; - if (indexTerms.length == enumOffset // but before end of block - || term.compareTo(indexTerms[enumOffset]) < 0) { - // no need to seek - - int numScans = enumerator.scanTo(term); - if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) { - ti = enumerator.termInfo(); - if (cache != null && numScans > 1) { - // we only want to put this TermInfo into the cache if - // scanEnum skipped more than one dictionary entry. - // This prevents RangeQueries or WildcardQueries to - // wipe out the cache when they iterate over a large numbers - // of terms in order - cache.put(term, ti); - } - } else { - ti = null; - } - - return ti; - } - } - - // random-access: must seek - seekEnum(enumerator, getIndexOffset(term)); - enumerator.scanTo(term); - if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) { - ti = enumerator.termInfo(); - if (cache != null) { - cache.put(term, ti); - } - } else { - ti = null; - } - return ti; - } - - /** Returns the nth term in the set. */ - final Term get(int position) throws IOException { - if (size == 0) return null; - - SegmentTermEnum enumerator = getThreadResources().termEnum; - if (enumerator != null && enumerator.term() != null && - position >= enumerator.position && - position < (enumerator.position + totalIndexInterval)) - return scanEnum(enumerator, position); // can avoid seek - - seekEnum(enumerator, position/totalIndexInterval); // must seek - return scanEnum(enumerator, position); - } - - private final Term scanEnum(SegmentTermEnum enumerator, int position) throws IOException { - while(enumerator.position < position) - if (!enumerator.next()) - return null; - - return enumerator.term(); - } - - /** Returns the position of a Term in the set or -1. */ - final long getPosition(Term term) throws IOException { - if (size == 0) return -1; - - ensureIndexIsRead(); - int indexOffset = getIndexOffset(term); - - SegmentTermEnum enumerator = getThreadResources().termEnum; - seekEnum(enumerator, indexOffset); - - while(term.compareTo(enumerator.term()) > 0 && enumerator.next()) {} - - if (term.compareTo(enumerator.term()) == 0) - return enumerator.position; - else - return -1; - } - - /** Returns an enumeration of all the Terms and TermInfos in the set. */ - public SegmentTermEnum terms() { - return (SegmentTermEnum)origEnum.clone(); - } - - /** Returns an enumeration of terms starting at or after the named term. */ - public SegmentTermEnum terms(Term term) throws IOException { - // don't use the cache in this call because we want to reposition the - // enumeration - get(term, false); - return (SegmentTermEnum)getThreadResources().termEnum.clone(); - } -} diff --git a/src/main/java/org/apache/lucene/search/TermQuery.java b/src/main/java/org/apache/lucene/search/TermQuery.java deleted file mode 100644 index 2c243236ec..0000000000 --- a/src/main/java/org/apache/lucene/search/TermQuery.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.util.ToStringUtils; - -/** A Query that matches documents containing a term. - This may be combined with other terms with a {@link BooleanQuery}. - */ -public class TermQuery extends Query { - private Term term; - - private class TermWeight implements Weight { - private Similarity similarity; - private float value; - private float idf; - private float queryNorm; - private float queryWeight; - - public TermWeight(Searcher searcher) - throws IOException { - this.similarity = getSimilarity(searcher); - idf = similarity.idf(term, searcher); // compute idf - } - - public String toString() { return "weight(" + TermQuery.this + ")"; } - - public Query getQuery() { return TermQuery.this; } - public float getValue() { return value; } - - public float sumOfSquaredWeights() { - queryWeight = idf * getBoost(); // compute query weight - return queryWeight * queryWeight; // square it - } - - public void normalize(float queryNorm) { - this.queryNorm = queryNorm; - queryWeight *= queryNorm; // normalize query weight - value = queryWeight * idf; // idf for document - } - - public Scorer scorer(IndexReader reader) throws IOException { - TermDocs termDocs = reader.termDocs(term); - - if (termDocs == null) - return null; - - String field = term.field(); - return new TermScorer(this, termDocs, similarity, - reader.hasNorms(field) ? reader.norms(field) : null); - } - - public Explanation explain(IndexReader reader, int doc) - throws IOException { - - ComplexExplanation result = new ComplexExplanation(); - result.setDescription("weight("+getQuery()+" in "+doc+"), product of:"); - - Explanation idfExpl = - new Explanation(idf, "idf(docFreq=" + reader.docFreq(term) + - ", numDocs=" + reader.numDocs() + ")"); - - // explain query weight - Explanation queryExpl = new Explanation(); - queryExpl.setDescription("queryWeight(" + getQuery() + "), product of:"); - - Explanation boostExpl = new Explanation(getBoost(), "boost"); - if (getBoost() != 1.0f) - queryExpl.addDetail(boostExpl); - queryExpl.addDetail(idfExpl); - - Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm"); - queryExpl.addDetail(queryNormExpl); - - queryExpl.setValue(boostExpl.getValue() * - idfExpl.getValue() * - queryNormExpl.getValue()); - - result.addDetail(queryExpl); - - // explain field weight - String field = term.field(); - ComplexExplanation fieldExpl = new ComplexExplanation(); - fieldExpl.setDescription("fieldWeight("+term+" in "+doc+ - "), product of:"); - - Explanation tfExpl = scorer(reader).explain(doc); - fieldExpl.addDetail(tfExpl); - fieldExpl.addDetail(idfExpl); - - Explanation fieldNormExpl = new Explanation(); - byte[] fieldNorms = reader.norms(field); - float fieldNorm = - fieldNorms!=null ? Similarity.decodeNorm(fieldNorms[doc]) : 0.0f; - fieldNormExpl.setValue(fieldNorm); - fieldNormExpl.setDescription("fieldNorm(field="+field+", doc="+doc+")"); - fieldExpl.addDetail(fieldNormExpl); - - fieldExpl.setMatch(Boolean.valueOf(tfExpl.isMatch())); - fieldExpl.setValue(tfExpl.getValue() * - idfExpl.getValue() * - fieldNormExpl.getValue()); - - result.addDetail(fieldExpl); - result.setMatch(fieldExpl.getMatch()); - - // combine them - result.setValue(queryExpl.getValue() * fieldExpl.getValue()); - - if (queryExpl.getValue() == 1.0f) - return fieldExpl; - - return result; - } - } - - /** Constructs a query for the term t. */ - public TermQuery(Term t) { - term = t; - } - - /** Returns the term of this query. */ - public Term getTerm() { return term; } - - protected Weight createWeight(Searcher searcher) throws IOException { - return new TermWeight(searcher); - } - - public void extractTerms(Set terms) { - terms.add(getTerm()); - } - - /** Prints a user-readable version of this query. */ - public String toString(String field) { - StringBuffer buffer = new StringBuffer(); - if (!term.field().equals(field)) { - buffer.append(term.field()); - buffer.append(":"); - } - buffer.append(term.text()); - buffer.append(ToStringUtils.boost(getBoost())); - return buffer.toString(); - } - - /** Returns true iff o is equal to this. */ - public boolean equals(Object o) { - if (!(o instanceof TermQuery)) - return false; - TermQuery other = (TermQuery)o; - return (this.getBoost() == other.getBoost()) - && this.term.equals(other.term); - } - - /** Returns a hash code value for this object.*/ - public int hashCode() { - return Float.floatToIntBits(getBoost()) ^ term.hashCode(); - } - -} diff --git a/src/main/java/org/apache/lucene/search/TermScorer.java b/src/main/java/org/apache/lucene/search/TermScorer.java deleted file mode 100644 index 53e4b736b4..0000000000 --- a/src/main/java/org/apache/lucene/search/TermScorer.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.TermDocs; - -/** Expert: A Scorer for documents matching a Term. - */ -final class TermScorer extends Scorer { - private Weight weight; - private TermDocs termDocs; - private byte[] norms; - private float weightValue; - private int doc; - - private final int[] docs = new int[32]; // buffered doc numbers - private final int[] freqs = new int[32]; // buffered term freqs - private int pointer; - private int pointerMax; - - private static final int SCORE_CACHE_SIZE = 32; - private float[] scoreCache = new float[SCORE_CACHE_SIZE]; - private static final byte DEFAULT_NORM = DefaultSimilarity.encodeNorm(1.0f); - - /** Construct a TermScorer. - * @param weight The weight of the Term in the query. - * @param td An iterator over the documents matching the Term. - * @param similarity The Similarity implementation to be used for score computations. - * @param norms The field norms of the document fields for the Term. - */ - TermScorer(Weight weight, TermDocs td, Similarity similarity, - byte[] norms) { - super(similarity); - this.weight = weight; - this.termDocs = td; - this.norms = norms; - this.weightValue = weight.getValue(); - - for (int i = 0; i < SCORE_CACHE_SIZE; i++) - scoreCache[i] = getSimilarity().tf(i) * weightValue; - } - - public void score(HitCollector hc) throws IOException { - next(); - score(hc, Integer.MAX_VALUE); - } - - protected boolean score(HitCollector c, int end) throws IOException { - Similarity similarity = getSimilarity(); // cache sim in local - float[] normDecoder = Similarity.getNormDecoder(); - while (doc < end) { // for docs in window - int f = freqs[pointer]; - float score = // compute tf(f)*weight - f < SCORE_CACHE_SIZE // check cache - ? scoreCache[f] // cache hit - : similarity.tf(f)*weightValue; // cache miss - - score *= normDecoder[(norms == null ? DEFAULT_NORM : norms[doc]) & 0xFF]; // normalize for field - - c.collect(doc, score); // collect score - - if (++pointer >= pointerMax) { - pointerMax = termDocs.read(docs, freqs); // refill buffers - if (pointerMax != 0) { - pointer = 0; - } else { - termDocs.close(); // close stream - doc = Integer.MAX_VALUE; // set to sentinel value - return false; - } - } - doc = docs[pointer]; - } - return true; - } - - /** Returns the current document number matching the query. - * Initially invalid, until {@link #next()} is called the first time. - */ - public int doc() { return doc; } - - /** Advances to the next document matching the query. - *
The iterator over the matching documents is buffered using - * {@link TermDocs#read(int[],int[])}. - * @return true iff there is another document matching the query. - */ - public boolean next() throws IOException { - pointer++; - if (pointer >= pointerMax) { - pointerMax = termDocs.read(docs, freqs); // refill buffer - if (pointerMax != 0) { - pointer = 0; - } else { - termDocs.close(); // close stream - doc = Integer.MAX_VALUE; // set to sentinel value - return false; - } - } - doc = docs[pointer]; - return true; - } - - public float score() { - int f = freqs[pointer]; - float raw = // compute tf(f)*weight - f < SCORE_CACHE_SIZE // check cache - ? scoreCache[f] // cache hit - : getSimilarity().tf(f)*weightValue; // cache miss - - return raw * Similarity.decodeNorm(norms == null ? DEFAULT_NORM : norms[doc]); // normalize for field - } - - /** Skips to the first match beyond the current whose document number is - * greater than or equal to a given target. - *
The implementation uses {@link TermDocs#skipTo(int)}. - * @param target The target document number. - * @return true iff there is such a match. - */ - public boolean skipTo(int target) throws IOException { - // first scan in cache - for (pointer++; pointer < pointerMax; pointer++) { - if (docs[pointer] >= target) { - doc = docs[pointer]; - return true; - } - } - - // not found in cache, seek underlying stream - boolean result = termDocs.skipTo(target); - if (result) { - pointerMax = 1; - pointer = 0; - docs[pointer] = doc = termDocs.doc(); - freqs[pointer] = termDocs.freq(); - } else { - doc = Integer.MAX_VALUE; - } - return result; - } - - /** Returns an explanation of the score for a document. - *
When this method is used, the {@link #next()} method - * and the {@link #score(HitCollector)} method should not be used. - * @param doc The document number for the explanation. - */ - public Explanation explain(int doc) throws IOException { - TermQuery query = (TermQuery)weight.getQuery(); - Explanation tfExplanation = new Explanation(); - int tf = 0; - while (pointer < pointerMax) { - if (docs[pointer] == doc) - tf = freqs[pointer]; - pointer++; - } - if (tf == 0) { - if (termDocs.skipTo(doc)) - { - if (termDocs.doc() == doc) - { - tf = termDocs.freq(); - } - } - } - termDocs.close(); - tfExplanation.setValue(getSimilarity().tf(tf)); - tfExplanation.setDescription("tf(termFreq("+query.getTerm()+")="+tf+")"); - - return tfExplanation; - } - - /** Returns a string representation of this TermScorer. */ - public String toString() { return "scorer(" + weight + ")"; } -} diff --git a/src/main/java/org/apache/lucene/store/AlfrescoFSDirectory.java b/src/main/java/org/apache/lucene/store/AlfrescoFSDirectory.java deleted file mode 100644 index 01cb672b38..0000000000 --- a/src/main/java/org/apache/lucene/store/AlfrescoFSDirectory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.apache.lucene.store; - -public class AlfrescoFSDirectory extends FSDirectory -{ - -} diff --git a/src/main/java/org/apache/lucene/store/FSDirectory.java b/src/main/java/org/apache/lucene/store/FSDirectory.java deleted file mode 100644 index 94da18c7aa..0000000000 --- a/src/main/java/org/apache/lucene/store/FSDirectory.java +++ /dev/null @@ -1,749 +0,0 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.lucene.index.IndexFileNameFilter; -import org.apache.lucene.index.IndexWriter; - -/** - * Straightforward implementation of {@link Directory} as a directory of files. - * Locking implementation is by default the {@link SimpleFSLockFactory}, but - * can be changed either by passing in a {@link LockFactory} instance to - * getDirectory, or specifying the LockFactory class by setting - * org.apache.lucene.store.FSDirectoryLockFactoryClass Java system - * property, or by calling {@link #setLockFactory} after creating - * the Directory. - - *

Directories are cached, so that, for a given canonical - * path, the same FSDirectory instance will always be - * returned by getDirectory. This permits - * synchronization on directories.

- * - * @see Directory - */ -public class FSDirectory extends Directory { - - /** This cache of directories ensures that there is a unique Directory - * instance per path, so that synchronization on the Directory can be used to - * synchronize access between readers and writers. We use - * refcounts to ensure when the last use of an FSDirectory - * instance for a given canonical path is closed, we remove the - * instance from the cache. See LUCENE-776 - * for some relevant discussion. - */ - private static final Map DIRECTORIES = new HashMap(); - - private static boolean disableLocks = false; - - // TODO: should this move up to the Directory base class? Also: should we - // make a per-instance (in addition to the static "default") version? - - /** - * Set whether Lucene's use of lock files is disabled. By default, - * lock files are enabled. They should only be disabled if the index - * is on a read-only medium like a CD-ROM. - */ - public static void setDisableLocks(boolean doDisableLocks) { - FSDirectory.disableLocks = doDisableLocks; - } - - /** - * Returns whether Lucene's use of lock files is disabled. - * @return true if locks are disabled, false if locks are enabled. - */ - public static boolean getDisableLocks() { - return FSDirectory.disableLocks; - } - - /** - * Directory specified by org.apache.lucene.lockDir - * or java.io.tmpdir system property. - - * @deprecated As of 2.1, LOCK_DIR is unused - * because the write.lock is now stored by default in the - * index directory. If you really want to store locks - * elsewhere you can create your own {@link - * SimpleFSLockFactory} (or {@link NativeFSLockFactory}, - * etc.) passing in your preferred lock directory. Then, - * pass this LockFactory instance to one of - * the getDirectory methods that take a - * lockFactory (for example, {@link #getDirectory(String, LockFactory)}). - */ - public static final String LOCK_DIR = System.getProperty("org.apache.lucene.lockDir", - System.getProperty("java.io.tmpdir")); - - /** The default class which implements filesystem-based directories. */ - private static Class IMPL; - static { - try { - String name = - System.getProperty("org.apache.lucene.FSDirectory.class", - FSDirectory.class.getName()); - IMPL = Class.forName(name); - } catch (ClassNotFoundException e) { - throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e); - } catch (SecurityException se) { - try { - IMPL = Class.forName(FSDirectory.class.getName()); - } catch (ClassNotFoundException e) { - throw new RuntimeException("cannot load default FSDirectory class: " + e.toString(), e); - } - } - } - - private static MessageDigest DIGESTER; - - static { - try { - DIGESTER = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e.toString(), e); - } - } - - /** A buffer optionally used in renameTo method */ - private byte[] buffer = null; - - /** Returns the directory instance for the named location. - * @param path the path to the directory. - * @return the FSDirectory for the named file. */ - public static FSDirectory getDirectory(String path) - throws IOException { - return getDirectory(new File(path), null); - } - - /** Returns the directory instance for the named location. - * @param path the path to the directory. - * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. - * @return the FSDirectory for the named file. */ - public static FSDirectory getDirectory(String path, LockFactory lockFactory) - throws IOException { - return getDirectory(new File(path), lockFactory); - } - - /** Returns the directory instance for the named location. - * @param file the path to the directory. - * @return the FSDirectory for the named file. */ - public static FSDirectory getDirectory(File file) - throws IOException { - return getDirectory(file, null); - } - - /** Returns the directory instance for the named location. - * @param file the path to the directory. - * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. - * @return the FSDirectory for the named file. */ - public static FSDirectory getDirectory(File file, LockFactory lockFactory) - throws IOException - { - file = new File(file.getCanonicalPath()); - - if (file.exists() && !file.isDirectory()) - throw new IOException(file + " not a directory"); - - if (!file.exists()) - if (!file.mkdirs()) - throw new IOException("Cannot create directory: " + file); - - FSDirectory dir; - synchronized (DIRECTORIES) { - dir = (FSDirectory)DIRECTORIES.get(file); - if (dir == null) { - try { - dir = (FSDirectory)IMPL.newInstance(); - } catch (Exception e) { - throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e); - } - dir.init(file, lockFactory); - DIRECTORIES.put(file, dir); - } else { - // Catch the case where a Directory is pulled from the cache, but has a - // different LockFactory instance. - if (lockFactory != null && lockFactory != dir.getLockFactory()) { - throw new IOException("Directory was previously created with a different LockFactory instance; please pass null as the lockFactory instance and use setLockFactory to change it"); - } - } - } - synchronized (dir) { - dir.refCount++; - } - return dir; - } - - - /** Returns the directory instance for the named location. - * - * @deprecated Use IndexWriter's create flag, instead, to - * create a new index. - * - * @param path the path to the directory. - * @param create if true, create, or erase any existing contents. - * @return the FSDirectory for the named file. */ - public static FSDirectory getDirectory(String path, boolean create) - throws IOException { - return getDirectory(new File(path), create); - } - - /** Returns the directory instance for the named location. - * - * @deprecated Use IndexWriter's create flag, instead, to - * create a new index. - * - * @param file the path to the directory. - * @param create if true, create, or erase any existing contents. - * @return the FSDirectory for the named file. */ - public static FSDirectory getDirectory(File file, boolean create) - throws IOException - { - FSDirectory dir = getDirectory(file, null); - - // This is now deprecated (creation should only be done - // by IndexWriter): - if (create) { - dir.create(); - } - - return dir; - } - - private void create() throws IOException { - if (directory.exists()) { - String[] files = directory.list(IndexFileNameFilter.getFilter()); // clear old files - if (files == null) - throw new IOException("cannot read directory " + directory.getAbsolutePath() + ": list() returned null"); - for (int i = 0; i < files.length; i++) { - File file = new File(directory, files[i]); - if (!file.delete()) - throw new IOException("Cannot delete " + file); - } - } - lockFactory.clearLock(IndexWriter.WRITE_LOCK_NAME); - } - - private File directory = null; - private int refCount; - - protected FSDirectory() {}; // permit subclassing - - private void init(File path, LockFactory lockFactory) throws IOException { - - // Set up lockFactory with cascaded defaults: if an instance was passed in, - // use that; else if locks are disabled, use NoLockFactory; else if the - // system property org.apache.lucene.store.FSDirectoryLockFactoryClass is set, - // instantiate that; else, use SimpleFSLockFactory: - - directory = path; - - boolean doClearLockID = false; - - if (lockFactory == null) { - - if (disableLocks) { - // Locks are disabled: - lockFactory = NoLockFactory.getNoLockFactory(); - } else { - String lockClassName = System.getProperty("org.apache.lucene.store.FSDirectoryLockFactoryClass"); - - if (lockClassName != null && !lockClassName.equals("")) { - Class c; - - try { - c = Class.forName(lockClassName); - } catch (ClassNotFoundException e) { - throw new IOException("unable to find LockClass " + lockClassName); - } - - try { - lockFactory = (LockFactory) c.newInstance(); - } catch (IllegalAccessException e) { - throw new IOException("IllegalAccessException when instantiating LockClass " + lockClassName); - } catch (InstantiationException e) { - throw new IOException("InstantiationException when instantiating LockClass " + lockClassName); - } catch (ClassCastException e) { - throw new IOException("unable to cast LockClass " + lockClassName + " instance to a LockFactory"); - } - - if (lockFactory instanceof NativeFSLockFactory) { - ((NativeFSLockFactory) lockFactory).setLockDir(path); - } else if (lockFactory instanceof SimpleFSLockFactory) { - ((SimpleFSLockFactory) lockFactory).setLockDir(path); - } - } else { - // Our default lock is SimpleFSLockFactory; - // default lockDir is our index directory: - lockFactory = new SimpleFSLockFactory(path); - doClearLockID = true; - } - } - } - - setLockFactory(lockFactory); - - if (doClearLockID) { - // Clear the prefix because write.lock will be - // stored in our directory: - lockFactory.setLockPrefix(null); - } - } - - /** Returns an array of strings, one for each Lucene index file in the directory. */ - public String[] list() { - ensureOpen(); - return directory.list(IndexFileNameFilter.getFilter()); - } - - /** Returns true iff a file with the given name exists. */ - public boolean fileExists(String name) { - ensureOpen(); - File file = new File(directory, name); - return file.exists(); - } - - /** Returns the time the named file was last modified. */ - public long fileModified(String name) { - ensureOpen(); - File file = new File(directory, name); - return file.lastModified(); - } - - /** Returns the time the named file was last modified. */ - public static long fileModified(File directory, String name) { - File file = new File(directory, name); - return file.lastModified(); - } - - /** Set the modified time of an existing file to now. */ - public void touchFile(String name) { - ensureOpen(); - File file = new File(directory, name); - file.setLastModified(System.currentTimeMillis()); - } - - /** Returns the length in bytes of a file in the directory. */ - public long fileLength(String name) { - ensureOpen(); - File file = new File(directory, name); - return file.length(); - } - - /** Removes an existing file in the directory. */ - public void deleteFile(String name) throws IOException { - ensureOpen(); - File file = new File(directory, name); - if (!file.delete()) - throw new IOException("Cannot delete " + file); - } - - /** Renames an existing file in the directory. - * Warning: This is not atomic. - * @deprecated - */ - public synchronized void renameFile(String from, String to) - throws IOException { - ensureOpen(); - File old = new File(directory, from); - File nu = new File(directory, to); - - /* This is not atomic. If the program crashes between the call to - delete() and the call to renameTo() then we're screwed, but I've - been unable to figure out how else to do this... */ - - if (nu.exists()) - if (!nu.delete()) - throw new IOException("Cannot delete " + nu); - - // Rename the old file to the new one. Unfortunately, the renameTo() - // method does not work reliably under some JVMs. Therefore, if the - // rename fails, we manually rename by copying the old file to the new one - if (!old.renameTo(nu)) { - java.io.InputStream in = null; - java.io.OutputStream out = null; - try { - in = new FileInputStream(old); - out = new FileOutputStream(nu); - // see if the buffer needs to be initialized. Initialization is - // only done on-demand since many VM's will never run into the renameTo - // bug and hence shouldn't waste 1K of mem for no reason. - if (buffer == null) { - buffer = new byte[1024]; - } - int len; - while ((len = in.read(buffer)) >= 0) { - out.write(buffer, 0, len); - } - - // delete the old file. - old.delete(); - } - catch (IOException ioe) { - IOException newExc = new IOException("Cannot rename " + old + " to " + nu); - newExc.initCause(ioe); - throw newExc; - } - finally { - try { - if (in != null) { - try { - in.close(); - } catch (IOException e) { - throw new RuntimeException("Cannot close input stream: " + e.toString(), e); - } - } - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - throw new RuntimeException("Cannot close output stream: " + e.toString(), e); - } - } - } - } - } - } - - /** Creates a new, empty file in the directory with the given name. - Returns a stream writing this file. */ - public IndexOutput createOutput(String name) throws IOException { - ensureOpen(); - File file = new File(directory, name); - if (file.exists() && !file.delete()) // delete existing, if any - throw new IOException("Cannot overwrite: " + file); - - return new FSIndexOutput(file); - } - - public void sync(String name) throws IOException { - ensureOpen(); - File fullFile = new File(directory, name); - boolean success = false; - int retryCount = 0; - IOException exc = null; - while(!success && retryCount < 5) { - retryCount++; - RandomAccessFile file = null; - try { - try { - file = new RandomAccessFile(fullFile, "rw"); - file.getFD().sync(); - success = true; - } finally { - if (file != null) - file.close(); - } - } catch (IOException ioe) { - if (exc == null) - exc = ioe; - try { - // Pause 5 msec - Thread.sleep(5); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - } - } - } - if (!success) - // Throw original exception - throw exc; - } - - // Inherit javadoc - public IndexInput openInput(String name) throws IOException { - ensureOpen(); - return openInput(name, BufferedIndexInput.BUFFER_SIZE); - } - - // Inherit javadoc - public IndexInput openInput(String name, int bufferSize) throws IOException { - ensureOpen(); - return new FSIndexInput(new File(directory, name), bufferSize); - } - - /** - * So we can do some byte-to-hexchar conversion below - */ - private static final char[] HEX_DIGITS = - {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; - - - public String getLockID() { - ensureOpen(); - String dirName; // name to be hashed - try { - dirName = directory.getCanonicalPath(); - } catch (IOException e) { - throw new RuntimeException(e.toString(), e); - } - - byte digest[]; - synchronized (DIGESTER) { - digest = DIGESTER.digest(dirName.getBytes()); - } - StringBuffer buf = new StringBuffer(); - buf.append("lucene-"); - for (int i = 0; i < digest.length; i++) { - int b = digest[i]; - buf.append(HEX_DIGITS[(b >> 4) & 0xf]); - buf.append(HEX_DIGITS[b & 0xf]); - } - - return buf.toString(); - } - - /** Closes the store to future operations. */ - public synchronized void close() { - if (isOpen && --refCount <= 0) { - isOpen = false; - synchronized (DIRECTORIES) { - DIRECTORIES.remove(directory); - } - } - } - - public File getFile() { - ensureOpen(); - return directory; - } - - /** For debug output. */ - public String toString() { - return this.getClass().getName() + "@" + directory; - } - - protected static class FSIndexInput extends BufferedIndexInput { - - protected static class Descriptor implements Cloneable{ - // remember if the file is open, so that we don't try to close it - // more than once - private boolean isOpen; - final long length; - final Map fileMap = new TreeMap(); - final File file; - final String mode; - final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - - public Descriptor(File file, String mode) throws IOException { - this.file = file; - this.mode = mode; - isOpen=true; - RandomAccessFile raf = new RandomAccessFile(file, mode); - length=raf.length(); - fileMap.put(Thread.currentThread().getName(), raf); - } - - private RandomAccessFile getFile() { - String threadKey = Thread.currentThread().getName(); - lock.readLock().lock(); - RandomAccessFile file = fileMap.get(threadKey); - if (file == null) { - lock.readLock().unlock(); - lock.writeLock().lock(); - try { - file = fileMap.get(threadKey); - if (file == null) { - file = new RandomAccessFile(this.file, mode); - fileMap.put(threadKey, file); - } - } catch (FileNotFoundException e) { - throw new RuntimeException(e); - } finally { - lock.writeLock().unlock(); - } - } else { - lock.readLock().unlock(); - } - return file; - } - - public void close() throws IOException { - lock.readLock().lock(); - if (isOpen) { - lock.readLock().unlock(); - lock.writeLock().lock(); - try { - if (isOpen) { - for (RandomAccessFile file : fileMap.values()) - { - file.close(); - } - fileMap.clear(); - isOpen=false; - } - } - finally - { - lock.writeLock().unlock(); - } - } - else { - lock.readLock().unlock(); - } - } - - protected void finalize() throws Throwable { - try { - close(); - } finally { - super.finalize(); - } - } - } - - protected final Descriptor file; - boolean isClone; - - public FSIndexInput(File path) throws IOException { - this(path, BufferedIndexInput.BUFFER_SIZE); - } - - public FSIndexInput(File path, int bufferSize) throws IOException { - super(bufferSize); - file = new Descriptor(path, "r"); - } - - /** IndexInput methods */ - protected void readInternal(byte[] b, int offset, int len) - throws IOException { - int total = 0; - do { - RandomAccessFile raf = file.getFile(); - raf.seek(getFilePointer()); - int i = raf.read(b, offset+total, len-total); - if (i == -1) - throw new IOException("read past EOF"); - total += i; - } while (total < len); - } - - public void close() throws IOException { - // only close the file if this is not a clone - if (!isClone) file.close(); - } - - protected void seekInternal(long position) { - } - - public long length() { - return file.length; - } - - public Object clone() { - FSIndexInput clone = (FSIndexInput)super.clone(); - clone.isClone = true; - return clone; - } - - /** Method used for testing. Returns true if the underlying - * file descriptor is valid. - */ - boolean isFDValid() throws IOException { - return file.getFile().getFD().valid(); - } - } - - protected static class FSIndexOutput extends BufferedIndexOutput { - RandomAccessFile file = null; - - // remember if the file is open, so that we don't try to close it - // more than once - private volatile boolean isOpen; - - public FSIndexOutput(File path) throws IOException { - file = new RandomAccessFile(path, "rw"); - file.getChannel(); - isOpen = true; - } - - /** output methods: */ - public void flushBuffer(byte[] b, int offset, int size) throws IOException { - file.write(b, offset, size); - } - public void close() throws IOException { - // only close the file if it has not been closed yet - if (isOpen) { - boolean success = false; - try { - super.close(); - success = true; - } finally { - isOpen = false; - if (!success) { - try { - file.close(); - } catch (Throwable t) { - // Suppress so we don't mask original exception - } - } else - file.close(); - } - } - } - - /** Random-access methods */ - public void seek(long pos) throws IOException { - super.seek(pos); - file.seek(pos); - } - public long length() throws IOException { - return file.length(); - } - public void setLength(long length) throws IOException { - file.setLength(length); - } - } -} diff --git a/src/main/resources/alfresco/core-services-context.xml b/src/main/resources/alfresco/core-services-context.xml index 24086d45e9..5f81ba196b 100644 --- a/src/main/resources/alfresco/core-services-context.xml +++ b/src/main/resources/alfresco/core-services-context.xml @@ -1263,20 +1263,6 @@ - - - - - - search.adm.luceneQueryEngineImpl - - - - org.alfresco.repo.search.impl.querymodel.QueryEngine - - - - diff --git a/src/main/resources/alfresco/extension/index-tracking-context.xml.sample b/src/main/resources/alfresco/extension/index-tracking-context.xml.sample deleted file mode 100644 index db0df81b52..0000000000 --- a/src/main/resources/alfresco/extension/index-tracking-context.xml.sample +++ /dev/null @@ -1,4 +0,0 @@ - -Index tracking is now controlled using core properties. -See 'alfresco/repository.properties' and 'alfresco/extension/custom-repository.properties.sample'. - diff --git a/src/main/resources/alfresco/extension/language-specific-index-and-search-context.xml.sample b/src/main/resources/alfresco/extension/language-specific-index-and-search-context.xml.sample deleted file mode 100644 index b083c06fad..0000000000 --- a/src/main/resources/alfresco/extension/language-specific-index-and-search-context.xml.sample +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - EXACT_LANGUAGE - - - - - - - - \ No newline at end of file diff --git a/src/test/java/org/alfresco/AllUnitTestsSuite.java b/src/test/java/org/alfresco/AllUnitTestsSuite.java index c5d8a016fd..6840d989e1 100644 --- a/src/test/java/org/alfresco/AllUnitTestsSuite.java +++ b/src/test/java/org/alfresco/AllUnitTestsSuite.java @@ -186,7 +186,7 @@ org.alfresco.repo.version.common.versionlabel.SerialVersionLabelPolicyTest.class, org.alfresco.repo.workflow.activiti.WorklfowObjectFactoryTest.class, org.alfresco.repo.workflow.WorkflowSuiteContextShutdownTest.class, - org.alfresco.repo.search.impl.lucene.analysis.PathTokenFilterTest.class, + org.alfresco.repo.search.LuceneUtilsTest.class, org.alfresco.heartbeat.HBDataCollectorServiceImplTest.class, org.alfresco.heartbeat.jobs.LockingJobTest.class, diff --git a/src/test/java/org/alfresco/opencmis/search/OpenCmisQueryTest.java b/src/test/java/org/alfresco/opencmis/search/OpenCmisQueryTest.java index 5ce345773d..add709f96b 100644 --- a/src/test/java/org/alfresco/opencmis/search/OpenCmisQueryTest.java +++ b/src/test/java/org/alfresco/opencmis/search/OpenCmisQueryTest.java @@ -3364,11 +3364,11 @@ public void testPropertyToSelectorBinding() throws Exception testQuery("SELECT * FROM cmis:document where cmis:parentId <> 'woof://woof/woof'", 10, false, "cmis:objectId", new String(), true, CMISQueryMode.CMS_STRICT); testQuery("SELECT D.*, O.cmis:name FROM CMIS:DOCUMENT AS D JOIN CM:OWNABLE AS O ON D.cmis:objectId = O.cmis:objectId", 1, false, "cmis:objectId", new String(), true, CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); - testQuery("SELECT D.*, O.cmis:name FROM CMIS:DOCUMENT AS D LEFT OUTER JOIN CM:OWNABLE AS O ON D.cmis:objectId = O.cmis:objectId where O.cm:owner is not null", 1, false, "cmis:objectId", new String(), true, - CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); - testQuery("SELECT D.*, O.cmis:name FROM CMIS:DOCUMENT AS D LEFT OUTER JOIN CM:OWNABLE AS O ON D.cmis:objectId = O.cmis:objectId where O.cm:owner is null", 1, false, "cmis:objectId", new String(), true, - CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); - testQuery("SELECT D.*, O.cmis:name FROM CMIS:DOCUMENT AS D LEFT OUTER JOIN CM:OWNABLE AS O ON D.cmis:objectId = O.cmis:objectId", 20, false, "cmis:objectId", new String(), true, + testQuery("SELECT D.*, O.cmis:name FROM CMIS:DOCUMENT AS D LEFT OUTER JOIN CM:OWNABLE AS O ON D.cmis:objectId = O.cmis:objectId where O.cm:owner is not null", 1, false, "cmis:objectId", new String(), true, + CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); + testQuery("SELECT D.*, O.cmis:name FROM CMIS:DOCUMENT AS D LEFT OUTER JOIN CM:OWNABLE AS O ON D.cmis:objectId = O.cmis:objectId where O.cm:owner is null", 1, false, "cmis:objectId", new String(), true, + CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); + testQuery("SELECT D.*, O.cmis:name FROM CMIS:DOCUMENT AS D LEFT OUTER JOIN CM:OWNABLE AS O ON D.cmis:objectId = O.cmis:objectId", 20, false, "cmis:objectId", new String(), true, CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); testQuery("SELECT * FROM cmis:document order by cmis:parentId", 10, false, "cmis:objectId", new String(), true, CMISQueryMode.CMS_STRICT); testQuery("SELECT * FROM cmis:document where CONTAINS('cmis:parentId:*')", 10, false, "cmis:objectId", new String(), true, CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); @@ -3786,17 +3786,17 @@ public void testAspectJoin() throws Exception testQuery( "select o.*, t.* from ( cm:ownable o join cm:titled t on o.cmis:objectId = t.cmis:objectId JOIN cmis:document AS D ON D.cmis:objectId = o.cmis:objectId ) where o.cm:owner = 'andy' and t.cm:title = 'Alfresco tutorial' and CONTAINS(D, 'jumped') and D.cmis:contentStreamLength <> 2", 1, false, "cmis:objectId", new String(), false, CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); - - // LOJ - testQuery("SELECT * FROM cmis:document", 11, false, "cmis:objectId", new String(), false, - CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); - testQuery("SELECT D.*, O.* FROM cmis:document AS D LEFT JOIN cm:ownable AS O ON D.cmis:objectId = O.cmis:objectId", 11, false, "cmis:objectId", new String(), false, - CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); - testQuery("SELECT D.*, T.* FROM cmis:document AS D LEFT JOIN cm:titled AS T ON D.cmis:objectId = T.cmis:objectId", 11, false, "cmis:objectId", new String(), false, - CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); - testQuery("SELECT T.*, O.* FROM cm:titled AS T LEFT JOIN cm:ownable AS O ON O.cmis:objectId = T.cmis:objectId", 11, false, "cmis:objectId", new String(), false, - CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); - testQuery("SELECT T.*, O.* FROM cm:ownable AS O LEFT JOIN cm:titled AS T ON O.cmis:objectId = T.cmis:objectId", 1, false, "cmis:objectId", new String(), false, + + // LOJ + testQuery("SELECT * FROM cmis:document", 11, false, "cmis:objectId", new String(), false, + CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); + testQuery("SELECT D.*, O.* FROM cmis:document AS D LEFT JOIN cm:ownable AS O ON D.cmis:objectId = O.cmis:objectId", 11, false, "cmis:objectId", new String(), false, + CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); + testQuery("SELECT D.*, T.* FROM cmis:document AS D LEFT JOIN cm:titled AS T ON D.cmis:objectId = T.cmis:objectId", 11, false, "cmis:objectId", new String(), false, + CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); + testQuery("SELECT T.*, O.* FROM cm:titled AS T LEFT JOIN cm:ownable AS O ON O.cmis:objectId = T.cmis:objectId", 11, false, "cmis:objectId", new String(), false, + CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); + testQuery("SELECT T.*, O.* FROM cm:ownable AS O LEFT JOIN cm:titled AS T ON O.cmis:objectId = T.cmis:objectId", 1, false, "cmis:objectId", new String(), false, CMISQueryMode.CMS_WITH_ALFRESCO_EXTENSIONS); } diff --git a/src/test/java/org/alfresco/repo/model/filefolder/HiddenAspectTest.java b/src/test/java/org/alfresco/repo/model/filefolder/HiddenAspectTest.java index c44897c08c..6decdcc32f 100644 --- a/src/test/java/org/alfresco/repo/model/filefolder/HiddenAspectTest.java +++ b/src/test/java/org/alfresco/repo/model/filefolder/HiddenAspectTest.java @@ -54,7 +54,6 @@ import org.alfresco.repo.imap.AlfrescoImapUser; import org.alfresco.repo.imap.ImapService; import org.alfresco.repo.model.filefolder.HiddenAspect.Visibility; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParser; import org.alfresco.repo.security.authentication.AuthenticationUtil; import org.alfresco.service.ServiceRegistry; import org.alfresco.service.cmr.coci.CheckOutCheckInService; diff --git a/src/test/java/org/alfresco/repo/search/LuceneUtilsTest.java b/src/test/java/org/alfresco/repo/search/LuceneUtilsTest.java new file mode 100644 index 0000000000..9c83c64ea7 --- /dev/null +++ b/src/test/java/org/alfresco/repo/search/LuceneUtilsTest.java @@ -0,0 +1,62 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.repo.search; + +import java.util.Calendar; +import java.util.Date; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Unit tests for {@link org.alfresco.repo.search.impl.lucene.LuceneUtils}. + * + * @author Neil Mc Erlean + * @since 4.0 + */ +public class LuceneUtilsTest +{ + @Test public void convertSimpleDate() throws Exception + { + Calendar cal = Calendar.getInstance(); + + // November 12th, 1955. 10:04 pm exactly. :) + final int year = 1955; + final int month = 10; // 0-based + final int day = 12; + final int hours = 22; + final int minutes = 04; + final int seconds = 00; + cal.set(year, month, day, hours, minutes, seconds); + + Date testDate = cal.getTime(); + + String dateString = LuceneUtils.getLuceneDateString(testDate); + final String expectedString = "1955\\-11\\-12T22:04:00"; + + assertEquals("Incorrect data string.", expectedString, dateString); + } +} diff --git a/src/test/java/org/alfresco/repo/search/impl/solr/SolrSQLHttpClientTest.java b/src/test/java/org/alfresco/repo/search/impl/solr/SolrSQLHttpClientTest.java index d2a769f310..50dde88226 100644 --- a/src/test/java/org/alfresco/repo/search/impl/solr/SolrSQLHttpClientTest.java +++ b/src/test/java/org/alfresco/repo/search/impl/solr/SolrSQLHttpClientTest.java @@ -46,7 +46,7 @@ import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.repo.admin.RepositoryState; -import org.alfresco.repo.search.impl.lucene.LuceneQueryParserException; +import org.alfresco.repo.search.QueryParserException; import org.alfresco.repo.search.impl.lucene.SolrJsonProcessor; import org.alfresco.repo.tenant.TenantService; import org.alfresco.service.cmr.repository.StoreRef; @@ -177,7 +177,7 @@ public void testExecuteQuery_connectException() throws Exception solrSQLHttpClient.executeQuery(mockSearchParameters, LANGUAGE); fail("Expected exception to be thrown due to failed connection."); } - catch (LuceneQueryParserException e) + catch (QueryParserException e) { assertTrue("Expected message to mention InsightEngine.", e.getMessage().contains("InsightEngine")); } From 0e032481662c9fae3b427830f1ce494c8a63ae10 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Thu, 24 Sep 2020 15:50:24 +0200 Subject: [PATCH 2/3] SEARCH-2434: Remove lucene dependency from repository --- .../java/org/alfresco/opencmis/search/OpenCmisQueryTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/org/alfresco/opencmis/search/OpenCmisQueryTest.java b/src/test/java/org/alfresco/opencmis/search/OpenCmisQueryTest.java index add709f96b..2d1ef97a78 100644 --- a/src/test/java/org/alfresco/opencmis/search/OpenCmisQueryTest.java +++ b/src/test/java/org/alfresco/opencmis/search/OpenCmisQueryTest.java @@ -74,7 +74,6 @@ import org.alfresco.repo.dictionary.M2Model; import org.alfresco.repo.node.BaseNodeServiceTest; import org.alfresco.repo.search.MLAnalysisMode; -import org.alfresco.repo.search.impl.lucene.analysis.DateTimeAnalyser; import org.alfresco.repo.search.impl.parsers.CMISLexer; import org.alfresco.repo.search.impl.parsers.CMISParser; import org.alfresco.repo.search.impl.parsers.FTSQueryException; @@ -259,7 +258,6 @@ public void setUp() throws Exception DataTypeDefinition dataType = dictionaryService.getDataType(DataTypeDefinition.DATETIME); String analyserClassName = dataType.resolveAnalyserClassName(); - usesDateTimeAnalyser = analyserClassName.equals(DateTimeAnalyser.class.getCanonicalName()); base = nodeService.createNode(rootNodeRef, ContentModel.ASSOC_CHILDREN, QName.createQName("cm", "Base Folder", namespaceService), ContentModel.TYPE_FOLDER).getChildRef(); nodeService.setProperty(base, ContentModel.PROP_NAME, "Base Folder"); From 0344912f3a953078151f587c9f7a8cfca8b58f03 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Thu, 24 Sep 2020 16:49:51 +0100 Subject: [PATCH 3/3] Update data-model to search-2434-2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 358b1cae1b..a8ddc9ff02 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ 11 - search-2434-1 + search-2434-2 8.49 6.2