Skip to content

Commit

Permalink
Merge branch 'trunk' of github.com:apache/jackrabbit-oak into trunk
Browse files Browse the repository at this point in the history
  • Loading branch information
reschke committed Sep 13, 2021
2 parents 1eecb92 + 0558bf2 commit c37f1d4
Show file tree
Hide file tree
Showing 8 changed files with 232 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@

import com.google.common.collect.Iterables;

import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
Expand Down Expand Up @@ -73,6 +74,14 @@ public class IndexUpdate implements Editor, PathSource {
private static final Logger log = LoggerFactory.getLogger(IndexUpdate.class);
private static final String TYPE_ELASTICSEARCH = "elasticsearch";

//This is used so that wrong index definitions are sparsely logged. After every 1000 indexing cycles, index definitions
// with wrong nodetype will be logged.
public static final long INDEX_JCR_TYPE_INVALID_LOG_LIMITER = Long.parseLong(System.getProperty("oak.indexer.indexJcrTypeInvalidLogLimiter", "1000"));

// Initial value is set at indexJcrTypeInvalidLogLimiter so that first logging start on first cycle/update itself.
// This counter is cyclically incremented till indexJcrTypeInvalidLogLimiter and then reset to 0
private static volatile long cyclicExecutionCount = INDEX_JCR_TYPE_INVALID_LOG_LIMITER;

/**
* <p>
* The value of this flag determines the behavior of the IndexUpdate when
Expand Down Expand Up @@ -124,7 +133,6 @@ public class IndexUpdate implements Editor, PathSource {
*/
private final Map<String, Editor> reindex = new HashMap<String, Editor>();


public IndexUpdate(
IndexEditorProvider provider, String async,
NodeState root, NodeBuilder builder,
Expand Down Expand Up @@ -280,10 +288,25 @@ private void collectIndexEditors(NodeBuilder definitions,
NodeBuilder definition = definitions.getChildNode(name);
if (isIncluded(rootState.async, definition)) {
String type = definition.getString(TYPE_PROPERTY_NAME);
String primaryType = definition.getName(JcrConstants.JCR_PRIMARYTYPE);
if (type == null) {
// probably not an index def
continue;
}
/*
Log a warning after every indexJcrTypeInvalidLogLimiter cycles of indexer where nodeState changed.
and skip further execution for invalid nodetype of index definition.
*/
if (!IndexConstants.INDEX_DEFINITIONS_NODE_TYPE.equals(primaryType)) {
// It is a cyclic counter which reset back to 0 after INDEX_JCR_TYPE_INVALID_LOG_LIMITER
// This is to sparsely log this warning.
if ((cyclicExecutionCount >= INDEX_JCR_TYPE_INVALID_LOG_LIMITER)) {
log.warn("jcr:primaryType of index {} should be {} instead of {}", name, IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, primaryType);
cyclicExecutionCount = 0;
}
cyclicExecutionCount++;
continue;
}

boolean shouldReindex = shouldReindex(definition, before, name);
String indexPath = getIndexPath(getPath(), name);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2107,7 +2107,8 @@ public void indexCommitCallback() throws Exception {

// Make index
NodeBuilder builder = store.getRoot().builder();
builder.child("oak:index").child("fooIndex").setProperty("async", "async").setProperty("type", "foo");
builder.child("oak:index").child("fooIndex").setProperty("async", "async").setProperty("type", "foo")
.setProperty("jcr:primaryType", "oak:QueryIndexDefinition", Type.NAME);
store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);

// reset stats
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.jackrabbit.oak.plugins.index.lucene;

import ch.qos.logback.classic.Level;
import org.apache.jackrabbit.oak.InitialContent;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.*;
import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdate;
import org.apache.jackrabbit.oak.plugins.index.TrackingCorruptIndexHandler;
import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexProvider;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.spi.commit.Observer;
import org.apache.jackrabbit.oak.spi.nodetype.NodeTypeConstants;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.util.List;
import java.util.concurrent.TimeUnit;

import static com.google.common.collect.Lists.newArrayList;
import static org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider.compose;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;

/**
* Tests: Logging of invalid jcr:primaryType of index.
*/
public class InvalidIndexTest {

private final long INDEX_CORRUPT_INTERVAL_IN_MILLIS = 100;
private MemoryBlobStore blobStore;

protected Root root;

private AsyncIndexUpdate asyncIndexUpdate;

private static final String invalidJcrPrimaryTypeIndexName = "myTestIndexWithWrongJcrPrimaryType";
private static final String invalidJcrPrimaryTypeForIndex = NodeTypeConstants.NT_OAK_UNSTRUCTURED;

private NodeStore nodeStore;
private LogCustomizer customLogger;


// Setting contentCount to DEFAULT_indexJcrTypeInvalidLogLimiter + 1, so that invalid indexes are logged atleast once
private final long contentCount = IndexUpdate.INDEX_JCR_TYPE_INVALID_LOG_LIMITER + 1;

@Before
public void before() throws Exception {
ContentSession session = createRepository().login(null, null);
root = session.getLatestRoot();
customLogger = LogCustomizer
.forLogger(IndexUpdate.class.getName())
.enable(Level.WARN).create();
customLogger.starting();
}

@After
public void after() {
customLogger.finished();
}

private ContentRepository createRepository() {
nodeStore = new MemoryNodeStore();
blobStore = new MemoryBlobStore();
blobStore.setBlockSizeMin(48);//make it as small as possible

LuceneIndexEditorProvider luceneIndexEditorProvider = new LuceneIndexEditorProvider();
LuceneIndexProvider provider = new LuceneIndexProvider();
luceneIndexEditorProvider.setBlobStore(blobStore);

asyncIndexUpdate = new AsyncIndexUpdate("async", nodeStore, compose(newArrayList(
luceneIndexEditorProvider,
new NodeCounterEditorProvider()
)));
TrackingCorruptIndexHandler trackingCorruptIndexHandler = new TrackingCorruptIndexHandler();
trackingCorruptIndexHandler.setCorruptInterval(INDEX_CORRUPT_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS);
asyncIndexUpdate.setCorruptIndexHandler(trackingCorruptIndexHandler);
return new Oak(nodeStore)
.with(new InitialContent())
.with(new OpenSecurityProvider())
.with((QueryIndexProvider) provider)
.with((Observer) provider)
.with(luceneIndexEditorProvider)
.with(new PropertyIndexEditorProvider())
.with(new NodeTypeIndexProvider())
.with(new PropertyIndexProvider())
.with(new PropertyIndexEditorProvider())
.createContentRepository();
}

private void runEmptyAsyncIndexerCyclesWithoutNewContent(long count) {
for (int i = 0; i < count; i++) {
asyncIndexUpdate.run();
}
}

private void runIndexerCyclesAfterEachNodeCommit(long count, boolean async) throws CommitFailedException {
for (int i = 0; i < count; i++) {
root.getTree("/").addChild("testNode" + i);
root.commit();
if (async) {
asyncIndexUpdate.run();
}
}
}

private boolean assertionLogPresent(List<String> logs, String assertionLog) {
for (String log : logs) {
if (log.equals(assertionLog)) {
return true;
}
}
return false;
}

private String invalidJcrPrimaryTypeLog() {
return "jcr:primaryType of index " + invalidJcrPrimaryTypeIndexName + " should be oak:QueryIndexDefinition instead of " + invalidJcrPrimaryTypeForIndex;
}

/**
* Logs warning as index is not a valid index definition
*/
@Test
public void loggingInvalidJcrPrimaryType() throws Exception {
Tree test1 = root.getTree("/").addChild(INDEX_DEFINITIONS_NAME).addChild(invalidJcrPrimaryTypeIndexName);
test1.setProperty("jcr:primaryType", invalidJcrPrimaryTypeForIndex, Type.NAME);
test1.setProperty("type", "lucene");
test1.setProperty("reindex", true);
test1.setProperty("async", "async");
root.commit();
runIndexerCyclesAfterEachNodeCommit(contentCount, true);
runEmptyAsyncIndexerCyclesWithoutNewContent(contentCount);

List<String> logs = customLogger.getLogs();
assertTrue(assertionLogPresent(logs, invalidJcrPrimaryTypeLog()));
}

@Test
public void noLoggingIfNodeIsNotIndexDefinition() throws Exception {
// This is not a valid index definition. Refer method: IndexUpdate.isIncluded
Tree test1 = root.getTree("/").addChild(INDEX_DEFINITIONS_NAME).addChild(invalidJcrPrimaryTypeIndexName);
test1.setProperty("jcr:primaryType", invalidJcrPrimaryTypeForIndex, Type.NAME);
// Here index definition itself is not valid as there is no type property
// test1.setProperty("type", "lucene");
test1.setProperty("reindex", true);
test1.setProperty("async", "async");
root.commit();

runIndexerCyclesAfterEachNodeCommit(contentCount, true);
runEmptyAsyncIndexerCyclesWithoutNewContent(contentCount);

List<String> logs = customLogger.getLogs();
assertFalse(assertionLogPresent(logs, invalidJcrPrimaryTypeLog()));
}

@Test
public void loggingInCaseOfSyncLuceneIndexDefinition() throws Exception {
Tree test1 = root.getTree("/").addChild(INDEX_DEFINITIONS_NAME).addChild(invalidJcrPrimaryTypeIndexName);
test1.setProperty("jcr:primaryType", invalidJcrPrimaryTypeForIndex, Type.NAME);
test1.setProperty("type", "lucene");
test1.setProperty("reindex", true);
// test1.setProperty("async", "async");
root.commit();

runIndexerCyclesAfterEachNodeCommit(contentCount, false);
List<String> logs = customLogger.getLogs();
assertTrue(assertionLogPresent(logs, invalidJcrPrimaryTypeLog()));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ private Tree createSuggestIndex(String indexedNodeType)
String indexName = "lucene-suggest";
Tree def = root.getTree("/" + INDEX_DEFINITIONS_NAME)
.addChild(indexName);
def.setProperty(JcrConstants.JCR_PRIMARYTYPE, INDEX_DEFINITIONS_NODE_TYPE);
def.setProperty(JcrConstants.JCR_PRIMARYTYPE, INDEX_DEFINITIONS_NODE_TYPE, Type.NAME);
def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE);
def.setProperty(REINDEX_PROPERTY_NAME, true);
def.setProperty("name", indexName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import static org.apache.jackrabbit.commons.JcrUtils.getOrCreateByPath;
import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FACETS;
import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_REFRESH_DEFN;
import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS;
Expand Down Expand Up @@ -159,7 +160,7 @@ void initialize(String nodeType) {

String build() throws RepositoryException {
final String indexName = UUID.randomUUID().toString();
indexDefinitionBuilder.build(adminSession.getRootNode().getNode(INDEX_DEFINITIONS_NAME).addNode(indexName));
indexDefinitionBuilder.build(adminSession.getRootNode().getNode(INDEX_DEFINITIONS_NAME).addNode(indexName, INDEX_DEFINITIONS_NODE_TYPE));
return indexName;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import static org.apache.jackrabbit.commons.JcrUtils.getOrCreateByPath;
import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
import static org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition.BULK_FLUSH_INTERVAL_MS_DEFAULT;
import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_ANALYZED;
import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_USE_IN_SPELLCHECK;
Expand Down Expand Up @@ -138,7 +139,7 @@ void initialize(String nodeType) {

String build() throws RepositoryException {
final String indexName = UUID.randomUUID().toString();
indexDefinitionBuilder.build(adminSession.getRootNode().getNode(INDEX_DEFINITIONS_NAME).addNode(indexName));
indexDefinitionBuilder.build(adminSession.getRootNode().getNode(INDEX_DEFINITIONS_NAME).addNode(indexName, INDEX_DEFINITIONS_NODE_TYPE));
return indexName;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import javax.jcr.Session;

import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;

public abstract class IndexOptions {

Expand All @@ -35,7 +36,7 @@ protected Tree setIndex(Root root, String idxName, IndexDefinitionBuilder builde
}

protected Node setIndex(Session session, String idxName, IndexDefinitionBuilder builder) throws RepositoryException {
return builder.build(session.getRootNode().getNode(INDEX_DEFINITIONS_NAME).addNode(idxName));
return builder.build(session.getRootNode().getNode(INDEX_DEFINITIONS_NAME).addNode(idxName, INDEX_DEFINITIONS_NODE_TYPE));
}

protected Node getIndexNode(Session session, String idxName) throws RepositoryException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ public void createIndex() throws RepositoryException {
String indexName = UUID.randomUUID().toString();
IndexDefinitionBuilder builder = indexOptions.createIndex(indexOptions.createIndexDefinitionBuilder(), false);
builder.noAsync();
builder.getBuilderTree().setProperty("jcr:primaryType", "oak:QueryIndexDefinition", Type.NAME);
IndexDefinitionBuilder.IndexRule indexRule = builder.indexRule(JcrConstants.NT_BASE);
indexRule.property("cons").propertyIndex();
indexRule.property("foo").propertyIndex().getBuilderTree().setProperty(FACET_PROP, true, Type.BOOLEAN);
Expand Down

0 comments on commit c37f1d4

Please sign in to comment.