diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc index a1f7e9824..b26f31e09 100644 --- a/docs/reference/mapping/fields.asciidoc +++ b/docs/reference/mapping/fields.asciidoc @@ -21,6 +21,8 @@ include::fields/boost-field.asciidoc[] include::fields/parent-field.asciidoc[] +include::fields/field-names-field.asciidoc[] + include::fields/routing-field.asciidoc[] include::fields/index-field.asciidoc[] diff --git a/docs/reference/mapping/fields/field-names-field.asciidoc b/docs/reference/mapping/fields/field-names-field.asciidoc new file mode 100644 index 000000000..606b5655d --- /dev/null +++ b/docs/reference/mapping/fields/field-names-field.asciidoc @@ -0,0 +1,11 @@ +[[mapping-field-names-field]] +=== `_field_names` + +coming[1.3.0] + +The `_field_names` field indexes the field names of a document, which can later +be used to search for documents based on the fields that they contain typically +using the `exists` and `missing` filters. + +`_field_names` is indexed by default for indices that have been created after +Elasticsearch 1.3.0. diff --git a/src/main/java/org/elasticsearch/Version.java b/src/main/java/org/elasticsearch/Version.java index 7dcace852..c416e7111 100644 --- a/src/main/java/org/elasticsearch/Version.java +++ b/src/main/java/org/elasticsearch/Version.java @@ -19,12 +19,14 @@ package org.elasticsearch; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.monitor.jvm.JvmInfo; import java.io.IOException; @@ -344,6 +346,15 @@ public static Version fromId(int id) { } } + /** + * Return the {@link Version} of Elasticsearch that has been used to create an index given its settings. + */ + public static Version indexCreated(Settings indexSettings) { + assert indexSettings.get(IndexMetaData.SETTING_UUID) == null // if the UUDI is there the index has actually been created otherwise this might be a test + || indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null : IndexMetaData.SETTING_VERSION_CREATED + " not set in IndexSettings"; + return indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); + } + public static void writeVersion(Version version, StreamOutput out) throws IOException { out.writeVInt(version.id); } diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index a8171cf45..9bcee201f 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -180,6 +180,8 @@ public Builder(String index, @Nullable Settings indexSettings, RootObjectMapper. this.rootMappers.put(TTLFieldMapper.class, new TTLFieldMapper()); this.rootMappers.put(VersionFieldMapper.class, new VersionFieldMapper()); this.rootMappers.put(ParentFieldMapper.class, new ParentFieldMapper()); + // _field_names last so that it can see all other fields + this.rootMappers.put(FieldNamesFieldMapper.class, new FieldNamesFieldMapper(indexSettings)); } public Builder meta(ImmutableMap meta) { diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 398b8cc9e..a925885d3 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -21,9 +21,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; @@ -51,7 +49,6 @@ import java.util.Iterator; import java.util.Map; -import java.util.Set; import static org.elasticsearch.index.mapper.MapperBuilders.doc; @@ -122,10 +119,9 @@ public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, .put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser()) .put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser()) .put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()) + .put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser()) .immutableMap(); - assert indexSettings.get(IndexMetaData.SETTING_UUID) == null // if the UUDI is there the index has actually been created otherwise this might be a test - || indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null : IndexMetaData.SETTING_VERSION_CREATED + " not set in IndexSettings"; - indexVersionCreated = indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); + indexVersionCreated = Version.indexCreated(indexSettings); } public void putTypeParser(String type, Mapper.TypeParser typeParser) { diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java b/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java index 320bb08be..3972e2c3d 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java @@ -74,6 +74,10 @@ public static TypeFieldMapper.Builder type() { return new TypeFieldMapper.Builder(); } + public static FieldNamesFieldMapper.Builder fieldNames() { + return new FieldNamesFieldMapper.Builder(); + } + public static IndexFieldMapper.Builder index() { return new IndexFieldMapper.Builder(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java new file mode 100644 index 000000000..74cc13d3c --- /dev/null +++ b/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java @@ -0,0 +1,248 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.internal; + +import com.google.common.collect.UnmodifiableIterator; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.XStringField; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatProvider; +import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider; +import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.core.AbstractFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.index.mapper.MapperBuilders.fieldNames; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseField; + +/** + * A mapper that indexes the field names of a document under _field_names. This mapper is typically useful in order + * to have fast exists and missing queries/filters. + * + * Added in Elasticsearch 1.3. + */ +public class FieldNamesFieldMapper extends AbstractFieldMapper implements InternalMapper, RootMapper { + + public static final String NAME = "_field_names"; + + public static final String CONTENT_TYPE = "_field_names"; + + public static class Defaults extends AbstractFieldMapper.Defaults { + public static final String NAME = FieldNamesFieldMapper.NAME; + public static final String INDEX_NAME = FieldNamesFieldMapper.NAME; + + public static final FieldType FIELD_TYPE = new FieldType(AbstractFieldMapper.Defaults.FIELD_TYPE); + public static final FieldType FIELD_TYPE_PRE_1_3_0; + + static { + FIELD_TYPE.setIndexed(true); + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setStored(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_ONLY); + FIELD_TYPE.freeze(); + FIELD_TYPE_PRE_1_3_0 = new FieldType(FIELD_TYPE); + FIELD_TYPE_PRE_1_3_0.setIndexed(false); + FIELD_TYPE_PRE_1_3_0.freeze(); + } + } + + public static class Builder extends AbstractFieldMapper.Builder { + + private boolean indexIsExplicit; + + public Builder() { + super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE)); + indexName = Defaults.INDEX_NAME; + } + + @Override + public Builder index(boolean index) { + indexIsExplicit = true; + return super.index(index); + } + + @Override + public FieldNamesFieldMapper build(BuilderContext context) { + if ((context.indexCreatedVersion() == null || context.indexCreatedVersion().before(Version.V_1_3_0)) && !indexIsExplicit) { + fieldType.setIndexed(false); + } + return new FieldNamesFieldMapper(name, indexName, boost, fieldType, postingsProvider, docValuesProvider, fieldDataSettings, context.indexSettings()); + } + } + + public static class TypeParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + FieldNamesFieldMapper.Builder builder = fieldNames(); + parseField(builder, builder.name, node, parserContext); + return builder; + } + } + + private final FieldType defaultFieldType; + + private static FieldType defaultFieldType(Settings indexSettings) { + return indexSettings != null && Version.indexCreated(indexSettings).onOrAfter(Version.V_1_3_0) ? Defaults.FIELD_TYPE : Defaults.FIELD_TYPE_PRE_1_3_0; + } + + public FieldNamesFieldMapper(Settings indexSettings) { + this(Defaults.NAME, Defaults.INDEX_NAME, indexSettings); + } + + protected FieldNamesFieldMapper(String name, String indexName, Settings indexSettings) { + this(name, indexName, Defaults.BOOST, new FieldType(defaultFieldType(indexSettings)), null, null, null, indexSettings); + } + + public FieldNamesFieldMapper(String name, String indexName, float boost, FieldType fieldType, PostingsFormatProvider postingsProvider, + DocValuesFormatProvider docValuesProvider, @Nullable Settings fieldDataSettings, Settings indexSettings) { + super(new Names(name, indexName, indexName, name), boost, fieldType, null, Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings); + this.defaultFieldType = defaultFieldType(indexSettings); + } + + @Override + public FieldType defaultFieldType() { + return defaultFieldType; + } + + @Override + public FieldDataType defaultFieldDataType() { + return new FieldDataType("string"); + } + + @Override + public String value(Object value) { + if (value == null) { + return null; + } + return value.toString(); + } + + @Override + public boolean useTermQueryWithQueryString() { + return true; + } + + @Override + public void preParse(ParseContext context) throws IOException { + } + + @Override + public void postParse(ParseContext context) throws IOException { + super.parse(context); + } + + @Override + public void parse(ParseContext context) throws IOException { + // we parse in post parse + } + + @Override + public boolean includeInObject() { + return false; + } + + static Iterable extractFieldNames(final String fullPath) { + return new Iterable() { + @Override + public Iterator iterator() { + return new UnmodifiableIterator() { + + int endIndex = nextEndIndex(0); + + private int nextEndIndex(int index) { + while (index < fullPath.length() && fullPath.charAt(index) != '.') { + index += 1; + } + return index; + } + + @Override + public boolean hasNext() { + return endIndex <= fullPath.length(); + } + + @Override + public String next() { + final String result = fullPath.substring(0, endIndex); + endIndex = nextEndIndex(endIndex + 1); + return result; + } + + }; + } + }; + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + if (!fieldType.indexed() && !fieldType.stored() && !hasDocValues()) { + return; + } + for (ParseContext.Document document : context.docs()) { + final List paths = new ArrayList<>(); + for (IndexableField field : document.getFields()) { + paths.add(field.name()); + } + for (String path : paths) { + for (String fieldName : extractFieldNames(path)) { + if (fieldType.indexed() || fieldType.stored()) { + document.add(new XStringField(names().indexName(), fieldName, fieldType)); + } + if (hasDocValues()) { + document.add(new SortedSetDocValuesField(names().indexName(), new BytesRef(fieldName))); + } + } + } + } + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + XContentBuilder json = XContentFactory.jsonBuilder(); + super.toXContent(json, params); + if (json.string().equals("\"" + NAME + "\"{\"type\":\"" + CONTENT_TYPE + "\"}")) { + return builder; + } + return super.toXContent(builder, params); + } +} diff --git a/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java b/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java index 63bc895bb..6ee9d65c7 100644 --- a/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java @@ -27,7 +27,9 @@ import org.elasticsearch.common.lucene.search.XBooleanFilter; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.cache.filter.support.CacheKeyFilter; +import org.elasticsearch.index.mapper.FieldMappers; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; import java.io.IOException; import java.util.Set; @@ -81,6 +83,8 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar } public static Filter newFilter(QueryParseContext parseContext, String fieldPattern, String filterName) { + final FieldMappers fieldNamesMapper = parseContext.mapperService().indexName(FieldNamesFieldMapper.CONTENT_TYPE); + MapperService.SmartNameObjectMapper smartNameObjectMapper = parseContext.smartObjectMapper(fieldPattern); if (smartNameObjectMapper != null && smartNameObjectMapper.hasMapper()) { // automatic make the object mapper pattern @@ -101,7 +105,17 @@ public static Filter newFilter(QueryParseContext parseContext, String fieldPatte nonNullFieldMappers = smartNameFieldMappers; } Filter filter = null; - if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + if (fieldNamesMapper!= null && fieldNamesMapper.mapper().fieldType().indexed()) { + final String f; + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + f = smartNameFieldMappers.mapper().names().indexName(); + } else { + f = field; + } + filter = fieldNamesMapper.mapper().termFilter(f, parseContext); + } + // if _field_names are not indexed, we need to go the slow way + if (filter == null && smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true, parseContext); } if (filter == null) { diff --git a/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java b/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java index fa27253e4..01eaeb1b2 100644 --- a/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/MissingFilterParser.java @@ -28,7 +28,9 @@ import org.elasticsearch.common.lucene.search.XBooleanFilter; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.cache.filter.support.CacheKeyFilter; +import org.elasticsearch.index.mapper.FieldMappers; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; import java.io.IOException; import java.util.Set; @@ -94,6 +96,7 @@ public static Filter newFilter(QueryParseContext parseContext, String fieldPatte throw new QueryParsingException(parseContext.index(), "missing must have either existence, or null_value, or both set to true"); } + final FieldMappers fieldNamesMapper = parseContext.mapperService().indexName(FieldNamesFieldMapper.NAME); MapperService.SmartNameObjectMapper smartNameObjectMapper = parseContext.smartObjectMapper(fieldPattern); if (smartNameObjectMapper != null && smartNameObjectMapper.hasMapper()) { // automatic make the object mapper pattern @@ -122,7 +125,17 @@ public static Filter newFilter(QueryParseContext parseContext, String fieldPatte nonNullFieldMappers = smartNameFieldMappers; } Filter filter = null; - if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + if (fieldNamesMapper != null && fieldNamesMapper.mapper().fieldType().indexed()) { + final String f; + if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { + f = smartNameFieldMappers.mapper().names().indexName(); + } else { + f = field; + } + filter = fieldNamesMapper.mapper().termFilter(f, parseContext); + } + // if _field_names are not indexed, we need to go the slow way + if (filter == null && smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) { filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true, parseContext); } if (filter == null) { diff --git a/src/test/java/org/elasticsearch/VersionTests.java b/src/test/java/org/elasticsearch/VersionTests.java index b54e912fa..cc6045f57 100644 --- a/src/test/java/org/elasticsearch/VersionTests.java +++ b/src/test/java/org/elasticsearch/VersionTests.java @@ -19,6 +19,8 @@ package org.elasticsearch; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.test.ElasticsearchTestCase; import org.junit.Test; @@ -94,4 +96,12 @@ public void testTooShortVersionFromString() { public void testWrongVersionFromString() { Version.fromString("WRONG.VERSION"); } + + public void testVersion() { + // test scenario + assertEquals(Version.CURRENT, Version.indexCreated(ImmutableSettings.builder().build())); + // an actual index has a IndexMetaData.SETTING_UUID + final Version version = randomFrom(Version.V_0_18_0, Version.V_0_90_13, Version.V_1_3_0); + assertEquals(version, Version.indexCreated(ImmutableSettings.builder().put(IndexMetaData.SETTING_UUID, "foo").put(IndexMetaData.SETTING_VERSION_CREATED, version).build())); + } } \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java new file mode 100644 index 000000000..f16f862ff --- /dev/null +++ b/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.internal; + +import com.google.common.collect.ImmutableSet; +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperTestUtils; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.test.ElasticsearchTestCase; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class FieldNamesFieldMapperTests extends ElasticsearchTestCase { + + private static Set extract(String path) { + return ImmutableSet.builder().addAll(FieldNamesFieldMapper.extractFieldNames(path)).build(); + } + + private static Set set(T... values) { + return new HashSet(Arrays.asList(values)); + } + + public void testExtractFieldNames() { + assertEquals(set("abc"), extract("abc")); + assertEquals(set("a", "a.b"), extract("a.b")); + assertEquals(set("a", "a.b", "a.b.c"), extract("a.b.c")); + // and now corner cases + assertEquals(set("", ".a"), extract(".a")); + assertEquals(set("a", "a."), extract("a.")); + assertEquals(set("", ".", ".."), extract("..")); + } + + public void test() throws Exception { + DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string()); + + ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("a", "100") + .startObject("b") + .field("c", 42) + .endObject() + .endObject() + .bytes()); + + final Set fieldNames = new HashSet<>(); + for (IndexableField field : doc.rootDoc().getFields()) { + if (FieldNamesFieldMapper.CONTENT_TYPE.equals(field.name())) { + fieldNames.add(field.stringValue()); + } + } + assertEquals(new HashSet<>(Arrays.asList("a", "b", "b.c", "_uid", "_type", "_version", "_source", "_all")), fieldNames); + } +} diff --git a/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsTests.java b/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsTests.java index fe002b141..d4cda692f 100644 --- a/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsTests.java +++ b/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.internal.IndexFieldMapper; import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -1265,5 +1266,14 @@ public void indexMetaField() throws Exception { assertThat(bucket.getDocCount(), equalTo(i == 0 ? 5L : 2L)); i++; } + + response = client().prepareSearch("idx", "empty_bucket_idx").setTypes("type") + .addAggregation(terms("terms") + .executionHint(randomExecutionHint()) + .field(FieldNamesFieldMapper.NAME) + ).execute().actionGet(); + assertSearchResponse(response); + terms = response.getAggregations().get("terms"); + assertEquals(5L, terms.getBucketByKey("i").getDocCount()); } } diff --git a/src/test/java/org/elasticsearch/search/query/ExistsMissingTests.java b/src/test/java/org/elasticsearch/search/query/ExistsMissingTests.java new file mode 100644 index 000000000..853d11529 --- /dev/null +++ b/src/test/java/org/elasticsearch/search/query/ExistsMissingTests.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.query; + +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.test.ElasticsearchIntegrationTest; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; + + +public class ExistsMissingTests extends ElasticsearchIntegrationTest { + + public void testExistsMissing() throws Exception { + assertAcked(client().admin().indices().prepareCreate("idx").addMapping("type", XContentBuilder.builder(JsonXContent.jsonXContent) + .startObject() + .startObject("type") + .startObject(FieldNamesFieldMapper.NAME) + // by setting randomly index to no we also test the pre-1.3 behavior + .field("index", randomFrom("no", "not_analyzed")) + .field("store", randomFrom("no", "yes")) + .endObject() + .endObject() + .endObject())); + @SuppressWarnings("unchecked") + Map[] sources = new Map[] { + // simple property + ImmutableMap.of("foo", "bar"), + // object fields + ImmutableMap.of("bar", ImmutableMap.of("foo", "bar", "bar", ImmutableMap.of("bar", "foo"))), + ImmutableMap.of("bar", ImmutableMap.of("baz", 42)), + // empty doc + ImmutableMap.of() + }; + List reqs = new ArrayList(); + for (Map source : sources) { + reqs.add(client().prepareIndex("idx", "type").setSource(source)); + } + indexRandom(true, reqs); + + final Map expected = new LinkedHashMap(); + expected.put("foo", 1); + expected.put("f*", 2); // foo and bar.foo, that's how the expansion works + expected.put("bar", 2); + expected.put("bar.*", 2); + expected.put("bar.foo", 1); + expected.put("bar.bar", 1); + expected.put("bar.bar.bar", 1); + expected.put("baz", 1); + expected.put("foobar", 0); + + final long numDocs = client().prepareSearch("idx").execute().actionGet().getHits().totalHits(); + + for (Map.Entry entry : expected.entrySet()) { + final String fieldName = entry.getKey(); + final int count = entry.getValue(); + // exists + SearchResponse resp = client().prepareSearch("idx").setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.existsFilter(fieldName))).execute().actionGet(); + assertSearchResponse(resp); + assertEquals(count, resp.getHits().totalHits()); + + // missing + resp = client().prepareSearch("idx").setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.missingFilter(fieldName))).execute().actionGet(); + assertSearchResponse(resp); + assertEquals(numDocs - count, resp.getHits().totalHits()); + } + } + +} diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java index 0ce7c69db..78241edeb 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java @@ -71,6 +71,7 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapper.Loading; +import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.internal.IdFieldMapper; import org.elasticsearch.index.merge.policy.*; import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider; @@ -333,6 +334,11 @@ private void randomIndexTemplate() throws IOException { .field("index", randomFrom("not_analyzed", "no")) .endObject(); } + mappings.startObject(FieldNamesFieldMapper.NAME) + .startObject("fielddata") + .field(FieldDataType.FORMAT_KEY, randomFrom("paged_bytes", "fst", "doc_values")) + .endObject() + .endObject(); mappings.startArray("dynamic_templates") .startObject() .startObject("template-strings")