HUE-2127 [indexer] Select a default field

When creating a collection, you can choose the default field. The edit view doesn't show the default field because there didn't seem to be a good way to get this information.
ManS · Jun 3, 2014 · 86f0395 · 86f0395
1 parent 777646d
commit 86f0395
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 34 deletions.
diff --git a/desktop/libs/indexer/src/indexer/api.py b/desktop/libs/indexer/src/indexer/api.py
@@ -139,7 +139,7 @@ def collections_create(request):
     searcher = CollectionManagerController(request.user)
 
     # Create instance directory, collection, and add fields
-    searcher.create_collection(collection.get('name'), collection.get('fields', []), collection.get('uniqueKeyField'))
+    searcher.create_collection(collection.get('name'), collection.get('fields', []), collection.get('uniqueKeyField'), collection.get('df'))
 
     try:
       if request.POST.get('source') == 'file':

diff --git a/desktop/libs/indexer/src/indexer/controller.py b/desktop/libs/indexer/src/indexer/controller.py
@@ -78,13 +78,13 @@ def get_fields(self, collection_or_core_name):
 
     return uniquekey, fields
 
-  def create_collection(self, name, fields, unique_key_field='id'):
+  def create_collection(self, name, fields, unique_key_field='id', df='text'):
     """
     Create solr collection and instance dir.
     Create schema.xml file so that we can set UniqueKey field.
     """
     # Need to remove path afterwards
-    tmp_path, solr_config_path = utils.copy_config_with_fields_and_unique_key(fields, unique_key_field)
+    tmp_path, solr_config_path = utils.copy_configs(fields, unique_key_field, df)
 
     # Create instance directory.
     process = subprocess.Popen([conf.SOLRCTL_PATH.get(), "instancedir", "--create", name, solr_config_path],

diff --git a/desktop/libs/indexer/src/indexer/management/commands/indexer_setup.py b/desktop/libs/indexer/src/indexer/management/commands/indexer_setup.py
@@ -46,7 +46,8 @@ def handle_noargs(self, **options):
     self._setup_collection_from_csv({
       'name': 'twitter_demo',
       'fields': self._parse_fields(path),
-      'uniqueKeyField': 'id'
+      'uniqueKeyField': 'id',
+      'df': 'text'
     }, path)
     LOG.info(_("Twitter collection successfully installed"))
 
@@ -55,7 +56,8 @@ def handle_noargs(self, **options):
     self._setup_collection_from_csv({
       'name': 'yelp_demo',
       'fields': self._parse_fields(path),
-      'uniqueKeyField': 'id'
+      'uniqueKeyField': 'id',
+      'df': 'text'
     }, path)
     LOG.info(_("Yelp collection successfully installed"))
 
@@ -64,7 +66,8 @@ def handle_noargs(self, **options):
     self._setup_collection_from_csv({
       'name': 'jobs_demo',
       'fields': self._parse_fields(path),
-      'uniqueKeyField': 'id'
+      'uniqueKeyField': 'id',
+      'df': 'description'
     }, path)
     LOG.info(_("Jobs collection successfully installed"))
 
@@ -76,13 +79,14 @@ def handle_noargs(self, **options):
         'region_code': 'string',
         'referer': 'string'
       }),
-      'uniqueKeyField': 'id'
+      'uniqueKeyField': 'id',
+      'df': 'record'
     }, path)
     LOG.info(_("Logs collection successfully installed"))
 
   def _setup_collection_from_csv(self, collection, path, separator=',', quote_character='"'):
     if not self.searcher.collection_exists(collection['name']):
-      self.searcher.create_collection(collection['name'], collection['fields'], collection['uniqueKeyField'])
+      self.searcher.create_collection(collection['name'], collection['fields'], collection['uniqueKeyField'], collection['df'])
 
     try:
       hdfs_path = '/tmp/%s' % uuid.uuid4()

diff --git a/desktop/libs/indexer/src/indexer/templates/collections.mako b/desktop/libs/indexer/src/indexer/templates/collections.mako
@@ -294,6 +294,7 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
         <th width="0%" class="nowrap">${_('Indexed')}</th>
         <th width="0%" class="nowrap">${_('Stored')}</th>
         <th width="0%" class="nowrap">${_('Unique Key')}</th>
+        <th width="0%" class="nowrap">${_('Default Field')}</th>
         <th width="50%"></th>
       </tr>
     </thead>
@@ -317,6 +318,12 @@ ${ commonheader(_('Collection Manager'), "indexer", user, "29px") | n,unicode }
             <span class="fa" data-bind="css: {'fa-check': uniqueKeyField}">
           </p>
         </td>
+        <td>
+          <p class="text-center">
+            <input data-bind="checked: df, visible: !df()" name="unique-key" type="checkbox" />
+            <span class="fa" data-bind="css: {'fa-check': df}">
+          </p>
+        </td>
         <td>
           <a data-bind="click: remove, visible: editable" href="javascript:void(0)" class="btn"><i class="fa fa-minus"></i></a>
         </td>

diff --git a/desktop/libs/indexer/src/indexer/utils.py b/desktop/libs/indexer/src/indexer/utils.py
@@ -47,38 +47,56 @@
   'required': 'true'
 }
 
-def schema_xml_with_fields(schema_xml, fields):
-  fields_xml = ''
-  for field in fields:
-    field_dict = DEFAULT_FIELD.copy()
-    field_dict.update(field)
-    fields_xml += FIELD_XML_TEMPLATE % field_dict + '\n'
-  return force_unicode(force_unicode(schema_xml).replace(u'<!-- REPLACE FIELDS -->', force_unicode(fields_xml)))
-
-def schema_xml_with_unique_key_field(schema_xml, unique_key_field):
-  return force_unicode(force_unicode(schema_xml).replace(u'<!-- REPLACE UNIQUE KEY -->', force_unicode(unique_key_field)))
-
-def schema_xml_with_fields_and_unique_key(schema_xml, fields, unique_key_field):
-  return schema_xml_with_unique_key_field(schema_xml_with_fields(schema_xml, fields), unique_key_field)
-
-def example_schema_xml_with_fields_and_unique_key(fields, unique_key_field):
-  # Get complete schema.xml
-  with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/schema.xml')) as f:
-    return schema_xml_with_fields_and_unique_key(f.read(), fields, unique_key_field)
-
-def copy_config_with_fields_and_unique_key(fields, unique_key_field):
-  # Get complete schema.xml
-  with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/schema.xml')) as f:
-    schema_xml = schema_xml_with_fields_and_unique_key(f.read(), fields, unique_key_field)
+class SchemaXml(object):
+  def __init__(self, xml):
+    self.xml = xml
 
+  def uniqueKeyField(self, unique_key_field):
+    self.xml = force_unicode(force_unicode(self.xml).replace(u'<!-- REPLACE UNIQUE KEY -->', force_unicode(unique_key_field)))
+
+  def fields(self, fields):
+    fields_xml = ''
+    for field in fields:
+      field_dict = DEFAULT_FIELD.copy()
+      field_dict.update(field)
+      fields_xml += FIELD_XML_TEMPLATE % field_dict + '\n'
+    self.xml = force_unicode(force_unicode(self.xml).replace(u'<!-- REPLACE FIELDS -->', force_unicode(fields_xml)))
+
+
+class SolrConfigXml(object):
+  def __init__(self, xml):
+    self.xml = xml
+
+  def defaultField(self, df):
+    self.xml = force_unicode(force_unicode(self.xml).replace(u'<str name="df">text</str>', u'<str name="df">%s</str>' % force_unicode(df)))
+
+
+def copy_configs(fields, unique_key_field, df):
   # Create temporary copy of solr configs
   tmp_path = tempfile.mkdtemp()
   solr_config_path = os.path.join(tmp_path, os.path.basename(conf.CONFIG_TEMPLATE_PATH.get()))
   shutil.copytree(conf.CONFIG_TEMPLATE_PATH.get(), solr_config_path)
 
-  # Write complete schema.xml to copy
-  with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
-    f.write(smart_str(schema_xml))
+  if fields or unique_key_field:
+    # Get complete schema.xml
+    with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/schema.xml')) as f:
+      schemaxml = SchemaXml(f.read())
+      schemaxml.uniqueKeyField(unique_key_field)
+      schemaxml.fields(fields)
+
+    # Write complete schema.xml to copy
+    with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
+      f.write(smart_str(schemaxml.xml))
+
+  if df:
+    # Get complete solrconfig.xml
+    with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/solrconfig.xml')) as f:
+      solrconfigxml = SolrConfigXml(f.read())
+      solrconfigxml.defaultField(df)
+
+    # Write complete solrconfig.xml to copy
+    with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
+      f.write(smart_str(solrconfigxml.xml))
 
   return tmp_path, solr_config_path
 

diff --git a/desktop/libs/indexer/static/js/collections.js b/desktop/libs/indexer/static/js/collections.js
@@ -224,6 +224,7 @@ var CreateCollectionViewModel = function() {
         if (data.status == 0) {
           self.collection.fields(inferFields(data.data, self.collection));
           chooseUniqueKey(self.collection);
+          chooseDefaultField(self.collection);
         } else {
           $(document).trigger("error", data.message);
         }

diff --git a/desktop/libs/indexer/static/js/lib.js b/desktop/libs/indexer/static/js/lib.js
@@ -24,6 +24,7 @@ var Collection = function(name) {
 
   // Metadata
   self.uniqueKeyField = ko.observable().extend({'errors': null});
+  self.df = ko.observable().extend({'errors': null});
   self.hasHueCollection = ko.observable(true).extend({'errors': null});
   self.hasSolrCollection = ko.observable(true).extend({'errors': null});
 
@@ -71,6 +72,15 @@ var Field = function(collection, name, type, required, indexed, stored) {
       self.indexed(true);
     }
   });
+  self.df = ko.computed({
+    'read': function() {
+      return collection.df() == self.name();
+    },
+    'write': function(value) {
+      collection.df(self.name());
+      self.indexed(true);
+    }
+  });
   self.editable = ko.computed(function() {
     return !self.uniqueKeyField() && self.name() != '_version_';
   });
@@ -331,6 +341,39 @@ function chooseUniqueKey(collection) {
   fieldChooser(collection.fields());
 }
 
+function chooseDefaultField(collection) {
+  function fieldChooser(fields) {
+    if (fields.length > 0) {
+      fields[0].df(true);
+      return true;
+    }
+    return false;
+  }
+
+  // Find a field named "text"
+  if (fieldChooser(ko.utils.arrayFilter(collection.fields(), function(field) {
+    return field.name().toLowerCase() == 'text';
+  }))) return;
+
+  // Find a text field
+  if (fieldChooser(ko.utils.arrayFilter(collection.fields(), function(field) {
+    return field.type().toLowerCase().substring(0, 4) == 'text';
+  }))) return;
+
+  // Find a string field
+  if (fieldChooser(ko.utils.arrayFilter(collection.fields(), function(field) {
+    return field.type().toLowerCase() == 'string';
+  }))) return;
+
+  // Find first indexed field
+  if (fieldChooser(ko.utils.arrayFilter(collection.fields(), function(field) {
+    return field.indexed();
+  }))) return;
+
+  // Choose a field
+  fieldChooser(collection.fields());
+}
+
 function getCharacterLabel(character) {
   var LABELS = {
     '\t': '\\t'