[ADD] allow creating missing indexes and purging properties (OCA#736)

* [ADD] allow creating missing indexes

* [FIX] tests; installation

* [ADD] allow purging properties

* [ADD] missing file

* [ADD] test purging properties

* [ADD] missing parent_id for menu entry

* [FIX] don't delete too many and wrong properties

database_cleanup/__manifest__.py

@@ -16,6 +16,8 @@
         'views/purge_columns.xml',
         'views/purge_tables.xml',
         'views/purge_data.xml',
+        "views/create_indexes.xml",
+        'views/purge_properties.xml',
         'views/menu.xml',
     ],
     'installable': False,

database_cleanup/models/__init__.py

@@ -6,3 +6,5 @@
 from . import purge_data
 from . import purge_menus
 from . import ir_model_fields
+from . import create_indexes
+from . import purge_properties

database_cleanup/models/create_indexes.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+# © 2017 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+from ..identifier_adapter import IdentifierAdapter
+from openerp import api, fields, models
+
+
+class CreateIndexesLine(models.TransientModel):
+    _inherit = 'cleanup.purge.line'
+    _name = 'cleanup.create_indexes.line'
+
+    purged = fields.Boolean('Created')
+    wizard_id = fields.Many2one('cleanup.create_indexes.wizard')
+    field_id = fields.Many2one('ir.model.fields', required=True)
+
+    @api.multi
+    def purge(self):
+        tables = set()
+        for field in self.mapped('field_id'):
+            model = self.env[field.model]
+            name = '%s_%s_index' % (model._table, field.name)
+            self.env.cr.execute(
+                'create index %s ON %s (%s)',
+                (
+                    IdentifierAdapter(name, quote=False),
+                    IdentifierAdapter(model._table),
+                    IdentifierAdapter(field.name),
+                ),
+            )
+            tables.add(model._table)
+        for table in tables:
+            self.env.cr.execute(
+                'analyze %s', (IdentifierAdapter(model._table),)
+            )
+        self.write({
+            'purged': True,
+        })
+
+
+class CreateIndexesWizard(models.TransientModel):
+    _inherit = 'cleanup.purge.wizard'
+    _name = 'cleanup.create_indexes.wizard'
+    _description = 'Create indexes'
+
+    purge_line_ids = fields.One2many(
+        'cleanup.create_indexes.line', 'wizard_id',
+    )
+
+    @api.multi
+    def find(self):
+        for field in self.env['ir.model.fields'].search([
+                ('index', '=', True),
+        ]):
+            if field.model not in self.env.registry:
+                continue
+            model = self.env[field.model]
+            name = '%s_%s_index' % (model._table, field.name)
+            self.env.cr.execute(
+                'select indexname from pg_indexes '
+                'where indexname=%s and tablename=%s',
+                (name, model._table)
+            )
+            if self.env.cr.rowcount:
+                continue
+
+            self.env.cr.execute(
+                'select a.attname '
+                'from pg_attribute a '
+                'join pg_class c on a.attrelid=c.oid '
+                'join pg_tables t on t.tablename=c.relname '
+                'where attname=%s and c.relname=%s',
+                (field.name, model._table,)
+            )
+            if not self.env.cr.rowcount:
+                continue
+
+            yield (0, 0, {
+                'name': '%s.%s' % (field.model, field.name),
+                'field_id': field.id,
+            })

database_cleanup/models/purge_properties.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# © 2017 Therp BV <http://therp.nl>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+from openerp import api, models, fields
+REASON_DUPLICATE = 1
+REASON_DEFAULT = 2
+
+
+class CleanupPurgeLineProperty(models.TransientModel):
+    _inherit = 'cleanup.purge.line'
+    _name = 'cleanup.purge.line.property'
+    _description = 'Purge properties'
+
+    wizard_id = fields.Many2one(
+        'cleanup.purge.wizard.property', 'Purge Wizard', readonly=True)
+    property_id = fields.Many2one('ir.property')
+    reason = fields.Selection([
+        (REASON_DUPLICATE, 'Duplicated property'),
+        (REASON_DEFAULT, 'Same value as default'),
+    ])
+
+    @api.multi
+    def purge(self):
+        """Delete properties"""
+        self.write({'purged': True})
+        return self.mapped('property_id').unlink()
+
+
+class CleanupPurgeWizardProperty(models.TransientModel):
+    _inherit = 'cleanup.purge.wizard'
+    _name = 'cleanup.purge.wizard.property'
+    _description = 'Purge properties'
+
+    @api.model
+    def find(self):
+        """
+        Search property records which are duplicated or the same as the default
+        """
+        result = []
+        default_properties = self.env['ir.property'].search([
+            ('res_id', '=', False),
+        ])
+        handled_field_ids = []
+        for prop in default_properties:
+            if prop.fields_id.id in handled_field_ids:
+                continue
+            domain = [
+                ('id', '!=', prop.id),
+                ('fields_id', '=', prop.fields_id.id),
+                # =? explicitly tests for None or False, not falsyness
+                ('value_float', '=?', prop.value_float or False),
+                ('value_integer', '=?', prop.value_integer or False),
+                ('value_text', '=?', prop.value_text or False),
+                ('value_binary', '=?', prop.value_binary or False),
+                ('value_reference', '=?', prop.value_reference or False),
+                ('value_datetime', '=?', prop.value_datetime or False),
+            ]
+            if prop.company_id:
+                domain.append(('company_id', '=', prop.company_id.id))
+            else:
+                domain.extend([
+                    '|',
+                    ('company_id', '=', False),
+                    (
+                        'company_id', 'in', self.env['res.company'].search([
+                            (
+                                'id', 'not in', default_properties.filtered(
+                                    lambda x: x.company_id and
+                                    x.fields_id == prop.fields_id
+                                ).ids,
+                            )
+                        ]).ids
+                    ),
+                ])
+
+            for redundant_property in self.env['ir.property'].search(domain):
+                result.append({
+                    'name': '%s@%s: %s' % (
+                        prop.name, prop.res_id, prop.get_by_record(prop)
+                    ),
+                    'property_id': redundant_property.id,
+                    'reason': REASON_DEFAULT,
+                })
+            handled_field_ids.append(prop.fields_id.id)
+        self.env.cr.execute(
+            '''
+            with grouped_properties(ids, cnt) as (
+                select array_agg(id), count(*)
+                from ir_property group by res_id, company_id, fields_id
+            )
+            select ids from grouped_properties where cnt > 1
+            '''
+        )
+        for ids, in self.env.cr.fetchall():
+            # odoo uses the first property found by search
+            for prop in self.env['ir.property'].search([
+                    ('id', 'in', ids)
+            ])[1:]:
+                result.append({
+                    'name': '%s@%s: %s' % (
+                        prop.name, prop.res_id, prop.get_by_record(prop)
+                    ),
+                    'property_id': prop.id,
+                    'reason': REASON_DUPLICATE,
+                })
+
+        return result
+
+    purge_line_ids = fields.One2many(
+        'cleanup.purge.line.property', 'wizard_id', 'Properties to purge')

database_cleanup/tests/test_database_cleanup.py

@@ -17,6 +17,21 @@ def setUp(self):
         self.model = None
 
     def test_database_cleanup(self):
+        # delete some index and check if our module recreated it
+        self.env.cr.execute('drop index res_partner_name_index')
+        create_indexes = self.env['cleanup.create_indexes.wizard'].create({})
+        create_indexes.purge_all()
+        self.env.cr.execute(
+            'select indexname from pg_indexes '
+            "where indexname='res_partner_name_index' and "
+            "tablename='res_partner'"
+        )
+        self.assertEqual(self.env.cr.rowcount, 1)
+        # duplicate a property
+        duplicate_property = self.env['ir.property'].search([], limit=1).copy()
+        purge_property = self.env['cleanup.purge.wizard.property'].create({})
+        purge_property.purge_all()
+        self.assertFalse(duplicate_property.exists())
         # create an orphaned column
         self.cr.execute(
             'alter table res_partner add column database_cleanup_test int')

database_cleanup/views/create_indexes.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<odoo>
+    <record id="cleanup_create_indexes_wizard_view_form" model="ir.ui.view">
+        <field name="model">cleanup.create_indexes.wizard</field>
+        <field name="inherit_id" ref="form_purge_wizard" />
+        <field name="mode">primary</field>
+        <field name="arch" type="xml">
+            <button name="purge_all" position="attributes">
+                <attribute name="string">Create all</attribute>
+            </button>
+            <button name="purge" position="attributes">
+                <attribute name="string">Create</attribute>
+            </button>
+        </field>
+    </record>
+
+    <record id="cleanup_create_indexes_wizard_action" model="ir.actions.server">
+        <field name="name">Create missing indexes</field>
+        <field name="type">ir.actions.server</field>
+        <field name="state">code</field>
+        <field name="model_id" ref="database_cleanup.model_cleanup_create_indexes_wizard" />
+        <field name="code">action = self.get_wizard_action(cr, uid, context=context)</field>
+    </record>
+
+    <record id="cleanup_create_indexes_line_view_tree" model="ir.ui.view">
+        <field name="model">cleanup.create_indexes.line</field>
+        <field name="inherit_id" ref="tree_purge_line" />
+        <field name="mode">primary</field>
+        <field name="arch" type="xml">
+            <button name="purge" position="attributes">
+                <attribute name="string">Create this index</attribute>
+                <attribute name="icon">gtk-add</attribute>
+            </button>
+        </field>
+    </record>
+
+    <record id="cleanup_create_indexes_line_action" model="ir.actions.server">
+        <field name="name">Create</field>
+        <field name="type">ir.actions.server</field>
+        <field name="state">code</field>
+        <field name="model_id" ref="database_cleanup.model_cleanup_create_indexes_line" />
+        <field name="code">self.purge(cr, uid, context.get('active_ids', []), context)</field>
+    </record>
+
+    <record id="cleanup_create_indexes_line_action_value" model="ir.values">
+        <field name="name">Create indexes</field>
+        <field name="key">action</field>
+        <field name="key2">client_action_multi</field>
+        <field name="model">cleanup.create_indexes.line</field>
+        <field name="value" eval="'ir.actions.server,%d' % ref('database_cleanup.cleanup_create_indexes_line_action')" />
+    </record>
+</odoo>

database_cleanup/views/menu.xml

@@ -52,5 +52,19 @@
             <field name="parent_id" ref="menu_database_cleanup"/>
         </record>
 
+        <record model="ir.ui.menu" id="menu_create_indexes">
+            <field name="name">Create missing indexes</field>
+            <field name="sequence" eval="70" />
+            <field name="action" ref="cleanup_create_indexes_wizard_action" />
+            <field name="parent_id" ref="menu_database_cleanup"/>
+        </record>
+
+        <record model="ir.ui.menu" id="menu_purge_property">
+            <field name="name">Purge obsolete properties</field>
+            <field name="sequence" eval="80" />
+            <field name="action" ref="action_purge_property" />
+            <field name="parent_id" ref="menu_database_cleanup"/>
+        </record>
+
     </data>
 </openerp>

database_cleanup/views/purge_properties.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="utf-8"?>
+<openerp>
+    <data>
+        <record id="purge_property_view" model="ir.ui.view">
+            <field name="model">cleanup.purge.wizard.property</field>
+            <field name="inherit_id" ref="form_purge_wizard" />
+            <field name="mode">primary</field>
+            <field name="arch" type="xml">
+                <data />
+            </field>
+        </record>
+
+        <record id="action_purge_property" model="ir.actions.server">
+            <field name="name">Purge properties</field>
+            <field name="type">ir.actions.server</field>
+            <field name="state">code</field>
+            <field name="model_id" ref="database_cleanup.model_cleanup_purge_wizard_property" />
+            <field name="code">action = self.get_wizard_action(cr, uid, context=context)</field>
+        </record>
+
+        <record id="purge_property_line_tree" model="ir.ui.view">
+            <field name="model">cleanup.purge.line.property</field>
+            <field name="inherit_id" ref="tree_purge_line" />
+            <field name="mode">primary</field>
+            <field name="arch" type="xml">
+                <field name="name" position="after">
+                    <field name="reason" />
+                </field>
+            </field>
+        </record>
+
+        <record id="action_purge_property_line" model="ir.actions.server">
+            <field name="name">Purge</field>
+            <field name="type">ir.actions.server</field>
+            <field name="state">code</field>
+            <field name="model_id" ref="database_cleanup.model_cleanup_purge_line_property" />
+            <field name="code">self.purge(cr, uid, context.get('active_ids', []), context)</field>
+        </record>
+
+        <record id="action_purge_property_line_value" model="ir.values">
+            <field name="name">Purge</field>
+            <field name="key">action</field>
+            <field name="key2">client_action_multi</field>
+            <field name="model">cleanup.purge.line.property</field>
+            <field name="value" eval="'ir.actions.server,%d' % ref('database_cleanup.action_purge_property_line')" />
+        </record>
+
+    </data>
+</openerp>

database_cleanup/views/purge_wizard.xml

@@ -9,7 +9,7 @@
                         <button type="object" name="purge_all" string="Purge all" class="oe_highlight" />
                         <button type="object" name="select_lines" string="Select lines" />
                     </header>
-                    <field name="purge_line_ids">
+                    <field name="purge_line_ids" attrs="{'invisible': [('purge_line_ids', '=', [])]}">
                         <form>
                             <group>
                                 <field name="name" />
@@ -22,13 +22,16 @@
                             </footer>
                         </form>
                     </field>
+                    <div attrs="{'invisible': [('purge_line_ids', '!=', [])]}">
+                        Nothing found to clean up.
+                    </div>
                 </form>
             </field>
         </record>
         <record id="tree_purge_line" model="ir.ui.view">
             <field name="model">cleanup.purge.line</field>
             <field name="arch" type="xml">
-                <tree string="Purge models" delete="false">
+                <tree string="Purge models" delete="false" create="false">
                     <field name="name" />
                     <field name="purged" />
                     <button type="object" name="purge"