Remove the ecosystems and backends tables

These tables are in the database for historical reasons, but they don't
need to remain there. This removes the tables with a migration, removes
the models, and adjusts the code that used the models previously.

Signed-off-by: Jeremy Cline <[email protected]>

alembic/versions/b9201d816075_remove_the_backends_and_ecosystems_.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""Remove the backends and ecosystems tables
+
+Revision ID: b9201d816075
+Revises: 9c29da0af3af
+Create Date: 2017-03-22 18:46:06.281100
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+from anitya.lib import plugins
+
+# revision identifiers, used by Alembic.
+revision = 'b9201d816075'
+down_revision = '9c29da0af3af'
+
+
+def upgrade():
+    """Drop the Backends and Ecosystems tables and remove foreign keys."""
+    op.drop_constraint(u'projects_backend_fkey', 'projects', type_='foreignkey')
+    op.drop_constraint(u'FK_ECOSYSTEM_FOR_PROJECT', 'projects', type_='foreignkey')
+    op.create_index(
+        op.f('ix_projects_ecosystem_name'), 'projects', ['ecosystem_name'], unique=False)
+    op.drop_table('ecosystems')
+    op.drop_table('backends')
+
+
+def downgrade():
+    """Restore the Backends and Ecosystems tables."""
+    op.drop_index(op.f('ix_projects_ecosystem_name'), table_name='projects')
+    op.create_table(
+        'backends',
+        sa.Column('name', sa.VARCHAR(length=200), autoincrement=False, nullable=False),
+        sa.PrimaryKeyConstraint('name', name=u'backends_pkey'),
+        postgresql_ignore_search_path=False
+    )
+    # We have to populate the backends table before we can add the ecosystems
+    # table with its foreign key constraint.
+    for backend in plugins.BACKEND_PLUGINS.get_plugins():
+        op.execute("INSERT INTO backends (name) VALUES ('{}');".format(backend.name))
+
+    op.create_table(
+        'ecosystems',
+        sa.Column('name', sa.VARCHAR(length=200), autoincrement=False, nullable=False),
+        sa.Column(
+            'default_backend_name',
+            sa.VARCHAR(length=200),
+            autoincrement=False,
+            nullable=True
+        ),
+        sa.ForeignKeyConstraint(
+            ['default_backend_name'],
+            [u'backends.name'],
+            name=u'ecosystems_default_backend_name_fkey',
+            onupdate=u'CASCADE',
+            ondelete=u'CASCADE'
+        ),
+        sa.PrimaryKeyConstraint('name', name=u'ecosystems_pkey'),
+        sa.UniqueConstraint('default_backend_name', name=u'ecosystems_default_backend_name_key')
+    )
+    for ecosystem in plugins.ECOSYSTEM_PLUGINS.get_plugins():
+        op.execute("""
+            INSERT INTO ecosystems (name, default_backend_name)
+            VALUES ('{name}', '{default}');""".format(
+                name=ecosystem.name, default=ecosystem.default_backend))
+
+    op.create_foreign_key(
+        u'FK_ECOSYSTEM_FOR_PROJECT',
+        'projects',
+        'ecosystems',
+        ['ecosystem_name'],
+        ['name'],
+        onupdate=u'CASCADE',
+        ondelete=u'SET NULL'
+    )
+    op.create_foreign_key(
+        u'projects_backend_fkey',
+        'projects', 'backends',
+        ['backend'],
+        ['name'],
+        onupdate=u'CASCADE',
+        ondelete=u'CASCADE'
+    )

anitya/lib/__init__.py

@@ -79,14 +79,15 @@ def create_project(
 
     """
     # Set the ecosystem if there's one associated with the given backend
-    backend_ref = anitya.lib.model.Backend.by_name(session, name=backend)
-    ecosystem_ref = backend_ref.default_ecosystem
+    ecosystems = [e for e in anitya.lib.plugins.ECOSYSTEM_PLUGINS.get_plugins()
+                  if e.default_backend == backend]
+    ecosystem_name = ecosystems[0].name if len(ecosystems) == 1 else None
 
     project = anitya.lib.model.Project(
         name=name,
         homepage=homepage,
         backend=backend,
-        ecosystem=ecosystem_ref,
+        ecosystem_name=ecosystem_name,
         version_url=version_url,
         regex=regex,
         version_prefix=version_prefix,

anitya/lib/model.py

@@ -17,8 +17,8 @@
 import pkg_resources  # NOQA
 
 import sqlalchemy as sa
+from sqlalchemy.orm import validates
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship
 from sqlalchemy.orm.exc import NoResultFound
 
 import anitya
@@ -137,56 +137,6 @@ def search(cls, session, project_name=None, from_date=None, user=None,
         return query.all()
 
 
-class Backend(BASE):
-    __tablename__ = 'backends'
-
-    name = sa.Column(sa.String(200), primary_key=True)
-    default_ecosystem = relationship("Ecosystem", uselist=False,
-                                     back_populates="default_backend")
-
-    @classmethod
-    def all(cls, session):
-        query = session.query(cls).order_by(cls.name)
-        return query.all()
-
-    @classmethod
-    def by_name(cls, session, name):
-        return session.query(cls).filter_by(name=name).first()
-
-    get = by_name
-
-
-class Ecosystem(BASE):
-    __tablename__ = 'ecosystems'
-
-    name = sa.Column(sa.String(200), primary_key=True)
-    default_backend_name = sa.Column(
-        sa.String(200),
-        sa.ForeignKey(
-            "backends.name",
-            ondelete="cascade",
-            onupdate="cascade"),
-        unique=True
-    )
-    default_backend = relationship("Backend",
-                                   back_populates="default_ecosystem")
-    projects = relationship("Project", back_populates="ecosystem")
-
-    @classmethod
-    def all(cls, session):
-        query = session.query(cls).order_by(cls.name)
-        return query.all()
-
-    @classmethod
-    def by_name(cls, session, name):
-        try:
-            return session.query(cls).filter_by(name=name).one()
-        except NoResultFound:
-            return None
-
-    get = by_name
-
-
 class Distro(BASE):
     __tablename__ = 'distros'
 
@@ -330,25 +280,8 @@ class Project(BASE):
     name = sa.Column(sa.String(200), nullable=False, index=True)
     homepage = sa.Column(sa.String(200), nullable=False)
 
-    # TODO: Define ORM forward/backward references for backend as for ecosystem
-    backend = sa.Column(
-        sa.String(200),
-        sa.ForeignKey(
-            "backends.name",
-            ondelete="cascade",
-            onupdate="cascade"),
-        default='custom',
-    )
-    ecosystem_name = sa.Column(
-        sa.String(200),
-        sa.ForeignKey(
-            "ecosystems.name",
-            ondelete="set null",
-            onupdate="cascade",
-            name="FK_ECOSYSTEM_FOR_PROJECT"),
-        nullable=True
-    )
-    ecosystem = relationship("Ecosystem", back_populates="projects")
+    backend = sa.Column(sa.String(200), default='custom')
+    ecosystem_name = sa.Column(sa.String(200), nullable=True, index=True)
     version_url = sa.Column(sa.String(200), nullable=True)
     regex = sa.Column(sa.String(200), nullable=True)
     version_prefix = sa.Column(sa.String(200), nullable=True)
@@ -369,6 +302,26 @@ class Project(BASE):
                             name="UNIQ_PROJECT_NAME_PER_ECOSYSTEM"),
     )
 
+    @validates('backend')
+    def validate_backend(self, key, value):
+        # At the moment I have to stash this here because there's a circular
+        # import. It can be resolved after the config is decoupled from Flask:
+        # https://github.com/release-monitoring/anitya/pull/450
+        from .plugins import BACKEND_PLUGINS
+        if value not in BACKEND_PLUGINS.get_plugin_names():
+            raise ValueError('Backend "{}" is not supported.'.format(value))
+        return value
+
+    @validates('ecosystem_name')
+    def validate_ecosystem_name(self, key, value):
+        # At the moment I have to stash this here because there's a circular
+        # import. It can be resolved after the config is decoupled from Flask:
+        # https://github.com/release-monitoring/anitya/pull/450
+        from .plugins import ECOSYSTEM_PLUGINS
+        if value and value not in ECOSYSTEM_PLUGINS.get_plugin_names():
+            raise ValueError('Ecosystem "{}" is not supported.'.format(value))
+        return value
+
     @property
     def versions(self):
         ''' Return list of all versions stored, sorted from newest to oldest.
@@ -401,13 +354,6 @@ def __json__(self, detailed=False):
     def get_or_create(cls, session, name, homepage, backend='custom'):
         project = cls.by_name_and_homepage(session, name, homepage)
         if not project:
-            # Before creating, make sure the backend already exists
-            backend_obj = Backend.get(session, name=backend)
-            if not backend_obj:
-                # We don't want to automatically create these.  They must have
-                # code associated with them in anitya.lib.backends
-                raise ValueError("No such backend %r" % backend)
-
             project = cls(name=name, homepage=homepage, backend=backend)
             session.add(project)
             session.flush()
@@ -441,15 +387,8 @@ def by_name_and_homepage(cls, session, name, homepage):
     @classmethod
     def by_name_and_ecosystem(cls, session, name, ecosystem):
         try:
-            query = session.query(
-                cls
-            ).filter(
-                cls.name == name
-            ).join(
-                Project.ecosystem
-            ).filter(
-                Ecosystem.name == ecosystem
-            )
+            query = session.query(cls)
+            query = query.filter(cls.name == name, cls.ecosystem_name == ecosystem)
             return query.one()
         except NoResultFound:
             return None

anitya/lib/plugins.py

@@ -11,11 +11,8 @@
 
 import logging
 
-from sqlalchemy.exc import SQLAlchemyError
-
 from straight.plugin import load
 
-import anitya.lib.model as model
 from anitya.lib.backends import BaseBackend
 from anitya.lib.ecosystems import BaseEcosystem
 
@@ -52,45 +49,13 @@ def get_plugin(self, plugin_name):
 
 def _load_backend_plugins(session):
     """Load any new backend plugins into the DB"""
-    backends = [bcke.name for bcke in model.Backend.all(session)]
     plugins = list(BACKEND_PLUGINS.get_plugins())
-    # Add any new Backend definitions
-    plugin_names = [plugin.name for plugin in plugins]
-    for backend in set(backends).symmetric_difference(set(plugin_names)):
-        _log.info("Registering backend %r", backend)
-        bcke = model.Backend(name=backend)
-        session.add(bcke)
-        try:
-            session.commit()
-        except SQLAlchemyError as err:  # pragma: no cover
-            # We cannot test this as it would come from a defective DB
-            print(err)
-            session.rollback()
     return plugins
 
 
 def _load_ecosystem_plugins(session):
     """Load any new ecosystem plugins into the DB"""
-    ecosystems = [ecosystem.name for ecosystem in model.Ecosystem.all(session)]
     plugins = list(ECOSYSTEM_PLUGINS.get_plugins())
-    # Add any new Ecosystem definitions
-    backends_by_ecosystem = dict((plugin.name, plugin.default_backend)
-                                 for plugin in plugins)
-    eco_names = set(ecosystems).symmetric_difference(
-        set(backends_by_ecosystem))
-    for eco_name in eco_names:
-        backend = backends_by_ecosystem[eco_name]
-        bcke = model.Backend.by_name(session, backend)
-        _log.info("Registering ecosystem %r with default backend %r",
-                  eco_name, backend)
-        ecosystem = model.Ecosystem(name=eco_name, default_backend=bcke)
-        session.add(ecosystem)
-        try:
-            session.commit()
-        except SQLAlchemyError as err:  # pragma: no cover
-            # We cannot test this as it would come from a defective DB
-            print(err)
-            session.rollback()
     return plugins