Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

execute_many insert sql do not have high preformance? #152

Open
ShichaoMa opened this issue Oct 24, 2019 · 1 comment
Open

execute_many insert sql do not have high preformance? #152

ShichaoMa opened this issue Oct 24, 2019 · 1 comment

Comments

@ShichaoMa
Copy link

ShichaoMa commented Oct 24, 2019

    async def execute_many(self, queries: typing.List[ClauseElement]) -> None:
        assert self._connection is not None, "Connection is not acquired"
        cursor = await self._connection.cursor()
        try:
            for single_query in queries:
                single_query, args, context = self._compile(single_query)
                await cursor.execute(single_query, args)
        finally:
            await cursor.close()

execute single sql many times?

aiomysql sames work better

    async def executemany(self, query, args):
        """Execute the given operation multiple times

        The executemany() method will execute the operation iterating
        over the list of parameters in seq_params.

        Example: Inserting 3 new employees and their phone number

            data = [
                ('Jane','555-001'),
                ('Joe', '555-001'),
                ('John', '555-003')
                ]
            stmt = "INSERT INTO employees (name, phone) VALUES ('%s','%s')"
            await cursor.executemany(stmt, data)

        INSERT or REPLACE statements are optimized by batching the data,
        that is using the MySQL multiple rows syntax.

        :param query: `str`, sql statement
        :param args: ``tuple`` or ``list`` of arguments for sql query
        """
        if not args:
            return

        if self._echo:
            logger.info("CALL %s", query)
            logger.info("%r", args)

        m = RE_INSERT_VALUES.match(query)
        if m:
            q_prefix = m.group(1)
            q_values = m.group(2).rstrip()
            q_postfix = m.group(3) or ''
            assert q_values[0] == '(' and q_values[-1] == ')'
            return (await self._do_execute_many(
                q_prefix, q_values, q_postfix, args, self.max_stmt_length,
                self._get_db().encoding))
        else:
            rows = 0
            for arg in args:
                await self.execute(query, arg)
                rows += self._rowcount
            self._rowcount = rows
        return self._rowcount
@ljluestc
Copy link

import typing
from sqlalchemy.sql import ClauseElement

async def execute_many(self, queries: typing.List[ClauseElement]) -> None:
    assert self._connection is not None, "Connection is not acquired"
    cursor = await self._connection.cursor()
    try:
        prepared_queries = []
        for single_query in queries:
            single_query, args, context = self._compile(single_query)
            prepared_queries.append((single_query, args))
        await cursor.executemany(prepared_queries)
    finally:
        await cursor.close()

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

2 participants