chore: bump pyarrow (apache#10733)

* chore: bump pandas an pyarrow * remove unsupported kwargs * fix types * move pandas back to 1.0 branch due to possible MultiIndex regression
xxiao23 · Aug 31, 2020 · 4392c46 · 4392c46
1 parent 234b6bb
commit 4392c46
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 23 deletions.
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -9,16 +9,16 @@
 aiohttp==3.6.2            # via slackclient
 alembic==1.4.2            # via flask-migrate
 amqp==2.6.1               # via kombu
-apispec[yaml]==3.3.1      # via flask-appbuilder
+apispec[yaml]==3.3.2      # via flask-appbuilder
 async-timeout==3.0.1      # via aiohttp
 attrs==20.1.0             # via aiohttp, jsonschema
 babel==2.8.0              # via flask-babel
 backoff==1.10.0           # via apache-superset
 billiard==3.6.3.0         # via celery
 bleach==3.1.5             # via apache-superset
-boto3==1.14.48            # via tabulator
-botocore==1.17.48         # via boto3, s3transfer
-brotli==1.0.7             # via flask-compress
+boto3==1.14.51            # via tabulator
+botocore==1.17.51         # via boto3, s3transfer
+brotli==1.0.9             # via flask-compress
 cached-property==1.5.1    # via tableschema
 cachelib==0.1.1           # via apache-superset
 celery==4.4.7             # via apache-superset
@@ -29,7 +29,7 @@ click==7.1.2              # via apache-superset, flask, flask-appbuilder, tables
 colorama==0.4.3           # via apache-superset, flask-appbuilder
 contextlib2==0.6.0.post1  # via apache-superset
 croniter==0.3.34          # via apache-superset
-cryptography==3.0         # via apache-superset
+cryptography==3.1         # via apache-superset
 dataclasses==0.6          # via apache-superset
 decorator==4.4.2          # via retry
 defusedxml==0.6.0         # via python3-openid
@@ -41,7 +41,7 @@ flask-appbuilder==3.0.1   # via apache-superset
 flask-babel==1.0.0        # via flask-appbuilder
 flask-caching==1.9.0      # via apache-superset
 flask-compress==1.5.0     # via apache-superset
-flask-cors==3.0.8         # via apache-superset
+flask-cors==3.0.9         # via apache-superset
 flask-jwt-extended==3.24.1  # via flask-appbuilder
 flask-login==0.4.1        # via flask-appbuilder
 flask-migrate==2.5.3      # via apache-superset
@@ -88,7 +88,7 @@ polyline==1.4.0           # via apache-superset
 prison==0.1.3             # via flask-appbuilder
 psycopg2-binary==2.8.5    # via apache-superset
 py==1.9.0                 # via retry
-pyarrow==0.17.1           # via apache-superset
+pyarrow==1.0.1            # via apache-superset
 pycparser==2.20           # via cffi
 pydruid==0.6.1            # via apache-superset
 pyhive[hive]==0.6.3       # via apache-superset

diff --git a/requirements/testing.txt b/requirements/testing.txt
@@ -22,13 +22,13 @@ isort==5.4.2              # via pylint
 jedi==0.17.2              # via ipython
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
-more-itertools==8.4.0     # via pytest
+more-itertools==8.5.0     # via pytest
 openapi-spec-validator==0.2.9  # via -r requirements/testing.in
 parameterized==0.7.4      # via -r requirements/testing.in
 parso==0.7.1              # via jedi
 pexpect==4.8.0            # via ipython
 pickleshare==0.7.5        # via ipython
-prompt-toolkit==3.0.6     # via ipython
+prompt-toolkit==3.0.7     # via ipython
 ptyprocess==0.6.0         # via pexpect
 pygments==2.6.1           # via ipython
 pyhive[hive,presto]==0.6.3  # via -r requirements/testing.in, apache-superset

diff --git a/setup.py b/setup.py
@@ -90,14 +90,14 @@ def get_git_sha():
         "isodate",
         "markdown>=3.0",
         "msgpack>=1.0.0, <1.1",
-        "pandas>=1.0.3, <1.1",
+        "pandas>=1.0.5, <1.1",
         "parsedatetime",
         "pathlib2",
         "polyline",
         "python-dateutil",
         "python-dotenv",
         "python-geohash",
-        "pyarrow>=0.17.0, <0.18",
+        "pyarrow>=1.0.1, <1.1",
         "pyyaml>=5.1",
         "retry>=0.9.2",
         "selenium>=3.141.0",

diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py
@@ -441,17 +441,6 @@ def set_or_update_query_limit(cls, sql: str, limit: int) -> str:
         parsed_query = sql_parse.ParsedQuery(sql)
         return parsed_query.set_or_update_query_limit(limit)
 
-    @staticmethod
-    def excel_to_df(**kwargs: Any) -> pd.DataFrame:
-        """ Read excel into Pandas DataFrame
-           :param kwargs: params to be passed to DataFrame.read_excel
-           :return: Pandas DataFrame containing data from excel
-        """
-        kwargs["encoding"] = "utf-8"
-        kwargs["iterator"] = True
-        df = pd.read_excel(**kwargs)
-        return df
-
     @staticmethod
     def csv_to_df(**kwargs: Any) -> pd.DataFrame:
         """ Read csv into Pandas DataFrame
@@ -521,7 +510,7 @@ def create_table_from_excel(  # pylint: disable=too-many-arguments
         Create table from contents of a excel. Note: this method does not create
         metadata for the table.
         """
-        df = cls.excel_to_df(io=filename, **excel_to_df_kwargs,)
+        df = pd.read_excel(io=filename, **excel_to_df_kwargs)
         engine = cls.get_engine(database)
         if table.schema:
             # only add schema when it is preset and non empty