pyproject.toml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
[tool.poetry]
name = "pyiceberg"
version = "0.6.1"
readme = "README.md"
homepage = "https://py.iceberg.apache.org/"
repository = "https://github.com/apache/iceberg-python"
description = "Apache Iceberg is an open table format for huge analytic datasets"
authors = ["Apache Software Foundation <dev@iceberg.apache.org>"]
license = "Apache License 2.0"
classifiers = [
  "License :: OSI Approved :: Apache Software License",
  "Operating System :: OS Independent",
  "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11"
]
packages = [
  { include = "pyiceberg" },
  { from = "vendor", include = "fb303" },
  { from = "vendor", include = "hive_metastore" },
  { include = "tests", format = "sdist" },
  { include = "Makefile", format = "sdist" },
  { include = "NOTICE", format = ["sdist", "wheel"] }
]
include = [
  { path = "dev", format = "sdist" },
  { path = "pyiceberg/**/*.so", format = "wheel" },
  { path = "pyiceberg/**/*.pyd", format = "wheel" },
]

[tool.poetry.dependencies]
python = "^3.8"
mmh3 = ">=4.0.0,<5.0.0"
requests = ">=2.20.0,<3.0.0"
click = ">=7.1.1,<9.0.0"
rich = ">=10.11.0,<14.0.0"
strictyaml = ">=1.7.0,<2.0.0" # CVE-2020-14343 was fixed in 5.4.
pydantic = ">=2.0,<3.0,!=2.4.0,!=2.4.1" # 2.4.0, 2.4.1 has a critical bug
sortedcontainers = "2.4.0"
fsspec = ">=2023.1.0,<2025.1.0"
pyparsing = ">=3.1.0,<4.0.0"
zstandard = ">=0.13.0,<1.0.0"
tenacity = ">=8.2.3,<9.0.0"
pyarrow = { version = ">=9.0.0,<16.0.0", optional = true }
pandas = { version = ">=1.0.0,<3.0.0", optional = true }
duckdb = { version = ">=0.5.0,<1.0.0", optional = true }
ray = { version = ">=2.0.0,<2.10.0", optional = true }
python-snappy = { version = ">=0.6.0,<1.0.0", optional = true }
thrift = { version = ">=0.13.0,<1.0.0", optional = true }
mypy-boto3-glue = { version = ">=1.28.18", optional = true }
boto3 = { version = ">=1.24.59", optional = true }
s3fs = { version = ">=2023.1.0,<2024.1.0", optional = true }
adlfs = { version = ">=2023.1.0,<2024.3.0", optional = true }
gcsfs = { version = ">=2023.1.0,<2024.1.0", optional = true }
psycopg2-binary = { version = ">=2.9.6", optional = true }
sqlalchemy = { version = "^2.0.18", optional = true }
getdaft = { version = ">=0.2.12", optional = true }

[tool.poetry.group.dev.dependencies]
pytest = "7.4.4"
pytest-checkdocs = "2.10.1"
pytest-lazy-fixture = "0.6.3"
pre-commit = "3.5.0"
fastavro = "1.9.4"
coverage = { version = "^7.4.2", extras = ["toml"] }
requests-mock = "1.11.0"
moto = { version = "^5.0.2", extras = ["server"] }
typing-extensions = "4.9.0"
pytest-mock = "3.12.0"
pyspark = "3.5.0"
cython = "3.0.8"
deptry = "^0.14.0"

[[tool.mypy.overrides]]
module = "pytest_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "numpy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "numpy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "numpy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[tool.poetry.scripts]
pyiceberg = "pyiceberg.cli.console:run"

[build-system]
requires = ["poetry-core>=1.0.0", "wheel", "Cython>=3.0.0", "setuptools"]
build-backend = "poetry.core.masonry.api"

[tool.poetry.build]
generate-setup-file = false
script = "build-module.py"

[tool.poetry.extras]
pyarrow = ["pyarrow"]
pandas = ["pandas", "pyarrow"]
duckdb = ["duckdb", "pyarrow"]
ray = ["ray", "pyarrow", "pandas"]
daft = ["getdaft"]
snappy = ["python-snappy"]
hive = ["thrift"]
s3fs = ["s3fs"]
glue = ["boto3", "mypy-boto3-glue"]
adlfs = ["adlfs"]
dynamodb = ["boto3"]
zstandard = ["zstandard"]
sql-postgres = ["sqlalchemy", "psycopg2-binary"]
sql-sqlite = ["sqlalchemy"]
gcsfs = ["gcsfs"]

[tool.pytest.ini_options]
markers = [
  "unmarked: marks a test as a unittest",
  "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint args)",
  "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)",
  "integration: marks integration tests against Apache Spark",
  "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint)"
]

# Turns a warning into an error
#filterwarnings = [
#    "error"
#]

[tool.black]
line-length = 130
target-version = ['py38']

[tool.pycln]
all = true

[tool.mypy]
mypy_path = "python"
no_implicit_optional = true
namespace_packages = false
warn_redundant_casts = true
warn_unreachable = true
warn_unused_ignores = true
disallow_any_generics = true
disallow_untyped_defs = true
implicit_reexport = false

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "numpy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[tool.coverage.run]
source = ['pyiceberg/']

[tool.ruff]
src = ['pyiceberg','tests']
extend-exclude = ["dev/provision.py"]
lint.select = [
    "E", # pycodestyle
    "W", # pycodestyle
    "F", # Pyflakes
    "B", # flake8-bugbear
    "PIE", # flake8-pie
    "C4", # flake8-comprehensions
    "I", # isort
    "UP", # pyupgrade
]
lint.ignore = ["E501","E203","B024","B028","UP037"]

# Allow autofix for all enabled rules (when `--fix`) is provided.
lint.fixable = ["ALL"]
lint.unfixable = []

# Exclude a variety of commonly ignored directories.
exclude = [
    ".bzr",
    ".direnv",
    ".eggs",
    ".git",
    ".git-rewrite",
    ".hg",
    ".mypy_cache",
    ".nox",
    ".pants.d",
    ".pytype",
    ".ruff_cache",
    ".svn",
    ".tox",
    ".venv",
    "__pypackages__",
    "_build",
    "buck-out",
    "build",
    "dist",
    "node_modules",
    "venv",
]
lint.per-file-ignores = {}
# Ignore _all_ violations.
# Same as Black.
line-length = 130

# Allow unused variables when underscore-prefixed.
lint.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"

[tool.ruff.lint.pyupgrade]
# Preserve types, even if a file imports `from __future__ import annotations`.
keep-runtime-typing = true

[tool.ruff.lint.isort]
detect-same-package = true
lines-between-types = 0
known-first-party = ["pyiceberg", "tests"]
section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]

[tool.ruff.format]
quote-style = "preserve"