diff --git a/.darglint b/.darglint new file mode 100644 index 00000000000..68a1086042b --- /dev/null +++ b/.darglint @@ -0,0 +1,2 @@ +[darglint] +docstring_style=google diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65ad4292e58..9a94aa8a480 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,5 +85,8 @@ jobs: markdown-link-check: uses: ./.github/workflows/markdown-link-check.yml + docstring-check: + uses: ./.github/workflows/docstring.yml + spell-check: uses: ./.github/workflows/spellcheck.yml diff --git a/.github/workflows/docstring.yml b/.github/workflows/docstring.yml new file mode 100644 index 00000000000..95412470642 --- /dev/null +++ b/.github/workflows/docstring.yml @@ -0,0 +1,25 @@ +name: Docstring Check + +on: workflow_call + +jobs: + docstring: + name: docstringcheck + runs-on: ubuntu-latest + env: + ZENML_DEBUG: 1 + ZENML_ANALYTICS_OPT_IN: false + PYTHONIOENCODING: "utf-8" + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Install Dependencies + run: | + python -m pip install --upgrade pip setuptools + python -m pip install darglint + - name: Docstring Check + run: bash scripts/docstring.sh diff --git a/docs/mkdocstrings_helper.py b/docs/mkdocstrings_helper.py index d59ae90c091..0ab872c1d4b 100644 --- a/docs/mkdocstrings_helper.py +++ b/docs/mkdocstrings_helper.py @@ -205,7 +205,7 @@ def generate_docs( ) integration_file_contents = [INTEGRATION_DOCS_TITLE] - integrations_file_content = create_entity_docs( + create_entity_docs( api_doc_file_dir=integrations_dev_doc_file_dir, ignored_modules=["__init__.py", "__pycache__"], sources_path=path / "integrations", diff --git a/pyproject.toml b/pyproject.toml index 04028d9f644..c1b473cf4d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,14 +86,14 @@ server = ["fastapi", "uvicorn", "python-multipart", "python-jose", "fastapi-util black = "^22.3.0" pytest = "^6.2.4" mypy = "^0.991" -ruff = "^0.0.223" +ruff = "^0.0.238" coverage = { extras = ["toml"], version = "^5.5" } pre-commit = "^2.14.0" -autoflake = "^1.4" pyment = "^0.3.3" tox = "^3.24.3" hypothesis = "^6.43.1" typing-extensions = ">=3.7.4" +darglint = "^1.8.1" # pytest dev dependencies pytest-randomly = "^3.10.1" @@ -204,7 +204,7 @@ exclude = [ ] per-file-ignores = {} select = ["D", "E", "F", "I", "I001", "Q"] -ignore = ["E501", "F401", "F403", "D301", "D403", "D407", "D213", "D203", "S101", "S104", "S105", "S106", "S107"] +ignore = ["E501", "F401", "F403", "D301", "D401", "D403", "D407", "D213", "D203", "S101", "S104", "S105", "S106", "S107"] src = ["src", "test"] # use Python 3.7 as the minimum version for autofixing target-version = "py37" diff --git a/scripts/docstring.sh b/scripts/docstring.sh new file mode 100644 index 00000000000..8ec0e99f337 --- /dev/null +++ b/scripts/docstring.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e +set -x + +DOCSTRING_SRC=${1:-"src/zenml tests/harness"} + +export ZENML_DEBUG=1 +export ZENML_ANALYTICS_OPT_IN=false + +darglint -v 2 $DOCSTRING_SRC diff --git a/scripts/format.sh b/scripts/format.sh index 33821e46b76..4340c150ed3 100755 --- a/scripts/format.sh +++ b/scripts/format.sh @@ -5,7 +5,10 @@ SRC=${1:-"src/zenml tests examples docs/mkdocstrings_helper.py"} export ZENML_DEBUG=1 export ZENML_ANALYTICS_OPT_IN=false -autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place $SRC --exclude=__init__.py + +# autoflake replacement: removes unused imports and variables +ruff $SRC --select F401,F841 --fix --exclude "__init__.py" --isolated + # sorts imports ruff $SRC --select I --fix --ignore D black $SRC diff --git a/scripts/lint.sh b/scripts/lint.sh index 67dfc513220..89ac8220bef 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -13,8 +13,8 @@ ruff $SRC_NO_TESTS # TODO: Fix docstrings in tests and examples and remove the `--extend-ignore D` flag ruff $TESTS_EXAMPLES --extend-ignore D -# TODO: remove this once ruff implements the feature -autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place $SRC --exclude=__init__.py --check | ( grep -v "No issues detected" || true ) +# autoflake replacement: checks for unused imports and variables +ruff $SRC --select F401,F841 --exclude "__init__.py" --isolated black $SRC --check diff --git a/src/zenml/cli/artifact.py b/src/zenml/cli/artifact.py index cad5e569303..4e7aa6e5512 100644 --- a/src/zenml/cli/artifact.py +++ b/src/zenml/cli/artifact.py @@ -36,7 +36,11 @@ def artifact() -> None: @cli_utils.list_options(ArtifactFilterModel) @artifact.command("list", help="List all artifacts.") def list_artifacts(**kwargs: Any) -> None: - """List all artifacts.""" + """List all artifacts. + + Args: + **kwargs: Keyword arguments to filter artifacts. + """ cli_utils.print_active_config() artifacts = Client().list_artifacts(**kwargs) diff --git a/src/zenml/cli/pipeline.py b/src/zenml/cli/pipeline.py index 6b7e6bebf21..f772250dd16 100644 --- a/src/zenml/cli/pipeline.py +++ b/src/zenml/cli/pipeline.py @@ -58,7 +58,11 @@ def cli_pipeline_run(python_file: str, config_path: str) -> None: @pipeline.command("list", help="List all registered pipelines.") @list_options(PipelineFilterModel) def list_pipelines(**kwargs: Any) -> None: - """List all registered pipelines.""" + """List all registered pipelines. + + Args: + **kwargs: Keyword arguments to filter pipelines. + """ cli_utils.print_active_config() client = Client() with console.status("Listing pipelines...\n"): @@ -180,7 +184,11 @@ def runs() -> None: @runs.command("list", help="List all registered pipeline runs.") @list_options(PipelineRunFilterModel) def list_pipeline_runs(**kwargs: Any) -> None: - """List all registered pipeline runs for the filter.""" + """List all registered pipeline runs for the filter. + + Args: + **kwargs: Keyword arguments to filter pipeline runs. + """ cli_utils.print_active_config() client = Client() diff --git a/src/zenml/cli/role.py b/src/zenml/cli/role.py index d8ce61c50b4..f7bb43412a7 100644 --- a/src/zenml/cli/role.py +++ b/src/zenml/cli/role.py @@ -35,7 +35,11 @@ def role() -> None: @role.command("list") @list_options(RoleFilterModel) def list_roles(**kwargs: Any) -> None: - """List all roles that fulfill the filter requirements.""" + """List all roles that fulfill the filter requirements. + + Args: + **kwargs: Keyword arguments to filter the list of roles. + """ cli_utils.print_active_config() client = Client() with console.status("Listing roles...\n"): @@ -287,7 +291,11 @@ def assignment() -> None: @assignment.command("list") @list_options(UserRoleAssignmentFilterModel) def list_role_assignments(**kwargs: Any) -> None: - """List all user role assignments that fulfill the filter requirements.""" + """List all user role assignments that fulfill the filter requirements. + + Args: + kwargs: Keyword arguments. + """ cli_utils.print_active_config() client = Client() with console.status("Listing roles...\n"): diff --git a/src/zenml/cli/stack.py b/src/zenml/cli/stack.py index e001c2bb880..e4194ddcb97 100644 --- a/src/zenml/cli/stack.py +++ b/src/zenml/cli/stack.py @@ -685,7 +685,11 @@ def rename_stack( @stack.command("list") @list_options(StackFilterModel) def list_stacks(**kwargs: Any) -> None: - """List all stacks that fulfill the filter requirements.""" + """List all stacks that fulfill the filter requirements. + + Args: + kwargs: Keyword arguments to filter the stacks. + """ client = Client() with console.status("Listing stacks...\n"): stacks = client.list_stacks(**kwargs) diff --git a/src/zenml/cli/stack_components.py b/src/zenml/cli/stack_components.py index b1e78f142b4..d39b36924e5 100644 --- a/src/zenml/cli/stack_components.py +++ b/src/zenml/cli/stack_components.py @@ -136,7 +136,11 @@ def generate_stack_component_list_command( @list_options(ComponentFilterModel) def list_stack_components_command(**kwargs: Any) -> None: - """Prints a table of stack components.""" + """Prints a table of stack components. + + Args: + kwargs: Keyword arguments to filter the components. + """ client = Client() with console.status(f"Listing {component_type.plural}..."): kwargs["type"] = component_type diff --git a/src/zenml/cli/user_management.py b/src/zenml/cli/user_management.py index df8e1753f1d..2d57c8f12fe 100644 --- a/src/zenml/cli/user_management.py +++ b/src/zenml/cli/user_management.py @@ -75,7 +75,11 @@ def describe_user(user_name_or_id: Optional[str] = None) -> None: @user.command("list") @list_options(UserFilterModel) def list_users(**kwargs: Any) -> None: - """List all users.""" + """List all users. + + Args: + kwargs: Keyword arguments to filter the list of users. + """ cli_utils.print_active_config() client = Client() with console.status("Listing stacks...\n"): @@ -251,7 +255,11 @@ def team() -> None: @team.command("list") @list_options(TeamFilterModel) def list_teams(**kwargs: Any) -> None: - """List all teams that fulfill the filter requirements.""" + """List all teams that fulfill the filter requirements. + + Args: + kwargs: The filter options. + """ cli_utils.print_active_config() client = Client() diff --git a/src/zenml/cli/utils.py b/src/zenml/cli/utils.py index 8def6dd04ec..6b33e593f1f 100644 --- a/src/zenml/cli/utils.py +++ b/src/zenml/cli/utils.py @@ -1064,7 +1064,11 @@ def warn_unsupported_non_default_workspace() -> None: def print_page_info(page: Page[T]) -> None: - """Print all information pertaining to a page to show the amount of items and pages.""" + """Print all page information showing the number of items and pages. + + Args: + page: The page to print the information for. + """ declare( f"Page `({page.page}/{page.total_pages})`, `{page.total}` items " f"found for the applied filters." @@ -1174,6 +1178,9 @@ def list_options(filter_model: Type[BaseFilterModel]) -> Callable[[F], F]: Args: filter_model: The filter model based on which to decorate the function. + + Returns: + The inner decorator. """ def inner_decorator(func: F) -> F: diff --git a/src/zenml/cli/workspace.py b/src/zenml/cli/workspace.py index bb61f17234e..e78080837bd 100644 --- a/src/zenml/cli/workspace.py +++ b/src/zenml/cli/workspace.py @@ -38,7 +38,11 @@ def workspace() -> None: @workspace.command("list", hidden=True) @list_options(WorkspaceFilterModel) def list_workspaces(**kwargs: Any) -> None: - """List all workspaces.""" + """List all workspaces. + + Args: + **kwargs: Keyword arguments to filter the list of workspaces. + """ warn_unsupported_non_default_workspace() cli_utils.print_active_config() client = Client() diff --git a/src/zenml/client.py b/src/zenml/client.py index 056bc7ff50b..c86ae3a93d5 100644 --- a/src/zenml/client.py +++ b/src/zenml/client.py @@ -789,6 +789,7 @@ def list_teams( created: Use to filter by time of creation updated: Use the last updated date for filtering name: Use the team name for filtering + Returns: The Team """ @@ -947,6 +948,7 @@ def list_roles( created: Use to filter by time of creation updated: Use the last updated date for filtering name: Use the role name for filtering + Returns: The Role """ @@ -1073,9 +1075,6 @@ def get_user_role_assignment( Returns: The role assignment. - - Raises: - RuntimeError: If the role assignment does not exist. """ return self.zen_store.get_user_role_assignment( user_role_assignment_id=role_assignment_id @@ -1182,9 +1181,6 @@ def get_team_role_assignment( Returns: The role assignment. - - Raises: - RuntimeError: If the role assignment does not exist. """ return self.zen_store.get_team_role_assignment( team_role_assignment_id=team_role_assignment_id @@ -1362,6 +1358,7 @@ def list_workspaces( created: Use to filter by time of creation updated: Use the last updated date for filtering name: Use the team name for filtering + Returns: The Team """ @@ -1773,6 +1770,7 @@ def activate_stack( Raises: KeyError: If the stack is not registered. + ZenKeyError: If the stack is not registered. """ # Make sure the stack is registered try: @@ -2797,6 +2795,7 @@ def list_run_steps( cache_key: The cache_key of the run to filter by. status: The name of the run to filter by. num_outputs: The number of outputs for the step run + Returns: A page with Pipeline fitting the filter description """ diff --git a/src/zenml/integrations/great_expectations/visualizers/ge_visualizer.py b/src/zenml/integrations/great_expectations/visualizers/ge_visualizer.py index 94bee94d095..da643ee5679 100644 --- a/src/zenml/integrations/great_expectations/visualizers/ge_visualizer.py +++ b/src/zenml/integrations/great_expectations/visualizers/ge_visualizer.py @@ -16,7 +16,7 @@ from typing import Any, cast -import great_expectations as ge # type: ignore[import] +import great_expectations as ge # type: ignore[import] # noqa from great_expectations.checkpoint.types.checkpoint_result import ( # type: ignore[import] CheckpointResult, ) diff --git a/src/zenml/materializers/pandas_materializer.py b/src/zenml/materializers/pandas_materializer.py index 406a6ddd6f1..7ab4cfc2f17 100644 --- a/src/zenml/materializers/pandas_materializer.py +++ b/src/zenml/materializers/pandas_materializer.py @@ -45,7 +45,7 @@ def __init__(self, uri: str): """ super().__init__(uri) try: - import pyarrow # type: ignore + import pyarrow # type: ignore # noqa self.pyarrow_exists = True except ImportError: diff --git a/src/zenml/models/component_models.py b/src/zenml/models/component_models.py index 465e8d8f999..b5452bf7014 100644 --- a/src/zenml/models/component_models.py +++ b/src/zenml/models/component_models.py @@ -128,7 +128,11 @@ class ComponentFilterModel(ShareableWorkspaceScopedFilterModel): user_id: Union[UUID, str] = Field(None, description="User of the stack") def set_scope_type(self, component_type: str) -> None: - """Set the type of component on which to perform the filtering to scope the response.""" + """Set the type of component on which to perform the filtering to scope the response. + + Args: + component_type: The type of component to scope the query to. + """ self.scope_type = component_type def generate_filter( diff --git a/src/zenml/models/filter_models.py b/src/zenml/models/filter_models.py index 5b5a547ca50..bce62f0d111 100644 --- a/src/zenml/models/filter_models.py +++ b/src/zenml/models/filter_models.py @@ -78,6 +78,12 @@ def validate_operation(cls, op: str) -> str: Args: op: The operation of this filter. + + Returns: + The operation if it is valid. + + Raises: + ValueError: If the operation is not valid for this field type. """ if op not in cls.ALLOWED_OPS: raise ValueError( @@ -279,7 +285,18 @@ class BaseFilterModel(BaseModel): @validator("sort_by", pre=True) def validate_sort_by(cls, v: str) -> str: - """Validate that the sort_column is a valid column with a valid operand.""" + """Validate that the sort_column is a valid column with a valid operand. + + Args: + v: The sort_by field value. + + Returns: + The validated sort_by field value. + + Raises: + ValidationError: If the sort_by field is not a string. + ValueError: If the resource can't be sorted by this field. + """ # Somehow pydantic allows you to pass in int values, which will be # interpreted as string, however within the validator they are still # integers, which don't have a .split() method @@ -316,20 +333,35 @@ def validate_sort_by(cls, v: str) -> str: @root_validator(pre=True) def filter_ops(cls, values: Dict[str, Any]) -> Dict[str, Any]: - """Parse incoming filters to ensure all filters are legal.""" + """Parse incoming filters to ensure all filters are legal. + + Args: + values: The values of the class. + + Returns: + The values of the class. + """ cls._generate_filter_list(values) return values @property def list_of_filters(self) -> List[Filter]: - """Converts the class variables into a list of usable Filter Models.""" + """Converts the class variables into a list of usable Filter Models. + + Returns: + A list of Filter models. + """ return self._generate_filter_list( {key: getattr(self, key) for key in self.__fields__} ) @property def sorting_params(self) -> Tuple[str, SorterOps]: - """Converts the class variables into a list of usable Filter Models.""" + """Converts the class variables into a list of usable Filter Models. + + Returns: + A tuple of the column to sort by and the sorting operand. + """ column = self.sort_by # The default sorting operand is asc operator = SorterOps.ASCENDING @@ -468,27 +500,62 @@ def _define_filter( @classmethod def is_datetime_field(cls, k: str) -> bool: - """Checks if it's a datetime field.""" + """Checks if it's a datetime field. + + Args: + k: The key to check. + + Returns: + True if the field is a datetime field, False otherwise. + """ return issubclass(datetime, get_args(cls.__fields__[k].type_)) @classmethod def is_uuid_field(cls, k: str) -> bool: - """Checks if it's a uuid field.""" + """Checks if it's a uuid field. + + Args: + k: The key to check. + + Returns: + True if the field is a uuid field, False otherwise. + """ return issubclass(UUID, get_args(cls.__fields__[k].type_)) @classmethod def is_int_field(cls, k: str) -> bool: - """Checks if it's a int field.""" + """Checks if it's a int field. + + Args: + k: The key to check. + + Returns: + True if the field is a int field, False otherwise. + """ return issubclass(int, get_args(cls.__fields__[k].type_)) @classmethod def is_bool_field(cls, k: str) -> bool: - """Checks if it's a bool field.""" + """Checks if it's a bool field. + + Args: + k: The key to check. + + Returns: + True if the field is a bool field, False otherwise. + """ return issubclass(bool, get_args(cls.__fields__[k].type_)) @classmethod def is_str_field(cls, k: str) -> bool: - """Checks if it's a string field.""" + """Checks if it's a string field. + + Args: + k: The key to check. + + Returns: + True if the field is a string field, False otherwise. + """ return ( issubclass(str, get_args(cls.__fields__[k].type_)) or cls.__fields__[k].type_ == str @@ -595,7 +662,11 @@ def _define_bool_filter( @property def offset(self) -> int: - """Returns the offset needed for the query on the data persistence layer.""" + """Returns the offset needed for the query on the data persistence layer. + + Returns: + The offset for the query. + """ return self.size * (self.page - 1) def generate_filter( @@ -608,6 +679,9 @@ def generate_filter( Returns: The filter expression for the query. + + Raises: + RuntimeError: If a valid logical operator is not supplied. """ from sqlalchemy import and_ from sqlmodel import or_ @@ -642,7 +716,11 @@ class WorkspaceScopedFilterModel(BaseFilterModel): ) def set_scope_workspace(self, workspace_id: UUID) -> None: - """Set the workspace to scope this response.""" + """Set the workspace to scope this response. + + Args: + workspace_id: The workspace to scope this response to. + """ self.scope_workspace = workspace_id def generate_filter( @@ -687,7 +765,11 @@ class ShareableWorkspaceScopedFilterModel(WorkspaceScopedFilterModel): ) def set_scope_user(self, user_id: UUID) -> None: - """Set the user that is performing the filtering to scope the response.""" + """Set the user that is performing the filtering to scope the response. + + Args: + user_id: The user ID to scope the response to. + """ self.scope_user = user_id def generate_filter( diff --git a/src/zenml/models/page_model.py b/src/zenml/models/page_model.py index 48b6291a648..0215759b472 100644 --- a/src/zenml/models/page_model.py +++ b/src/zenml/models/page_model.py @@ -66,13 +66,31 @@ class Page(GenericModel, Generic[B]): __params_type__ = BaseFilterModel def __len__(self) -> int: - """Return the length of the page.""" + """Return the length of the page. + + Returns: + The length of the page. + """ return len(self.items) def __getitem__(self, index: int) -> B: - """Return the item at the given index.""" + """Return the item at the given index. + + Args: + index: The index to get the item from. + + Returns: + The item at the given index. + """ return self.items[index] def __contains__(self, item: B) -> bool: - """Returns whether the page contains a specific item.""" + """Returns whether the page contains a specific item. + + Args: + item: The item to check for. + + Returns: + Whether the item is in the page. + """ return item in self.items diff --git a/src/zenml/post_execution/base_view.py b/src/zenml/post_execution/base_view.py index c1bdd43b013..105d4a26323 100644 --- a/src/zenml/post_execution/base_view.py +++ b/src/zenml/post_execution/base_view.py @@ -37,6 +37,10 @@ def __init__(self, model: BaseResponseModel): Args: model: The model to create a view for. + + Raises: + TypeError: If the model is not of the correct type. + ValueError: If any of the `REPR_KEYS` are not valid. """ # Check that the model is of the correct type. if not isinstance(model, self.MODEL_CLASS): diff --git a/src/zenml/steps/base_step.py b/src/zenml/steps/base_step.py index 0eaf269605d..15d085514d4 100644 --- a/src/zenml/steps/base_step.py +++ b/src/zenml/steps/base_step.py @@ -767,10 +767,6 @@ def configure( Returns: The step instance that this method was called on. - - Raises: - StepInterfaceError: If a materializer or artifact for a non-existent - output name are configured. """ def _resolve_if_necessary(value: Union[str, Type[Any]]) -> str: diff --git a/src/zenml/zen_stores/rest_zen_store.py b/src/zenml/zen_stores/rest_zen_store.py index e3bdd169621..61de9a2fba2 100644 --- a/src/zenml/zen_stores/rest_zen_store.py +++ b/src/zenml/zen_stores/rest_zen_store.py @@ -600,8 +600,7 @@ def list_flavors( Args: flavor_filter_model: All filter parameters including pagination - params - + params Returns: List of all the stack component flavors matching the given criteria. @@ -1011,9 +1010,6 @@ def get_team_role_assignment( Returns: The requested role assignment. - - Raises: - KeyError: If no role assignment with the given ID exists. """ return self._get_resource( resource_id=team_role_assignment_id, diff --git a/src/zenml/zen_stores/sql_zen_store.py b/src/zenml/zen_stores/sql_zen_store.py index 3eba9174bd7..44bc5d31720 100644 --- a/src/zenml/zen_stores/sql_zen_store.py +++ b/src/zenml/zen_stores/sql_zen_store.py @@ -644,6 +644,10 @@ def filter_and_paginate( Returns: The Domain Model representation of the DB resource + + Raises: + ValueError: if the filtered page number is out of bounds. + RuntimeError: if the schema does not have a `to_model` method. """ # Filtering filters = filter_model.generate_filter(table=table) @@ -1554,8 +1558,7 @@ def list_flavors( Args: flavor_filter_model: All filter parameters including pagination - params - + params Returns: List of all the stack component flavors matching the given criteria. @@ -2134,7 +2137,7 @@ def create_user_role_assignment( The created role assignment. Raises: - ValueError: If neither a user nor a team is specified. + EntityExistsError: if the role assignment already exists. """ with Session(self.engine) as session: role = self._get_role_schema( @@ -2245,6 +2248,9 @@ def create_team_role_assignment( Returns: The newly created role assignment. + + Raises: + EntityExistsError: If the role assignment already exists. """ with Session(self.engine) as session: role = self._get_role_schema( @@ -2321,6 +2327,9 @@ def delete_team_role_assignment( Args: team_role_assignment_id: The ID of the specific role assignment + + Raises: + KeyError: If the role assignment does not exist. """ with Session(self.engine) as session: team_role = session.exec(