Skip to content

Commit

Permalink
Improve test failure logging (ivy-llc#10562)
Browse files Browse the repository at this point in the history
Co-authored-by: CatB1t <[email protected]>
  • Loading branch information
hello-fri-end and CatB1t authored Feb 23, 2023
1 parent aebbc36 commit a67f563
Show file tree
Hide file tree
Showing 6 changed files with 32 additions and 9 deletions.
3 changes: 2 additions & 1 deletion ivy/functional/ivy/general.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
trace_mode_dict["frontend"] = "ivy/functional/frontends"
trace_mode_dict["ivy"] = "ivy/"
trace_mode_dict["full"] = ""
trace_mode_dict["none"] = ""
show_func_wrapper_trace_mode_stack = list()


Expand Down Expand Up @@ -406,7 +407,7 @@ def set_exception_trace_mode(mode: str) -> None:
global exception_trace_mode_stack
trace_modes = list(trace_mode_dict.keys())
ivy.utils.assertions.check_elem_in_list(
mode, trace_modes, "trace mode must be one of {}".format(trace_modes)
mode, trace_modes, False, "trace mode must be one of {}".format(trace_modes)
)
exception_trace_mode_stack.append(mode)

Expand Down
4 changes: 4 additions & 0 deletions ivy/utils/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ def _print_new_stack_trace(old_stack_trace, trace_mode, func_wrapper_trace_mode)
def _custom_exception_handle(type, value, tb_history):
trace_mode = ivy.get_exception_trace_mode()
func_wrapper_trace_mode = ivy.get_show_func_wrapper_trace_mode()
if trace_mode == 'none':
return
if trace_mode == "full" and func_wrapper_trace_mode:
print("".join(tb.format_tb(tb_history)))
else:
Expand All @@ -52,6 +54,8 @@ def _custom_exception_handle(type, value, tb_history):
def _print_traceback_history():
trace_mode = ivy.get_exception_trace_mode()
func_wrapper_trace_mode = ivy.get_show_func_wrapper_trace_mode()
if trace_mode == 'none':
return
if trace_mode == "full" and func_wrapper_trace_mode:
print("".join(tb.format_tb(sys.exc_info()[2])))
else:
Expand Down
7 changes: 7 additions & 0 deletions ivy_tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ def pytest_addoption(parser):
type=int,
help="set deadline for testing one example",
)
parser.addoption(
"--ivy-tb",
action="store",
default="full",
type=str,
help="ivy traceback",
)


def pytest_configure(config):
Expand Down
10 changes: 8 additions & 2 deletions ivy_tests/test_ivy/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
# local
import ivy_tests.test_ivy.helpers.test_parameter_flags as pf
from ivy import DefaultDevice
from ivy import set_exception_trace_mode
from ivy_tests.test_ivy.helpers import globals as test_globals
from ivy_tests.test_ivy.helpers.available_frameworks import available_frameworks

Expand All @@ -38,6 +39,13 @@

def pytest_configure(config):
global available_frameworks

# Ivy Exception traceback
set_exception_trace_mode(config.getoption("--ivy-tb"))

# Pytest traceback
config.option.tbstyle = config.getoption("--tb")

# device
raw_value = config.getoption("--device")
if raw_value == "all":
Expand All @@ -53,9 +61,7 @@ def pytest_configure(config):
backend_strs = raw_value.split(",")

# frontend

frontend = config.getoption("--frontend")

if frontend:
frontend_strs = frontend.split(",")
for i in frontend_strs:
Expand Down
15 changes: 9 additions & 6 deletions ivy_tests/test_ivy/helpers/assertions.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
def assert_all_close(
ret_np, ret_from_gt_np, rtol=1e-05, atol=1e-08, ground_truth_backend="TensorFlow"
):
"""Matches the ret_np and ret_from_np inputs element-by-element to ensure that
"""Matches the ret_np and ret_from_gt_np inputs element-by-element to ensure that
they are the same.
Parameters
Expand All @@ -36,8 +36,8 @@ def assert_all_close(
ret_dtype = str(ret_np.dtype)
ret_from_gt_dtype = str(ret_from_gt_np.dtype).replace("longlong", "int64")
assert ret_dtype == ret_from_gt_dtype, (
"the return with a {} backend produced data type of {}, while the return with"
" a {} backend returned a data type of {}.".format(
"the ground truth framework {} returned a {} datatype while "
"the backend {} returned a {} datatype".format(
ground_truth_backend,
ret_from_gt_dtype,
ivy.current_backend_str(),
Expand All @@ -52,7 +52,9 @@ def assert_all_close(
ret_from_gt_np = ret_from_gt_np.astype("float64")
assert np.allclose(
np.nan_to_num(ret_np), np.nan_to_num(ret_from_gt_np), rtol=rtol, atol=atol
), "{} != {}".format(ret_np, ret_from_gt_np)
), f" the results from backend {ivy.current_backend_str()} " \
f"and ground truth framework {ground_truth_backend} " \
f"do not match\n {ret_np}!={ret_from_gt_np} \n\n"


def assert_same_type_and_shape(values, this_key_chain=None):
Expand All @@ -73,7 +75,7 @@ def value_test(
ground_truth_backend="TensorFlow",
):
"""Performs a value test for matching the arrays in ret_np_flat and
ret_from_np_flat.
ret_from_np_gt_flat.
Parameters
----------
Expand Down Expand Up @@ -101,9 +103,10 @@ def value_test(
if type(ret_np_from_gt_flat) != list:
ret_np_from_gt_flat = [ret_np_from_gt_flat]
assert len(ret_np_flat) == len(ret_np_from_gt_flat), (
"The length of results from backend {} and ground truth framework {} does not match\n\n"
"len(ret_np_flat) != len(ret_np_from_gt_flat):\n\n"
"ret_np_flat:\n\n{}\n\nret_np_from_gt_flat:\n\n{}".format(
ret_np_flat, ret_np_from_gt_flat
ivy.current_backend_str(), ground_truth_backend, ret_np_flat, ret_np_from_gt_flat
)
)
# value tests, iterating through each array in the flattened returns
Expand Down
2 changes: 2 additions & 0 deletions ivy_tests/test_ivy/helpers/function_testing.py
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,8 @@ def test_function(
container_flags=container_flags,
)

if ('out' in kwargs or test_flags.with_out) and 'out' not in inspect.signature(fn).parameters:
raise Exception(f"Function {fn_name} does not have an out parameter")
# run either as an instance method or from the API directly
instance = None
if instance_method:
Expand Down

0 comments on commit a67f563

Please sign in to comment.