Improve noop test templates for reuse. (IBM#555)

* update noop tests to use standardized way of loading test data for not transforms and use empty ignore columns list. Signed-off-by: David Wood <[email protected]> * minor cleanups in noop tests Signed-off-by: David Wood <[email protected]> --------- Signed-off-by: David Wood <[email protected]>
pankajskku · Sep 3, 2024 · 87d6295 · 87d6295
1 parent 9700d95
commit 87d6295
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 14 deletions.
diff --git a/transforms/universal/noop/python/test/test_noop.py b/transforms/universal/noop/python/test/test_noop.py
@@ -9,8 +9,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ################################################################################
+import os
 
 import pyarrow as pa
+from data_processing.test_support import get_tables_in_folder
 from data_processing.test_support.transform.table_transform_test import (
  AbstractTableTransformTest,
 )
@@ -29,7 +31,14 @@ class TestNOOPTransform(AbstractTableTransformTest):
  """
 
  def get_test_transform_fixtures(self) -> list[tuple]:
+ src_file_dir = os.path.abspath(os.path.dirname(__file__))
+ input_dir = os.path.join(src_file_dir, "../test-data/input")
+ expected_dir = os.path.join(src_file_dir, "../test-data/expected")
+ input_tables = get_tables_in_folder(input_dir)
+ expected_tables = get_tables_in_folder(expected_dir)
+ expected_metadata_list = [{"nfiles": 1, "nrows": 7}, {}]
+ config = {sleep_key: 0}
  fixtures = [
- (NOOPTransform({sleep_key: 0}), [table], [expected_table], expected_metadata_list),
+ (NOOPTransform(config), input_tables, expected_tables, expected_metadata_list),
  ]
  return fixtures
diff --git a/transforms/universal/noop/python/test/test_noop_python.py b/transforms/universal/noop/python/test/test_noop_python.py
@@ -27,9 +27,21 @@ class TestPythonNOOPTransform(AbstractTransformLauncherTest):
  """
 
  def get_test_transform_fixtures(self) -> list[tuple]:
- basedir = "../test-data"
- basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), basedir))
+ src_file_dir = os.path.abspath(os.path.dirname(__file__))
  fixtures = []
+
  launcher = PythonTransformLauncher(NOOPPythonTransformConfiguration())
- fixtures.append((launcher, {sleep_cli_param: 0}, basedir + "/input", basedir + "/expected"))
+ input_dir = os.path.join(src_file_dir, "../test-data/input")
+ expected_dir = os.path.join(src_file_dir, "../test-data/expected")
+ transform_config = {sleep_cli_param: 0}
+ fixtures.append(
+ (
+ launcher,
+ transform_config,
+ input_dir,
+ expected_dir,
+ [], # optional list of column names to ignore in comparing test-generated with expected.
+ )
+ )
+
  return fixtures
diff --git a/transforms/universal/noop/ray/test/test_noop_ray.py b/transforms/universal/noop/ray/test/test_noop_ray.py
@@ -27,12 +27,21 @@ class TestRayNOOPTransform(AbstractTransformLauncherTest):
  """
 
  def get_test_transform_fixtures(self) -> list[tuple]:
- basedir = "../test-data"
- basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), basedir))
+ src_file_dir = os.path.abspath(os.path.dirname(__file__))
  fixtures = []
- # launcher = NOOPRayLauncher()
+
  launcher = RayTransformLauncher(NOOPRayTransformConfiguration())
+ input_dir = os.path.join(src_file_dir, "../test-data/input")
+ expected_dir = os.path.join(src_file_dir, "../test-data/expected")
+ runtime_config = {"run_locally": True}
+ transform_config = {sleep_cli_param: 0}
  fixtures.append(
- (launcher, {sleep_cli_param: 0, "run_locally": True}, basedir + "/input", basedir + "/expected")
+ (
+ launcher,
+ transform_config | runtime_config,
+ input_dir,
+ expected_dir,
+ [], # optional list of column names to ignore in comparing test-generated with expected.
+ )
  )
  return fixtures
diff --git a/transforms/universal/noop/spark/test/test_noop_spark.py b/transforms/universal/noop/spark/test/test_noop_spark.py
@@ -29,19 +29,26 @@ class TestSparkNOOPTransform(AbstractSparkTransformLauncherTest):
  """
 
  def get_test_transform_fixtures(self) -> list[tuple]:
- proj_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
- basedir = os.path.join(proj_dir, "test-data")
- config_file_path = os.path.join(proj_dir, "config", "spark_profile_local.yml")
+
+ src_file_dir = os.path.abspath(os.path.dirname(__file__))
  fixtures = []
+
  launcher = SparkTransformLauncher(NOOPSparkRuntimeConfiguration())
+ input_dir = os.path.join(src_file_dir, "../test-data/input")
+ expected_dir = os.path.join(src_file_dir, "../test-data/expected")
+ config_file_path = os.path.join(src_file_dir, "../config/spark_profile_local.yml")
+ runtime_config = {local_config_path_cli: config_file_path}
+ transform_config = {sleep_cli_param: 0}
  fixtures.append(
  (
  launcher,
- {sleep_cli_param: 0, local_config_path_cli: config_file_path},
- basedir + "/input",
- basedir + "/expected",
+ transform_config | runtime_config,
+ input_dir,
+ expected_dir,
+ [], # optional list of column names to ignore in comparing test-generated with expected.
  )
  )
+
  return fixtures
 
  def _validate_metadata_content(self, test_generated: dict, expected: dict, ignore_columns: list[str] = []):