Merge pull request #619 from roytman/python3.12

support python 3.12
IBM · Oct 5, 2024 · 75e3364 · 75e3364
2 parents 45bb6e3 + 0964481
commit 75e3364
Show file tree

Hide file tree

Showing 54 changed files with 76 additions and 56 deletions.
diff --git a/.make.defaults b/.make.defaults
@@ -170,8 +170,8 @@ __check_defined = \
  @version=$$(echo $(PYTHON_VERSION) | sed -e 's/Python[ ]*//'); \
  major=$$(echo $$version | awk -F. '{print $$1}'); \
  minor=$$(echo $$version | awk -F. '{print $$2}'); \
- if [ $$major -lt 3 -o $$minor -lt 10 -o $$minor -gt 11 ]; then \
- echo Python 3.10 or 3.11 is required, but got $$version; \
+ if [ $$major -lt 3 -o $$minor -lt 10 -o $$minor -gt 12 ]; then \
+ echo Python 3.10 - 3.12 is required, but got $$version; \
  echo Try overriding PYTHON=$(PYTHON). For example, "'"make PYTHON=python3.10" ...'"; \
  exit 1; \
  fi
@@ -553,12 +553,15 @@ MINIO_ADMIN_PWD= localminiosecretkey
 # Changes the version field of the pyproject.toml file to the given version
 # and update the referenced library versions as defined in .make.versions.
 # Expects TOML_VERSION
+
+
 .PHONY: .defaults.update-toml
 .defaults.update-toml:
  $(call check_defined, TOML_VERSION)
  if [ -e pyproject.toml ]; then \
  $(MAKE) TOML_VERSION=$(TOML_VERSION) .defaults.__set-toml-version; \
  $(MAKE) .defaults.__update-toml-lib-dep-versions; \
+ $(MAKE) .defaults.__update-toml-python-versions; \
  fi
 
 # Changes the version field of the pyproject.toml file to the given version
@@ -573,6 +576,18 @@ MINIO_ADMIN_PWD= localminiosecretkey
  mv tt.toml pyproject.toml; \
  fi
 
+# Updates the Python supported versions field of the pyproject.toml file to the given versions
+# Expects REQUIRED_PYTHON_VERSIONS
+.PHONY: .defaults.__update-toml-python-versions
+.defaults.__update-toml-python-versions:
+ @# Help: Set the version= field of pyproject.toml
+ if [ -e pyproject.toml ]; then \
+ cat pyproject.toml | sed -e \
+ 's/^requires-python[ ]*=.*"/requires-python = "'${REQUIRED_PYTHON_VERSIONS}'"/' \
+ > tt.toml; \
+ mv tt.toml pyproject.toml; \
+ fi
+
 # Updates the versions references to our repo source as defined in .make.versions
 .PHONY: .defaults.__update-toml-lib-dep-versions
 .defaults.__update-toml-lib-dep-versions:

diff --git a/.make.versions b/.make.versions
@@ -118,9 +118,11 @@ KFP_v2=2.2.0
 KFP_v2_SDK=2.8.0
 KFP_v1=1.8.5
 KFP_v1_SDK=1.8.22
-RAY=2.24.0
+RAY=2.36.1
 SPARK_VERSION=3.5.2
 
+REQUIRED_PYTHON_VERSIONS=">=3.10,<3.13"
+
 ifeq ($(KFPv2), 1)
  WORKFLOW_SUPPORT_LIB=kfp_v2_workflow_support
 else

diff --git a/data-processing-lib/python/pyproject.toml b/data-processing-lib/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Library"
 license = {text = "Apache-2.0"}

diff --git a/data-processing-lib/ray/Makefile b/data-processing-lib/ray/Makefile
@@ -17,6 +17,9 @@ setup::
 
 set-versions: .check-env
  $(MAKE) TOML_VERSION=$(DPK_LIB_VERSION) .defaults.update-toml
+ cat pyproject.toml | sed -e 's/"ray[default]==.*",/"ray[default]==$(RAY)",/' > tt.toml
+ mv tt.toml pyproject.toml
+
 
 build:: build-dist 
 

diff --git a/data-processing-lib/ray/pyproject.toml b/data-processing-lib/ray/pyproject.toml
@@ -2,7 +2,7 @@
 name = "data_prep_toolkit_ray"
 version = "0.2.2.dev0"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Data Preparation Toolkit Library for Ray"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -12,7 +12,7 @@ authors = [
 ]
 dependencies = [
  "data-prep-toolkit>=0.2.2.dev0",
- "ray[default]==2.24.0",
+ "ray[default]==2.36.1",
  # These two are to fix security issues identified by quay.io
  "fastapi>=0.110.2",
  "pillow>=10.3.0",

diff --git a/data-processing-lib/spark/pyproject.toml b/data-processing-lib/spark/pyproject.toml
@@ -2,7 +2,7 @@
 name = "data_prep_toolkit_spark"
 version = "0.2.2.dev0"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Data Preparation Toolkit Library for Spark"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/kfp/kfp_ray_components/Dockerfile b/kfp/kfp_ray_components/Dockerfile
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=docker.io/rayproject/ray:2.24.0-py310
+ARG BASE_IMAGE=docker.io/rayproject/ray:2.36.1-py312
 
 FROM ${BASE_IMAGE}
 

diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit_kfp_v1"
 version = "0.2.2.dev0"
-requires-python = ">=3.10,<3.12"
+requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/kfp/kfp_support_lib/kfp_v2_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v2_workflow_support/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit_kfp_v2"
 version = "0.2.2.dev0"
-requires-python = ">=3.10,<3.12"
+requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml b/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit_kfp_shared"
 version = "0.2.2.dev0"
-requires-python = ">=3.10,<3.12"
+requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/code2parquet/python/pyproject.toml b/transforms/code/code2parquet/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_code2parquet_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "code2parquet Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/code2parquet/ray/pyproject.toml b/transforms/code/code2parquet/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_code2parquet_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "code2parquet Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/code_quality/python/pyproject.toml b/transforms/code/code_quality/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_code_quality_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Code Quality Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/code_quality/ray/pyproject.toml b/transforms/code/code_quality/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_code_quality_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Code Quality Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/header_cleanser/python/pyproject.toml b/transforms/code/header_cleanser/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_header_cleanser_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "License and Copyright Removal Transform for Python"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/header_cleanser/ray/pyproject.toml b/transforms/code/header_cleanser/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_header_cleanser_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "License and copyright removal Transform for Ray"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/malware/python/pyproject.toml b/transforms/code/malware/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_malware_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Malware Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/malware/ray/pyproject.toml b/transforms/code/malware/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_malware_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Malware Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/proglang_select/python/pyproject.toml b/transforms/code/proglang_select/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_proglang_select_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Programming Language Selection Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/proglang_select/ray/pyproject.toml b/transforms/code/proglang_select/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_proglang_select_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Programming Language Selection Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py b/transforms/code/repo_level_ordering/kfp_ray/repo_level_order_wf.py
@@ -24,7 +24,7 @@
 EXEC_SCRIPT_NAME: str = "repo_level_order_transform_ray.py"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing_v2:latest"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"
@@ -138,7 +138,7 @@ def repo_level_order(
  repo_lvl_grouping_column: str = "repo_name",
  repo_lvl_store_type: str = "ray",
  repo_lvl_store_backend_dir: str = "",
- repo_lvl_store_ray_cpus: float = "0.5",
+ repo_lvl_store_ray_cpus: float = 0.5,
  repo_lvl_store_ray_nworkers: int = 1,
  repo_lvl_sorting_enabled: bool = False,
  repo_lvl_sorting_algo: str = "SORT_BY_PATH",

diff --git a/transforms/code/repo_level_ordering/ray/pyproject.toml b/transforms/code/repo_level_ordering/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_repo_level_order_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "repo_level_order Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/doc_chunk/python/pyproject.toml b/transforms/language/doc_chunk/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_chunk_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "chunk documents Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/doc_chunk/ray/pyproject.toml b/transforms/language/doc_chunk/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_chunk_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "chunk documents Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/doc_quality/python/pyproject.toml b/transforms/language/doc_quality/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_quality_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Document Quality Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/doc_quality/ray/pyproject.toml b/transforms/language/doc_quality/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_quality_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Document Quality Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/html2parquet/python/pyproject.toml b/transforms/language/html2parquet/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_html2parquet_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "HTML2PARQUET Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/lang_id/python/pyproject.toml b/transforms/language/lang_id/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_lang_id_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Language Identification Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/lang_id/ray/pyproject.toml b/transforms/language/lang_id/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_lang_id_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Language Identification Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/pdf2parquet/python/pyproject.toml b/transforms/language/pdf2parquet/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_pdf2parquet_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "PDF2PARQUET Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/pdf2parquet/ray/pyproject.toml b/transforms/language/pdf2parquet/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_pdf2parquet_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "PDF2PARQUET Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/pii_redactor/python/pyproject.toml b/transforms/language/pii_redactor/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_pii_redactor_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "PII redactor Transform for Python"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/pii_redactor/ray/pyproject.toml b/transforms/language/pii_redactor/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_pii_redactor_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "PII Redactor Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/text_encoder/python/pyproject.toml b/transforms/language/text_encoder/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_text_encoder_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Text Encoder Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/language/text_encoder/ray/pyproject.toml b/transforms/language/text_encoder/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_text_encoder_transform_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "Text Encoder Ray Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}

diff --git a/transforms/packaging/python/pyproject.toml b/transforms/packaging/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit_transforms"
 version = "0.2.2.dev0"
-requires-python = ">=3.10,<3.12"
+requires-python = ">=3.10,<3.13"
 keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Transforms"
 license = {text = "Apache-2.0"}

diff --git a/transforms/packaging/ray/pyproject.toml b/transforms/packaging/ray/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "data_prep_toolkit_transforms_ray"
 version = "0.2.2.dev0"
-requires-python = ">=3.10,<3.12"
+requires-python = ">=3.10,<3.13"
 keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Transforms using Ray"
 license = {text = "Apache-2.0"}

diff --git a/transforms/universal/doc_id/python/pyproject.toml b/transforms/universal/doc_id/python/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "dpk_doc_id_transform_python"
 version = "0.2.2.dev0"
-requires-python = ">=3.10"
+requires-python = ">=3.10,<3.13"
 description = "ededup Python Transform"
 license = {text = "Apache-2.0"}
 readme = {file = "README.md", content-type = "text/markdown"}